通过抓取页面生成 RSS

 1    const axios = require('axios');
 2    const cheerio = require('cheerio');
 3    const fs = require('fs');
 4
 5    // for blogs.json
 6    const url = 'https://www.yidajiabei.xyz/blog/index.html';
 7
 8    async function scrapeData() {
 9      try {
10        // Fetch HTML of the page we want to scrape
11        const { data } = await axios.get(url);
12        // Load HTML we fetched in the previous line
13        const $ = cheerio.load(data);
14        // Select all the list items in plainlist class
15        const listItems = $('.org-ul li');
16        // Stores data for all countries
17        const countries = [];
18        // Use .each method to loop through the li we selected
19        listItems.each((idx, el) => {
20          // Object holding data for each country/jurisdiction
21          const country = { title: '', link: '', date: '', desp: '' }; // , link: ""
22          // Select the text content of a and span elements
23          // Store the textcontent in the above object
24          country.title = $(el).children('a').text();
25          country.link = $(el).children('a').attr('href');
26          country.date = $(el)
27            .text()
28            .replace(/[^0-9][^\s]+/g, '')
29            .split('-')
30            .join('')
31            .match(/.{1,2}/g)
32            .join('-');
33          if (country.link.split('/')[0] === '2022') {
34            country.date = '2022-' + country.date;
35          } else if (country.link.split('/')[0] === '2021') {
36            country.date = '2021-' + country.date;
37          } else if (country.link.split('/')[0] === '2020') {
38            country.date = '2020-' + country.date;
39          } else if (country.link.split('/')[0] === '2019') {
40            country.date = '2019-' + country.date;
41          } else if (country.link.split('/')[0] === '2018') {
42            country.date = '2018-' + country.date;
43          }
44          // console.log(country.title);
45          // Populate countries array with country data
46          country.desp = fs
47            .readFileSync('/home/archie/repo/blog/blog/' + country.link)
48            .toString()
49            .match(/<body[^>]*>([\w|\W]*)<\/body>/im)[1]
50            .replaceAll('<', '&lt;')
51            .replaceAll('>', '&gt;')
52            .trim();
53          countries.push(country);
54        });
55        // Logs countries array to the console
56        // console.dir(countries);
57        // Write countries array in countries.json file
58        fs.writeFile('blogs.json', JSON.stringify(countries, null, 2), (err) => {
59          if (err) {
60            console.error(err);
61            return;
62          }
63          console.log('Successfully written data to blogs.json');
64        });
65      } catch (err) {
66        console.error(err);
67      }
68    }
69    // Invoke the above function
70    scrapeData();