怎樣實(shí)現(xiàn)百度指數(shù)爬蟲功能

npm install --save puppeteer

npm config set PUPPETEER_DOWNLOAD_HOST=https://npm.taobao.org/mirrors
npm install --save puppeteer

// npm
npm install --save puppeteer --ignore-scripts
// node
puppeteer.launch({ executablePath: '/path/to/Chrome' });

// 啟動瀏覽器，
// headless參數(shù)如果設(shè)置為true，Puppeteer將在后臺操作你Chromium，換言之你將看不到瀏覽器的操作過程
// 設(shè)為false則相反，會在你電腦上打開瀏覽器，顯示瀏覽器每一操作。
const browser = await puppeteer.launch({headless:false});
const page = await browser.newPage();
// 打開百度指數(shù)
await page.goto(BAIDU_INDEX_URL);
// 模擬登陸
await page.click('...');
await page.waitForSelecto('...');
// 輸入百度賬號密碼然后登錄
await page.type('...','username');
await page.type('...','password');
await page.click('...');
await page.waitForNavigation();
console.log(':white_check_mark: 登錄成功');

// 獲取chart第一天的坐標(biāo)
const position = await page.evaluate(() => {
 const $image = document.querySelector('...');
 const $area = document.querySelector('...');
 const areaRect = $area.getBoundingClientRect();
 const imageRect = $image.getBoundingClientRect();
 // 滾動到圖表可視化區(qū)域
 window.scrollBy(0, areaRect.top);
 return { x: imageRect.x, y: 200 }；
});
// 移動鼠標(biāo)，觸發(fā)tooltip
await page.mouse.move(position.x, position.y);
await page.waitForSelector('...');
// 獲取tooltip信息
const tooltipInfo = await page.evaluate(() => {
 const $tooltip = document.querySelector('...');
 const $title = $tooltip.querySelector('...');
 const $value = $tooltip.querySelector('...');
 const valueRect = $value.getBoundingClientRect();
 const padding = 5;
 return {
 title: $title.textContent.split(' ')[0],
 x: valueRect.x - padding,
 y: valueRect.y,
 width: valueRect.width + padding * 2,
 height: valueRect.height
 }
});

await page.screenshot({ path: imgPath });
// 對圖片進(jìn)行裁剪，只保留數(shù)字部分
const img = await jimp.read(imgPath);
await img.crop(tooltipInfo.x, tooltipInfo.y, tooltipInfo.width, tooltipInfo.height);
// 將圖片放大一些，識別準(zhǔn)確率會有提升
await img.scale(5);
await img.write(imgPath);

Tesseract.process(imgPath, (err, val) => {
if (err || val == null) {
 console.error(':x: 識別失敗：' + imgPath);
 return;
}
console.log(val);

const recognition = require('./src/recognition');
const Spider = require('./src/spider');
module.exports = {
 async run (word, options, puppeteerOptions = { headless: true }) {
 const spider = new Spider({ 
 imgDir, 
 ...options 
 }, puppeteerOptions);
 // 抓取數(shù)據(jù)
 await spider.run(word);
 // 讀取抓取到的截圖，做圖像識別
 const wordDir = path.resolve(imgDir, word);
 const imgNames = fs.readdirSync(wordDir);
 const result = [];
 imgNames = imgNames.filter(item => path.extname(item) === '.png');
 for (let i = 0; i < imgNames.length; i++) {
 const imgPath = path.resolve(wordDir, imgNames[i]);
 const val = await recognition.run(imgPath);
 result.push(val);
 }
 return result;
 }
}