首页 > 代码库 > node小爬虫

node小爬虫

这一章主利用node的http模块制作一个网页的小爬虫来爬去网页信息,其中对于后端html的节点的获取采用了cheerio模块,这

/** * Created by Administrator on 2016/9/16. */var http = require(‘http‘);var cheerio = require(‘cheerio‘);var url = ‘http://www.imooc.com/learn/348‘;function filterChapters(html){      var $ = cheerio.load(html);// 要使用cheerio模块先要用npm install cheerio加载进来,然后再前面引入(var cheerio = require(‘cheerio‘);)      var chapters = $(‘.chapter‘);    var courseData =http://www.mamicode.com/ [];    chapters.each(function(item){        var chapter = $(this);        var chapterTitle = chapter.find(‘strong‘).text();        var videos = chapter.find(‘.video‘).children(‘li‘);        var chapterData =http://www.mamicode.com/ {            chapterTitle:chapterTitle,            videos:[]        }        videos.each(function(item){            var video = $(this).find(‘.J-media-item‘);            var videoTitle = video.text();            var id = video.attr(‘href‘).split(‘video/‘)[1];            chapterData.videos.push({                title:videoTitle,                id:id,            })        })        courseData.push(chapterData);    })    return courseData;}function printCourseInfo(courseData){    courseData.forEach(function(item){         var chapterTitle = item.chapterTitle;        console.log(chapterTitle );        item.videos.forEach(function(video){            console.log(video.id)            //console.log(‘ 【‘+ video.id + ‘】 ‘+ video.title + ‘\n‘);        })    })}http.get(url,function(res){    var html = ‘‘;    res.on(‘data‘,function(data){ res会监听data事件的发生        html += data;    });    res.on(‘end‘,function(){        var courseData =http://www.mamicode.com/ filterChapters(html);        printCourseInfo(courseData);    })}).on(‘error‘,function(){    console.log(‘获取课程出错!‘)})

 

个模块可以在后端获取html页面的元素

,获取方法类似于jquery

代码如下

node小爬虫