首页 > 代码库 > NODE学习:利用nodeJS去抓网页的信息
NODE学习:利用nodeJS去抓网页的信息
1:引用模块"http" (执行命令node app.js "http://www.baidu.com")
//app.js
var http = require(‘http‘);var url = require(‘url‘);function spider(u,cb){ http.get( url.parse(u), function(res){ var d = ‘‘ res.on(‘data‘,function(chunk){ d += chunk; }) res.on(‘end‘,function(){ console.log(‘spider_end && do cb‘); cb(d); }) });};var u = "";if( require.main === module ) { u = process.argv[2]};spider(u,function( data ){
//这个返回的是网页内容的信息; console.log( data );});
2:引用模块nodegrass: (执行命令node app.js "http://www.baidu.com")
//app.js
var url = "http://www.cnblogs.com/xiaochao12345/archive/2014/10/23/4044950.html";var ng = require(‘nodegrass‘);if(require.main === module) { console.log( process.argv ); url = process.argv[2];}ng.get(url,function(data){
//这个返回的是网页内容的信息;
console.log(data); },‘utf8‘);
3:引用模块superagent: (执行命令node app.js "http://www.baidu.com")
//app.js
var url = "http://www.cnblogs.com/xiaochao12345/archive/2014/10/23/4044950.html";var superagent = require("superagent");if(require.main === module) { console.log( process.argv ); url = process.argv[2];};superagent.get(url) .end(function (err, res) { console.log(‘fetch successful‘); console.log(res); });
4:使用curl模块:(执行命令node app.js "http://www.baidu.com")
//app.js
var curl = require("curl");var u = "";if( require.main === module ) { u = process.argv[2]};curl.get(u,function( data ){ console.log( arguments );});
______________________________________________________________________________________________________________
NODEJS中处理dom节点的方式(接口都和jq是统一的);
1:引用cheerio
var cheerio = require("cheerio");var html = "<html><body><div id=\"div1\">text</div></body></html>";var $ = cheerio.load(html);console.log( "html" );console.log( $.html() );console.log( "#div1————〉html" );console.log( $("#div1").html() );console.log( "#div1----〉text" );console.log( $("#div1").text() );
2:引用jquery
var $ = require("jquery");var $dom = $("<html><body><div id=\"div1\">text</div></body></html>");console.log( $dom.find("#div1").text() );
3:引用jsdom
var jsdom = require(‘jsdom‘);var curl = require("curl");var u = "https://github.com";if( require.main === module ) { u = process.argv[2];};curl.get(u,function( arg0 , html ){ //jsdom相当于打开了一个页面,在这个页面里面运行js; var document = jsdom.jsdom( html.body ); for(var a in html) console.log(a); var window = document.createWindow(); var script = document.createElement(‘script‘); //script.src = http://www.mamicode.com/‘http://code.jquery.com/jquery-1.4.2.js‘;"http://127.0.0.1:81/js/jquery.min.js"; script.onload = function(){ console.log(1); console.log( window.jQuery("body").text() ); }; document.head.appendChild( script );});
_________________________________________________________________________________________________________________
文件保存的话直接用nodeJS中的fs模块:
var fs = require("fs");fs.appendFile(‘file-name‘, "text_text_text_text" ,function(err){ if(err)throw err; console.log(‘done‘)});
NODE学习:利用nodeJS去抓网页的信息
声明:以上内容来自用户投稿及互联网公开渠道收集整理发布,本网站不拥有所有权,未作人工编辑处理,也不承担相关法律责任,若内容有误或涉及侵权可进行投诉: 投诉/举报 工作人员会在5个工作日内联系你,一经查实,本站将立刻删除涉嫌侵权内容。