var moment = require(‘moment‘);exports.inc = function(n, callback, timeout) {    timeout = timeout || 200;    setTimeout(function() {        callback(null, n+1);    }, timeout);};exports.fire = function(obj, callback, timeout) {    timeout = timeout || 200;    setTimeout(function() {        callback(null, obj);    }, timeout);};exports.err = function(errMsg, callback, timeout) {    timeout = timeout || 200;    setTimeout(function() {        callback(errMsg);    }, timeout);};// utilsexports.log = function(msg, obj) {    process.stdout.write(moment().format(‘ss.SSS‘)+‘> ‘);    if(obj!==undefined) {        process.stdout.write(msg);        console.log(obj);    } else {        console.log(msg);    }};exports.wait = function(mils) {    var now = new Date;    while(new Date - now <= mils);}

//获得页面数据var http = require("http");//分析页面数据，提前内容var cheerio = require("cheerio");//过滤没用的数据 如空格等var sanitize = require("validator");//操作文件，保存结果var fs = require(‘fs‘);var scrapy = {};scrapy.get = function(url, callback) {  http.get(url, function(res) {    var size = 0;    var chunks = [];    res.on(‘data‘, function(chunk) {      size += chunk.length;      chunks.push(chunk);    });    res.on(‘end‘, function() {      var data =http://www.mamicode.com/ Buffer.concat(chunks, size);      callback(null, data);    });  }).on(‘error‘, function(e) {    callback(e, null);  });}var getPage = function(pageUrl, callback){  scrapy.get(pageUrl, function(err, data){    if(err){      callback(err);    }    var html = data.toString();    $ = cheerio.load(html);      //title link, link to detail page      var news = $(‘.cell .topic_title_wrapper a‘);         callback(null, news);    });}var getDetail = function(detailUrl, callback){    scrapy.get(detailUrl, function(err, data){    if(err){      callback(err);    }    var html = data.toString();    $ = cheerio.load(html);      var item = {};    item.href = detailUrl;    $(‘.header .topic_full_title .put_top‘).remove(); //删除 “置顶”    item.title = sanitize.escape(sanitize.trim($(‘.header .topic_full_title‘).text()));    item.content = sanitize.escape(sanitize.trim($(‘.inner.topic .topic_content‘).text()));    callback(null, item);  });}var save = function(fileName, data) {  var result = JSON.stringify(data);  fs.writeFileSync(fileName, result);}exports.getUrl = scrapy.get;exports.getPage = getPage;exports.getDetail = getDetail;exports.save = save;

//自定义console.log 加入了输出时间var utils = require("./utils");var log = utils.log;//异步操作 如each， filtervar async = require("async");//计算花费时间var ts = require("timespans")//过滤没用的数据 如空格等var sanitize = require("validator");var url = require("./module_url")var baseUrl = ‘http://cnodejs.org‘;var pageUrl = baseUrl + ‘/?page=‘;var isOnlyTitle = true;var pages = [];for (var i = 1; i < 4; i++) {    pages.push(i);};ts.start();var titles = {};//page 之间并行async.forEach(pages, function(page, callback_each){    titles[page] = [];    url.getPage(pageUrl + page, function(err, news){        if(err){            log("page error");            return;        }        if (news.length === 0) {              log("no data for the page:" + page);              return;        }                async.filter(news, function(index, callback){            var detailUrl = baseUrl + news[index].attribs[‘href‘];            if(isOnlyTitle){                var curNew = news[index];                var item = {};                item.href = detailUrl;                        $(curNew).find(".put_top").remove();    //删除 “置顶”                                item.title = sanitize.escape(sanitize.trim($(curNew).text()));                titles[page][index] = item;                callback(true);            }            else{                url.getDetail(detailUrl, function(err, item){                    if(err){                        log("detail error");                        return;                    }                    titles[page][index] = item;                    //titles[page].push(item);                    callback(true);                });            }        }, function(result){            //log("filter news:", result);            callback_each(null);        });            });}, function(err){    ts.stop();    //ts.pause();    ---   ts.continue();    console.log(‘total: %s pause: %s used: %s‘, ts.elapsedtime(), ts.pausetime(), ts.usedtime());    log(titles);    //url.save("cnodejs.json", titles);});