首页 > 代码库 > 幽灵蛛(pholcus)规则(二)

幽灵蛛(pholcus)规则(二)


此文档主要涉及到页面解析,goquery的使用方法,我觉得可以专门写篇文档。
package pholcus_lib

// 基础包
import (
    "github.com/henrylee2cn/pholcus/common/goquery"                          //DOM解析
    "github.com/henrylee2cn/pholcus/app/downloader/request" //必需
    . "github.com/henrylee2cn/pholcus/app/spider"           //必需
    // . "github.com/henrylee2cn/pholcus/app/spider/common" //选用
    // "github.com/henrylee2cn/pholcus/logs"
    // net包
     "net/http" //设置http.Header
    // "net/url"
    // 编码包
    // "encoding/xml"
    //"encoding/json"
    // 字符串处理包
    //"regexp"
    // "strconv"
        "strings"
    // 其他包
     "fmt"
    // "math"
    // "time"
)

func init() {
    FileTest1.Register()
}

var FileTest1 = &Spider{
    Name:        "meizitu图片下载",//url:http://www.meizitu.com/
    Description: "meizitu图片下载",
    // Pausetime: 300,
    // Keyin:   KEYIN,
    // Limit:        LIMIT,
    EnableCookie: false,
    RuleTree: &RuleTree{
        Root: func(ctx *Context) {
            ctx.AddQueue(&request.Request{
                Url:          "http://www.meizitu.com/",
                Rule:         "meizitu",
                ConnTimeout:  -1,
                DownloaderID: 0, //图片等多媒体文件必须使用0(surfer surf go原生下载器)
            })
            
        },

        Trunk: map[string]*Rule{
            "meizitu":{
                ParseFunc: func(ctx *Context) {
                    query := ctx.GetDom()
                    query.Find("#picture p a").Each(func(i int, s *goquery.Selection) { //其中picture是id,
                    //p是tagname a也是tagname,这里就找到了a,而且是好多个,所以用了each进行循环
                        //s.Find(‘p>a‘).
                        fmt.Println("打印一下")
                        fmt.Println(s.Html())//s对象的内部HTML代码
                        fmt.Println("-----------")
                        url1,_:=s.Attr("href")//s的href属性值
                        fmt.Println(url1)
                        fmt.Println("??????????????")
                        //t:=s.Find(‘a‘).Eq(0)
                        //fmt.Println("A的html",t.Html())
                        if href, ok := s.Attr("href"); ok {
                                ctx.AddQueue(&request.Request{
                                    Url:    href,
                                    Header: http.Header{"Content-Type": []string{"text/html; charset=gbk"}},
                                    Rule:   "图片URL",
                                })
                            }
                        })
                },
            },

            "图片URL": {
                ParseFunc: func(ctx *Context) {
                    query := ctx.GetDom()
                    query.Find("#picture p img").Each(func(i int, s *goquery.Selection) {//同上
                        //s.Find(‘p>a‘).
                        fmt.Println("二级页面开始!打印一下????????????????????????????????????")
                        fmt.Println(s.Html())
                        fmt.Println("-----------")
                        url1,_:=s.Attr("src")
                        fmt.Println(url1)
                        fmt.Println("??????????????")
                        fmt.Println("二级页面结束!")
                        //t:=s.Find(‘a‘).Eq(0)
                        //fmt.Println("A的html",t.Html())
                        if href, ok := s.Attr("src"); ok {
                                ctx.AddQueue(&request.Request{
                                    Url:    href,
                                    Header: http.Header{"Content-Type": []string{"text/html; charset=gbk"}},
                                    Rule:   "图片下载",
                                })
                            }
                        
                        })
                },
            },
            "图片下载": {
                ParseFunc: func(ctx *Context) {
                    fmt.Println("图片链接URL:",ctx.GetUrl())
                    picurl:=ctx.GetUrl()//获取链接值,用于为图片命名,这里可能还会涉及新建文件夹,所以后面还需要想想怎么传参数?????
                    picname:=strings.Replace(picurl,"http://mm.howkuai.com/wp-content/uploads/","",-1)//替换值
                    picname=strings.Replace(picname,"/","-",-1)//替换值
                    ctx.FileOutput(picname) // 等价于ctx.AddFile("baidu")
                },
            },
            
        },
    },
}

幽灵蛛(pholcus)规则(二)