首页 > 代码库 > scrapy爬虫之爬取汽车信息
scrapy爬虫之爬取汽车信息
scrapy爬虫还是很简单的，主要是三部分：spider，item，pipeline
其中后面两个也是通用套路，需要详细解析的也就是spider。
具体如下：
在网上找了几个汽车网站，后来敲定，以易车网作为爬取站点
原因在于，其数据源实在是太方便了。
看这个页面，左边按照品牌排序，搜索子品牌，再挨个查看信息即可
按照通常的思路，是需要手动解析左边这列表
找出每个品牌的链接页面
结果分析源码发现，网站直接通过js生成的导航栏，直接通过这个链接生成的json即可获得所有的信息
http://api.car.bitauto.com/CarInfo/getlefttreejson.ashx?tagtype=baojia&pagetype=masterbrand&objid=2
直接解析其中需要的数据即可
如下图
可以用json解析，我没尝试，我采用最简单的正则匹配提取
代码很简单
json_str = """
上面全部的数据
JsonpCallBack({char:{A:1,B:1,C:1,D:1,E:0,F:1,G:1,H:1,I:0,J:1,K:1,L:1,M:1,
N:1,O:1,P:1,Q:1,R:1,S:1,T:1,U:0,V:0,W:1,X:1,Y:1,Z:1}
,brand:{A:[{type:"mb",id:9,name:"奥迪",url:"/mb9/",cur:0,num:95546}
~~~~~~~~~~太长 剩余的代码中我省略了
"""
import re
result = re.findall(r‘\/mb\d+\/‘, json_str)
print result
#mb_pages = [‘/mb9/‘, ‘/mb97/‘, ] #192条
所以，品牌页的代码：
def parse(self, response):
page_root = ‘price.bitauto.com‘ #response.url.split(‘/‘)[2] #‘price.bitauto.com‘
mb_pages = [‘/mb2/‘, ‘/mb3/‘,‘/mb9/‘,
] #/mb9/audi, /mb2/benz, /mb3/bmw
for info in mb_pages:
page_href = http://www.mamicode.com/info>
品牌页面下面的子品牌
元素定位，爬取，代码如下
def parse_brand_page(self,response):
#命令行测试 scrapy shell http://price.bitauto.com/mb196/
page_xpath = ‘//div[@id="c_result"]/div[@class="carpic_list"]/ul/li‘
page_root = ‘price.bitauto.com‘ #response.url.split(‘/‘)[2] #‘price.bitauto.com‘
treeurl = response.meta[‘treeurl‘]
brand = response.xpath(‘//div[@class="tree_navigate"]/div/strong/text()‘).extract()[0]
for info in response.xpath(page_xpath):
page_href = http://www.mamicode.com/info.xpath(‘a/attribute::href‘).extract()[0]>
到款型详情页，然而，需要的是参数页，，继续request跳转
def parse_car_page(self,response):
peizhi_xpath = ‘//a[@id="linkOutCs"]/@href‘
page_url = response.xpath(peizhi_xpath).extract()[0]
treeurl = response.meta[‘treeurl‘]
brand = response.meta[‘brand‘]
yield scrapy.Request(url=page_url, meta={‘treeurl‘: treeurl, ‘brand‘: brand}, callback=self.parse_detail_page)
最后到参数页，这才是我们需要的数据啊
分析源代码发现，所有的数据依旧是在js代码以字符串形式存在的，这种代码，正是正则的用武之地啊
数据字符串是三个[[[]]]嵌套的字符串，处理方式为
ff = re.search(‘\[\[\[.*\]\]\]‘,response.body).group() #str
infos = eval(ff)
ff获得的是整个[[[]]]的内容，然后用eval转化成python的值，然后再用循环取对应位置的数据即可。代码如下
def parse_detail_page(self,response):
#命令行测试 scrapy shell http://car.bitauto.com/changchengh5/peizhi/
ff = re.search(‘\[\[\[.*\]\]\]‘,response.body).group() #str
infos = eval(ff)
for s_second in infos:
item = BitautoCarItem()
item[‘carid‘] =s_second[0][0] #"117388"
item[‘url‘] = response.url
item[‘brand‘] = response.meta[‘brand‘] ###
item[‘treeurl‘] = response.meta[‘treeurl‘] ###
item[‘brandurl‘] = s_second[0][6] ##changchengh5,benchieji
item[‘brandmodel4‘] = s_second[0][4] #"哈弗H5" "奔驰E级"
item[‘brandmodel5‘] = s_second[0][5] ###
item[‘version‘] = s_second[0][1] #"经典版 2.0T 手动 两驱 精英型",
item[‘image‘] = s_second[0][2]
item[‘cyear‘] = s_second[0][7]
item[‘ctype‘] = s_second[0][12] #"SUV"
item[‘color‘] = s_second[0][13]
item[‘price1‘] = s_second[1][0] # 厂家指导价
item[‘price2‘] = s_second[1][1] # 商家报价
item[‘displacement‘] = s_second[1][5] #"2.0", 排量(L)
item[‘shiftgears‘] = s_second[1][6] # "6"
item[‘shifttype‘] = s_second[1][7] # "手动"
item[‘clength‘] = s_second[2][0] # 长宽高，为了清楚表示，加了前缀c
item[‘cwidth‘] = s_second[2][1] # 长宽高，为了清楚表示，加了前缀c
item[‘cheight‘] = s_second[2][2] # 长宽高，为了清楚表示，加了前缀c
item[‘wheelbase‘] = s_second[2][3] #轴距
item[‘mingrounddistance‘] = s_second[2][8] #最小离地间隙
item[‘motor‘] = s_second[3][1] # 发动机型号
item[‘intaketype‘] = s_second[3][5] # 进气形式
item[‘maxhorsepower‘] = s_second[3][13] # 最大马力(Ps)
item[‘maxpower‘] = s_second[3][14] # 最大功率(kW)
item[‘maxrpm‘] = s_second[3][15] # 最大功率转速(rpm)
item[‘oiltype‘] = s_second[3][19] # 燃料类型
item[‘oilsupply‘] = s_second[3][21] # 供油方式
item[‘tankvolume‘] = s_second[3][22] # 燃油箱容积(L)
item[‘drivetype‘] = s_second[5][6] # 驱动方式
item[‘braketype‘] = s_second[5][5] # 驻车制动类型
item[‘frontwheel‘] = s_second[7][0] # 前轮
item[‘backwheel‘] = s_second[7][1] # 后轮
yield item
以上，整个爬取代码，为：
#!/usr/bin/env python
# coding=utf-8
import scrapy
import re
from Car_spider.items import BitautoCarItem
class BitautoSpider(scrapy.Spider):
name = ‘bitauto‘
allowed_domains = [‘bitauto.com‘]
start_urls = [‘http://price.bitauto.com/mb2/‘,]
def parse(self, response):
page_root = ‘price.bitauto.com‘ #response.url.split(‘/‘)[2] #‘price.bitauto.com‘
mb_pages = [‘/mb2/‘, ‘/mb3/‘,‘/mb9/‘,
] #/mb9/audi, /mb2/benz, /mb3/bmw
for info in mb_pages:
page_href = http://www.mamicode.com/info"c_result"]/div[@class="carpic_list"]/ul/li‘
page_root = ‘price.bitauto.com‘ #response.url.split(‘/‘)[2] #‘price.bitauto.com‘
treeurl = response.meta[‘treeurl‘]
brand = response.xpath(‘//div[@class="tree_navigate"]/div/strong/text()‘).extract()[0]
for info in response.xpath(page_xpath):
page_href = http://www.mamicode.com/info.xpath(‘a/attribute::href‘).extract()[0]"linkOutCs"]/@href‘
page_url = response.xpath(peizhi_xpath).extract()[0]
treeurl = response.meta[‘treeurl‘]
brand = response.meta[‘brand‘]
yield scrapy.Request(url=page_url, meta={‘treeurl‘: treeurl, ‘brand‘: brand}, callback=self.parse_detail_page)
def parse_detail_page(self,response):
#命令行测试 scrapy shell http://car.bitauto.com/changchengh5/peizhi/
ff = re.search(‘\[\[\[.*\]\]\]‘,response.body).group() #str
infos = eval(ff)
for s_second in infos:
item = BitautoCarItem()
item[‘carid‘] =s_second[0][0] #"117388"
item[‘url‘] = response.url
item[‘brand‘] = response.meta[‘brand‘] ###
item[‘treeurl‘] = response.meta[‘treeurl‘] ###
item[‘brandurl‘] = s_second[0][6] ##changchengh5,benchieji
item[‘brandmodel4‘] = s_second[0][4] #"哈弗H5" "奔驰E级"
item[‘brandmodel5‘] = s_second[0][5] ###
item[‘version‘] = s_second[0][1] #"经典版 2.0T 手动 两驱 精英型",
item[‘image‘] = s_second[0][2]
item[‘cyear‘] = s_second[0][7]
item[‘ctype‘] = s_second[0][12] #"SUV"
item[‘color‘] = s_second[0][13]
item[‘price1‘] = s_second[1][0] # 厂家指导价
item[‘price2‘] = s_second[1][1] # 商家报价
item[‘displacement‘] = s_second[1][5] #"2.0", 排量(L)
item[‘shiftgears‘] = s_second[1][6] # "6"
item[‘shifttype‘] = s_second[1][7] # "手动"
item[‘clength‘] = s_second[2][0] # 长宽高，为了清楚表示，加了前缀c
item[‘cwidth‘] = s_second[2][1] # 长宽高，为了清楚表示，加了前缀c
item[‘cheight‘] = s_second[2][2] # 长宽高，为了清楚表示，加了前缀c
item[‘wheelbase‘] = s_second[2][3] #轴距
item[‘mingrounddistance‘] = s_second[2][8] #最小离地间隙
item[‘motor‘] = s_second[3][1] # 发动机型号
item[‘intaketype‘] = s_second[3][5] # 进气形式
item[‘maxhorsepower‘] = s_second[3][13] # 最大马力(Ps)
item[‘maxpower‘] = s_second[3][14] # 最大功率(kW)
item[‘maxrpm‘] = s_second[3][15] # 最大功率转速(rpm)
item[‘oiltype‘] = s_second[3][19] # 燃料类型
item[‘oilsupply‘] = s_second[3][21] # 供油方式
item[‘tankvolume‘] = s_second[3][22] # 燃油箱容积(L)
item[‘drivetype‘] = s_second[5][6] # 驱动方式
item[‘braketype‘] = s_second[5][5] # 驻车制动类型
item[‘frontwheel‘] = s_second[7][0] # 前轮
item[‘backwheel‘] = s_second[7][1] # 后轮
yield item
前面只是从页面层次去parse，没叙述item，因为这个也简单，没啥需要叙述的。其定义代码为
class BitautoCarItem(scrapy.Item):
carid = scrapy.Field()
url = scrapy.Field()
treeurl = scrapy.Field()
brand = scrapy.Field() ###
brandurl = scrapy.Field() ###
brandmodel4 = scrapy.Field() #"哈弗H5"
brandmodel5 = scrapy.Field() #"哈弗H5"
version = scrapy.Field() #"经典版 2.0T 手动 两驱 精英型",
image = scrapy.Field()
cyear = scrapy.Field()
ctype = scrapy.Field() #"SUV"
color = scrapy.Field()
price1 = scrapy.Field() # 厂家指导价
price2 = scrapy.Field() # 商家报价
displacement = scrapy.Field() # "2.0", 排量(L)
shiftgears = scrapy.Field() # "6"
shifttype = scrapy.Field() # "手动"
clength = scrapy.Field() # 长宽高，为了清楚表示，加了前缀c
cwidth = scrapy.Field() # 长宽高，为了清楚表示，加了前缀c
cheight = scrapy.Field() # 长宽高，为了清楚表示，加了前缀c
wheelbase = scrapy.Field() #轴距
mingrounddistance = scrapy.Field() #最小离地间隙
motor = scrapy.Field() # 发动机型号
intaketype = scrapy.Field() # 进气形式
maxhorsepower = scrapy.Field() # 最大马力(Ps)
maxpower = scrapy.Field() # 最大功率(kW)
maxrpm = scrapy.Field() # 最大功率转速(rpm)
oiltype = scrapy.Field() # 燃料类型
oilsupply = scrapy.Field() # 供油方式
tankvolume = scrapy.Field() # 燃油箱容积(L)
drivetype = scrapy.Field() # 驱动方式
braketype = scrapy.Field() # 驻车制动类型
frontwheel = scrapy.Field() # 前轮胎规格
backwheel = scrapy.Field() # 后轮胎规格
至于Pipeline，随意写即可，也是套路而已，mongodb的pipeline如下
class MongoDBPipeline(object):
def __init__(self):
connection = MongoClient(
settings[‘MONGODB_SERVER‘],
settings[‘MONGODB_PORT‘]
)
db=connection[settings[‘MONGODB_DB‘]]
self.collection = db[settings[‘MONGODB_COLLECTION‘]]
def process_item(self, item, spider):
self.collection.insert(dict(item))
也可以是关系型数据库，如postgresql
class CarsPgPipeline(object):
def __init__(self):
#reload(sys)
#sys.setdefaultencoding(‘utf-8‘)
self.connection = psycopg2.connect(
database= settings[‘POSTGRES_DB‘],
user= settings[‘POSTGRES_USER‘],
password= settings[‘POSTGRES_PW‘],
host= settings[‘POSTGRES_SERVER‘],
port= settings[‘POSTGRES_PORT‘],
)
self.cursor = self.connection.cursor()
def process_item(self,item,spider):
if instance(item, BitautoCarItem):
_sql = """INSERT INTO BitautoCar(carid,url,treeurl,brand,brandurl,brandmodel4,brandmodel5,version,image,cyear,ctype,color,price1,price2,displacement,shiftgears,shifttype,clength,cwidth,cheight,wheelbase,mingrounddistance,motor,intaketype,maxhorsepower,maxpower,maxrpm,oiltype,oilsupply,tankvolume,drivetype,braketype,frontwheel,backwheel) VALUES (‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘,‘%s‘);"""%( item[‘carid‘],item[‘url‘],item[‘treeurl‘],item[‘brand‘],item[‘brandurl‘],item[‘brandmodel4‘],item[‘brandmodel5‘],item[‘version‘],item[‘image‘],item[‘cyear‘],item[‘ctype‘],item[‘color‘],item[‘price1‘],item[‘price2‘],item[‘displacement‘],item[‘shiftgears‘],item[‘shifttype‘],item[‘clength‘],item[‘cwidth‘],item[‘cheight‘],item[‘wheelbase‘],item[‘mingrounddistance‘],item[‘motor‘],item[‘intaketype‘],item[‘maxhorsepower‘],item[‘maxpower‘],item[‘maxrpm‘],item[‘oiltype‘],item[‘oilsupply‘],item[‘tankvolume‘],item[‘drivetype‘],item[‘braketype‘],item[‘frontwheel‘],item[‘backwheel‘])
try:
self.cursor.execute(self.cursor.mogrify(_sql) )
self.connection.commit()
except Exception, e:
self.connection.rollback()
print "Error: %s" % e
return item
项目setting部分
BOT_NAME = ‘Car_spider‘
SPIDER_MODULES = [‘Car_spider.spiders‘]
NEWSPIDER_MODULE = ‘Car_spider.spiders‘
ITEM_PIPELINES = {
‘Car_spider.pipelines.CarsPgPipeline‘ : 1000,
}
MONGODB_SERVER = ‘localhost‘
MONGODB_PORT = 27017
MONGODB_DB = ‘car‘
MONGODB_COLLECTION = ‘kache360‘ #‘bitantotest‘
POSTGRES_SERVER = ‘localhost‘
POSTGRES_PORT = 5432
POSTGRES_DB = ‘yourdb‘
POSTGRES_USER = ‘yourname‘
POSTGRES_PW = ‘123456‘
ROBOTSTXT_OBEY = True
DOWNLOAD_DELAY = 3
RANDOMIZE_DOWNLOAD_DELAY = True
scrapy爬虫之爬取汽车信息
声明：以上内容来自用户投稿及互联网公开渠道收集整理发布，本网站不拥有所有权，未作人工编辑处理，也不承担相关法律责任，若内容有误或涉及侵权可进行投诉： 投诉/举报 工作人员会在5个工作日内联系你，一经查实，本站将立刻删除涉嫌侵权内容。