【原创】Python 二手车之家车辆档案数据爬虫

2022-05-13 00:48 作者:拉灯的小手 0人读过 | 我要投稿

本文仅供学习交流使用，如侵立删！

二手车之家车辆档案数据爬虫

先上效果图

环境

win10
python3.9
lxml、retrying、requests

需求分析

需求：主要是需要车辆详情页中车辆档案的数据

先抓包分析一波，网页抓包没有什么有用的，转战APP

拿到数据接口就简单了，直接构造请求保存数据即可

获取车辆信息列表

def _get_car_list(self, _url: str):
"""
获取二手车信息列表
"""
res = self._parse_url(_url=_url)
ret = res.text # 解析获得字符串类型数据
result = etree.HTML(ret) # 转换数据类型为HTML,方便使用xpath
url_list = result.xpath('//*[@id="goodStartSolrQuotePriceCore0"]/ul/li/a/@href')
if not url_list:
print('获取完成！')
return
for i in url_list:
# 有些车型url直接是带域名的
if 'www.che168.com/' in i:
yield 'https://' + i[2:]
else:
yield 'https://www.che168.com' + i

获取车辆详情信息

def _get_car_info(self, _url: str):
"""
获取车辆详情信息
"""
res = self._parse_url(_url=_url)
ret = res.text # 解析获得字符串类型数据
result = etree.HTML(ret) # 转换数据类型为HTML,方便使用xpath

# 标题
title = result.xpath('//div[@class="car-box"]/h3//text()')
title = title[1].strip() if len(title) > 1 else title[0].strip()
# 上牌时间
play_time = result.xpath('//*[@id="nAV1"]/div[1]/ul[1]/li[1]/text()')
play_time = play_time[0].strip() if play_time else '-'
# 表显里程
display_mileage = result.xpath('//*[@id="nAV1"]/div[1]/ul[1]/li[2]/text()')
display_mileage = display_mileage[0].strip() if display_mileage else '-'
# 变速箱
gearbox = result.xpath('//*[@id="nAV1"]/div[1]/ul[1]/li[3]/text()')
gearbox = gearbox[0].strip() if gearbox else '-'
# 排放标准
emission_standards = result.xpath('//*[@id="nAV1"]/div[1]/ul[1]/li[4]/text()')
emission_standards = emission_standards[0].strip() if emission_standards else '-'
# 排量
displacement = result.xpath('//*[@id="nAV1"]/div[1]/ul[1]/li[5]/text()')
displacement = displacement[0].strip() if displacement else '-'
# 发布时间
release_time = result.xpath('//*[@id="nAV1"]/div[1]/ul[1]/li[6]/text()')
release_time = release_time[0].strip() if release_time else '-'
# 年检到期
annual_inspection_expires = result.xpath('//*[@id="nAV1"]/div[1]/ul[2]/li[1]/text()')
annual_inspection_expires = annual_inspection_expires[0].strip() if annual_inspection_expires else '-'
# 保险到期
insurance_expires = result.xpath('//*[@id="nAV1"]/div[1]/ul[2]/li[2]/text()')
insurance_expires = insurance_expires[0].strip() if insurance_expires else '-'
# 质保到期
warranty_expires = result.xpath('//*[@id="nAV1"]/div[1]/ul[2]/li[3]/text()')
warranty_expires = warranty_expires[0].strip() if warranty_expires else '-'
# 过户次数
number_of_transfers = result.xpath('//*[@id="nAV1"]/div[1]/ul[2]/li[5]/text()')
number_of_transfers = number_of_transfers[0].strip() if number_of_transfers else '-'
# 所在地
location = result.xpath('//*[@id="nAV1"]/div[1]/ul[2]/li[6]/text()')
location = location[0].strip() if location else '-'
# 发动机
engine = result.xpath('//*[@id="nAV1"]/div[1]/ul[3]/li[1]/text()')
engine = engine[0].strip() if engine else '-'
# 车辆级别
vehicle = result.xpath('//*[@id="nAV1"]/div[1]/ul[3]/li[2]/text()')
vehicle = vehicle[0].strip() if vehicle else '-'
# 车身颜色
car_color = result.xpath('//*[@id="nAV1"]/div[1]/ul[3]/li[3]/text()')
car_color = car_color[0].strip() if car_color else '-'
# 燃油标号
fuel_label = result.xpath('//*[@id="nAV1"]/div[1]/ul[3]/li[4]/text()')
fuel_label = fuel_label[0].strip() if fuel_label else '-'
# 驱动方式
drive_mode = result.xpath('//*[@id="nAV1"]/div[1]/ul[3]/li[5]/text()')
drive_mode = drive_mode[0].strip() if drive_mode else '-'
data = [[title, play_time, display_mileage, gearbox, emission_standards, displacement, release_time, annual_inspection_expires,
insurance_expires, warranty_expires, number_of_transfers, location, engine, vehicle, car_color, fuel_label, drive_mode, _url]]
print(data)
self._save_csv(data=data)