爬虫——xpath、peewee 和 Selenium

关注

发布于: 2020 年 09 月 19 日

xpath语法
谓语
选取所有带属性的值
﻿
peewee将表直接映射到class类之中
'''
char类型，设置最大长度
'''
from peewee import *
db = MySQLDatabase("spider", host="127.0.0.1", port=3306, user="root", password="199704")
class BaseModel(Model):
    class Meta:
        database = db
class Good(BaseModel):
    id = IntegerField(primary_key=True, verbose_name="商品id")
    name = CharField(max_length=500, verbose_name="商品名称")
    content = TextField(default="", verbose_name="商品描述")
    supplier = CharField(max_length=500, default="")
    ggbz = TextField(default="", verbose_name="规格和包装")
    image_list = TextField(default="", verbose_name="商品的轮播图")
    price = FloatField(default=0.0, verbose_name="商品价格")
    good_rate = IntegerField(default=0, verbose_name="好评率")
    comments_nums = IntegerField(default=0, verbose_name="评论数")
    has_image_comment_nums = IntegerField(default=0, verbose_name="晒图数")
    has_video_comment_nums = IntegerField(default=0, verbose_name="视频晒单数")
    has_add_comment_nums = IntegerField(default=0, verbose_name="追评数")
    well_comment_nums = IntegerField(default=0, verbose_name="好评数")
    middle_comment_nums = IntegerField(default=0, verbose_name="中评数")
    bad_comment_nums = IntegerField(default=0, verbose_name="差评数")
class GoodEvaluate(BaseModel):
    id = CharField(primary_key=True)
    good = ForeignKeyField(Good,verbose_name="商品")
    user_head_url = CharField(verbose_name="用户头像")
    user_name = CharField(verbose_name="用户名")
    good_info = CharField(max_length=500, verbose_name="购买的商品的信息")
    evaluate_time = DateTimeField(verbose_name="评价时间")
    content = TextField(default="", verbose_name="评论内容")
    star = IntegerField(default=0, verbose_name="评分")
    comment_nums = IntegerField(default=0, verbose_name="评论数")
    praised_nums = IntegerField(default=0, verbose_name="点赞数")
    image_list = TextField(default="", verbose_name="图片")
    video_list = TextField(default="", verbose_name="视频")
class GoodEvaluateSummary(BaseModel):
    good = ForeignKeyField(Good, verbose_name="商品")
    tag = CharField(max_length=20, verbose_name="标签")
    num = IntegerField(default=0, verbose_name="数量")
if __name__ == "__main__":
    db.create_tables([Good, GoodEvaluate, GoodEvaluateSummary])
selenium selenium 是一个用于web应用程序测试工具
selenium 可以直接操作浏览器，就像真正的用户在操作一样
selenium支持多种语言的测试，是一种工具集，可以自动化测试
﻿
import time
from selenium import webdriver
from scrapy import Selector
chrome_browser = webdriver.Chrome(executable_path="E:/installation_code/chromedrive/chromedriver.exe")
chrome_browser.get("https://item.jd.com/100004770235.html")
sel =  Selector(text=chrome_browser.page_source)
print(sel.xpath("//span[@class='price J-p-100004770235']/text()").extract_first())
# 将浏览器进行关闭
chrome_browser.close()
﻿