爬虫 120 例之第 17 例，用 Python 面向对象的思路，采集各种精彩句子

http://www.xuejuzi.cn/gaoxiaohttp://www.xuejuzi.cn/gaoxiao/2.htmlhttp://www.xuejuzi.cn/gaoxiao/3.html

import requestsfrom lxml import etreeimport random

class Spider16:    def __init__(self):
        self.wait_urls = ["http://www.xuejuzi.cn/gaoxiao/"]        self.url_template = "http://www.xuejuzi.cn/gaoxiao/{num}.html"        self.details = []
    def get_headers(self):        uas = [            "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",            "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)",            "Baiduspider-image+(+http://www.baidu.com/search/spider.htm)",            "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36",            "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",            "Mozilla/5.0 (compatible; Googlebot-Image/1.0; +http://www.google.com/bot.html)",            "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",            "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",            "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0);",            "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",            "Sosospider+(+http://help.soso.com/webspider.htm)",            "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"        ]        ua = random.choice(uas)        headers = {            "user-agent": ua,            "referer": "https://www.baidu.com"        }        return headers
    # 生成待爬取页面    def create_urls(self):        headers = self.get_headers()        page_url = self.wait_urls[0]        res = requests.get(url=page_url, headers=headers, timeout=5)        html = etree.HTML(res.text)        # 提取总页码        last_page = html.xpath("//div[@class='page']/a[last()]/@href")        if len(last_page) > 0:            last_page = int(last_page[0].split(".")[0])
        # 生成待爬取页面        for i in range(1, last_page + 1):            self.wait_urls.append(self.url_template.format(num=i))
    def get_html(self):        for url in self.wait_urls:            headers = self.get_headers()            res = requests.get(url, headers=headers, timeout=5)            if res:                html = etree.HTML(res.text)                detail_link = html.xpath("//dl/dd[1]/a/@href")                self.details.extend(detail_link)
    def get_detail(self):        for url in self.details:            headers = self.get_headers()            res = requests.get(url, headers=headers, timeout=5)            res.encoding = "gb2312"            if res:                html = etree.HTML(res.text)                sentences = html.xpath("//div[@class='content']/p/text()")                # 打印句子                long_str = "\n".join(sentences)
                with open("sentences.txt","a+",encoding="utf-8") as f:                    f.write(long_str)
    def run(self):        self.create_urls()        self.get_html()        self.get_detail()
if __name__ == '__main__':    s = Spider16()    s.run()

1，时间真的很宝贵，就差一秒厕所就被其他人抢了。2，我要给我未来婆婆一个差评，发货太慢。3，爱上你，疼死了自己。4，戒烟了，再抽真就腾云驾雾了！5，我发现这么多年我就是一个裤衩，什么屁都得接着。6，祝我生日快乐！愿我未来的媳妇找到我，我们赶紧登记结婚生孩子。