【Python 技能树共建】requests-html 库初识

from requests_html import HTMLSessionsession = HTMLSession()
r = session.get('https://python.org/')
print(r)

print(dir(r.html))# 输出如下内容：['__aiter__', '__anext__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__','__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__lt__', '__module__', '__ne__','__new__', '__next__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__','__weakref__', '_async_render', '_encoding', '_html', '_lxml', '_make_absolute', '_pq', 'absolute_links', 'add_next_symbol','arender', 'base_url', 'default_encoding', 'element', 'encoding', 'find', 'full_text', 'html', 'links', 'lxml', 'next','next_symbol', 'page', 'pq', 'raw_html', 'render', 'search', 'search_all', 'session', 'skip_anchors', 'text', 'url', 'xpath']

DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8'
def user_agent(style=None) -> _UserAgent:    """Returns an apparently legit user-agent, if not requested one of a specific    style. Defaults to a Chrome-style User-Agent.    """    global useragent    if (not useragent) and style:        useragent = UserAgent()
    return useragent[style] if style else DEFAULT_USER_AGENT

www.world68.com/top.asp?t=5star&page=1www.world68.com/top.asp?t=5star&page=2

from requests_html import HTMLSession
session = HTMLSession()
page_size = int(input("请输入总页码："))for page in range(1, page_size + 1):
    world = session.get(f'http://www.world68.com/top.asp?t=5star&page={page}')    world.encoding = 'gb2312'    # world.html.encoding = "gb2312"    # print(world.text)    print("正在采集数据", world.url)    title_a = world.html.find('dl>dt>a')    for item in title_a:        name = item.text        url = item.attrs['href']        with open('webs.txt', "a+", encoding="utf-8") as f:            f.write(f"{name},{url}\n")

import requests_htmlimport threadingimport timeimport fcntl

class MyThread(threading.Thread):    def __init__(self):        threading.Thread.__init__(self)
    def run(self):        global page, lock, page_size        while True:            lock.acquire(True)            if page >= page_size:                lock.release()                break            else:                page += 1                lock.release()                requests_html.DEFAULT_ENCODING = "gb18030"                session = requests_html.HTMLSession()
                print("正在采集第{}页".format(page), "*" * 50)                try:                    page_url = f'http://www.world68.com/top.asp?t=5star&page={page}'                    world = session.get(page_url, timeout=10)                    print("正在采集数据", world.url)                    # print(world.html)                    title_a = world.html.find('dl>dt>a')                    print(title_a)                    my_str = ""
                    for item in title_a:                        name = item.text                        url = item.attrs['href']                        my_str += f"{name.encode('utf-8').decode('utf-8')},{url}\n"
                    with open('thread_webs.txt', "a+", encoding="utf-8") as f:                        fcntl.flock(f.fileno(), fcntl.LOCK_EX)  # 文件加锁                        f.write(f"{my_str}")
                except Exception as e:                    print(e, page_url)

if "__main__" == __name__:    page_size = int(input("请输入总页码："))    page = 0    thread_list = []
    # 获取开始时间    start = time.perf_counter()
    lock = threading.Lock()    for i in range(1, 5):        t = MyThread()        thread_list.append(t)    for t in thread_list:        t.start()    for t in thread_list:        t.join()    # 获取时间间隔    elapsed = (time.perf_counter() - start)    print("程序运行完毕，总耗时为：", elapsed)

encoding error : input conversion failed due to input error, bytes 0x81 0xE3 0xD3 0xAAencoding error : input conversion failed due to input error, bytes 0x81 0xE3 0xD3 0xAAencoding error : input conversion failed due to input error, bytes 0x81 0xE3 0xD3 0xAAI/O error : encoder error

def __init__(self, *, session: Union['HTMLSession', 'AsyncHTMLSession'] = None, url: str = DEFAULT_URL, html: _HTML, default_encoding: str = DEFAULT_ENCODING, async_: bool = False) -> None:  # 修改本部分代码    # Convert incoming unicode HTML into bytes.    # if isinstance(html, str):    html = html.decode(DEFAULT_ENCODING,'replace')
    super(HTML, self).__init__(        # Convert unicode HTML to bytes.        element=PyQuery(html)('html') or PyQuery(f'<html>{html}</html>')('html'),        html=html,        url=url,        default_encoding=default_encoding    )

requests_html.DEFAULT_ENCODING = "gb18030"

# 获取开始时间start = time.perf_counter()# 执行代码的部分# 获取时间间隔elapsed = (time.perf_counter() - start)print("程序运行完毕，总耗时为：", elapsed)