Python 爬取微信公众号文章保存到数据库

wjchenge

关注

发布于: 2020 年 08 月 07 日

抓包工具 Charles 安装

下载 Charles 进行安装

配置 Charles

本文环境

Windows 10Charles v4.5.6电脑版微信 2.9.5.41

复制代码

抓取配置

打开 Charles
打开需要抓取的微信公众号

https 配置 SSL 代理

Charles提示信息

Charles配置SSL代理流程

鼠标往下滑动微信公众号文章，同时观察 Charles

MySql 表结构脚本

CREATE TABLE `wjchenge` (  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,  `title` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,  `digest` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,  `content_url` varchar(255) CHARACTER SET utf8 DEFAULT NULL,  `cover` varchar(255) CHARACTER SET utf8 DEFAULT NULL,  `datetime` int(11) DEFAULT NULL,  PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

复制代码

Python 脚本

# -*- coding: utf-8 -*-import requestsimport timeimport randomfrom mysql.connector import connectimport emoji
# 目标urlurl = "https://mp.weixin.qq.com/mp/profile_ext"
headers = {    "Host": "mp.weixin.qq.com",    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0",    "Accept": "*/*",    "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",    "Accept-Encoding": "gzip, deflate, br",    "X-Requested-With": "XMLHttpRequest",    "DNT": "1",    "Connection": "keep-alive",    "Referer": "https://mp.weixin.qq.com/cgi-bin/appmsg?t=media/appmsg_edit&action=edit&type=10&isMul=1&lang=zh_CN&token=1043021413",    "Pragma": "no-cache",    "Cache-Control": "no-cache",    "TE": "Trailers",}
# 每个公众号固定参数biz = 'MzU3MDc4MDk0Mw=='
# 隔段时间就需要更新的参数key = '10b5f81a683662236f8e935602059f39ece87145d94e5d3f1f859ccae3578becca7c902d7c8d10c80b451121638d632b36a8e0bb91e7caeb7127eef53a20adc1ff331bb1b2208f770a07bf8d2638a6e6'pass_ticket = 'bfo9QKq00RxUloXOxKcYnoFLtyVNyCJQNYipZgleCL+fo36wPTHriFEASbENfJM3'appmsg_token = '1073_dDGYR6k4L8i8jn93AlZZsHhVaoMQGQ56DlYznA~~'
"""需要提交的data以下个别字段是否一定需要还未验证。注意修改yourtoken,numbernumber表示从第number页开始爬取，为5的倍数，从0开始。如0、5、10……token可以使用Chrome自带的工具进行获取fakeid是公众号独一无二的一个id，等同于后面的__biz"""def getData(number):    data = {        'action': 'getmsg',        '__biz': biz,        'f': 'json',        'offset': number,        'count': 10,        'is_ok': 1,        'scene': 124,        'uin': 'Mjc1ODc2ODgyMA==',        'key': key,        'pass_ticket': pass_ticket,        'wxtoken': '',        'appmsg_token': appmsg_token,        'x5': 0,        'f': 'json',
    }
    # 使用get方法进行提交    content_json = requests.get(url, headers=headers, params=data).json()    general_msg_list = eval(content_json["general_msg_list"])    # 返回了一个json，里面是每一页的数据    for item in general_msg_list["list"]:        comm_msg_info = item["comm_msg_info"]        app_msg_ext_info = item["app_msg_ext_info"]        # 爬取到2019-12-01号的文章        if comm_msg_info["datetime"] < 1575161671:            return 0        # 提取每页文章的标题及对应的url        print(number, "||", app_msg_ext_info["title"], "||", app_msg_ext_info["digest"], "||",              app_msg_ext_info["content_url"], "||", app_msg_ext_info["cover"], "||", comm_msg_info["datetime"])        insertMsqlDb(app_msg_ext_info["title"], app_msg_ext_info["digest"], app_msg_ext_info["content_url"],                     app_msg_ext_info["cover"], comm_msg_info["datetime"])        # 多级嵌套        for item2 in app_msg_ext_info["multi_app_msg_item_list"]:            print(number, "||", item2["title"], "||", item2["digest"], "||", item2["content_url"], "||", item2["cover"],                  "||", comm_msg_info["datetime"])            insertMsqlDb(item2["title"], item2["digest"], item2["content_url"],                         item2["cover"], comm_msg_info["datetime"])    return content_json["can_msg_continue"]

# 数据写入数据库def insertMsqlDb(title, digest, content_url, cover, datetime):    try:        conn = connect(host='127.0.0.1', port='3306', user='root', password='123456', database='test')        cursor = conn.cursor()        # 微信表情字符存入mysql不兼容报错，通过emoji处理一下        cursor.execute(            'insert into wjchenge (title, digest, content_url, cover, datetime) values (%s, %s, %s, %s, %s)',            [emoji.demojize(title), emoji.demojize(digest), content_url, cover, datetime])        conn.commit()    except Exception as e:        print('Error:', e)        raise    finally:        cursor.close()        conn.close

# 下一页指针位置next_offset = 0# 1 表示有下一页 0表示没有下一页can_msg_continue = getData(next_offset)while can_msg_continue > 0:    next_offset += 10    # 每次间隔5-9秒    time.sleep(random.randint(5, 9))    can_msg_continue = getData(next_offset)