Python 抓取 B 站"卡塔尔 世界杯"的视频数据

pip3 install selenium==3.8.0 ## 4.0版本之后不支持plantomJspip3 install beautifulsoup4pip3 install lxmlpip3 install xlwt

# 创建chrome浏览器驱动，加载上面下载的驱动安装路径driver = webdriver.Chrome(executable_path = '/Users/warrior/opt/chromedriver')# 打开网址driver.get('https://www.bilibili.com/')

# 取得浏览器中的搜索框 并输入查询关键字input = WAIT.until(EC.presence_of_element_located((By.XPATH,"//div[@class='nav-search-content']/input[@class='nav-search-input']")))input.send_keys(key_words)
# 点击查询按钮submit = WAIT.until(EC.element_to_be_clickable((By.XPATH,"//div[@class='nav-search-btn']")))submit.click()
# 选中第二个标签页all_h = driver.window_handlesdriver.switch_to_window(all_h[1])
## 取最大页码maxbtn = WAIT.until(EC.presence_of_element_located((By.XPATH, "//div[@class='vui_pagenation--btns']//button[last()-1]")))return int(maxbtn.text)

# 等待页面加载完成(判断是否有下一页按钮)WAIT.until(EC.presence_of_element_located((By.XPATH, "//div[@class='vui_pagenation--btns']//button[contains(text(), '下一页')]")))
# 获得页面源代码html = driver.page_source

soup = BeautifulSoup(html, 'lxml')# 得到视频listvidos = soup.find(class_='video-list').find_all(class_='bili-video-card')# 全局list变量，用于存储抓取的数据global vlist
for item in vidos:  # 取标题  title = item.find("h3",class_="bili-video-card__info--tit").get('title')  # 取视频链接  link = item.find("a", class_="").get('href')  link = "https://" + link  # 播放数  view_num = item.find("span",class_="bili-video-card__stats--item").span.string  # 弹幕数  danmu = item.find_all("span",class_="bili-video-card__stats--item")[1].span.string  # UP主  author = item.find("span", class_="bili-video-card__info--author").string  # 日期  datee = item.find("span",class_="bili-video-card__info--date").string  datee = datee.strip()[1:]  print(title + " " + link + " " + view_num)  # 将数据放入到vlist中  vlist.append([title, link, view_num, danmu, author, datee])

# 取得下一页按钮nextbtn = WAIT.until(EC.element_to_be_clickable((By.XPATH, "//div[@class='vui_pagenation--btns']//button[contains(text(), '下一页')]")))# 点击nextbtn.click()

# 创建excelwb = xlwt.Workbook(encoding='utf-8', style_compression=0)
sheet = wb.add_sheet('b站搜索结果',cell_overwrite_ok=True)sheet.write(0,0,'序号')sheet.write(0,1,'名称')sheet.write(0,2,'链接')sheet.write(0,3,'观看次数')sheet.write(0,4,'弹幕数')sheet.write(0,5,'作者')sheet.write(0,6,'发布时间')
global index# 遍历并且填入数据for m in vlist:  sheet.write(index, 0, index)  sheet.write(index, 1, m[0])  sheet.write(index, 2, m[1])  sheet.write(index, 3, m[2])  sheet.write(index, 4, m[3])  sheet.write(index, 5, m[4])  sheet.write(index, 6, m[5])  index = index + 1
# 保存输出Excel 文件wb.save(key_words + "b站搜索结果.xlsx")

if __name__ == '__main__':  try:    # 打开页面并搜索，返回最大页码    total = first_search(key_words)    # 循环    for i in range(2, total+1):      # 点击下一页      next_page(i)  except:    print("异常")  finally:    # 关闭浏览器    driver.close()  ## 保存数据到Excel  save_to_excel(vlist)