爬取微博热搜榜 - 李白之死 - Python

本文主要是介绍爬取微博热搜榜 - 李白之死 - Python，对大家解决编程问题具有一定的参考价值，需要的程序猿们随着小编来一起学习吧！

最近有关中国传统文化的内容频频登上热搜，就比如最近的李白之死，今天换一种方式爬取，以前爬取微博评论是网址里一大串参数，今天把参数提出来做一个字典，然后请求的时候再构造url。

 1 """
 2     就爬取李白之死的评论
 3 """
 4 import requests
 5 import re
 6 import openpyxl as op
 7 
 8 wb = op.Workbook()
 9 ws = wb.create_sheet(index=0)
10 # 表头
11 ws.cell(row=1, column=1, value='评论者id') # 第一行第一列userId
12 ws.cell(row=1, column=2, value='评论者昵称') # 第一行第一列userId
13 ws.cell(row=1, column=3, value='获赞数') # 第一行第一列userId
14 ws.cell(row=1, column=4, value='创建时间') # 第一行第一列userId
15 ws.cell(row=1, column=5, value='评论内容') # 第一行第一列userId
16 
17 headers = {
18     "cookie": "cookie",
19     "referer": "https://m.weibo.cn/status/L690FmKXW?jumpfrom=weibocom",
20     "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
21 }
22 page = 1
23 while page < 100 + 1:
24     url = 'https://m.weibo.cn/comments/hotflow'
25     if page == 1:
26         params = {
27                     'id': '4714646055423756',
28                     'mid': '4714646055423756',
29                     'max_id_type':0,
30         }
31     else:
32         params = {
33                     'id': '4714646055423756',
34                     'mid': '4714646055423756',
35                     'max_id': max_id,
36                     'max_id_type':max_id_type,
37         }
38     response = requests.get(url=url, headers=headers, params=params)
39     max_id = response.json()['data']['max_id']
40     max_id_type = response.json()['data']['max_id_type']
41     results = response.json()['data']['data'] # 获取到评论列表
42     for item in results:
43         userId = item['user']['id']
44         userName = item['user']['screen_name']
45         likeCount = item['like_count']
46         timeCreated = item['created_at'] # 创建时间
47         commentContent = item['text'] # 评论内容
48         print(userId, userName, likeCount, timeCreated, commentContent, response.url, sep=' | ')
49         ws.append([userId, userName, likeCount, timeCreated, commentContent])
50     page += 1
51 
52 wb.save('李白之死.xlsx')
53 wb.close()

保存方式有很多，前面也有过一篇关于python爬虫数据保存方式的。但是今天只要爬评论内容来做个词频。

 1 """
 2     就爬取李白之死的评论
 3 """
 4 import requests
 5 import re
 6 import openpyxl as op
 7 
 8 # wb = op.Workbook()
 9 # ws = wb.create_sheet(index=0)
10 # 表头
11 # ws.cell(row=1, column=1, value='评论者id') # 第一行第一列userId
12 # ws.cell(row=1, column=2, value='评论者昵称') # 第一行第一列userId
13 # ws.cell(row=1, column=3, value='获赞数') # 第一行第一列userId
14 # ws.cell(row=1, column=4, value='创建时间') # 第一行第一列userId
15 # ws.cell(row=1, column=5, value='评论内容') # 第一行第一列userId
16 
17 headers = {
18     "cookie": "cookie",
19     "referer": "https://m.weibo.cn/status/L690FmKXW?jumpfrom=weibocom",
20     "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
21 }
22 page = 1
23 while page < 100 + 1:
24     url = 'https://m.weibo.cn/comments/hotflow'
25     if page == 1:
26         params = {
27                     'id': '4714646055423756',
28                     'mid': '4714646055423756',
29                     'max_id_type':0,
30         }
31     else:
32         params = {
33                     'id': '4714646055423756',
34                     'mid': '4714646055423756',
35                     'max_id': max_id,
36                     'max_id_type':max_id_type,
37         }
38     response = requests.get(url=url, headers=headers, params=params)
39     max_id = response.json()['data']['max_id']
40     max_id_type = response.json()['data']['max_id_type']
41     results = response.json()['data']['data'] # 获取到评论列表
42     for item in results:
43         """
44         这一次只爬评论内容
45         """
46         commentContent = re.sub(r'<[^>]*>', '', item['text']) # 将评论内容里的特殊字符用正则替换掉
47         print(commentContent)
48         with open('libazhisi.txt', mode='a', encoding='utf-8') as f:
49             f.write(f'{commentContent}\n') # 换行写入
50     page += 1
51 
52 # wb.save('李白之死.xlsx')
53 # wb.close()

词频展示：

 1 """
 2     做个词频
 3 """
 4 # 打开文档
 5 import re
 6 from collections import Counter
 7 import jieba
 8 from pyecharts.charts import Bar
 9 import pyecharts.options as opts
10 from pyecharts.globals import ThemeType
11 
12 def replaceSth(sth):
13     pattern = re.compile(r'[a-zA-Z0-9…，\@”！[\\]_]。')
14     new = re.sub(pattern, '', sth)
15     return new
16 
17 with open('李白之死.txt', mode='r', encoding='utf-8') as f:
18     reader = f.read()
19     new_reader = re.sub('[”0-9a-zA-Z！

	

这篇关于爬取微博热搜榜 - 李白之死 - Python的文章就介绍到这儿，希望我们推荐的文章对大家有所帮助，也希望大家多多支持为之网！