1,微博网页版爬取难度较大，本次爬取手机版评论数据：;
2,评论网页为动态的，查看方式network----XHR----请求对应的url
注：url中的id,mid与微博地址中的后缀一致，评论往下滑，20条之后，会出现新的url,新的url中多了max_id,该参数类似分页参数，在preview的data下面有，该max_id即下一条url中的max_id
import requestsimport jsonimport pprintimport pandas as pdimport re#服务器返回的内容用变量接收response=requests.get('/comments/hotflow?id=4664928428097689&mid=4664928428097689&max_id_type=0')print(response.text)print(json.loads(response.text))pprint.pprint(json.loads(response.text))#字典数据data=json.loads(response.text)pprint.pprint(data)users=data['data']['data']lname=[]ltext=[]for user in users:#print(user['user']['id'])#print(user['user']['screen_name'])#print(user['text'])text=user['text']text=re.sub(r'<[^>]*>','',user['text']) #去除图片name=user['user']['screen_name']print(name)print(text)lname.append(name)ltext.append(text)df=pd.DataFrame({'name':lname,'comment':ltext})df=df[['name','comment']]df.to_excel(r'D:\comment.xlsx')
