爬虫:新浪微博爬虫的最简单办法

VeUB7M · 发表于 2023-1-4 00:27:57

媒介：原文主要实质是介绍怎样用最简朴的法子来收罗新浪微专的数据，主要是收罗指定微专用户公布的微专和微专支到的复兴等外容，能够颠末设置项去调解爬与的微专用户列表和其余属性。
既然道是最简朴的法子，那末咱们便患上先阐发微专爬虫可以挑选的多少个目标网址，起首必然是最多见的web网站了

爬虫:新浪微专爬虫的最简朴法子-1.jpg

另有即是m站，也即是挪动端网页

爬虫:新浪微专爬虫的最简朴法子-2.jpg

和一个没法旧版原的会见进口了，起首能够拂拭web站了，那个是最省事的，它的恳求是被js减稀过，处置起去很省事
这咱们为什么没有进而供其次呢，咱们察看下那个m站的恳求

爬虫:新浪微专爬虫的最简朴法子-3.jpg

能够收现在某个恳求里能够发明咱们需要的微专疑息，既然如许便佳办了，咱们就能够动手咱们的代码了
起首是获得用户疑息，颠末用户疑息获得用户的微专总额，如许就能够明白统共几页的数据了，代码以下所示

def get_json(self, params):
&#34;&#34;&#34;获得网页中json数据&#34;&#34;&#34;
url = &#39;https://m.weibo.cn/api/container/getIndex?&#39;
r = requests.get(url, params=params, cookies=self.cookie)
return r.json()
def get_page_count(self):
&#34;&#34;&#34;获得微专页数&#34;&#34;&#34;
try:
weibo_count = self.user[&#39;statuses_count&#39;]
page_count = int(math.ceil(weibo_count / 10.0))
return page_count
except KeyError:
sys.exit(u&#39;法式堕落&#39;)
def get_user_info(self):
&#34;&#34;&#34;获得用户疑息&#34;&#34;&#34;
params = {&#39;containerid&#39;: &#39;100505&#39; + str(weibo_config[&#39;user_id&#39;])}
js = self.get_json(params)
if js[&#39;ok&#39;]:
info = js[&#39;data&#39;][&#39;userInfo&#39;]
user_info = {}
user_info[&#39;id&#39;] = weibo_config[&#39;user_id&#39;]
user_info[&#39;screen_name&#39;] = info.get(&#39;screen_name&#39;, &#39;&#39;)
user_info[&#39;gender&#39;] = info.get(&#39;gender&#39;, &#39;&#39;)
user_info[&#39;statuses_count&#39;] = info.get(&#39;statuses_count&#39;, 0)
user_info[&#39;followers_count&#39;] = info.get(&#39;followers_count&#39;, 0)
user_info[&#39;follow_count&#39;] = info.get(&#39;follow_count&#39;, 0)
user_info[&#39;description&#39;] = info.get(&#39;description&#39;, &#39;&#39;)
user_info[&#39;profile_url&#39;] = info.get(&#39;profile_url&#39;, &#39;&#39;)
user_info[&#39;profile_image_url&#39;] = info.get(&#39;profile_image_url&#39;, &#39;&#39;)
user_info[&#39;avatar_hd&#39;] = info.get(&#39;avatar_hd&#39;, &#39;&#39;)
user_info[&#39;urank&#39;] = info.get(&#39;urank&#39;, 0)
user_info[&#39;mbrank&#39;] = info.get(&#39;mbrank&#39;, 0)
user_info[&#39;verified&#39;] = info.get(&#39;verified&#39;, False)
user_info[&#39;verified_type&#39;] = info.get(&#39;verified_type&#39;, 0)
user_info[&#39;verified_reason&#39;] = info.get(&#39;verified_reason&#39;, &#39;&#39;)
user = self.standardize_info(user_info)
self.user = user

复造代码

分页收罗数据

page1 = 0
random_pages = random.randint(1, 5)
self.start_date = datetime.now().strftime(&#39;%Y-%m-%d&#39;)
for page in tqdm(range(1, page_count + 1), desc=&#39;Progress&#39;):
is_end = self.get_one_page(page)
if is_end:
break
if page % 20 == 0: # 每一爬20页写进一次文献
self.weibo_to_mysql(wrote_count)
wrote_count = self.got_count
# 颠末参加随机等候制止被限定。爬虫速率过快简单被体系限定(一段时间后限
# 造会主动消除)，参加随机等候模仿人的操纵，可低落被体系限定的危急。默
# 认是每一爬与1到5页随机等候6到10秒，假设依旧被限，可恰当增加sleep时间
if (page - page1) % random_pages == 0 and page < page_count:
sleep(random.randint(6, 10))
page1 = page
random_pages = random.randint(1, 5)
self.weibo_to_mysql(wrote_count) # 将盈余不敷20页的微专写进文献
print(u&#39;微专爬与完毕，同爬与%d条微专&#39; % self.got_count)

复造代码

具体收罗单页微专代码以下

def get_one_page(self, page):
&#34;&#34;&#34;获得一页的局部微专&#34;&#34;&#34;
try:
js = self.get_weibo_json(page)
if js[&#39;ok&#39;]:
weibos = js[&#39;data&#39;][&#39;cards&#39;]
for w in weibos:
if w[&#39;card_type&#39;] == 9:
wb = self.get_one_weibo(w)
if wb:
if wb[&#39;id&#39;] in self.weibo_id_list:
continue
created_at = datetime.strptime(
wb[&#39;created_at&#39;], &#39;%Y-%m-%d&#39;)
since_date = datetime.strptime(
self.since_date, &#39;%Y-%m-%d&#39;)
if created_at < since_date:
if self.is_pinned_weibo(w):
continue
else:
print(u&#39;{}已经获得{}({})的第{}页微专{}&#39;.format(
&#39;-&#39; * 30, self.user[&#39;screen_name&#39;],
self.user[&#39;id&#39;], page, &#39;-&#39; * 30))
return True
if (&#39;retweet&#39; not in wb.keys()):
self.weibo.append(wb)
self.weibo_id_list.append(wb[&#39;id&#39;])
self.got_count += 1
print(u&#39;{}已经获得{}({})的第{}页微专{}&#39;.format(&#39;-&#39; * 30,
self.user[&#39;screen_name&#39;],
self.user[&#39;id&#39;], page,
&#39;-&#39; * 30))
except Exception as e:
print(&#34;Error: &#34;, e)
traceback.print_exc()

复造代码

获得具体微专疑息的代码

def get_one_weibo(self, info):
&#34;&#34;&#34;获得一条微专的局部疑息&#34;&#34;&#34;
try:
weibo_info = info[&#39;mblog&#39;]
weibo_id = weibo_info[&#39;id&#39;]
retweeted_status = weibo_info.get(&#39;retweeted_status&#39;)
is_long = weibo_info.get(&#39;isLongText&#39;)
if retweeted_status: # 转收
retweet_id = retweeted_status.get(&#39;id&#39;)
is_long_retweet = retweeted_status.get(&#39;isLongText&#39;)
if is_long:
weibo = self.get_long_weibo(weibo_id)
if not weibo:
weibo = self.parse_weibo(weibo_info)
else:
weibo = self.parse_weibo(weibo_info)
if is_long_retweet:
retweet = self.get_long_weibo(retweet_id)
if not retweet:
retweet = self.parse_weibo(retweeted_status)
else:
retweet = self.parse_weibo(retweeted_status)
retweet[&#39;created_at&#39;] = self.standardize_date(
retweeted_status[&#39;created_at&#39;])
weibo[&#39;retweet&#39;] = retweet
else: # 本创
if is_long:
weibo = self.get_long_weibo(weibo_id)
if not weibo:
weibo = self.parse_weibo(weibo_info)
else:
weibo = self.parse_weibo(weibo_info)
weibo[&#39;created_at&#39;] = self.standardize_date(
weibo_info[&#39;created_at&#39;])
return weibo
except Exception as e:
print(&#34;Error: &#34;, e)
traceback.print_exc()
def get_long_weibo(self, id):
&#34;&#34;&#34;获得少微专&#34;&#34;&#34;
for i in range(5):
url = &#39;https://m.weibo.cn/detail/%s&#39; % id
html = requests.get(url, cookies=self.cookie).text
html = html[html.find(&#39;&#34;status&#34;:&#39;):]
html = html[:html.rfind(&#39;&#34;hotScheme&#34;&#39;)]
html = html[:html.rfind(&#39;,&#39;)]
html = &#39;{&#39; + html + &#39;}&#39;
js = json.loads(html, strict=False)
weibo_info = js.get(&#39;status&#39;)
if weibo_info:
weibo = self.parse_weibo(weibo_info)
return weibo
sleep(random.randint(6, 10))

复造代码

以上即是中心的微专疑息收罗代码了，除微专疑息，咱们借需要收罗微专批评疑息，道理是一致的，找到数据滥觞

爬虫:新浪微专爬虫的最简朴法子-4.jpg

有了微专疑息收罗的经历，咱们很简单就能够找到咱们念要的谁人交心
具体代码以下

def add_co妹妹ents_json(self,jsondata):
for data in jsondata:
item = dict()
item[&#39;id&#39;] = data.get(&#39;id&#39;)
item[&#39;mid&#39;] = data.get(&#39;mid&#39;)
item[&#39;like_count&#39;] = data.get(&#34;like_count&#34;)
item[&#39;source&#39;] = data.get(&#34;source&#34;)
item[&#39;floor_number&#39;] = data.get(&#34;floor_number&#34;)
item[&#39;screen_name&#39;] = data.get(&#34;user&#34;).get(&#34;screen_name&#34;)
# 性别
item[&#39;gender&#39;] = data.get(&#34;user&#34;).get(&#34;gender&#34;)
if(item[&#39;gender&#39;] == &#39;m&#39;):
item[&#39;gender&#39;] = &#39;男&#39;
elif(item[&#39;gender&#39;] == &#39;f&#39;):
item[&#39;gender&#39;] = &#39;女&#39;
item[&#39;rootid&#39;] = data.get(&#34;rootid&#34;)
item[&#39;create_time&#39;] = data.get(&#34;created_at&#34;)
import time
item[&#39;create_time&#39;] = time.strptime(item[&#39;create_time&#39;], &#39;%a %b %d %H:%M:%S %z %Y&#39;)
item[&#39;create_time&#39;] = time.strftime(&#39;%Y-%m-%d&#39;,item[&#39;create_time&#39;])
item[&#39;co妹妹ent&#39;] = data.get(&#34;text&#34;)
item[&#39;co妹妹ent&#39;] = BeautifulSoup(item[&#39;co妹妹ent&#39;], &#39;html.parser&#39;).get_text()
item[&#39;co妹妹ent&#39;] = self.clear_character_chinese(item[&#39;co妹妹ent&#39;])
print(&#39;目前楼层{},批评{}&#39;.format(item[&#39;floor_number&#39;],item[&#39;co妹妹ent&#39;]))
# 批评那条批评的疑息
co妹妹ents = data.get(&#34;co妹妹ents&#34;)
if(co妹妹ents):
self.add_co妹妹ents_json(co妹妹ents)
# print jsondata.dumps(co妹妹ent, encoding=&#34;UTF-8&#34;, ensure_ascii=False)
self.co妹妹ents.append(item)
def get_co妹妹ents_page(self,max_id, id_type,mid):
from get_weibo_cookie import get_cookie
params = {
&#39;max_id&#39;: max_id,
&#39;max_id_type&#39;: id_type
}
try:
url = &#39;https://m.weibo.cn/co妹妹ents/hotflow?id={id}&amp;mid={mid}&amp;max_id=&#39;
headers = {
&#39;Cookie&#39;: &#39;T_WM=96849642965; __guid=52195957.2500582256236055600.1583058027995.9556; WEIBOCN_FROM=1110006030; SCF=Aimq85D9meHNU4Ip0PFUjYBTDjXFB0VtQr3EKoS8DHQDobRNUO3lDIufAcUg69h4J7BQWqryxQpuU3ReIHHxvQ4.; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W5H0p180lDMiCjNvXD_-uOh5JpX5KzhUgL.FoM0S0n0eo-0Sh.2dJLoI0qLxKqL1KMLBK-LxK-LBonLBonLxKMLB.-L12-LxK-LBK-LBoeLxK-L1hnL1hqLxKBLB.2LB-zt; XSRF-TOKEN=ca0a29; SUB=_2A25zWlwFDeRhGeFN7FoS8ivPzzWIHXVQpWRNrDV6PUJbkdANLW_9kW1NQ8CH90H5f8j5r1NA4GNPvu6__ERL-Jat; SUHB=0vJIkXXtLIIaZO; SSOLoginState=1583230037; MLOGIN=1; M_WEIBOCN_PARAMS=oid%3D4474164293517551%26luicode%3D20000174%26lfid%3D102803%26uicode%3D20000174; monitor_count=45&#39;,
&#39;User-Agent&#39;: &#39;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36&#39;,
&#39;X-Requested-With&#39;: &#39;XMLHttpRequest&#39;
}
r = requests.get(url.format(id=mid,mid=mid), params=params,headers=headers)
print(r.url)
if r.status_code == 200:
return r.json()
except requests.ConnectionError as e:
print(&#39;error&#39;, e.args)
def add_co妹妹ents(self,jsondata):
datas = jsondata.get(&#39;data&#39;).get(&#39;data&#39;)
for data in datas:
item = dict()
item[&#39;id&#39;] = data.get(&#39;id&#39;)
item[&#39;mid&#39;] = data.get(&#39;mid&#39;)
item[&#39;like_count&#39;] = data.get(&#34;like_count&#34;)
item[&#39;source&#39;] = data.get(&#34;source&#34;)
item[&#39;floor_number&#39;] = data.get(&#34;floor_number&#34;)
item[&#39;screen_name&#39;] = data.get(&#34;user&#34;).get(&#34;screen_name&#34;)
# 性别
item[&#39;gender&#39;] = data.get(&#34;user&#34;).get(&#34;gender&#34;)
if(item[&#39;gender&#39;] == &#39;m&#39;):
item[&#39;gender&#39;] = &#39;男&#39;
elif(item[&#39;gender&#39;] == &#39;f&#39;):
item[&#39;gender&#39;] = &#39;女&#39;
item[&#39;created_at&#39;] = self.standardize_date(
data.get([&#39;created_at&#39;]))
import time
item[&#39;create_time&#39;] = time.strptime(item[&#39;create_time&#39;], &#39;%a %b %d %H:%M:%S %z %Y&#39;)
item[&#39;create_time&#39;] = time.strftime(&#39;%Y-%m-%d&#39;,item[&#39;create_time&#39;])
item[&#39;rootid&#39;] = data.get(&#34;rootid&#34;)
item[&#39;co妹妹ent&#39;] = data.get(&#34;text&#34;)
item[&#39;co妹妹ent&#39;] = BeautifulSoup(item[&#39;co妹妹ent&#39;], &#39;html.parser&#39;).get_text()
item[&#39;co妹妹ent&#39;] = self.clear_character_chinese(item[&#39;co妹妹ent&#39;])
print(&#39;目前楼层{},批评{}&#39;.format(item[&#39;floor_number&#39;],item[&#39;co妹妹ent&#39;]))
# 批评那条批评的疑息
co妹妹ents = data.get(&#34;co妹妹ents&#34;)
# print jsondata.dumps(co妹妹ent, encoding=&#34;UTF-8&#34;, ensure_ascii=False)
self.co妹妹ents.append(item)