1 import requests
 2 import re
 3 # from bs4 import BeautifulSoup
 4 from urllib import request
 5 # import threading
 6 import gevent
 7 from gevent import monkey
 8 
 9 monkey.patch_all()
10 
11 def get_html_text(url):
12     try:
13         hd = {'User-Agent':'Mozilla/5.0'} #添加伪装浏览器头部信息 14         r = requests.get(url, timeout=10, headers = hd)
15         r.raise_for_status()
16         r.encoding = r.apparent_encoding
17         print(len(r.text))
18         return r.text
19     except Exception as result:
20         print('错误类型：', result)
21 
22 
23 def html_text_parser(img_list, html):
24 
25 # 下面是修改的重点部分，采用了正则表达式，没有采用bs4
26 # 技术大神可以研究一下斗鱼这个网页的源代码，真正的图片信息都存储在后面，不是传统的# # html,我还没怎么接触过前端的知识，不知怎么使用bs4,所以使用了正则表达式。
27     
28     img_pat = r'"rs\w+":"(.*?g)"'  
29     links = re.compile(img_pat, re.S).findall(html)
30     print(len(links))
31     print(links)
32     for link in links:
33         if link:
34             img_list.append(link)
35     return img_list
36 
37 
38 
39 
40 def get_douyu_img(Img_list):
41     for i,j in enumerate(Img_list):
42         # name = j.split('.')[-1]
43         try: #异常捕获，如果链接不能访问，退出当前一次循环，进入下一次循环 44             r = request.urlopen(j)
45             ima_content = r.read()
46             path = str(i)
47             with open(path, 'wb') as f:
48                 f.write(ima_content)
49         except:
50             continue
51 def main():
52     url = 'https://www.douyu.com/g_yz'
53     html = get_html_text(url)
54     img_list = list()
55     Img_list = html_text_parser(img_list, html)
56     # print(Img_list)
57     #t1 = threading.Thread(target=get_html_text, args=(url,))
58     #t2 = threading.Thread(target=html_text_parser, args=(img_list,html))
59     #t3 = threading.Thread(target=get_douyu_img, args=(Img_list,))
60     #t1.start()
61     #t2.start()
62     #t3.start()
63     gevent.joinall([
64         gevent.spawn(get_html_text, url),
65         gevent.spawn(html_text_parser, img_list, html),
66         gevent.spawn(get_douyu_img, Img_list)
67     ])
68 
69 
70 if __name__ == '__main__':
71     main()