最近百度云盘不提供搜索,闲来无事,玩玩python爬虫,爬一下百度云盘的资源

    最近百度云盘不知道为啥不提供资源检索,正好最近看了一下python,正好来练练手,写歌爬虫爬一下百度云盘的资源。

    分析了一下百度云盘的网友源码和js文件,里面有大量ajax的东西,利用json传输数据,前端显示。话说,这样数据爬去就方便多了,也不要用scrapy啥的,直接解析json数据就好。

    分析js文件提炼了下面三个链接:

   

URL_SHARE = 'http://yun.baidu.com/pcloud/feed/getsharelist?auth_type=1&start={start}&limit=20&query_uk={uk}&urlid={id}'
URL_FOLLOW = 'http://yun.baidu.com/pcloud/friend/getfollowlist?query_uk={uk}&limit=20&start={start}&urlid={id}'
#http://yun.baidu.com/pcloud/friend/getfanslist?query_uk=1327787586&limit=25&start=0
URL_FANS = 'http://yun.baidu.com/pcloud/friend/getfanslist?query_uk={uk}&limit=20&start={start}&urlid={id}'

整个数据爬取流程起到很重要的作用。

  爬虫分三步,一个是urlids 保存要爬取的网址,一个是user存放用户uk,另一个是share存放user分享的数据,包含任何你想要的数据。

  下面提供三个核心函数代码:

#演示站http://wwww.yunsou.me        
def response_worker():
    global news,totals
    dbconn = mdb.connect(DB_HOST, DB_USER, DB_PASS, 'baiduyun', charset='utf8')
    dbcurr = dbconn.cursor()
    dbcurr.execute('SET NAMES utf8')
    dbcurr.execute('set global wait_timeout=60000')
    while True:
        print "function response_worker",hc_r.qsize()
        # if hc_r.qsize()==0:
        #     print "continue"
        #     continue
        metadata, effective_url = hc_r.get()
        print "response_worker:", effective_url
        try:
            tnow = datetime.datetime.utcnow()
            date = (tnow + datetime.timedelta(hours=8))
            date = datetime.datetime(date.year, date.month, date.day)
            if news>=100:
                try:
                    dbcurr.execute('INSERT INTO spider_statusreport(date,new_hashes,total_requests)  VALUES(%s,%s,%s) ON DUPLICATE KEY UPDATE ' +'total_requests=total_requests+%s,new_hashes=new_hashes+%s',
                        (date, news,totals,totals,news))
                except Exception as ex:
                    print "E10", str(ex)
                news=0
            id = re_urlid.findall(effective_url)[0]
            start = re_start.findall(effective_url)[0]
            if True:
                if 'getfollowlist' in effective_url: #type = 1
                    follows = json.loads(metadata)
                    print "-------------------------------------follows-------------------------------\n"
                    uid = re_uid.findall(effective_url)[0]
                    if "total_count" in follows.keys() and follows["total_count"]>0 and str(start) == "0":
                        for i in range((follows["total_count"]-1)/ONEPAGE):
                            try:
                                dbcurr.execute('INSERT INTO urlids(uk, start, limited, type, status) VALUES(%s, %s, %s, 1, 0)' % (uid, str(ONEPAGE*(i+1)), str(ONEPAGE)))
                            except Exception as ex:
                                print "E1", str(ex)
                                pass
                    
                    if "follow_list" in follows.keys():
                        for item in follows["follow_list"]:
			    if item['pubshare_count']==0:
			        print "---------------------count ==0-------------------------------------------\n"
			        #continue
	
                            y = dbcurr.execute('SELECT id FROM user WHERE userid=%s', (item['follow_uk'],))
                            y = dbcurr.fetchone()
                            print "user uk",item['follow_uk']
                            if not y:
                                try:
                                    dbcurr.execute('INSERT INTO user(userid, username, files, status, downloaded, lastaccess,avatar_url,fans_count,follow_count,album_count) VALUES(%s, "%s", %s, 0, 0, "%s","%s",%s,%s,%s)' % (item['follow_uk'], item['follow_uname'],item['pubshare_count'],tnow,item['avatar_url'],item['fans_count'],item['follow_count'],item['album_count']))
                                except Exception as ex:
                                    print "E13", str(ex)
                                    pass
                            else:
                                print "-----------------userid exists---------------------------------\n"
                    else:
                        print "delete 1", uid, start
                        dbcurr.execute('delete from urlids where uk=%s and type=1 and start>%s' % (uid, start))
                elif 'getfanslist' in effective_url: #type = 2
                    fans = json.loads(metadata)
                    print "----------------------------------------fans----------------------------------\n"
                    uid = re_uid.findall(effective_url)[0]
                    if "total_count" in fans.keys() and fans["total_count"]>0 and str(start) == "0":
                        for i in range((fans["total_count"]-1)/ONEPAGE):
                            try:
                                dbcurr.execute('INSERT INTO urlids(uk, start, limited, type, status) VALUES(%s, %s, %s, 2, 0)' % (uid, str(ONEPAGE*(i+1)), str(ONEPAGE)))
                            except Exception as ex:
                                print "E2", str(ex)
                                pass
                    
                    if "fans_list" in fans.keys():
                        for item in fans["fans_list"]:
                            if item['pubshare_count']==0:
				print "---------------------count ==0-------------------------------------------\n"
                                #continue
                            y = dbcurr.execute('SELECT id FROM user WHERE userid=%s', (item['fans_uk'],))
                            y = dbcurr.fetchone()
                            print "user uk",item['fans_uk']
                            if not y:
                                try:
                                    dbcurr.execute('INSERT INTO user(userid, username, files, status, downloaded, lastaccess,avatar_url,fans_count,follow_count,album_count) VALUES(%s, "%s", %s, 0, 0, "%s","%s",%s,%s,%s)' % (item['fans_uk'], item['fans_uname'],item['pubshare_count'],tnow,item['avatar_url'],item['fans_count'],item['follow_count'],item['album_count']))
                                except Exception as ex:
                                    print "E23", str(ex)
                                    pass
                            else:
                                print "-----------------userid exists---------------------------------\n"
                            
                    else:
                        print "delete 2", uid, start
                        dbcurr.execute('delete from urlids where uk=%s and type=2 and start>%s' % (uid, start))
                else:
                    shares = json.loads(metadata)
                    print "shares"
                    uid = re_uid.findall(effective_url)[0]
                    totals+=1
                    if "total_count" in shares.keys() and shares["total_count"]>0 and str(start) == "0":
                        for i in range((shares["total_count"]-1)/ONESHAREPAGE):
                            try:
                                dbcurr.execute('INSERT INTO urlids(uk, start, limited, type, status) VALUES(%s, %s, %s, 0, 0)' % (uid, str(ONESHAREPAGE*(i+1)), str(ONESHAREPAGE)))
                            except Exception as ex:
                                print "E3", str(ex)
                                pass
                    if "records" in shares.keys():
                        for item in shares["records"]:
                            print "-------------------------------------filename------------------ ",item['title']
                            print "---------------------------------------------------------------\n"
                            try:
                                stamp_t=int(item["feed_time"])/1000
                                t= time.localtime(int(stamp_t))
                                share_time=time.strftime("%Y-%m-%d %H:%M:%S",t)
                                urls=""
                                if "shorturl" in item.keys():
                                    urls=item['shorturl']
                                news+=1
                                length=""
                                if "filelist" in item.keys():
                                    length=str(item['filelist'][0]['size'])
                                dbcurr.execute('INSERT INTO share(fid,userid, filename, shareid, status,filetype,share_time,create_time,urls,down,length) VALUES("%s",%s, "%s", %s, 0,"%s","%s","%s","%s",0,"%s")' % (sid(int(item['shareid'])),uid, item['title'], item['shareid'],get_category(get_ext(item['title'])),share_time,tnow,urls,length))
                                # time.sleep(10)
                                
                            except Exception as ex:
                                print "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>E33\n", str(ex)
                                print "item ---------------------------------------------\n"
                                # time.sleep(10)
                                pass
                    else:
                        print "delete 0", uid, start
                        dbcurr.execute('delete from urlids where uk=%s and type=0 and start>%s' % (uid, str(start)))
                    
                dbcurr.execute('delete from urlids where id=%s' % (id, ))
                dbconn.commit()
        except Exception as ex:
            print "E5", str(ex), id
        
        pid = re_pptt.findall(effective_url)
        
        if pid:
            print "pid>>>", pid
            ppid = int(pid[0])
            PROXY_LIST[ppid][6] -= 1
    dbcurr.close()
    dbconn.close()
#演示站http://wwww.yunsou.me        
def worker(k):
    global success, failed
    dbconn = mdb.connect(DB_HOST, DB_USER, DB_PASS, 'baiduyun', charset='utf8')
    dbcurr = dbconn.cursor()
    dbcurr.execute('SET NAMES utf8')
    dbcurr.execute('set global wait_timeout=60000')
    while True:
        #dbcurr.execute('select * from urlids where status=0 order by type limit 1')
        dbcurr.execute('select * from urlids where status=0 limit %s,1'%(str(k),))
        d = dbcurr.fetchall()
        #print d
        if d:
            id = d[0][0]
            uk = d[0][1]
            start = d[0][2]
            limit = d[0][3]
            type = d[0][4]
            dbcurr.execute('update urlids set status=1 where id=%s' % (str(id),))
            url = ""
            if type == 0:
                url = URL_SHARE.format(uk=uk, start=start, id=id).encode('utf-8')
            elif  type == 1:
                url = URL_FOLLOW.format(uk=uk, start=start, id=id).encode('utf-8')
            elif type == 2:
                url = URL_FANS.format(uk=uk, start=start, id=id).encode('utf-8')
            if url:
                hc_q.put((type, url))
        if len(d)==0:
            print "\ndata user uk\n "
            dbcurr.execute('select * from user where status=0 limit %s,100'%(str(k*100),))
            print "user "
            d = dbcurr.fetchall()
            #print "uk",d
            if d:
                for item in d:
                    try:
                        print "update user",item[1]
                        dbcurr.execute('insert into urlids(uk, start, limited, type, status) values("%s", 0, %s, 0, 0)' % (item[1], str(ONESHAREPAGE)))
                        dbcurr.execute('insert into urlids(uk, start, limited, type, status) values("%s", 0, %s, 1, 0)' % (item[1], str(ONEPAGE)))
                        dbcurr.execute('insert into urlids(uk, start, limited, type, status) values("%s", 0, %s, 2, 0)' % (item[1], str(ONEPAGE)))
                        dbcurr.execute('update user set status=1 where userid=%s and id=%s' % (item[1],item[6]))
                    except Exception as ex:
                        print "E6", str(ex)
            else:
                time.sleep(1)
        dbconn.commit()
    dbcurr.close()
    dbconn.close()
#演示站http://wwww.yunsou.me 
def req_worker(inx):
    s = requests.Session()
    while True:
        time.sleep(1)
        req_item = hc_q.get()
        
        req_type = req_item[0]
        url = req_item[1]
        try:
            r = s.get(url)
            hc_r.put((r.text, url))
        except:
            pass
for item in range(3):    
    t = threading.Thread(target = req_worker, args = (item,))
    t.setDaemon(True)
    t.start()
for item in range(2):     
    s = threading.Thread(target = worker, args = (item,))
    s.setDaemon(True)
    s.start()
for item in range(2):    
    t = threading.Thread(target = response_worker, args = ())
    t.setDaemon(True)
    t.start()
while 1:
    pass

   ok,完工,想看的可以来看下百度云搜

    原文作者:lonag
    原文地址: https://blog.51cto.com/lonag/1716517
    本文转自网络文章,转载此文章仅为分享知识,如有侵权,请联系博主进行删除。
点赞