scrapy+selenium爬取网易云音乐排行榜

2019年6月11日 278次阅读 来源: 朽小蜗

仔细研究网易云音乐排行榜html码源，发现榜单是iframe动态加载的，因此scrapy结合selenium来爬取

  • 在middlewares.py文件中建立中间件，结合selenium
from selenium import webdriver
import selenium.webdriver.support.ui as ui
from scrapy.http import HtmlResponse

#采用中间件结合selenium
class JavaScriptMiddleware(object):
    def process_request(self, request, spider):
        if spider.name == "wymusic":
            print "PhantomJS is starting..."
            driver = webdriver.PhantomJS() #指定使用的浏览器
            driver.get(request.url)
            driver.switch_to.frame('g_iframe') #移动到 iframe
            wait = ui.WebDriverWait(driver, 15)
            if wait.until(lambda driver: driver.find_element_by_id('toplist')):
                body = driver.page_source
            return HtmlResponse(driver.current_url, body=body, encoding='utf-8', request=request)
        else:
            return
  • 在settings.py中添加中间件
DOWNLOADER_MIDDLEWARES = {
    #'scrapySpider.middlewares.MyCustomDownloaderMiddleware': 543,
    'wymusicSpider.middlewares.JavaScriptMiddleware': 543,
}
  • 在items.py中定义要爬取的字段
class WymusicItem(scrapy.Item):
    rank = scrapy.Field()      # 排名
    title = scrapy.Field()     # 歌名
    duration = scrapy.Field()  # 时长
    author = scrapy.Field()    # 作者
  • 编写爬虫
import scrapy
from scrapy.http.request import Request
from wymusicSpider.items import WymusicItem
import sys
import lxml

reload(sys)
sys.setdefaultencoding('utf-8')

#html DOM tree to lxml 格式
def parse_from_unicode(unicode_str):  
        utf8_parser = lxml.etree.HTMLParser(encoding='utf-8')
        s = unicode_str.encode('utf-8')
        return lxml.etree.fromstring(s, parser=utf8_parser)

class WymusicSpider(scrapy.Spider):
    name = 'wymusic'
    allowed_domains = ['music.163.com']
    start_urls = ['http://music.163.com/#/discover/toplist']

    def parse(self, response):
        items = []
        dom = parse_from_unicode(response.body)
        
        lists = dom.xpath('//tbody/tr')
        for row in lists:
            td = row.xpath("td")
            rank = td[0].xpath("div/span[@class='num']/text()")[0]
            title = td[1].xpath("div/div/div/span/a/b//@title")[0]
            duration = td[2].xpath("span/text()")[0]
            author = td[3].xpath("div//@title")[0]
            
            item = WymusicItem()

            item['rank']  = rank
            item['title']  = title
            item['duration']  = duration
            item['author']  = author

            if item not in items:
                items.append(item)

        return items
    原文作者：朽小蜗
    原文地址: https://www.jianshu.com/p/66aacaaf6fa2
    本文转自网络文章，转载此文章仅为分享知识，如有侵权，请联系博主进行删除。
点赞