一、爬取的代码
1、网站地址
2、具体实现代码
import requests
from lxml import etree
class JobBole(object):
def __init__(self):
self.url = ‘http://python.jobbole.com/all-posts/‘
self.headers = {
‘User-Agent’: ‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36’,
}
def get_article_link(self):
“””
抓取网页
:return:
“””
response = requests.get(url=self.url, headers=self.headers)
if response.status_code == 200:
html = etree.HTML(response.text)
# 现在就取第一个的访问详情页面
article = html.xpath(‘//div[@id=”archive”]/div[@class=”post floated-thumb”]’)[0]
# 查找到当前标签上的a标签
a_link = article.xpath(‘./div[@class=”post-meta”]/p[@class=”align-right”]//a/@href‘)[0]
return a_link
def get_html(self):
“””
获取详情页面的数据
:return:
“””
if self.get_article_link:
response = requests.get(url=self.get_article_link, headers=self.headers)
if response.status_code == 200:
html = etree.HTML(response.text)
title = html.xpath(‘//div[@class=”entry-header”]/h1/text()’)[0]
article = html.xpath(‘//div[@id=”article_content”]’)[0]
print({“title”: title, “article”: etree.tostring(article, encoding=”utf8″).decode(“utf8”)})
if __name__ == ‘__main__’:
jobbole = JobBole()
jobbole.get_html()
