其他
猫眼电影TOP100爬取练习
(点击上方Python爱好者社区，可快速关注)
蒋蜀黍，Python爱好者社区专栏作者
博客地址：https://ask.hellobi.com/blog/JiangYiXin
首先说声抱歉，昨天小编在火车上，所以没法用电脑推送文章啦~
本文参考文末崔老师课程创作而来
本次练习使用到的知识点
Requsts 库的基本使用
正则表达式的使用
Python3写入CSV
1、项目流程分析
2、中心调度
# 主调度程序
def main():
# 起始URL
start_url = 'http://maoyan.com/board/4'
for i in range(0,100,10):
# 获取响应文本内容
html = get_one_page(url=start_url, offset=i)
if html is None:
print('链接:%s?offset=%s异常'.format(start_url,i))
continue
for item in parse_one_page(html=html):
store_data(item)
download_thumb(item['title'],item['thumb'])
3、页面内容获取
# 请求一个页面返回响应内容
def get_one_page(url, offset):
try:
response = requests.get(url=url,params={'offset':offset})
if response.status_code == 200:
return response.text
else:
return None
except RequestException as e:
return None
4、页面解析
# 解析一个页面
defparse_one_page(html):
pattern = '<dd>.*?board-index.*?">(\d+)</i>.*?data-src="(.*?)".*?/>.*?movie-item-info.*?title="(.*?)".*?star">'+\
'(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(\d+)</i>.*?</dd>'
# re.S匹配多行
regex = re.compile(pattern,re.S)
items = regex.findall(html)
foritem initems:
yield{
'index':item[0],
'thumb':get_large_thumb(item[1]),
'title':item[2],
'actors':item[3].strip()[3:],
'release_time':get_release_time(item[4].strip()[5:]),
'area':get_release_area(item[4].strip()[5:]),
'score':item[5]+item[6]
}
5、数据处理函数
# 获取上映时间
defget_release_time(data):
pattern = '^(.*?)(\(|$)'
regex = re.compile(pattern)
w = regex.search(data)
returnw.group(1)
# 获取上映地区
defget_release_area(data):
pattern = '.*\((.*)\)'
regex = re.compile(pattern)
w = regex.search(data)
ifw is None:
return'未知'
returnw.group(1)
# 获取封面大图
defget_large_thumb(url):
pattern = '(.*?)@.*?'
regex = re.compile(pattern)
w = regex.search(url)
returnw.group(1)6、数据存储
# 存储数据
defstore_data(item):
withopen('movie.csv','a',newline='',encoding='utf-8') asdata_csv:
# dialect为打开csv文件的方式，默认是excel，delimiter="\t"参数指写入的时候的分隔符
try:
csv_writer = csv.writer(data_csv)
csv_writer.writerow([item['index'], item['thumb'], item['title'], item['actors'],item['release_time'],item['area'],item['score']])
exceptException ase:
print(e)
print(item)
# 下载封面图
def download_thumb(title,url):
try:
response = requests.get(url=url)
# 获取二进制数据
with open('thumb/'+title+'.jpg', 'wb') as f:
f.write(response.content)
f.close()
except RequestException as e:
print(e)
pass7、完整运行代码
#!/usr/bin/python
# -*- coding: utf-8 -*-
importrequests
importre
importcsv
fromrequests.exceptions importRequestException
# 请求一个页面返回响应内容
defget_one_page(url, offset):
try:
response = requests.get(url=url,params={'offset':offset})
ifresponse.status_code == 200:
returnresponse.text
else:
return None
exceptRequestException ase:
return None
# 解析一个页面
defparse_one_page(html):
pattern = '<dd>.*?board-index.*?">(\d+)</i>.*?data-src="(.*?)".*?/>.*?movie-item-info.*?title="(.*?)".*?star">'+\
'(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(\d+)</i>.*?</dd>'
# re.S匹配多行
regex = re.compile(pattern,re.S)
items = regex.findall(html)
foritem initems:
yield{
'index':item[0],
'thumb':get_large_thumb(item[1]),
'title':item[2],
'actors':item[3].strip()[3:],
'release_time':get_release_time(item[4].strip()[5:]),
'area':get_release_area(item[4].strip()[5:]),
'score':item[5]+item[6]
}
# 获取上映时间
defget_release_time(data):
pattern = '^(.*?)(\(|$)'
regex = re.compile(pattern)
w = regex.search(data)
returnw.group(1)
# 获取上映地区
defget_release_area(data):
pattern = '.*\((.*)\)'
regex = re.compile(pattern)
w = regex.search(data)
ifw is None:
return'未知'
returnw.group(1)
# 获取封面大图
defget_large_thumb(url):
pattern = '(.*?)@.*?'
regex = re.compile(pattern)
w = regex.search(url)
returnw.group(1)
# 存储数据
defstore_data(item):
withopen('movie.csv','a',newline='',encoding='utf-8') asdata_csv:
# dialect为打开csv文件的方式，默认是excel，delimiter="\t"参数指写入的时候的分隔符
try:
csv_writer = csv.writer(data_csv)
csv_writer.writerow([item['index'], item['thumb'], item['title'], item['actors'],item['release_time'],item['area'],item['score']])
exceptException ase:
print(e)
print(item)
# 下载封面图
defdownload_thumb(title,url):
try:
response = requests.get(url=url)
# 获取二进制数据
withopen('thumb/'+title+'.jpg', 'wb') asf:
f.write(response.content)
f.close()
exceptRequestException ase:
print(e)
pass
# 主调度程序
defmain():
# 起始URL
start_url = 'http://maoyan.com/board/4'
fori inrange(0,100,10):
# 获取响应文本内容
html = get_one_page(url=start_url, offset=i)
ifhtml is None:
print('链接:%s?offset=%s异常'.format(start_url,i))
continue
foritem inparse_one_page(html=html):
store_data(item)
download_thumb(item['title'],item['thumb'])
if__name__=='__main__':
main()8、运行结果
点击文章末尾阅读原文即可购买
点击上图可查看详细课程说明信息
关注公众号，“Python爱好者社区”，回复“爬虫”即可获取崔老师爬虫免费学习视频。
为大家提供与Python相关的最新技术和资讯。
长按指纹 > 识别图中二维码 > 添加关注