Skip to content

Navigation Menu

Sign in
Sign up

Commit 395dfd9

Browse files
committed
锤神3
1 parent 819d0f9 commit 395dfd9

5 files changed

Lines changed: 194 additions & 1 deletion

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,4 @@ ENV/
9090
浏览器模拟爬虫/.DS_Store
9191
.DS_Store
9292
.vscode/settings.json
93+
豆瓣影评/锤神3/.vscode/settings.json

‎doubanmovie/doubanspider.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ def cached_url(url):
5454
else:
5555
# 建立 cached 文件夹
5656
if not os.path.exists(folder):
57-
os.makedirs(folder)
57+
os.makedir(folder)
5858
# 发送网络请求,把结果/二进制写入文件
5959
r = requests.get(url)
6060
with open(path, 'wb') as f:

‎豆瓣影评/锤神3/config.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
2+
EHCO_DB = {
3+
'host': '127.0.0.1',
4+
'user': 'root',
5+
'password': '19960202',
6+
'db': 'EhcoTestDb'
7+
}

‎豆瓣影评/锤神3/spider.py‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
'''
2+
爬取豆瓣影评
3+
雷神3的所有评价
4+
并存入数据库
5+
'''
6+
7+
import time
8+
import os
9+
10+
11+
import requests
12+
from bs4 import BeautifulSoup
13+
from http.cookies import SimpleCookie
14+
15+
from stroe import DbToMysql
16+
import config
17+
18+
request_url = 'https://movie.douban.com/subject/25821634/comments?start={}&limit=20'
19+
20+
HEADERS = {
21+
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36',
22+
}
23+
24+
COOKIES = '''
25+
bid=i2jh3YGuvEo; ll="118163"; gr_user_id=f52deadb-5f46-491c-9b52-4294ab176b90; viewed="1430904_25806793_25901403"; ps=y; dbcl2="169273073:UOYgsqmzhSs"; ck=m7xV; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1509972471%2C%22https%3A%2F%2Fwww.douban.com%2Fsearch%3Fsource%3Dsuggest%26q%3D%25E9%259B%25B7%25E7%25A5%259E3%22%5D; ct=y; _vwo_uuid_v2=4EE59BFD9BF5C48E3E9020C6DE3564D4|d2deb1b903e06e32351743192190c582; ap=1; _pk_id.100001.4cf6=0377929d7299aea4.1508405769.14.1509975906.1509934106.; _pk_ses.100001.4cf6=*; __utma=30149280.341450299.1508041022.1509932565.1509972306.17; __utmb=30149280.32.10.1509972306; __utmc=30149280; __utmz=30149280.1509972306.17.13.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); __utmv=30149280.16927; __utma=223695111.1000856564.1508405769.1509932565.1509972471.14; __utmb=223695111.0.10.1509972471; __utmc=223695111; __utmz=223695111.1509972471.14.10.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/search; push_noty_num=0; push_doumail_num=0
26+
'''
27+
28+
29+
def format_cookie(text):
30+
'''将字符串转换为字典形式的cookies'''
31+
cookie = SimpleCookie(text)
32+
return {i.key: i.value for i in cookie.values()}
33+
34+
35+
def get_html_text(url, header={}, cookies={}):
36+
'''
37+
下载网页数据
38+
返回文本文件
39+
'''
40+
try:
41+
# 使用Session来最会话管理
42+
s = requests.Session()
43+
s.headers.update(header)
44+
s.cookies.update(cookies)
45+
r = s.get(url)
46+
r.raise_for_status
47+
return r.content
48+
except:
49+
return -1
50+
51+
52+
def parse_detail(html):
53+
'''解析影评内容'''
54+
results = []
55+
try:
56+
soup = BeautifulSoup(html, 'lxml')
57+
comments = soup.find_all('div', class_='comment-item')
58+
for comment in comments:
59+
info = comment.find('span', class_='comment-info')
60+
name = info.contents[1].get_text().strip()
61+
try:
62+
# 针对没有评星的情况特殊处理
63+
star = info.contents[5]['title']
64+
time = info.contents[7].get_text().strip()
65+
except:
66+
star = '暂无评分'
67+
time = info.contents[5].get_text().strip()
68+
vote = comment.find('span', class_='votes').text.strip()
69+
content = comment.find('p').get_text().strip()
70+
results.append({
71+
'name': name, # 作者名
72+
'star': star, # 推荐程度
73+
'time': time, # 时间
74+
'vote': vote, # 赞同数
75+
'content': content # 影评内容
76+
})
77+
return results
78+
except:
79+
print('内容解析错误')
80+
return -1
81+
82+
83+
def cached_url(url):
84+
'''将访问过的url缓存到本地'''
85+
folder = 'cached_url'
86+
filename = url.split('?')[1].split('&')[0].split('=')[1] + '.html'
87+
path = os.path.join(folder, filename)
88+
89+
if os.path.exists(path):
90+
with open(path, 'rb') as f:
91+
s = f.read()
92+
return s
93+
else:
94+
if not os.path.exists(folder):
95+
os.mkdir(folder)
96+
html = get_html_text(url, HEADERS, format_cookie(COOKIES))
97+
if html != -1:
98+
with open(path, 'wb') as f:
99+
f.write(html)
100+
return html
101+
else:
102+
print('{}下载失败'.format(filename))
103+
return -1
104+
105+
106+
def main():
107+
store = DbToMysql(config.EHCO_DB)
108+
for i in range(14940, 20001, 20):
109+
html = cached_url(request_url.format(i))
110+
time.sleep(3)
111+
if html != -1:
112+
res_list = parse_detail(html)
113+
if res_list != -1:
114+
for data in res_list:
115+
store.save_one_data('GodOfHammer', data)
116+
print('第{}页保存完毕'.format(i))
117+
store.close()
118+
119+
120+
if __name__ == '__main__':
121+
main()

‎豆瓣影评/锤神3/stroe.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
'''
2+
将数据存入数据库模块
3+
'''
4+
5+
import pymysql.cursors
6+
7+
import config
8+
9+
10+
class DbToMysql():
11+
'''封装对数据库的操作'''
12+
13+
def __init__(self, configs):
14+
self.con = pymysql.connect(
15+
host=configs['host'],
16+
user=configs['user'],
17+
password=configs['password'],
18+
db=configs['db'],
19+
charset='utf8mb4',
20+
cursorclass=pymysql.cursors.DictCursor
21+
)
22+
23+
def close(self):
24+
'''关闭数据库链接'''
25+
self.con.close()
26+
27+
def save_one_data(self, table, data,):
28+
'''
29+
将一条记录保存到数据库
30+
Args:
31+
table: 表名字 str
32+
data: 记录 dict
33+
每条记录都以一个字典的形式传进来
34+
'''
35+
key_map = {}
36+
37+
if len(data) == 0:
38+
return -1
39+
40+
fields = ''
41+
values = ''
42+
datas = {}
43+
for k, v in data.items():
44+
# 防止sql注入
45+
datas.update({k: pymysql.escape_string(v)})
46+
47+
for d in datas:
48+
fields += "`{}`,".format(str(d))
49+
values += "'%s'," % (str(data[d]))
50+
if len(fields) <= 0 or len(values) <= 0:
51+
return -1
52+
# 生成sql语句
53+
sql = "insert ignore into {}({}) values({})".format(
54+
table, fields[:-1], values[:-1])
55+
56+
try:
57+
with self.con.cursor() as cursor:
58+
# 执行语句
59+
cursor.execute(sql)
60+
self.con.commit()
61+
res = cursor.fetchone()
62+
return res
63+
except:
64+
print('数据库保存错误')

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /