Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 743fddb

Browse files
feat: douban-spider-v & movie route
1 parent 49a35e1 commit 743fddb

File tree

16 files changed

+330
-10
lines changed

16 files changed

+330
-10
lines changed

‎front/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
node_modules
33
/dist/*
44
server/db/secret.js
5+
server/files/movies
56
package-lock.json
67

78
# Log files

‎front/package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
"cross-env": "^5.1.3",
6060
"cross-spawn": "^5.1.0",
6161
"css-loader": "^0.28.0",
62+
"douban-spider-v": "0.0.4",
6263
"ejs": "^2.5.7",
6364
"eslint": "4.13.0",
6465
"eslint-plugin-html": "4.0.1",
@@ -68,6 +69,7 @@
6869
"extract-text-webpack-plugin": "^3.0.0",
6970
"file-loader": "^1.1.4",
7071
"friendly-errors-webpack-plugin": "^1.6.1",
72+
"fs-extra": "^10.0.1",
7173
"html-webpack-plugin": "^2.30.1",
7274
"husky": "4.2.5",
7375
"js-md5": "^0.7.3",
@@ -81,6 +83,7 @@
8183
"morgan": "^1.9.0",
8284
"node-notifier": "^5.1.2",
8385
"node-sass": "^4.7.2",
86+
"node-schedule": "^2.1.0",
8487
"optimize-css-assets-webpack-plugin": "^3.2.0",
8588
"ora": "^1.2.0",
8689
"portfinder": "^1.0.13",

‎front/server.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ const ejs = require('ejs')
1010
const route = require('./server/api/')
1111
const compression = require('compression')
1212
const { createBundleRenderer } = require('vue-server-renderer')
13+
const { startSchedule } = require('./server/utils/schedule')
1314
const template = fs.readFileSync('./src/index.template.html', 'utf-8')
1415
const isProd = process.env.NODE_ENV === 'production'
1516
const server = express()
@@ -18,7 +19,8 @@ server.use(logger('dev')) //日志记录中间件,将请求信息打印在控
1819
server.use(bodyParser.json())
1920
server.use(bodyParser.urlencoded({ extended: true }))
2021
server.use(cookieParser())
21-
22+
// 开启定时任务
23+
startSchedule()
2224
//引入ejs模板引擎
2325
server.set('views', [path.join(__dirname, 'dist'), path.join(__dirname, 'static')])
2426
server.engine('.html', ejs.__express)

‎front/server/api/index.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ const news = require('./news')
1111
const reviseKey = require('./reviseKey')
1212
const donload = require('./donload')
1313
const category = require('./category')
14+
const movies = require('./movies')
1415

1516
// const confirmToken = require('../middleware/confirmToken')
1617

@@ -29,4 +30,5 @@ module.exports = app => {
2930
app.use(reviseKey)
3031
app.use(donload)
3132
app.use(category)
33+
app.use(movies)
3234
}

‎front/server/api/movies.js‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/**
2+
* @desc 豆瓣电影
3+
*/
4+
const express = require('express')
5+
const router = express.Router()
6+
const path = require('path')
7+
const fs = require('fs-extra')
8+
// const confirmToken = require('../middleware/confirmToken')
9+
10+
// 获取分类列表
11+
router.get('/api/front/douban/get', async (req, res) => {
12+
try {
13+
const page = req.query.page || 1
14+
const type = req.query.type || 'collect'
15+
const tp = path.join(__dirname, '..', `files/movies/${type}/${page}.json`)
16+
const exist = fs.existsSync(tp)
17+
18+
if (exist) {
19+
const doc = fs.readFileSync(tp, 'utf8')
20+
const pageTotal = fs.readFileSync(path.join(__dirname, '..', `files/movies/${type}/pageTotal.txt`), 'utf8')
21+
res.json({
22+
status: 200,
23+
data: JSON.parse(doc),
24+
pageTotal: parseInt(pageTotal) || 1,
25+
info: '获取影视记录成功'
26+
})
27+
} else {
28+
res.json({
29+
status: 404,
30+
data: [],
31+
info: '数据不存在'
32+
})
33+
}
34+
} catch (e) {
35+
console.log('指令错误--->>>', e)
36+
res.status(500).end()
37+
}
38+
})
39+
40+
module.exports = router

‎front/server/utils/schedule.js‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/**
2+
* @desc 定时任务 - 爬取豆瓣电影
3+
*/
4+
5+
const schedule = require('node-schedule')
6+
const DoubanSpider = require('douban-spider-v')
7+
const fs = require('fs-extra')
8+
const path = require('path')
9+
const moviesPath = {
10+
getMovieCollect: path.join(__dirname, '../files/movies/collect'),
11+
getMovieWish: path.join(__dirname, '../files/movies/wish'),
12+
getMovieDo: path.join(__dirname, '../files/movies/do')
13+
}
14+
15+
let cache = {
16+
getMovieCollect: [],
17+
getMovieWish: [],
18+
getMovieDo: []
19+
}
20+
const douban = new DoubanSpider({
21+
uid: 'tan-mu'
22+
})
23+
24+
function startSchedule() {
25+
// 每天凌晨1点进行爬取
26+
schedule.scheduleJob('0 0 1 * * *', async () => {
27+
// schedule.scheduleJob('0 50 18 * * *', async () => {
28+
console.log('定时任务触发------>>>>>>>')
29+
getMovies()
30+
})
31+
}
32+
async function getMovies() {
33+
await handleMovies('getMovieCollect')
34+
await sleep()
35+
await handleMovies('getMovieWish')
36+
await sleep()
37+
await handleMovies('getMovieDo')
38+
}
39+
40+
async function handleMovies(method) {
41+
try {
42+
const res = await douban[method]()
43+
cache[method].push(res.data)
44+
console.log('第1页爬取成功====>>>>>')
45+
if (res.page.totalPage > 1) {
46+
// 保存总页码数
47+
fs.writeFileSync(`${moviesPath[method]}/pageTotal.txt`, res.page.totalPage + '', 'utf8')
48+
49+
for (let i = 2; i <= res.page.totalPage; i++) {
50+
// for (let i = 2; i <= 3; i++) {
51+
// 爬取速度1分钟1页,避免触发反爬
52+
await sleep()
53+
const res = await douban[method](i)
54+
cache[method].push(res.data)
55+
console.log(`第${i}页爬取成功====>>>>>`)
56+
}
57+
}
58+
// 写入json文件
59+
cache[method].forEach((doc, index) => {
60+
fs.ensureDirSync(moviesPath[method])
61+
fs.writeFileSync(`${moviesPath[method]}/${index + 1}.json`, JSON.stringify(doc), 'utf8')
62+
})
63+
} catch (e) {
64+
console.log('爬虫解析错误---->>>>', e)
65+
cache = {
66+
getMovieCollect: [],
67+
getMovieWish: [],
68+
getMovieDo: []
69+
}
70+
}
71+
}
72+
async function sleep(ms = 1000 * 30) {
73+
await new Promise(resolve => setTimeout(resolve, ms))
74+
}
75+
exports.startSchedule = startSchedule

‎front/src/api/entertainment.js‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
/**
2+
* @desc 娱乐
3+
* @author Justjokee
4+
*/
5+
import http from '@/http/'
6+
export default {
7+
// 获取电影、电视剧等影视记录
8+
getMovies(payload) {
9+
return http.get('/api/front/douban/get', payload).then(data => {
10+
return data
11+
})
12+
}
13+
}

‎front/src/api/index.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@ import comments from './comments'
44
import messageBoard from './messageBoard'
55
import category from './category'
66
import tags from './tags'
7+
import entertainment from './entertainment'
78

89
export default {
910
...article,
1011
...visitor,
1112
...comments,
1213
...messageBoard,
1314
...category,
14-
...tags
15+
...tags,
16+
...entertainment
1517
}
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
<doc>
2+
@desc: 评分
3+
@author: justJokee
4+
</doc>
5+
<template>
6+
<div class="rating">
7+
<i :class="star.icon" v-for="(star, index) in crating" :key="index"></i>
8+
</div>
9+
</template>
10+
<script>
11+
export default {
12+
name: 'rating',
13+
props: {
14+
rating: {
15+
type: Number,
16+
default: 0
17+
}
18+
},
19+
data() {
20+
return {}
21+
},
22+
computed: {
23+
crating() {
24+
this.rating
25+
const arr = []
26+
for (let i = 0; i < 5; i++) arr.push({ icon: 'el-icon-star-off' })
27+
arr.some((item, index) => {
28+
if (index + 1 <= this.rating) {
29+
item.icon = 'el-icon-star-on'
30+
} else return true
31+
})
32+
return arr
33+
}
34+
}
35+
}
36+
</script>
37+
<style lang="scss">
38+
@import '~@/style/index.scss';
39+
.rating {
40+
.el-icon-star-on {
41+
font-size: 17px;
42+
transform: translateY(0.5px);
43+
color: rgb(236, 169, 68);
44+
}
45+
}
46+
</style>

‎front/src/router/index.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ const archives = () => import('@/views/archives/')
99
const tags = () => import('@/views/tags/')
1010
const articleFilter = () => import('@/views/article-filter/')
1111
const category = () => import('@/views/category/')
12+
const movies = () => import('@/views/movies/')
1213

1314
Vue.use(Router)
1415
Vue.use(Meta)
@@ -60,6 +61,11 @@ export function createRouter() {
6061
path: '/app/articles/:type/:param',
6162
name: 'articleFilter',
6263
component: articleFilter
64+
},
65+
{
66+
path: '/app/movies',
67+
name: 'movies',
68+
component: movies
6369
}
6470
],
6571
scrollBehavior(to, from, savedPosition) {

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /