diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2eea525 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/.env b/.env new file mode 100644 index 0000000..d3ea800 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +FLASK_ENV=production +DATABASE_URL=example_url +PUSHOVER_API_TOKEN=example_token +NAME=Watch \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/----.md similarity index 100% rename from .github/ISSUE_TEMPLATE/question.md rename to .github/ISSUE_TEMPLATE/----.md diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index ae5f271..15ebcea 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -30,7 +30,7 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} run: | docker buildx build \ - --platform linux/amd64,linux/arm64 \ + --platform=linux/amd64 \ --output "type=image,push=true" \ --file ./Dockerfile . \ --tag $(echo "${DOCKER_USERNAME}" | tr '[:upper:]' '[:lower:]')/webmonitor:latest diff --git a/.gitignore b/.gitignore index 53e9ed8..9b08107 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,14 @@ -.vscode/* -setting/management/__pycache__/* -db/db.sqlite3 -static/* -task/utils/notification/__pycache__/* -setting/management/commands/__pycache__/* -setting/migrations/__pycache__/* -task/migrations/__pycache__/* -task/utils/__pycache__/* -task/utils/selector/__pycache__/* -*/__pycache__ -.env -ghostdriver.log + +*.sqlite + +__pycache__/ + +log/ + +\.env + +.vscode/ + +ghostdriver\.log + +app/static/error/screenshot\.png diff --git a/Dockerfile b/Dockerfile index 3d6280b..fac914e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,50 @@ -FROM python:3.6-slim-buster +FROM ubuntu:16.04 ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 ENV PORT 5000 -ENV USERNAME admin -ENV PASSWORD admin -ENV OPENSSL_CONF /etc/ssl/ -COPY . /app +ADD . /app WORKDIR /app -RUN set -x; buildDeps='wget build-essential' \ -&& apt-get update && apt-get install -y ${buildDeps} \ -chrpath libssl-dev libxft-dev libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \ -&& rm -rf /var/lib/apt/lists/* \ -&& export OS_ARCH=$(uname -m) \ -&& wget https://github.com/mjysci/phantomjs/releases/download/v2.1.1/phantomjs-2.1.1-linux_${OS_ARCH}.tar.gz -O /tmp/phantomjs-2.1.1-linux_${OS_ARCH}.tar.gz \ -&& tar -xzvf /tmp/phantomjs-2.1.1-linux_${OS_ARCH}.tar.gz -C /usr/local/bin \ -&& rm /tmp/phantomjs-2.1.1-linux_${OS_ARCH}.tar.gz \ -&& pip install -r requirements.txt && pip cache purge \ -&& apt-get purge -y --auto-remove $buildDeps +# 安装 python3.6 +RUN apt-get update \ +&& apt-get install gcc -y\ +&& apt-get install g++ -y\ +&& apt-get install gdb -y\ +&& apt-get install libxml2-dev libxslt-dev -y\ +&& apt-get install python-software-properties -y\ +&& apt-get install software-properties-common -y\ +&& apt-get install libffi-dev -y\ +&& apt-get install libssl-dev -y\ +&& add-apt-repository ppa:deadsnakes/ppa -y\ +&& apt-get update \ +&& apt-get install python3.6-dev -y\ +&& apt-get install python3.6 -y\ +&& rm /usr/bin/python\ +&& ln -s /usr/bin/python3.6 /usr/bin/python\ +&& rm /usr/bin/python3\ +&& ln -s /usr/bin/python3.6 /usr/bin/python3\ +&& apt-get install python3-pip -y\ +&& pip3 install pip -U\ +&& rm /usr/bin/pip3 \ +&& ln -s -f /usr/local/bin/pip3 /usr/bin/pip3\ +&& ln -s -f /usr/local/bin/pip3 /usr/bin/pip + +RUN pip install -r requirements.txt + +RUN apt-get update\ +&& apt-get install wget -y\ +&& apt-get install build-essential chrpath libssl-dev libxft-dev -y\ +&& apt-get install libfreetype6 libfreetype6-dev -y\ +&& apt-get install libfontconfig1 libfontconfig1-dev -y\ +&& export PHANTOM_JS="phantomjs-2.1.1-linux-x86_64"\ +&& wget https://github.com/Medium/phantomjs/releases/download/v2.1.1/$PHANTOM_JS.tar.bz2 -O /tmp/$PHANTOM_JS.tar.bz2 \ +&& tar xvjf /tmp/$PHANTOM_JS.tar.bz2 -C /usr/local/share\ +&& ln -sf /usr/local/share/$PHANTOM_JS/bin/phantomjs /usr/local/bin\ +&& rm /tmp/$PHANTOM_JS.tar.bz2 EXPOSE $PORT -RUN chmod +x run.sh -CMD ./run.sh $PORT $USERNAME $PASSWORD \ No newline at end of file +CMD python -m flask run -h 0.0.0.0 -p $PORT diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..fc272ab --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: gunicorn wsgi:app \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..bf6d29c --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# @Author: LogicJake, Jacob +# @Date: 2019年02月15日 19:33:23 +# @Last Modified time: 2020年03月01日 14:59:53 +import os +import pymysql +from flask import Flask +from flask_admin import Admin, AdminIndexView +from flask_apscheduler import APScheduler +from flask_babelex import Babel +from flask_bootstrap import Bootstrap +from flask_login import LoginManager +from flask_sqlalchemy import SQLAlchemy + +from app.model_views.mail_setting_view import MailSettingView +from app.model_views.notification_view import NotificationView +from app.model_views.rss_task_view import RSSTaskView +from app.model_views.task_status_view import TaskStatusView +from app.model_views.task_view import TaskView +from app.model_views.user_view import UserView + +# 修复时区问题 +from apscheduler.schedulers.background import BackgroundScheduler +pymysql.install_as_MySQLdb() + +db = SQLAlchemy() +login = LoginManager() +siteName = os.getenv('NAME') +admin = Admin(name=siteName, template_mode='bootstrap3') +scheduler = APScheduler(BackgroundScheduler(timezone="Asia/Shanghai")) +app = Flask(__name__) +bootstrap = Bootstrap() + + +def create_app(config_name): + from config import config, logger + app.config.from_object(config[config_name]) + + # 中文化 + Babel(app) + + # 注册flask-login + login.init_app(app) + + # bootstrap + bootstrap.init_app(app) + + @login.user_loader + def load_user(user_id): + return User.query.get(user_id) + + # 注册蓝图 + from app.main.views import bp as main_bp + app.register_blueprint(main_bp) + + # 注册数据库 + db.init_app(app) + + # 注册flask-admin + admin.init_app(app, + index_view=AdminIndexView(template='admin/index.html', + url='/')) + + # 注册APScheduler + scheduler.init_app(app) + scheduler.start() + + # 视图 + from app.models.task import Task + from app.models.rss_task import RSSTask + from app.models.mail_setting import MailSetting + from app.models.notification import Notification + from app.models.task_status import TaskStatus + from app.models.user import User + + admin.add_view(TaskStatusView(TaskStatus, db.session, name='任务状态')) + admin.add_view(TaskView(Task, db.session, name='网页监控任务管理')) + admin.add_view(RSSTaskView(RSSTask, db.session, name='RSS监控任务管理')) + admin.add_view(NotificationView(Notification, db.session, name='通知方式管理')) + admin.add_view(MailSettingView(MailSetting, db.session, name='系统邮箱设置')) + admin.add_view(UserView(User, db.session, name='账号密码管理')) + + with app.test_request_context(): + db.create_all() + mail_setting = MailSetting.query.first() + # 插入默认邮箱配置 + if mail_setting is None: + mail_setting = MailSetting() + db.session.add(mail_setting) + db.session.commit() + + # 初始化账号密码 + user = User.query.first() + if user is None: + user = User('admin', 'admin') + db.session.add(user) + db.session.commit() + + # 插入默认通知方式 + notis = Notification.query.all() + mail_exist = False + wechat_exist = False + pushover_exist = False + + if len(notis) != 0: + for noti in notis: + if noti.type == 'mail': + mail_exist = True + if noti.type == 'wechat': + wechat_exist = True + if noti.type == 'pushover': + pushover_exist = True + + if not mail_exist: + mail_noti = Notification('mail') + db.session.add(mail_noti) + db.session.commit() + if not wechat_exist: + wechat_noti = Notification('wechat') + db.session.add(wechat_noti) + db.session.commit() + if not pushover_exist: + pushover_noti = Notification('pushover') + db.session.add(pushover_noti) + db.session.commit() + + # 在系统重启时重启任务 + from app.main.scheduler import add_job + task_statuss = TaskStatus.query.all() + count = 0 + for task_status in task_statuss: + if task_status.task_status == 'run': + count += 1 + task_id = task_status.task_id + if task_status.task_type == 'html': + task = Task.query.filter_by(id=task_id).first() + add_job(task.id, task.frequency) + logger.info('重启task_' + str(task_id)) + elif task_status.task_type == 'rss': + task = RSSTask.query.filter_by(id=task_id).first() + add_job(task_id, task.frequency, 'rss') + logger.info('重启task_rss' + str(task_id)) + logger.info('重启{}个任务'.format(count)) + return app diff --git a/app/main/__init__.py b/app/main/__init__.py new file mode 100644 index 0000000..465e41c --- /dev/null +++ b/app/main/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# @Author: LogicJake +# @Date: 2019年02月15日 19:39:29 +# @Last Modified time: 2019年02月15日 20:04:29 diff --git a/app/main/extract_info.py b/app/main/extract_info.py new file mode 100644 index 0000000..3cee93d --- /dev/null +++ b/app/main/extract_info.py @@ -0,0 +1,60 @@ +import re + +import feedparser +from func_timeout import func_set_timeout + +from app.main.selector.selector_handler import new_handler +from config import logger + + +def extract_by_re(conetnt, regular_expression): + m = re.search(regular_expression, conetnt) + + if m: + return m.groups()[0] + else: + logger.error('{} 无法使用正则提取'.format(regular_expression)) + raise Exception('无法使用正则提取') + + +def get_content(url, + is_chrome, + selector_type, + selector, + regular_expression=None, + headers=None, + debug=False): + if is_chrome == 'no': + selector_handler = new_handler('request', debug) + else: + selector_handler = new_handler('phantomjs', debug) + + if selector_type == 'xpath': + content = selector_handler.get_by_xpath(url, selector, headers) + elif selector_type == 'css': + content = selector_handler.get_by_css(url, selector, headers) + elif selector_type == 'json': + content = selector_handler.get_by_json(url, selector, headers) + else: + logger.error('无效选择器') + raise Exception('无效选择器') + + if regular_expression: + content = extract_by_re(content, regular_expression) + return content + + +@func_set_timeout(5) +def get_rss_content(url): + feeds = feedparser.parse(url) + + if len(feeds.entries) == 0: + raise Exception('无内容') + + single_post = feeds.entries[0] + item = {} + item['title'] = single_post.title + item['link'] = single_post.link + item['guid'] = single_post.id + + return item diff --git a/app/main/forms/login_form.py b/app/main/forms/login_form.py new file mode 100644 index 0000000..1c99335 --- /dev/null +++ b/app/main/forms/login_form.py @@ -0,0 +1,10 @@ +from flask_wtf import FlaskForm +from wtforms import StringField, PasswordField, BooleanField, SubmitField +from wtforms.validators import DataRequired + + +class LoginForm(FlaskForm): + username = StringField('用户名', validators=[DataRequired("请输入帐号!")]) + password = PasswordField('密码', validators=[DataRequired("请输入密码!")]) + remember_me = BooleanField('记住我', default=False) + submit = SubmitField('登录') diff --git a/app/main/forms/test_from.py b/app/main/forms/test_from.py new file mode 100644 index 0000000..8962cdd --- /dev/null +++ b/app/main/forms/test_from.py @@ -0,0 +1,26 @@ +import requests +from flask_wtf import FlaskForm +from wtforms import SelectField, StringField, SubmitField +from wtforms.validators import DataRequired, ValidationError + + +def check_url(form, field): + url = form.url.data + try: + requests.get(url, timeout=10) + except Exception as e: + raise ValidationError(repr(e)) + + +class TestForm(FlaskForm): + url = StringField('监控网址', validators=[DataRequired("请输入网址!"), check_url]) + selector_type = SelectField('元素选择器类型', + choices=[('xpath', 'xpath'), + ('css', 'css selector'), + ('json', 'Jsonpath')]) + selector = StringField('元素选择器', validators=[DataRequired("请输入元素选择器!")]) + is_chrome = SelectField('是否使用无头浏览器', + choices=[('no', 'no'), ('yes', 'yes')]) + regular_expression = StringField('正则表达式') + headers = StringField('自定义请求头') + submit = SubmitField('提取信息') diff --git a/setting/management/__init__.py b/app/main/notification/__init__.py similarity index 100% rename from setting/management/__init__.py rename to app/main/notification/__init__.py diff --git a/task/utils/notification/mail_notification.py b/app/main/notification/mail_notification.py similarity index 60% rename from task/utils/notification/mail_notification.py rename to app/main/notification/mail_notification.py index 0ba9326..5b44782 100644 --- a/task/utils/notification/mail_notification.py +++ b/app/main/notification/mail_notification.py @@ -1,30 +1,33 @@ -import logging +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake +@Date: 2019年03月24日 20:23:33 +@LastEditTime: 2019年03月31日 22:01:44 +''' import smtplib -import traceback from email.header import Header from email.mime.text import MIMEText -from setting.models import SystemMailSetting -from task.utils.notification.notification import Notification - -logger = logging.getLogger('main') +from app import db +from app.main.notification.notification import Notification +from app.models.mail_setting import MailSetting +from config import logger class MailNotification(Notification): def __init__(self): - try: - setting = SystemMailSetting.objects.first() - except Exception: + setting = db.session.query(MailSetting).first() + if setting.mail_sender != '默认用户名@mail.com': + self.mail_server = setting.mail_server + self.mail_port = setting.mail_port + self.mail_username = setting.mail_username + self.mail_sender = setting.mail_sender + self.mail_password = setting.mail_password + else: logger.error('没有设置系统邮箱,无法发送邮件通知') - logger.error(traceback.format_exc()) raise Exception('没有设置系统邮箱,无法发送邮件通知') - self.mail_server = setting.mail_server - self.mail_port = setting.mail_port - self.mail_username = setting.mail_username - self.mail_sender = setting.mail_sender - self.mail_password = setting.mail_password - def send(self, to, header, content): if to == '默认': logger.error('没有设置通知邮箱,无法发送邮件通知') diff --git a/task/utils/notification/notification.py b/app/main/notification/notification.py similarity index 100% rename from task/utils/notification/notification.py rename to app/main/notification/notification.py diff --git a/app/main/notification/notification_handler.py b/app/main/notification/notification_handler.py new file mode 100644 index 0000000..432100a --- /dev/null +++ b/app/main/notification/notification_handler.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake, Jacob +@Date: 2019年03月24日 20:27:55 +@LastEditTime: 2020年03月01日 15:00:54 +''' +from app.main.notification.mail_notification import MailNotification +from app.main.notification.wechat_notification import WechatNotification +from app.main.notification.pushover_notification import PushoverNotification +from config import logger + + +def new_handler(name): + if name == 'mail': + return MailNotification() + elif name == 'wechat': + return WechatNotification() + elif name == 'pushover': + return PushoverNotification() + else: + logger.error('通知方式错误') + raise Exception('通知方式错误') diff --git a/app/main/notification/pushover_notification.py b/app/main/notification/pushover_notification.py new file mode 100755 index 0000000..d814c9b --- /dev/null +++ b/app/main/notification/pushover_notification.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: Jacob +@Date: 2020年03月01日 15:01:07 +@LastEditTime: 2020年03月01日 15:01:10 +''' +import os +import json +import requests + +from app.main.notification.notification import Notification +from config import logger + + +class PushoverNotification(Notification): + def send(self, to, header, content): + if to == '默认': + logger.error('没有设置Prushover User Key,无法发送推送通知') + raise Exception('没有设置Prushover User Key,无法发送推送通知') + token = os.getenv('PUSHOVER_API_TOKEN') + sendData = { + 'token': token, # 监控猫 Api Token + 'user': to, + 'message': '【' + header + '】有更新!\n>>>新内容为:\n' + content, + } + pushoverApi = 'https://api.pushover.net/1/messages.json' + + try: + response = requests.post(pushoverApi, sendData, timeout=5) + except requests.exceptions.RequestException as e: + logger.error('请求错误') + raise Exception('Error: {}'.format(e)) + + res = json.loads(response.text) + + if res['status'] != 1: + raise Exception(res['errors']) diff --git a/task/utils/notification/wechat_notification.py b/app/main/notification/wechat_notification.py similarity index 56% rename from task/utils/notification/wechat_notification.py rename to app/main/notification/wechat_notification.py index cde6aa8..bbc6fc0 100644 --- a/task/utils/notification/wechat_notification.py +++ b/app/main/notification/wechat_notification.py @@ -1,12 +1,16 @@ - +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake +@Date: 2019年03月24日 20:23:33 +@LastEditTime: 2019年03月26日 21:32:59 +''' import json -import logging import requests -from task.utils.notification.notification import Notification - -logger = logging.getLogger('main') +from app.main.notification.notification import Notification +from config import logger class WechatNotification(Notification): @@ -15,9 +19,9 @@ def send(self, to, header, content): logger.error('没有设置Server酱 SCKEY,无法发送微信通知') raise Exception('没有设置Server酱 SCKEY,无法发送微信通知') data = {'text': header, 'desp': content} - url = 'https://sctapi.ftqq.com/{}.send'.format(to) + url = 'https://sc.ftqq.com/{}.send'.format(to) r = requests.post(url, data=data) res = json.loads(r.text) - if res['data']['errno'] != 0: - raise Exception(res['data']['errmsg']) + if res['errno'] != 0: + raise Exception(res['errmsg']) diff --git a/task/utils/rule.py b/app/main/rule.py similarity index 67% rename from task/utils/rule.py rename to app/main/rule.py index ac1fbb1..21ac8ec 100644 --- a/task/utils/rule.py +++ b/app/main/rule.py @@ -1,27 +1,15 @@ -def parse_without(args, content, last_content): - ''' - 新内容中是否不包含某个字符串 - -without 上架 - ''' - if args[0] != '-without': - return False - - value = args[1] - - if value not in content: - return True - return False - - def parse_contain(args, content, last_content): ''' 新内容中是否包含某个字符串 -contain 上架 ''' - if args[0] != '-contain': + try: + key_index = args.index('-contain') + except ValueError: return False - value = args[1] + value_index = key_index + 1 + value = args[value_index] if value in content: return True @@ -34,13 +22,13 @@ def parse_increase(args, content, last_content): -increase 3 content, last_content和参数值都应该为数值型,否则会抛出异常 ''' - if args[0] != '-increase': - return False - - if last_content == '': + try: + key_index = args.index('-increase') + except ValueError: return False - value = args[1] + value_index = key_index + 1 + value = args[value_index] last_content = float(last_content) content = float(content) @@ -57,13 +45,13 @@ def parse_decrease(args, content, last_content): -decrease 3 content, last_content和参数值都应该为数值型,否则会抛出异常 ''' - if args[0] != '-decrease': - return False - - if last_content == '': + try: + key_index = args.index('-decrease') + except ValueError: return False - value = args[1] + value_index = key_index + 1 + value = args[value_index] last_content = float(last_content) content = float(content) @@ -80,10 +68,13 @@ def parse_equal(args, content, last_content): -equal 3 content和参数值都应该为数值型,否则会抛出异常 ''' - if args[0] != '-equal': + try: + key_index = args.index('-equal') + except ValueError: return False - value = args[1] + value_index = key_index + 1 + value = args[value_index] content = float(content) value = float(value) @@ -99,10 +90,13 @@ def parse_less(args, content, last_content): -less 3 content和参数值都应该为数值型,否则会抛出异常 ''' - if args[0] != '-less': + try: + key_index = args.index('-less') + except ValueError: return False - value = args[1] + value_index = key_index + 1 + value = args[value_index] content = float(content) value = float(value) @@ -118,10 +112,13 @@ def parse_more(args, content, last_content): -less 3 content和参数值都应该为数值型,否则会抛出异常 ''' - if args[0] != '-more': + try: + key_index = args.index('-more') + except ValueError: return False - value = args[1] + value_index = key_index + 1 + value = args[value_index] content = float(content) value = float(value) @@ -132,8 +129,8 @@ def parse_more(args, content, last_content): rule_funs = [ - parse_without, parse_contain, parse_increase, parse_decrease, parse_equal, - parse_less, parse_more + parse_contain, parse_increase, parse_decrease, parse_equal, parse_less, + parse_more ] @@ -141,17 +138,15 @@ def parse_more(args, content, last_content): # 1 有变化但没有触发规则(更新content 但不发送) # 2 有变化且触发规则(更新content 发送) # 3 有变化没有设置规则(更新content 发送) -def is_changed(rules, content, last_content): +def is_changed(rule, content, last_content): if last_content is not None and last_content == content: return 0 else: - if rules: - rules = rules.split(';') - for rule in rules: - args = rule.split(' ') - for rule_fun in rule_funs: - if rule_fun(args, content, last_content): - return 2 + if rule: + args = rule.split(' ') + for rule_fun in rule_funs: + if rule_fun(args, content, last_content): + return 2 return 1 else: return 3 diff --git a/app/main/scheduler.py b/app/main/scheduler.py new file mode 100644 index 0000000..e5aea40 --- /dev/null +++ b/app/main/scheduler.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake +@Date: 2019年03月24日 14:32:34 +@LastEditTime: 2020年03月16日 13:41:46 +''' +import traceback +from datetime import datetime + +import markdown +from apscheduler.jobstores.base import JobLookupError +from func_timeout.exceptions import FunctionTimedOut + +from app import app, db, scheduler +from app.main.extract_info import get_content, get_rss_content +from app.main.rule import is_changed +from app.models.content import Content +from app.models.notification import Notification +from app.models.rss_task import RSSTask +from app.models.task import Task +from app.models.task_status import TaskStatus +from config import logger + + +# 部分通知方式出错异常 +class PartNotificationError(Exception): + pass + + +def wraper_rss_msg(item): + title = item['title'] + link = item['link'] + + res = '''[{}]({})'''.format(title, link) + return res + + +def wraper_msg(content, link): + res = '''[{}]({})'''.format(content, link) + return res + + +def send_message(content, header, mail, wechat, pushover): + from app.main.notification.notification_handler import new_handler + + total = 0 + fail = 0 + + exception_content = '' + try: + if mail == 'yes': + total += 1 + handler = new_handler('mail') + mail_info = Notification.query.filter_by(type='mail').first() + mail_address = mail_info.number + content = markdown.markdown(content, + output_format='html5', + extensions=['extra']) + handler.send(mail_address, header, content) + except Exception as e: + fail += 1 + exception_content += 'Mail Exception: {};'.format(repr(e)) + + try: + if wechat == 'yes': + total += 1 + handler = new_handler('wechat') + wechat_info = Notification.query.filter_by(type='wechat').first() + key = wechat_info.number + handler.send(key, header, content) + except Exception as e: + fail += 1 + exception_content += 'Wechat Exception: {};'.format(repr(e)) + + try: + if pushover == 'yes': + total += 1 + handler = new_handler('pushover') + pushover_info = Notification.query.filter_by( + type='pushover').first() + key = pushover_info.number + handler.send(key, header, content) + except Exception as e: + fail += 1 + exception_content += 'Pushover Exception: {};'.format(repr(e)) + + if fail> 0: + if fail < total: + raise PartNotificationError(exception_content) + else: + raise Exception(exception_content) + + +def monitor(id, type): + with app.app_context(): + status = '' + global_content = None + try: + if type == 'html': + task = Task.query.filter_by(id=id).first() + url = task.url + selector_type = task.selector_type + selector = task.selector + is_chrome = task.is_chrome + regular_expression = task.regular_expression + mail = task.mail + wechat = task.wechat + pushover = task.pushover + name = task.name + rule = task.rule + headers = task.headers + + last = Content.query.filter_by(task_id=id, + task_type=type).first() + if not last: + last = Content(id) + + last_content = last.content + content = get_content(url, is_chrome, selector_type, selector, + regular_expression, headers) + global_content = content + status_code = is_changed(rule, content, last_content) + logger.info( + 'rule: {}, content: {}, last_content: {}, status_code: {}'. + format(rule, content, last_content, status_code)) + if status_code == 1: + status = '监测到变化,但未命中规则,最新值为{}'.format(content) + last.content = content + db.session.add(last) + db.session.commit() + elif status_code == 2: + status = '监测到变化,且命中规则,最新值为{}'.format(content) + msg = wraper_msg(content, url) + send_message(msg, name, mail, wechat, pushover) + last.content = content + db.session.add(last) + db.session.commit() + elif status_code == 3: + status = '监测到变化,最新值为{}'.format(content) + msg = wraper_msg(content, url) + send_message(msg, name, mail, wechat, pushover) + last.content = content + db.session.add(last) + db.session.commit() + elif status_code == 0: + status = '成功执行但未监测到变化,当前值为{}'.format(content) + elif type == 'rss': + rss_task = RSSTask.query.filter_by(id=id).first() + url = rss_task.url + name = rss_task.name + mail = rss_task.mail + wechat = rss_task.wechat + pushover = rss_task.pushover + + last = Content.query.filter_by(task_id=id, + task_type=type).first() + if not last: + last = Content(id, 'rss') + + last_guid = last.content + item = get_rss_content(url) + global_content = item['guid'] + if item['guid'] != last_guid: + content = wraper_rss_msg(item) + send_message(content, name, mail, wechat, pushover) + last.content = item['guid'] + db.session.add(last) + db.session.commit() + status = '监测到变化,最新值:' + item['guid'] + status = '成功执行但未监测到变化,当前值为{}'.format(last_guid) + + except FunctionTimedOut: + logger.error(traceback.format_exc()) + status = '解析RSS超时' + except PartNotificationError as e: + logger.error(traceback.format_exc()) + status = repr(e) + last.content = global_content + db.session.add(last) + db.session.commit() + except Exception as e: + logger.error(traceback.format_exc()) + status = repr(e) + + task_status = TaskStatus.query.filter_by(task_id=id, + task_type=type).first() + task_status.last_run = datetime.now() + task_status.last_status = status + db.session.add(task_status) + db.session.commit() + + +def add_job(id, interval, type='html'): + if type == 'html': + task_id = id + elif type == 'rss': + task_id = 'rss{}'.format(id) + + scheduler.add_job(func=monitor, + args=( + id, + type, + ), + trigger='interval', + minutes=interval, + id='task_{}'.format(task_id), + replace_existing=True) + logger.info('添加task_{}'.format(task_id)) + + +def remove_job(id, type='html'): + if type == 'html': + task_id = id + elif type == 'rss': + task_id = 'rss{}'.format(id) + + try: + scheduler.remove_job('task_{}'.format(task_id)) + logger.info('删除task_{}'.format(task_id)) + except JobLookupError as e: + logger.info(e) + logger.info('task_{}不存在'.format(task_id)) diff --git a/task/utils/selector/phantomjs_selector.py b/app/main/selector/phantomjs_selector.py similarity index 56% rename from task/utils/selector/phantomjs_selector.py rename to app/main/selector/phantomjs_selector.py index ba10e1f..5ee8d6b 100644 --- a/task/utils/selector/phantomjs_selector.py +++ b/app/main/selector/phantomjs_selector.py @@ -1,13 +1,19 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake +@Date: 2019-03-24 11:52:35 +@LastEditTime: 2019-03-30 15:06:10 +''' import ast import warnings -from collections import OrderedDict +from scrapy.selector import Selector from selenium import webdriver -from task.utils.selector.selector import SelectorABC as FatherSelector -warnings.filterwarnings("ignore") +from app.main.selector.selector import Selector as FatherSelector -USERAGENT = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' +warnings.filterwarnings("ignore") class PhantomJSSelector(FatherSelector): @@ -17,7 +23,7 @@ def __init__(self, debug=False): def get_html(self, url, headers): # 默认userAgent webdriver.DesiredCapabilities.PHANTOMJS[ - 'phantomjs.page.settings.userAgent'] = USERAGENT + 'phantomjs.page.settings.userAgent'] = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' if headers: header_dict = ast.literal_eval(headers) @@ -44,29 +50,26 @@ def get_html(self, url, headers): driver.quit() return html - def get_by_xpath(self, url, selector_dict, headers=None): + def get_by_xpath(self, url, xpath, headers=None): html = self.get_html(url, headers) - - result = OrderedDict() - for key, xpath_ext in selector_dict.items(): - result[key] = self.xpath_parse(html, xpath_ext) - - return result - - def get_by_css(self, url, selector_dict, headers=None): + if 'string()' in xpath: + xpath = xpath.split('/') + xpath = '/'.join(xpath[:-1]) + res = Selector( + string(.)').extract()" + else: + res = Selector( + + if len(res) != 0: + return res[0] + else: + raise Exception('无法获取文本信息') + + def get_by_css(self, url, xpath, headers=None): html = self.get_html(url, headers) + res = Selector( - result = OrderedDict() - for key, css_ext in selector_dict.items(): - result[key] = self.css_parse(html, css_ext) - - return result - - def get_by_json(self, url, selector_dict, headers=None): - html = self.get_html(url, headers) - - result = OrderedDict() - for key, json_ext in selector_dict.items(): - result[key] = self.json_parse(html, json_ext) - - return result + if len(res) != 0: + return res[0] + else: + raise Exception('无法获取文本信息') diff --git a/app/main/selector/request_selector.py b/app/main/selector/request_selector.py new file mode 100644 index 0000000..8e57785 --- /dev/null +++ b/app/main/selector/request_selector.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake, Jacob +@Date: 2019-03-25 12:31:35 +@LastEditTime: 2020-03-01 14:53:38 +''' +import ast + +import requests +import json +import jsonpath + +from scrapy.selector import Selector + +from app.main.selector.selector import Selector as FatherSelector + + +class RequestsSelector(FatherSelector): + def __init__(self, debug=False): + self.debug = debug + + def get_html(self, url, headers): + if headers: + header_dict = ast.literal_eval(headers) + if type(header_dict) != dict: + raise Exception('必须是字典格式') + + r = requests.get(url, headers=header_dict, timeout=10) + else: + r = requests.get(url, timeout=10) + r.encoding = r.apparent_encoding + html = r.text + return html + + def get_by_xpath(self, url, xpath, headers=None): + html = self.get_html(url, headers) + if 'string()' in xpath: + xpath = xpath.split('/') + xpath = '/'.join(xpath[:-1]) + res = Selector( + string(.)').extract()" + else: + res = Selector( + + if len(res) != 0: + return res[0] + else: + raise Exception('无法获取文本信息') + + def get_by_css(self, url, xpath, headers=None): + html = self.get_html(url, headers) + res = Selector( + + if len(res) != 0: + return res[0] + else: + raise Exception('无法获取文本信息') + + def get_by_json(self, url, xpath, headers=None): + html = self.get_html(url, headers) + + try: + resJson = json.loads(html) + except Exception: + raise Exception('Json转换错误') + res = json.dumps(jsonpath.jsonpath(resJson, xpath), ensure_ascii=False) + + if len(res) != 0: + return res + else: + raise Exception('无法获取文本信息') diff --git a/app/main/selector/selector.py b/app/main/selector/selector.py new file mode 100644 index 0000000..e0ed278 --- /dev/null +++ b/app/main/selector/selector.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake, Jacob +@Date: 2019-03-25 12:23:59 +@LastEditTime: 2020-03-01 14:54:14 +''' +from abc import ABCMeta, abstractmethod + + +class Selector(): + __metaclass__ = ABCMeta + + @abstractmethod + def get_by_xpath(self): + pass + + @abstractmethod + def get_by_css(self): + pass + + @abstractmethod + def get_by_json(self): + pass \ No newline at end of file diff --git a/task/utils/selector/selector_handler.py b/app/main/selector/selector_handler.py similarity index 71% rename from task/utils/selector/selector_handler.py rename to app/main/selector/selector_handler.py index 8e9471d..506c2c5 100644 --- a/task/utils/selector/selector_handler.py +++ b/app/main/selector/selector_handler.py @@ -5,8 +5,8 @@ @Date: 2019-03-25 12:27:44 @LastEditTime: 2019-03-30 15:59:08 ''' -from task.utils.selector.phantomjs_selector import PhantomJSSelector -from task.utils.selector.request_selector import RequestsSelector +from app.main.selector.phantomjs_selector import PhantomJSSelector +from app.main.selector.request_selector import RequestsSelector def new_handler(name, debug=False): diff --git a/app/main/views.py b/app/main/views.py new file mode 100644 index 0000000..4ef6288 --- /dev/null +++ b/app/main/views.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# @Author: LogicJake +# @Date: 2019-02-15 20:04:12 +# @Last Modified time: 2019-03-14 21:17:03 +from flask import Blueprint, redirect, render_template, request +from flask_login import login_user, logout_user +from wtforms.validators import ValidationError + +from app.main.forms.login_form import LoginForm +from app.main.forms.test_from import TestForm +from app.models.user import User +from app.main.extract_info import get_content +bp = Blueprint('main', __name__) + + +@bp.route('/login', methods=['GET', 'POST']) +def login(): + error = None + form = LoginForm() + if form.validate_on_submit(): + user_name = request.form.get('username', None) + password = request.form.get('password', None) + remember_me = request.form.get('remember_me', False) + user = User.query.filter_by(name=user_name).first() + if user and password and user.password == password: + login_user(user, remember=remember_me) + return redirect('/') + else: + error = '账户名或密码错误' + return render_template('login.html', error=error, form=form) + + +@bp.route('/logout', methods=['GET', 'POST']) +def logout(): + logout_user() + return '注销成功' + + +@bp.route('/test', methods=['GET', 'POST']) +def test(): + error = None + content = None + show = False + form = TestForm() + if form.validate_on_submit(): + try: + url = request.form.get('url', None) + selector_type = request.form.get('selector_type', None) + selector = request.form.get('selector', None) + is_chrome = request.form.get('is_chrome', None) + regular_expression = request.form.get('regular_expression', None) + headers = request.form.get('headers', None) + + if is_chrome == 'yes': + show = True + content = get_content(url, + is_chrome, + selector_type, + selector, + regular_expression, + headers, + debug=True) + except ValidationError: + pass + except Exception as e: + error = repr(e) + + return render_template('test.html', + error=error, + form=form, + content=content, + show=show) diff --git a/setting/management/commands/__init__.py b/app/model_views/__init__.py similarity index 100% rename from setting/management/commands/__init__.py rename to app/model_views/__init__.py diff --git a/app/model_views/mail_setting_view.py b/app/model_views/mail_setting_view.py new file mode 100644 index 0000000..c373ef3 --- /dev/null +++ b/app/model_views/mail_setting_view.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake +@Date: 2019-03-24 18:41:42 +@LastEditTime: 2019-03-26 20:51:52 +''' +from flask_login import current_user +from flask import redirect, url_for +from flask_admin.contrib.sqla import ModelView + + +class MailSettingView(ModelView): + def is_accessible(self): + return current_user.is_authenticated + + def inaccessible_callback(self, name, **kwargs): + return redirect(url_for('main.login')) + + column_labels = { + 'mail_server': '邮箱服务器', + 'mail_port': '端口', + 'mail_username': '用户名', + 'mail_password': '密码', + 'mail_sender': '发件人' + } + + can_create = False + can_delete = False + + column_descriptions = {'mail_sender': '一般为邮箱地址', 'mail_password': '授权码'} diff --git a/app/model_views/notification_view.py b/app/model_views/notification_view.py new file mode 100644 index 0000000..02b643d --- /dev/null +++ b/app/model_views/notification_view.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake, Jacob +@Date: 2019-03-24 11:01:56 +@LastEditTime: 2020-03-01 15:01:28 +''' +from flask_admin.contrib.sqla import ModelView +from flask import redirect, url_for +from flask_login import current_user + + +class NotificationView(ModelView): + def is_accessible(self): + return current_user.is_authenticated + + def inaccessible_callback(self, name, **kwargs): + return redirect(url_for('main.login')) + + can_create = False + can_delete = False + + column_labels = { + 'type': '通知方式', + 'number': '邮箱地址/Server酱 SCKEY/Pushover User Key' + } + + form_widget_args = {'type': {'readonly': True}} diff --git a/app/model_views/rss_task_view.py b/app/model_views/rss_task_view.py new file mode 100644 index 0000000..ef0f93a --- /dev/null +++ b/app/model_views/rss_task_view.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake, Jacob +@Date: 2019-03-24 11:01:56 +@LastEditTime: 2020-03-01 15:01:39 +''' +import requests +from flask_admin.contrib.sqla import ModelView +from wtforms.validators import ValidationError +from flask_login import current_user +from flask import redirect, url_for + + +def check_url(form, field): + url = form.url.data + try: + requests.get(url, timeout=10) + except Exception as e: + raise ValidationError(repr(e)) + + +def check_noti(form, field): + is_wechat = form.wechat.data + is_mail = form.mail.data + is_pushover = form.pushover.data + + if is_wechat == 'no' and is_mail == 'no' and is_pushover == 'no': + raise ValidationError('必须选择一个通知方式') + + +class RSSTaskView(ModelView): + def is_accessible(self): + return current_user.is_authenticated + + def inaccessible_callback(self, name, **kwargs): + return redirect(url_for('main.login')) + + column_labels = { + 'id': '任务id', + 'name': '任务名称', + 'url': 'RSS地址', + 'create_time': '创建时间', + 'frequency': '频率(分钟)', + 'mail': '邮件提醒', + 'wechat': '微信提醒', + 'pushover': '推送提醒', + } + + column_list = [ + 'id', 'name', 'url', 'frequency', 'create_time', 'mail', 'wechat', + 'pushover' + ] + + form_args = { + 'url': { + 'validators': [check_url], + }, + 'wechat': { + 'validators': [check_noti] + }, + 'pushover': { + 'validators': [check_noti] + } + } + + form_choices = { + 'mail': [('no', 'no'), ('yes', 'yes')], + 'wechat': [('no', 'no'), ('yes', 'yes')], + 'pushover': [('no', 'no'), ('yes', 'yes')], + } + + form_excluded_columns = ('create_time') diff --git a/app/model_views/task_status_view.py b/app/model_views/task_status_view.py new file mode 100644 index 0000000..b958633 --- /dev/null +++ b/app/model_views/task_status_view.py @@ -0,0 +1,47 @@ +from flask_admin.contrib.sqla import ModelView +from flask import redirect, url_for +from flask_login import current_user + + +class TaskStatusView(ModelView): + def is_accessible(self): + return current_user.is_authenticated + + def inaccessible_callback(self, name, **kwargs): + return redirect(url_for('main.login')) + + can_create = False + can_delete = False + + column_labels = { + 'task_id': '任务id', + 'task_name': '任务名称', + 'last_run': '上次运行时间', + 'last_status': '上次运行结果', + 'task_status': '任务状态', + 'task_type': '监控任务类型' + } + + column_list = [ + 'task_name', 'last_run', 'last_status', 'task_status', 'task_type' + ] + + form_choices = {'work_status': [('run', 'run'), ('stop', 'stop')]} + + form_widget_args = { + 'task_id': { + 'readonly': True + }, + 'task_name': { + 'readonly': True + }, + 'task_type': { + 'readonly': True + } + } + + form_choices = { + 'task_status': [('run', 'run'), ('stop', 'stop')], + } + + form_excluded_columns = ('last_run', 'last_status') diff --git a/app/model_views/task_view.py b/app/model_views/task_view.py new file mode 100644 index 0000000..a7d798a --- /dev/null +++ b/app/model_views/task_view.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake, Jacob +@Date: 2019-03-24 11:01:56 +@LastEditTime: 2020-03-01 15:02:02 +''' +import requests +from flask_admin.contrib.sqla import ModelView +from wtforms.validators import ValidationError +from flask_login import current_user +from app.main.selector.selector_handler import new_handler +from flask import redirect, url_for + + +def check_url(form, field): + url = form.url.data + try: + requests.get(url, timeout=10) + except Exception as e: + raise ValidationError(repr(e)) + + +def check_noti(form, field): + is_wechat = form.wechat.data + is_mail = form.mail.data + is_pushover = form.pushover.data + + if is_wechat == 'no' and is_mail == 'no' and is_pushover == 'no': + raise ValidationError('必须选择一个通知方式') + + +def check_selector(form, field): + try: + selector_type = form.selector_type.data + selector = form.selector.data + url = form.url.data + is_chrome = form.is_chrome.data + headers = form.headers.data + + if is_chrome == 'no': + selector_handler = new_handler('request') + else: + selector_handler = new_handler('phantomjs') + + if selector_type == 'xpath': + selector_handler.get_by_xpath(url, selector, headers) + elif selector_type == 'css': + selector_handler.get_by_css(url, selector, headers) + elif selector_type == 'json': + selector_handler.get_by_json(url, selector, headers) + else: + raise Exception('无效选择器') + except Exception as e: + raise ValidationError(repr(e)) + + +class TaskView(ModelView): + def is_accessible(self): + return current_user.is_authenticated + + def inaccessible_callback(self, name, **kwargs): + return redirect(url_for('main.login')) + + column_labels = { + 'id': '任务id', + 'name': '任务名称', + 'url': '监控网址', + 'create_time': '创建时间', + 'selector_type': '元素选择器类型', + 'selector': '元素选择器', + 'is_chrome': '是否使用无头浏览器', + 'frequency': '频率(分钟)', + 'mail': '邮件提醒', + 'wechat': '微信提醒', + 'pushover': '推送提醒', + 'regular_expression': '正则表达式', + 'rule': '监控规则', + 'headers': '自定义请求头' + } + + column_descriptions = { + 'selector': '可以到测试页面测试是否能够提取出所需信息', + 'regular_expression': '使用正则表达式进一步提取信息,可以留空', + 'rule': + '规则写法参考文档,留空则只简单监控内容变化', + 'headers': '自定义请求头,如可以设置cookie获取登录后才能查看的页面' + } + + column_list = [ + 'id', 'name', 'url', 'frequency', 'create_time', 'mail', 'wechat', + 'pushover' + ] + + form_args = { + 'url': { + 'validators': [check_url], + }, + 'selector': { + 'validators': [check_selector] + }, + 'wechat': { + 'validators': [check_noti] + }, + 'pushover': { + 'validators': [check_noti] + } + } + + form_choices = { + 'selector_type': [('xpath', 'xpath'), ('css', 'css selector'), + ('json', 'Jsonpath')], + 'is_chrome': [('no', 'no'), ('yes', 'yes')], + 'mail': [('no', 'no'), ('yes', 'yes')], + 'wechat': [('no', 'no'), ('yes', 'yes')], + 'pushover': [('no', 'no'), ('yes', 'yes')], + } + + form_excluded_columns = ('create_time') diff --git a/app/model_views/user_view.py b/app/model_views/user_view.py new file mode 100644 index 0000000..b744f62 --- /dev/null +++ b/app/model_views/user_view.py @@ -0,0 +1,19 @@ +from flask_login import current_user +from flask import redirect, url_for +from flask_admin.contrib.sqla import ModelView + + +class UserView(ModelView): + def is_accessible(self): + return current_user.is_authenticated + + def inaccessible_callback(self, name, **kwargs): + return redirect(url_for('main.login')) + + column_labels = { + 'name': '用户名', + 'password': '密码', + } + + can_create = False + can_delete = False diff --git a/app/models/content.py b/app/models/content.py new file mode 100644 index 0000000..63b603f --- /dev/null +++ b/app/models/content.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake +@Date: 2019年03月26日 14:58:27 +@LastEditTime: 2019年03月31日 19:10:01 +''' + +from .. import db + + +class Content(db.Model): + id = db.Column(db.Integer, primary_key=True) + task_id = db.Column(db.Integer, nullable=False) + content = db.Column(db.String(512), nullable=False) + task_type = db.Column(db.String(32), nullable=False, default='html') + + def __init__(self, task_id, task_type='html'): + self.task_id = task_id + self.task_type = task_type + + +db.create_all() diff --git a/app/models/mail_setting.py b/app/models/mail_setting.py new file mode 100644 index 0000000..bd8e224 --- /dev/null +++ b/app/models/mail_setting.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake +@Date: 2019年03月24日 18:29:53 +@LastEditTime: 2019年03月25日 11:10:18 +''' +from .. import db + + +class MailSetting(db.Model): + id = db.Column(db.Integer, primary_key=True) + mail_server = db.Column(db.String(32), nullable=False, default='localhost') + mail_port = db.Column(db.Integer, nullable=False, default=25) + mail_username = db.Column(db.String(64), nullable=False, default='默认用户名') + mail_sender = db.Column(db.String(64), + nullable=False, + default='默认用户名@mail.com') + mail_password = db.Column(db.String(64), nullable=False, default='默认密码') diff --git a/app/models/notification.py b/app/models/notification.py new file mode 100644 index 0000000..427ae93 --- /dev/null +++ b/app/models/notification.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake +@Date: 2019年03月24日 18:03:07 +@LastEditTime: 2019年03月25日 18:47:02 +''' +from .. import db + + +class Notification(db.Model): + id = db.Column(db.Integer, primary_key=True) + type = db.Column(db.String(64), nullable=False) + number = db.Column(db.String(64), nullable=False, default='默认') + + def __init__(self, type): + self.type = type diff --git a/app/models/rss_task.py b/app/models/rss_task.py new file mode 100644 index 0000000..e06b0ef --- /dev/null +++ b/app/models/rss_task.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake, Jacob +@Date: 2019年03月24日 16:35:24 +@LastEditTime: 2020年03月01日 15:02:16 +''' +from datetime import datetime + +from sqlalchemy import event + +from config import logger + +from .. import db + + +class RSSTask(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(32), nullable=False) + url = db.Column(db.String(128), nullable=False) + frequency = db.Column(db.Integer, nullable=False, default='5') + create_time = db.Column(db.DateTime, nullable=False, default=datetime.now) + # 通知方式 + mail = db.Column(db.String(32), nullable=False, default='no') + wechat = db.Column(db.String(32), nullable=False, default='no') + pushover = db.Column(db.String(32), nullable=False, default='no') + + +def after_insert_listener(mapper, connection, target): + from app.main.scheduler import add_job + + add_job(target.id, target.frequency, 'rss') + + from app.models.task_status import TaskStatus + task_status = TaskStatus.__table__ + connection.execute(task_status.insert().values(task_id=target.id, + task_name=target.name, + task_type='rss')) + + +def after_update_listener(mapper, connection, target): + from app.main.scheduler import add_job, remove_job + + remove_job(target.id, 'rss') + + from app.models.task_status import TaskStatus + task_status = TaskStatus.__table__ + connection.execute(task_status.update().values( + last_status='更新任务成功', last_run=datetime.now(), + task_status='run').where(TaskStatus.task_id == target.id + and TaskStatus.task_type == 'rss')) + + add_job(target.id, target.frequency, 'rss') + logger.info('task_rss{}更新'.format(target.id)) + + +def after_delete_listener(mapper, connection, target): + from app.main.scheduler import remove_job + + remove_job(target.id, 'rss') + + from app.models.task_status import TaskStatus + task_status = TaskStatus.__table__ + connection.execute(task_status.delete().where( + TaskStatus.task_id == target.id and TaskStatus.task_type == 'rss')) + + from app.models.content import Content + content = Content.__table__ + connection.execute(content.delete().where(Content.task_id == target.id + and Content.task_type == 'rss')) + + logger.info('task_rss{}删除'.format(target.id)) + + +event.listen(RSSTask, 'after_insert', after_insert_listener) +event.listen(RSSTask, 'after_update', after_update_listener) +event.listen(RSSTask, 'after_delete', after_delete_listener) diff --git a/app/models/task.py b/app/models/task.py new file mode 100644 index 0000000..1155b9e --- /dev/null +++ b/app/models/task.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake, Jacob +@Date: 2019年03月24日 16:35:24 +@LastEditTime: 2020年03月01日 15:02:28 +''' +from datetime import datetime + +from sqlalchemy import event + +from config import logger + +from .. import db + + +class Task(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(32), nullable=False) + url = db.Column(db.String(128), nullable=False) + selector_type = db.Column(db.String(32), nullable=False, default='xpath') + selector = db.Column(db.String(128), nullable=False) + is_chrome = db.Column(db.String(32), nullable=False, default='no') + frequency = db.Column(db.Integer, nullable=False, default='5') + create_time = db.Column(db.DateTime, nullable=False, default=datetime.now) + # 通知方式 + mail = db.Column(db.String(32), nullable=False, default='no') + wechat = db.Column(db.String(32), nullable=False, default='no') + pushover = db.Column(db.String(32), nullable=False, default='no') + # 高级设置 + regular_expression = db.Column(db.String(128)) + rule = db.Column(db.String(128)) + headers = db.Column(db.String(1024)) + + +def after_insert_listener(mapper, connection, target): + from app.main.scheduler import add_job + + add_job(target.id, target.frequency) + + from app.models.task_status import TaskStatus + task_status = TaskStatus.__table__ + connection.execute(task_status.insert().values(task_id=target.id, + task_name=target.name)) + + +def after_update_listener(mapper, connection, target): + from app.main.scheduler import add_job, remove_job + + remove_job(target.id) + from app.models.task_status import TaskStatus + task_status = TaskStatus.__table__ + connection.execute(task_status.update().values( + task_name=target.name, last_status='更新任务成功', last_run=datetime.now(), + task_status='run').where(TaskStatus.task_id == target.id + and TaskStatus.task_type == 'html')) + + add_job(target.id, target.frequency) + logger.info('task_{}更新'.format(target.id)) + + +def after_delete_listener(mapper, connection, target): + from app.main.scheduler import remove_job + + remove_job(target.id) + + from app.models.task_status import TaskStatus + task_status = TaskStatus.__table__ + connection.execute(task_status.delete().where( + TaskStatus.task_id == target.id and TaskStatus.task_type == 'html')) + + from app.models.content import Content + content = Content.__table__ + connection.execute(content.delete().where(Content.task_id == target.id + and Content.task_type == 'html')) + + logger.info('task_{}删除'.format(target.id)) + + +event.listen(Task, 'after_insert', after_insert_listener) +event.listen(Task, 'after_update', after_update_listener) +event.listen(Task, 'after_delete', after_delete_listener) diff --git a/app/models/task_status.py b/app/models/task_status.py new file mode 100644 index 0000000..05c7c34 --- /dev/null +++ b/app/models/task_status.py @@ -0,0 +1,47 @@ +from .. import db +from datetime import datetime +from sqlalchemy import event + + +class TaskStatus(db.Model): + id = db.Column(db.Integer, primary_key=True) + task_id = db.Column(db.Integer) + task_name = db.Column(db.String(32), nullable=False) + last_run = db.Column(db.DateTime, nullable=False, default=datetime.now) + last_status = db.Column(db.String(64), nullable=False, default='创建任务成功') + task_status = db.Column(db.String(32), nullable=False, default='run') + task_type = db.Column(db.String(32), nullable=False, default='html') + + def __init__(self, task_id, task_name, task_status='html'): + self.task_id = task_id + self.task_name = task_name + self.task_status = task_status + + +def after_update_listener(mapper, connection, target): + from app.main.scheduler import add_job, remove_job + + if target.task_status == 'run': + if target.task_type == 'html': + from app.models.task import Task + task = Task.__table__ + select_res = connection.execute( + task.select().where(Task.id == target.task_id)) + + for t in select_res: + remove_job(target.task_id) + add_job(target.task_id, t[6]) + elif target.task_type == 'rss': + from app.models.rss_task import RSSTask + rss_task = RSSTask.__table__ + select_res = connection.execute( + rss_task.select().where(RSSTask.id == target.task_id)) + + for t in select_res: + remove_job(target.task_id, 'rss') + add_job(target.task_id, t[3], 'rss') + else: + remove_job(target.id) + + +event.listen(TaskStatus, 'after_update', after_update_listener) diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 0000000..4212d99 --- /dev/null +++ b/app/models/user.py @@ -0,0 +1,37 @@ +from flask import redirect +from flask_login import logout_user +from sqlalchemy import event + +from .. import db +from config import logger + + +class User(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(64)) + password = db.Column(db.String(64)) + + def __init__(self, name, password): + self.name = name + self.password = password + + def is_active(self): + return True + + def get_id(self): + return self.id + + def is_authenticated(self): + return True + + def is_anonymous(self): + return False + + +def after_update_listener(mapper, connection, target): + logout_user() + redirect('/') + logger.info('账号或密码修改,注销重新登录') + + +event.listen(User, 'after_update', after_update_listener) diff --git a/app/static/400.woff b/app/static/400.woff new file mode 100644 index 0000000..8b512d0 Binary files /dev/null and b/app/static/400.woff differ diff --git a/app/static/400i.woff b/app/static/400i.woff new file mode 100644 index 0000000..d6684e8 Binary files /dev/null and b/app/static/400i.woff differ diff --git a/app/static/700.woff b/app/static/700.woff new file mode 100644 index 0000000..29c4f31 Binary files /dev/null and b/app/static/700.woff differ diff --git a/app/static/700i.woff b/app/static/700i.woff new file mode 100644 index 0000000..2004dc9 Binary files /dev/null and b/app/static/700i.woff differ diff --git a/app/static/css/github.css b/app/static/css/github.css new file mode 100644 index 0000000..c6a8579 --- /dev/null +++ b/app/static/css/github.css @@ -0,0 +1,414 @@ +:root { + --side-bar-bg-color: #fafafa; + --control-text-color: #777; +} + +@include-when-export url(https://fonts.googleapis.com/css?family=Open+Sans:400italic,700italic,700,400&subset=latin,latin-ext); + +@font-face { + font-family: 'Open Sans'; + font-style: normal; + font-weight: normal; + src: local('Open Sans Regular'),url('../400.woff') format('woff') +} + +@font-face { + font-family: 'Open Sans'; + font-style: italic; + font-weight: normal; + src: local('Open Sans Italic'),url('../400i.woff') format('woff') +} + +@font-face { + font-family: 'Open Sans'; + font-style: normal; + font-weight: bold; + src: local('Open Sans Bold'),url('../700.woff') format('woff') +} + +@font-face { + font-family: 'Open Sans'; + font-style: italic; + font-weight: bold; + src: local('Open Sans Bold Italic'),url('../700i.woff') format('woff') +} + +html { + font-size: 16px; +} + +body { + font-family: "Open Sans","Clear Sans","Helvetica Neue",Helvetica,Arial,sans-serif; + color: rgb(51, 51, 51); + line-height: 1.6; +} + +#write{ + max-width: 860px; + margin: 0 auto; + padding: 20px 30px 40px 30px; + padding-top: 20px; + padding-bottom: 100px; +} +#write> ul:first-child, +#write> ol:first-child{ + margin-top: 30px; +} + +body> *:first-child { + margin-top: 0 !important; +} +body> *:last-child { + margin-bottom: 0 !important; +} +a { + color: #4183C4; +} +h1, +h2, +h3, +h4, +h5, +h6 { + position: relative; + margin-top: 1rem; + margin-bottom: 1rem; + font-weight: bold; + line-height: 1.4; + cursor: text; +} +h1:hover a.anchor, +h2:hover a.anchor, +h3:hover a.anchor, +h4:hover a.anchor, +h5:hover a.anchor, +h6:hover a.anchor { + text-decoration: none; +} +h1 tt, +h1 code { + font-size: inherit; +} +h2 tt, +h2 code { + font-size: inherit; +} +h3 tt, +h3 code { + font-size: inherit; +} +h4 tt, +h4 code { + font-size: inherit; +} +h5 tt, +h5 code { + font-size: inherit; +} +h6 tt, +h6 code { + font-size: inherit; +} +h1 { + padding-bottom: .3em; + font-size: 2.25em; + line-height: 1.2; + border-bottom: 1px solid #eee; +} +h2 { + padding-bottom: .3em; + font-size: 1.75em; + line-height: 1.225; + border-bottom: 1px solid #eee; +} +h3 { + font-size: 1.5em; + line-height: 1.43; +} +h4 { + font-size: 1.25em; +} +h5 { + font-size: 1em; +} +h6 { + font-size: 1em; + color: #777; +} +p, +blockquote, +ul, +ol, +dl, +table{ + margin: 0.8em 0; +} +li>ol, +li>ul { + margin: 0 0; +} +hr { + height: 2px; + padding: 0; + margin: 16px 0; + background-color: #e7e7e7; + border: 0 none; + overflow: hidden; + box-sizing: content-box; +} + +body> h2:first-child { + margin-top: 0; + padding-top: 0; +} +body> h1:first-child { + margin-top: 0; + padding-top: 0; +} +body> h1:first-child + h2 { + margin-top: 0; + padding-top: 0; +} +body> h3:first-child, +body> h4:first-child, +body> h5:first-child, +body> h6:first-child { + margin-top: 0; + padding-top: 0; +} +a:first-child h1, +a:first-child h2, +a:first-child h3, +a:first-child h4, +a:first-child h5, +a:first-child h6 { + margin-top: 0; + padding-top: 0; +} +h1 p, +h2 p, +h3 p, +h4 p, +h5 p, +h6 p { + margin-top: 0; +} +li p.first { + display: inline-block; +} +ul, +ol { + padding-left: 30px; +} +ul:first-child, +ol:first-child { + margin-top: 0; +} +ul:last-child, +ol:last-child { + margin-bottom: 0; +} +blockquote { + border-left: 4px solid #dfe2e5; + padding: 0 15px; + color: #777777; +} +blockquote blockquote { + padding-right: 0; +} +table { + padding: 0; + word-break: initial; +} +table tr { + border-top: 1px solid #dfe2e5; + margin: 0; + padding: 0; +} +table tr:nth-child(2n), +thead { + background-color: #f8f8f8; +} +table tr th { + font-weight: bold; + border: 1px solid #dfe2e5; + border-bottom: 0; + text-align: left; + margin: 0; + padding: 6px 13px; +} +table tr td { + border: 1px solid #dfe2e5; + text-align: left; + margin: 0; + padding: 6px 13px; +} +table tr th:first-child, +table tr td:first-child { + margin-top: 0; +} +table tr th:last-child, +table tr td:last-child { + margin-bottom: 0; +} + +.CodeMirror-lines { + padding-left: 4px; +} + +.code-tooltip { + box-shadow: 0 1px 1px 0 rgba(0,28,36,.3); + border-top: 1px solid #eef2f2; +} + +.md-fences, +code, +tt { + border: 1px solid #e7eaed; + background-color: #f8f8f8; + border-radius: 3px; + padding: 0; + padding: 2px 4px 0px 4px; + font-size: 0.9em; +} + +code { + background-color: #f3f4f4; + padding: 0 4px 2px 4px; +} + +.md-fences { + margin-bottom: 15px; + margin-top: 15px; + padding: 0.2em 1em; + padding-top: 8px; + padding-bottom: 6px; +} + + +.md-task-list-item> input { + margin-left: -1.3em; +} + +@media screen and (min-width: 914px) { + /*body { + width: 854px; + margin: 0 auto; + }*/ +} +@media print { + html { + font-size: 13px; + } + table, + pre { + page-break-inside: avoid; + } + pre { + word-wrap: break-word; + } +} + +.md-fences { + background-color: #f8f8f8; +} +#write pre.md-meta-block { + padding: 1rem; + font-size: 85%; + line-height: 1.45; + background-color: #f7f7f7; + border: 0; + border-radius: 3px; + color: #777777; + margin-top: 0 !important; +} + +.mathjax-block>.code-tooltip { + bottom: .375rem; +} + +.md-mathjax-midline { + background: #fafafa; +} + +#write>h3.md-focus:before{ + left: -1.5625rem; + top: .375rem; +} +#write>h4.md-focus:before{ + left: -1.5625rem; + top: .285714286rem; +} +#write>h5.md-focus:before{ + left: -1.5625rem; + top: .285714286rem; +} +#write>h6.md-focus:before{ + left: -1.5625rem; + top: .285714286rem; +} +.md-image>.md-meta { + /*border: 1px solid #ddd;*/ + border-radius: 3px; + padding: 2px 0px 0px 4px; + font-size: 0.9em; + color: inherit; +} + +.md-tag { + color: #a7a7a7; + opacity: 1; +} + +.md-toc { + margin-top:20px; + padding-bottom:20px; +} + +.sidebar-tabs { + border-bottom: none; +} + +#typora-quick-open { + border: 1px solid #ddd; + background-color: #f8f8f8; +} + +#typora-quick-open-item { + background-color: #FAFAFA; + border-color: #FEFEFE #e5e5e5 #e5e5e5 #eee; + border-style: solid; + border-width: 1px; +} + +/** focus mode */ +.on-focus-mode blockquote { + border-left-color: rgba(85, 85, 85, 0.12); +} + +header, .context-menu, .megamenu-content, footer{ + font-family: "Segoe UI", "Arial", sans-serif; +} + +.file-node-content:hover .file-node-icon, +.file-node-content:hover .file-node-open-state{ + visibility: visible; +} + +.mac-seamless-mode #typora-sidebar { + background-color: #fafafa; + background-color: var(--side-bar-bg-color); +} + +.md-lang { + color: #b4654d; +} + +.html-for-mac .context-menu { + --item-hover-bg-color: #E6F0FE; +} + +img { + width: 100%; + height: 100%; +} \ No newline at end of file diff --git a/app/templates/admin/index.html b/app/templates/admin/index.html new file mode 100644 index 0000000..fb3cdc6 --- /dev/null +++ b/app/templates/admin/index.html @@ -0,0 +1,17 @@ +{% extends 'admin/master.html' %} +{% block body %} + +

I Am Watching You

+

特性

+ +

使用指南

+请点击这里 +{% endblock %} \ No newline at end of file diff --git a/app/templates/login.html b/app/templates/login.html new file mode 100644 index 0000000..e16ac87 --- /dev/null +++ b/app/templates/login.html @@ -0,0 +1,23 @@ + + + + + +
+

登录

+ {% if error %} +

Error: {{ error }}

+ {% endif %} + {% import 'bootstrap/wtf.html' as wtf %} +
+ +
+ {{ form.csrf_token }} + {{ wtf.form_field(form.username) }} + {{ wtf.form_field(form.password) }} + {{ wtf.form_field(form.remember_me) }} + {{ wtf.form_field(form.submit) }} +
+
+ +

AltStyle によって変換されたページ (->オリジナル) /

\ No newline at end of file diff --git a/app/templates/test.html b/app/templates/test.html new file mode 100644 index 0000000..dcc638b --- /dev/null +++ b/app/templates/test.html @@ -0,0 +1,33 @@ + + + + + + +

页面信息提取测试

+ + {% import 'bootstrap/wtf.html' as wtf %} +
+
+ {{ form.csrf_token }} + {{ wtf.form_field(form.url) }} + {{ wtf.form_field(form.selector_type) }} + {{ wtf.form_field(form.selector) }} + {{ wtf.form_field(form.is_chrome) }} + {{ wtf.form_field(form.regular_expression) }} + {{ wtf.form_field(form.headers) }} + {{ wtf.form_field(form.submit) }} +
+
+ {% if error %} +

Error: {{ error }}

+ {% if show %} +
+ 你的图片被外星人劫持了〜〜 +
+ {% endif %} + {% endif %} + {% if content %} +

提取到信息:{{ content }}

+ {% endif %} + \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..172cb82 --- /dev/null +++ b/config.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# @Author: LogicJake, Jacob +# @Date: 2019年02月15日 19:35:17 +# @Last Modified time: 2020年03月01日 14:51:01 +import os +import logging +import logging.config + +basedir = os.path.abspath(os.path.dirname(__file__)) + +os.makedirs('log', exist_ok=True) +logging.config.fileConfig('log.conf') +logger = logging.getLogger() +logger.info('Finish loading config') + + +class BaseConfig: + SQLALCHEMY_TRACK_MODIFICATIONS = False + SQLALCHEMY_DATABASE_URI = os.getenv('DATABASE_URL') + SECRET_KEY = 'chinano.1' + BABEL_DEFAULT_LOCALE = 'zh_CN' + SCHEDULER_TIMEZONE = 'Asia/Shanghai' + + +class DevelopmentConfig(BaseConfig): + DEBUG = True + + +class TestingConfig(BaseConfig): + TESTING = True + + +class ProductionConfig(BaseConfig): + pass + + +config = { + 'development': DevelopmentConfig, + 'testing': TestingConfig, + 'production': ProductionConfig +} diff --git a/docs/README.md b/docs/README.md index 8d03c03..1ee807f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,21 +1,26 @@ -![[docker] CI for releases](https://github.com/LogicJake/WebMonitor/workflows/%5Bdocker%5D%20CI%20for%20releases/badge.svg?branch=master&event=push) -![Tests](https://github.com/LogicJake/WebMonitor/workflows/Tests/badge.svg?branch=master&event=push) -[![telegram](https://img.shields.io/badge/chat-telegram-brightgreen.svg?style=flat-square)](https://t.me/webmonitor_github) - -[中文文档](https://logicjake.github.io/WebMonitor) | [English Document](https://logicjake.github.io/WebMonitor/#/en/) | [Telegram Group](https://t.me/webmonitor_github) - - +![[docker] CI for releases](https://github.com/LogicJake/WebMonitor/workflows/%5Bdocker%5D%20CI%20for%20releases/badge.svg?branch=master&event=push) ![Tests](https://github.com/LogicJake/WebMonitor/workflows/Tests/badge.svg?branch=master&event=push) ## 特性 * 支持requests请求网页,支持使用PhantomJS抓取异步加载的网页 * 支持 xpath 和 css selector 选择器,支持 JsonPath 提取 json 数据 -* 支持邮件,pushover,微信提醒(support by server酱),Bark推送,自定义GET/POST通知, Slack 通知以及 Telegram 通知 -* 支持一个任务多个选择器提取信息 -* 支持自定义消息模板 +* 支持邮件,pushover 和微信提醒(support by server酱) * 简洁的UI,可视化操作 * 支持自定义请求头,抓取需要登录的网页 * 支持设置监控规则 * 监控RSS更新 -* 数据导入导出 -## Buy Me a Coffee -![](fig/donate_wechat.jpg) +## changelog +### 2019年3月31日 +* 修复 RSS 监控无法正常运行 bug +* 添加规则:equal, less, more + +### 2019年3月28日 +* xpath 支持非正式函数 string() 以获取元素及其子元素的所有文本信息 + +### 2019年3月16日 +* 支持 JsonPath 提取 json 数据 +* 支持 pushover 通知 + +### 2019年3月13日 +* 修正规则匹配逻辑 +* 修复多种通知方式下, 某一个出错导致的重复发送连锁反应。现在只要用一种方式通知成功, 系统将保存更新后的监控对象, 从而不会在下一次执行时重复发送 +* 展现更详细的任务执行状态 \ No newline at end of file diff --git a/docs/_navbar.md b/docs/_navbar.md deleted file mode 100644 index 9452847..0000000 --- a/docs/_navbar.md +++ /dev/null @@ -1,3 +0,0 @@ -- Translations - - [:uk: English](/en/) - - [:cn: 中文](/) \ No newline at end of file diff --git a/docs/_sidebar.md b/docs/_sidebar.md index bd0025f..39eb859 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -1,7 +1,6 @@ * [项目介绍](/) * [部署](install.md) * [使用方式](how.md) -* [更新日志](changelog.md) * **Links** -* [Github](https://github.com/LogicJake/WebMonitor) -* [Blog](https://logicjake.github.io) +* [![Github](https://icongram.jgog.in/devicon/github-original.svg?color=808080&size=16)Github](https://github.com/LogicJake/WebMonitor) +* [![Blog](https://icongram.jgog.in/clarity/pencil.svg?size=16)Blog](https://www.logicjake.xyz) diff --git a/docs/changelog.md b/docs/changelog.md deleted file mode 100644 index 81137f8..0000000 --- a/docs/changelog.md +++ /dev/null @@ -1,57 +0,0 @@ -## changelog - -### 2021年2月2日 -* 添加 Telegram 通知方式 -### 2021年2月1日 -* 添加系统保留选择器名称:url -* 添加 Slack 通知方式 - -### 2021年1月31日 -* **支持同时设置多个元素选择器,并安装自定义消息模板发送提醒消息** -* 修复规则 increase 和 decrease 在首次抓取时的错误 -* 添加新监控规则:-without - -### 2021年12月21日 -* 添加自定义 GET 和 POST 通知方式(#50) - -### 2021年12月20日 -* 添加 bark 通知方式(#48) - -### 2021年12月11日 -* docker 支持 arm64 系统(#45) - -### 2020年8月14日 -* 正则表达式,规则和元素选择器最大长度设为500 -* 支持多规则 - -### 2020年8月7日 -* header 字段不限长度 -* 任务频率可以设置为大于0的任意数 -* 元素选择器支持修改 - -### 2019年4月27日 -* 增加数据导入导出功能 -* fix bugs - -### 2019年4月18日 -***此版本改动较大,旧版本备份在 flask 分支*** - -* django 重构,样式更美观 -* 仅保留 sqlite 数据库连接方式 -* 通知方式可以预先设置无限多,不再限制各种方式各一个 - -### 2019年3月31日 -* 修复 RSS 监控无法正常运行 bug -* 添加规则:equal, less, more - -### 2019年3月28日 -* xpath 支持非正式函数 string() 以获取元素及其子元素的所有文本信息 - -### 2019年3月16日 -* 支持 JsonPath 提取 json 数据 -* 支持 pushover 通知 - -### 2019年3月13日 -* 修正规则匹配逻辑 -* 修复多种通知方式下, 某一个出错导致的重复发送连锁反应。现在只要用一种方式通知成功, 系统将保存更新后的监控对象, 从而不会在下一次执行时重复发送 -* 展现更详细的任务执行状态 \ No newline at end of file diff --git a/docs/en/README.md b/docs/en/README.md deleted file mode 100644 index 53f1017..0000000 --- a/docs/en/README.md +++ /dev/null @@ -1,16 +0,0 @@ -![[docker] CI for releases](https://github.com/LogicJake/WebMonitor/workflows/%5Bdocker%5D%20CI%20for%20releases/badge.svg?branch=master&event=push) -![Tests](https://github.com/LogicJake/WebMonitor/workflows/Tests/badge.svg?branch=master&event=push) -[![telegram](https://img.shields.io/badge/chat-telegram-brightgreen.svg?style=flat-square)](https://t.me/webmonitor_github) - -[中文文档](https://www.logicjake.xyz/WebMonitor) | [English Document](https://www.logicjake.xyz/WebMonitor/#/en/) | [Telegram Group](https://t.me/webmonitor_github) - - -## Features -* Support for 'requests' web pages and grabbing asynchronously loaded pages using PhantomJS -* Support for xpath and CSS selector, and JsonPath to extract JSON data -* Support email, pushover, and WeChat alerts (support by server酱) -* Simple UI, visual operation -* Support custom request headers to crawl pages that need to be logged in -* Support for setting monitoring rules -* Monitor RSS updates -* Data import and export \ No newline at end of file diff --git a/docs/en/_coverpage.md b/docs/en/_coverpage.md deleted file mode 100644 index d0dbaf0..0000000 --- a/docs/en/_coverpage.md +++ /dev/null @@ -1,5 +0,0 @@ -# I Am Watching You - -* Monitoring web page changes -* Monitoring RSS updates -* Multiple notification modes \ No newline at end of file diff --git a/docs/en/_sidebar.md b/docs/en/_sidebar.md deleted file mode 100644 index cf311e7..0000000 --- a/docs/en/_sidebar.md +++ /dev/null @@ -1,7 +0,0 @@ -* [Project introduction](/en/README.md) -* [Deploy](/en/install.md) -* [How to use](/en/how.md) -* [Changelog](/en/changelog.md) -* **Links** -* [Github](https://github.com/LogicJake/WebMonitor) -* [Blog](https://www.logicjake.xyz) diff --git a/docs/en/changelog.md b/docs/en/changelog.md deleted file mode 100644 index 56e1e6a..0000000 --- a/docs/en/changelog.md +++ /dev/null @@ -1,32 +0,0 @@ -## changelog -### 2020年8月7日 -* header 字段不限长度 -* 任务频率可以设置为大于0的任意数 -* 元素选择器支持修改 - -### 2019年4月27日 -* 增加数据导入导出功能 -* fix bugs - -### 2019年4月18日 -***此版本改动较大,旧版本备份在 flask 分支*** - -* django 重构,样式更美观 -* 仅保留 sqlite 数据库连接方式 -* 通知方式可以预先设置无限多,不再限制各种方式各一个 - -### 2019年3月31日 -* 修复 RSS 监控无法正常运行 bug -* 添加规则:equal, less, more - -### 2019年3月28日 -* xpath 支持非正式函数 string() 以获取元素及其子元素的所有文本信息 - -### 2019年3月16日 -* 支持 JsonPath 提取 json 数据 -* 支持 pushover 通知 - -### 2019年3月13日 -* 修正规则匹配逻辑 -* 修复多种通知方式下, 某一个出错导致的重复发送连锁反应。现在只要用一种方式通知成功, 系统将保存更新后的监控对象, 从而不会在下一次执行时重复发送 -* 展现更详细的任务执行状态 \ No newline at end of file diff --git a/docs/en/how.md b/docs/en/how.md deleted file mode 100644 index 7b7d792..0000000 --- a/docs/en/how.md +++ /dev/null @@ -1,122 +0,0 @@ -## Set notification mode -Supports three notification methods: email, pushover, and WeChat alerts. Email reminder only needs to set up the receiving mailbox, WeChat reminder needs to apply for SCKEY, search the Server 酱 to register, simple and free. The Pushover needs to fill in the User Key to register. - -### Set system mailbox -If you use mail reminders, you must set up the system mailbox, which is the sender of the reminder messages. According to the need to to find relevant settings, password generally refers to the authorization code. - -System mailbox configuration only needs to be set one, more than one default only takes effect the first. - -### Set the Pushover Application -If you use the Pushover alert, you must set the Pushover API Token. - -## Add a web monitoring task -Add a new task in 'task management> web monitoring management' - -* You must select a form of notification -* The default grab frequency is 5 minutes, adjust it according to the need, unit minutes, it is not recommended to adjust too fast, in order to prevent backcrawling - -![任务管理](../fig/task_manage.png) -![添加任务](../fig/task_setting.png) - -### selector -The element selector type can be Xpath, Css selector or JsonPath, and the first two selectors can be copied directly with the help of the browser F12. It's important to note that the browser often copies the element, rather than the text information. The following should be added: - -#### xpath -* Gets the element text information by adding ```/text()``` after the selector obtained by the browser, for example: -```//*[@id="id3"]/h3``` => ```//*[@id="id3"]/h3/text()``` - -* Gets the element attribute information, adding the ```/@attribute name``` after the selector obtained by the browser, if you want to get the element href value: -```//*[@id="id3"]/h3``` => ```//*[@id="id3"]/h3/@href``` - -* Gets all the text information for the element and its children, adding ```/string()``` after the selector obtained by the browser, for example: -```//*[@id="id3"]/h3``` => ```//*[@id="id3"]/h3/string()``` - -#### css selector -* Gets the element text information by adding ```::text``` after the selector obtained by the browser, for example: -```div#id3> h3``` => ```div#id3> h3::text``` - -* Gets the element attribute information, adding the ```::attr(href)``` after the selector obtained by the browser, if you want to get the element href value: -```div#id3> h3``` => ```div#id3> h3::attr(href)``` - -#### JsonPath -To return the json data interface, can use JsonPath extract data, specific reference https://goessner.net/articles/JsonPath/ - -### Whether to select a headless browser -If the source page is not asynchronously loaded, you can get the page without using a headless browser -``` -It is recommended that you choose not to use it first. If you are not prompted for text information when submitting, then try using a headless browser -``` - -### 正则表达式 -If the obtained text information is redundant, regular filtering can be used for further filtering, such as -```价格:1390``` Using the regular ```([1-9]\d*)``` to extract the pure number 1390 - -### Monitoring rules -The default is to send notifications when text changes. -Command format: - Command parameters. Support the following commands: - -#### -contain -For example, the text changes and the text content contains```上架``` -``` --contain 上架 -``` - -#### -increase -For example, the text changes and the value increases more than the old value```3``` -```如果文本内容不是纯数字,请用正则提取出纯数字,否则将会报错``` -``` --increase 3 -``` - -#### -decrease -For example, the text changes and the value decreases more than the old value```3``` -```如果文本内容不是纯数字,请用正则提取出纯数字,否则将会报错``` -``` --decrease 3 -``` - -#### -equal -For example, the text changes and equals a value, the value equals```3``` -```如果文本内容不是纯数字,请用正则提取出纯数字,否则将会报错``` -``` --equal 3 -``` - -#### -less -For example: text changes and is less than a value, the value is less than```3``` -```如果文本内容不是纯数字,请用正则提取出纯数字,否则将会报错``` -``` --less 3 -``` - -#### -more -For example, the text changes and is greater than a certain value, the value is greater than```3``` -```如果文本内容不是纯数字,请用正则提取出纯数字,否则将会报错``` -``` --more 3 -``` - -### Customize the request header -The request header at the time of request can be customized, which is mainly used to set cookies and obtain the page that can only be viewed by logging in, in the format of dictionary, such as -```{'Cookie':'Custom cookie values'}``` - -## Add RSS monitoring tasks -You can add new RSS monitoring tasks to task management> RSS monitoring task management - -![RSS](../fig/rss.png) - -![RSS设置](../fig/rss_setting.png) - -## Task status view -You can view all tasks under the task Status column, including task status (Run or Stop), last run time, last run results, and three types of running results: - -* Change detected, latest value: {latest value} -* Successful implementation but no changes were detected -* Error displays an exception message - -![任务状态](../fig/status.png) - -You can pause or restart a task by changing its status. - -## Data import and export -***WARNING: The notification mode of web monitoring task and RSS monitoring task is connected with the notification mode table through foreign keys. In case of changes in the data table, the foreign key ID may be invalid or cannot be consistent with that of export. It is recommended to check whether the notification mode is normal after importing the task data. *** \ No newline at end of file diff --git a/docs/en/install.md b/docs/en/install.md deleted file mode 100644 index 9a3934e..0000000 --- a/docs/en/install.md +++ /dev/null @@ -1,55 +0,0 @@ -## Manual deployment - -### Installation - -Download the code of [WebMonitor](https://github.com/LogicJake/WebMonitor). - -``` -git clone https://github.com/LogicJake/WebMonitor.git -cd WebMonitor -``` - -Install dependencies after download is complete. - -``` -pip install -r requirements.txt -``` - -If you need to use a headless browser, make sure 'phantomjs' is installed and added to the system path. - -For the first run, the database should be migrated and the admin account should be set, assuming the account is 'admin', password is 'password' and the port is '8000'. - -``` -python manage.py migrate -python manage.py initadmin --username admin --password password -python manage.py runserver 0.0.0.0:8000 --noreload -``` - -Not the first run, just specify the port. - -``` -python manage.py runserver 0.0.0.0:8000 --noreload -``` - -## Docker deployment - -### Installation - -Run the following command to download the WebMonitor image. - -``` -docker pull logicjake/webmonitor -``` - -Then run WebMonitor, assuming the account is 'admin', password is 'password', and the port is '8000'. -***It is strongly recommended to save the database file to the host machine through the docker folder mapping parameter -v, otherwise the database file will be lost after container reconstruction, assuming the mapped directory is /etc/webmonitor.*** - -``` -docker run -d --name webmonitor -v /etc/webmonitor:/app/db -p 8000:8000 -e PORT=8000 -e USERNAME=admin -e PASSWORD=password logicjake/webmonitor -``` - -You can use the following command to turn off WebMonitor. - -``` -docker stop webmonitor -``` diff --git a/docs/fig/donate_wechat.jpg b/docs/fig/donate_wechat.jpg deleted file mode 100644 index 28ff5e1..0000000 Binary files a/docs/fig/donate_wechat.jpg and /dev/null differ diff --git a/docs/fig/rss.png b/docs/fig/rss.png deleted file mode 100644 index 8502ff4..0000000 Binary files a/docs/fig/rss.png and /dev/null differ diff --git a/docs/fig/rss_setting.png b/docs/fig/rss_setting.png deleted file mode 100644 index 07e646f..0000000 Binary files a/docs/fig/rss_setting.png and /dev/null differ diff --git a/docs/fig/status.png b/docs/fig/status.png deleted file mode 100644 index 191143c..0000000 Binary files a/docs/fig/status.png and /dev/null differ diff --git a/docs/fig/task_manage.png b/docs/fig/task_manage.png deleted file mode 100644 index 8e437d3..0000000 Binary files a/docs/fig/task_manage.png and /dev/null differ diff --git a/docs/fig/task_setting.png b/docs/fig/task_setting.png deleted file mode 100644 index c66ca29..0000000 Binary files a/docs/fig/task_setting.png and /dev/null differ diff --git a/docs/how.md b/docs/how.md index c59a8f0..daad8df 100644 --- a/docs/how.md +++ b/docs/how.md @@ -1,60 +1,32 @@ -## 设置通知方式 -支持7种通知方式:邮件,pushover, Server 酱的微信提醒,Bark,自定义GET/POST通知, Slack 通知以及 Telegram 通知。邮件提醒只需要设置接收邮箱,微信提醒需要申请 SCKEY,自行搜索 Server 酱注册,简单免费。Pushover 需要填写注册就得到的 User Key。Bark需要安装[客户端](https://github.com/Finb/Bark)取得对应设备Key。Slack 需要填写"#"开头的 channel 名称,且需要保证 Slack app 已在该 channel 中。 +## 登录系统 +首次初始化数据库,系统会自动新建用户,默认用户名为```admin```,默认密码为```admin```,登录之后可以在账号密码管理中修改。进入```/login```,输入账号密码登录。 -### 设置系统邮箱 -如果采用邮件提醒,则必须设置"系统管理/系统邮箱",该邮箱为提醒邮件的发信人。自行根据需要使用的邮箱查找相关设置,密码一般指授权码。 +登录之后显示如下栏目 -系统邮箱配置只需设置一个,多于一个默认只生效第一条。 +![展示](https://github.com/LogicJake/WebMonitor/raw/master/fig/all.png) -### 设置 Pushover Application -如果采用 Pushover 提醒,则必须设置"系统管理/Pushover 设置"中的 Pushover api token。 +## 设置通知方式 +在通知方式管理中默认存在两种通知方式:邮件,pushover 和Server酱的微信提醒。邮件提醒只需要设置接收邮箱,微信提醒需要申请 SCKEY,自行搜索 Server 酱注册,简单免费。pushover 需要填写注册就得到的 User Key。 -### 设置 Slack -如果采用 Slack 提醒,则必须设置"系统管理/Slack 设置"中的 Slack OAuth Access Token。具体教程见:https://github.com/slackapi/python-slack-sdk/blob/main/tutorial/01-creating-the-slack-app.md +![通知方式](https://github.com/LogicJake/WebMonitor/raw/master/fig/noti.png) -### 设置 Telegram Bot -如果采用 Telegram 提醒,则必须设置"系统管理/Telegram Bot 设置"中的 Telegram Bot Token。 +### 设置系统邮箱 +如果采用邮件提醒,则必须设置系统邮箱设置,该邮箱为提醒邮件的发信人。自行根据需要使用的邮箱查找相关设置,密码一般指授权码。 -### 设置自定义GET/POST通知 -如果采用自定义通知,则必须设置自定义网址。 -#### GET -用`{header}`和`{content}`替换掉标题和内容的位置。以Bark为例,格式如下: -``` -https://api.day.app/yourkey/{header}/{content} -``` -#### POST -`发送网址{data=}`。将要发送的`body`内容放在`{data=}`内,其中`{header}`和`{content}`替换掉标题和内容的位置。以WxPusher为例,格式如下: -``` -http://wxpusher.zjiecode.com/api/send/message{data={ - "appToken":"AT_xxx", - "content":{content}, - "summary":{header}, - "contentType":3, - "uids":["UID_xxxx"], - "url":"http://wxpusher.zjiecode.com" -}} -``` +![系统邮件设置](https://github.com/LogicJake/WebMonitor/raw/master/fig/mail_setting.png) ## 添加网页监控任务 -在 任务管理> 网页监控管理 添加新任务 +在网页监控任务管理模块添加新任务 * 必须选择一种通知方式 * 默认抓取频率为5分钟,自行根据需要调整,单位分钟,不建议调太快,以防反爬 +![任务管理](https://github.com/LogicJake/WebMonitor/raw/master/fig/task_manage.png) +![添加任务](https://github.com/LogicJake/WebMonitor/raw/master/fig/task_setting.png) ### 选择器 -元素选择器类型可以选择 Xpath, Css selector 或 JsonPath。 - -一行一个元素选择器,每一行的格式为:选择器名称{选择器内容},例如: -``` -title{//*[@id="id3"]/h3/text()} -myurl{//*[@id="id3"]/h3/text()} -``` - -```以下字段为系统默认保留字段,请不要使用且无法被覆盖:``` -* url:该任务对应的监控网址 - -可以借助浏览器 F12 直接 copy 前两种选择器,需要注意的是,往往浏览器 copy 得到是元素,而不是文本信息,需要做以下补充: +提供测试页面```/test```,测试是否能够从页面提取所需信息,方便确认xpath或css selector是否填写正确;在测试页面下,使用无头浏览器获取网页,提取信息错误会展示页面截图。 +元素选择器类型可以选择xpath或css selector,可以借助浏览器F12直接copy两种选择器,需要注意的是,往往浏览器copy得到是元素,而不是文本信息,需要做以下补充: #### xpath * 获取元素文本信息,在浏览器得到的选择器后加```/text()```,如 @@ -75,19 +47,6 @@ myurl{//*[@id="id3"]/h3/text()} #### JsonPath 针对返回 json 数据的接口, 可以使用 JsonPath 提取数据, 具体教程参考 https://goessner.net/articles/JsonPath/ -在Chrome F12开发者工具中,也可以找到对应元素,然后右键该元素,选择"Copy Property Path"。 - -### 消息体模板 -消息体模板可为空,如果为空,则按照元素选择器的定义顺序以制表符为间隔拼接为字符串。下面介绍消息体模板的使用方式,如果元素选择器的设置为: -``` -title{//*[@id="id3"]/h3/text()} -myurl{//*[@id="id3"]/h3/text()} -``` -则消息体模板可以设置为: -``` -{title}的网址是{myurl} -``` -如果title对应的元素选择器提取的内容为"WebMonitor真棒",myurl对应的元素选择器提取的内容为"https://www.logicjake.xyz/WebMonitor",则得到的消息内容为"WebMonitor真棒的网址是https://www.logicjake.xyz/WebMonitor"。 ### 是否选择无头浏览器 如果源网页没有异步加载,可以不使用无头浏览器获取网页 @@ -100,15 +59,9 @@ myurl{//*[@id="id3"]/h3/text()} ```价格:1390```使用正则```([1-9]\d*)```提取到纯数字1390 ### 监控规则 -默认不填则文本发生变化就发通知,多规则请以';'分开。存在规则的情况下,如果文本发生变化,从前往后检查规则,若符合其中一项规则就发通知。 -规则格式:-规则 参数 -支持以下规则: - -#### -without -如:文本发生变化且文本内容不包含```上架``` -``` --without 上架 -``` +默认不填则文本发生变化就发通知 +命令格式:-命令 参数 +支持以下命令: #### -contain 如:文本发生变化且文本内容包含```上架``` ``` @@ -154,10 +107,6 @@ myurl{//*[@id="id3"]/h3/text()} 可以自定义请求时的请求头,主要用于设置Cookie,获取需要登录才能查看的页面,格式为字典,如 ```{'Cookie':'自定义cookie值'}``` -## 添加RSS监控任务 -可以在 任务管理> RSS监控任务管理 添加新RSS监控任务 - - ## 任务状态查看 可以在任务状态栏目下查看所有任务,包括任务状态(run or stop),上次运行时间,上次运行结果,运行结果包括三类: @@ -165,8 +114,13 @@ myurl{//*[@id="id3"]/h3/text()} * 成功执行但未监测到变化 * 出错显示异常信息 - +![任务状态](https://github.com/LogicJake/WebMonitor/raw/master/fig/status.png) 可以通过修改任务状态,暂停或重启任务 -## 数据导入导出 -***WARNING: 网页监控任务和RSS监控任务的通知方式是通过外键与通知方式表连接,在数据表发生变化的情况下,外键id可能失效或无法和导出时保持一致,建议每次导入任务数据后检查通知方式是否正常。*** +![状态设置](https://github.com/LogicJake/WebMonitor/raw/master/fig/status_setting.png) + +## 添加RSS监控任务 +可以在RSS监控任务管理模块添加新RSS监控任务 + +![RSS](https://github.com/LogicJake/WebMonitor/raw/master/fig/rss.png) +![RSS设置](https://github.com/LogicJake/WebMonitor/raw/master/fig/rss_setting.png) \ No newline at end of file diff --git a/docs/index.html b/docs/index.html index 1d2ebfb..4db2a30 100644 --- a/docs/index.html +++ b/docs/index.html @@ -18,13 +18,12 @@ window.$docsify = { name: 'I Am Watching You', repo: 'LogicJake/WebMonitor', - noEmoji: false, + noEmoji: true, search: 'auto', ga: 'UA-109200981-1', - coverpage: ['/', '/en/'], + coverpage: true, loadSidebar: true, subMaxLevel: 5, - loadNavbar: true, } diff --git a/docs/install.md b/docs/install.md index 041a737..821763b 100644 --- a/docs/install.md +++ b/docs/install.md @@ -13,21 +13,19 @@ cd WebMonitor ``` pip install -r requirements.txt -``` -如果需要使用无头浏览器,请确认已经安装 phantomjs,且 phantomjs 被添加到系统路径 +``` -首次运行需要迁移数据库且设置管理账号,假设账号为 admin,密码为 password,运行端口为 8000 +必须先设置环境变量 DATABASE_URL , 具体方法见"添加配置"。如果需要使用无头浏览器,请确认已经安装 phantomjs ,且 phantomjs 被添加到系统路径。然后在 WebMonitor 文件夹中运行下面的命令就可以启动 ``` -python manage.py migrate -python manage.py initadmin --username admin --password password -python manage.py runserver 0.0.0.0:8000 --noreload +python -m flask run -h 0.0.0.0 -p 5000 ``` -非首次运行,只需指定端口 +### 添加配置 +可以通过设置环境变量来配置 WebMonitor。在项目根目录新建 ```.env``` 文件,每行以 ```NAME=VALUE``` 格式添加环境变量,更多见配置说明。DATABASE_URL 必填,数据库字符集需设置为 ```utf8```,举例如下: ``` -python manage.py runserver 0.0.0.0:8000 --noreload +DATABASE_URL=mysql+pymysql://username:password@hostname/database ``` ## Docker 部署 @@ -40,11 +38,10 @@ python manage.py runserver 0.0.0.0:8000 --noreload docker pull logicjake/webmonitor ``` -然后运行 webmonitor 即可,假设账号为 admin,密码为 password,运行端口为 8000 -***强烈建议通过 docker 文件夹映射参数 -v,将数据库文件保存到主机,否则在容器重建之后会丢失数据库文件,假设映射的主机目录为 /etc/webmonitor*** +然后运行 webmonitor 即可 ``` -docker run -d --name webmonitor -v /etc/webmonitor:/app/db -p 8000:8000 -e PORT=8000 -e USERNAME=admin -e PASSWORD=password logicjake/webmonitor +docker run -d --name webmonitor -p 5000:5000 -e PORT=5000 -e DATABASE_URL=mysql+pymysql://username:password@hostname/database logicjake/webmonitor ``` 您可以使用下面的命令来关闭 webmonitor @@ -52,3 +49,20 @@ docker run -d --name webmonitor -v /etc/webmonitor:/app/db -p 8000:8000 -e PORT= ``` docker stop webmonitor ``` + +### 添加配置 +在运行时增加参数: -e NAME=VALUE,DATABASE_URL 必须添加,数据库字符集需设置为 ```utf8```。更多见配置说明 +``` +-e DATABASE_URL=mysql+pymysql://username:password@hostname/database +``` + +## 配置说明 +### DATABASE_URL +数据库也可以设置为无需安装的 sqlite,从而在本地新建文件存储数据。注意不要误删,否则会丢失数据。配置项示例: +``` +DATABASE_URL=sqlite:///test.db +``` +### PUSHOVER_API_TOKEN +新建 pushover application 得到的 API Token +### NAME +自定义网站名称 \ No newline at end of file diff --git a/fig/all.png b/fig/all.png new file mode 100644 index 0000000..955dbe9 Binary files /dev/null and b/fig/all.png differ diff --git a/fig/mail_setting.png b/fig/mail_setting.png new file mode 100644 index 0000000..c51d45e Binary files /dev/null and b/fig/mail_setting.png differ diff --git a/fig/noti.png b/fig/noti.png new file mode 100644 index 0000000..7ad0f98 Binary files /dev/null and b/fig/noti.png differ diff --git a/fig/rss.png b/fig/rss.png new file mode 100644 index 0000000..0591c20 Binary files /dev/null and b/fig/rss.png differ diff --git a/fig/rss_setting.png b/fig/rss_setting.png new file mode 100644 index 0000000..57394d9 Binary files /dev/null and b/fig/rss_setting.png differ diff --git a/fig/status.png b/fig/status.png new file mode 100644 index 0000000..e876d63 Binary files /dev/null and b/fig/status.png differ diff --git a/fig/status_setting.png b/fig/status_setting.png new file mode 100644 index 0000000..231bcd5 Binary files /dev/null and b/fig/status_setting.png differ diff --git a/fig/task_manage.png b/fig/task_manage.png new file mode 100644 index 0000000..ab78777 Binary files /dev/null and b/fig/task_manage.png differ diff --git a/fig/task_setting.png b/fig/task_setting.png new file mode 100644 index 0000000..4d64556 Binary files /dev/null and b/fig/task_setting.png differ diff --git a/log.conf b/log.conf new file mode 100644 index 0000000..1b92e53 --- /dev/null +++ b/log.conf @@ -0,0 +1,32 @@ +[loggers] +keys=root + +[logger_root] +level=INFO +handlers=filert + +############################################### + +[handlers] +keys=filert,stream + +[handler_stream] +class=StreamHandler +level=INFO +formatter=form +args=(sys.stdout,) + + +[handler_filert] +class=handlers.RotatingFileHandler +level=INFO +formatter=form +args=('log/log.txt', 'a', 10*1024*1024, 5) + +############################################### + +[formatters] +keys=form + +[formatter_form] +format= %(asctime)s [%(filename)s:%(lineno)d][%(levelname)s]: %(message)s \ No newline at end of file diff --git a/manage.py b/manage.py deleted file mode 100755 index 60c3a87..0000000 --- a/manage.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -"""Django's command-line utility for administrative tasks.""" -import os -import sys - - -def main(): - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'webmonitor.settings') - try: - from django.core.management import execute_from_command_line - except ImportError as exc: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) from exc - execute_from_command_line(sys.argv) - - -if __name__ == '__main__': - main() diff --git a/requirements.txt b/requirements.txt index 9ef6c6b..df2c0ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,24 @@ requests==2.21.0 Markdown==3.1 +Flask_BabelEx==0.9.3 +SQLAlchemy==1.3.1 +APScheduler==3.6.0 +Flask_APScheduler==1.11.0 +Flask==1.0.2 +Flask_Login==0.4.1 +WTForms==2.1 +Flask_WTF==0.14.2 Scrapy==1.6.0 +Flask_SQLAlchemy==2.3.2 +python-dotenv==0.10.1 +flask_admin==1.5.3 +flask_bootstrap==3.3.7.1 selenium==3.141.0 +Flask-WTF==0.14.2 html5lib==1.0.1 +PyMySQL==0.9.3 Twisted==19.7.0 feedparser==5.2.1 func_timeout==4.3.0 -jsonpath==0.82 -django==2.2.13 -django-simpleui==3.9.1 -django-apscheduler==0.3.0 -django-import-export==2.0.2 -slack-sdk==3.2.1 \ No newline at end of file +werkzeug==0.16.1 +jsonpath==0.82 \ No newline at end of file diff --git a/run.sh b/run.sh deleted file mode 100755 index baceda9..0000000 --- a/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -python manage.py migrate -python manage.py initadmin --username 2ドル --password 3ドル -python manage.py runserver 0.0.0.0:1ドル --noreload diff --git a/setting/__init__.py b/setting/__init__.py deleted file mode 100644 index 108dca8..0000000 --- a/setting/__init__.py +++ /dev/null @@ -1 +0,0 @@ -default_app_config = 'setting.apps.SettingConfig' diff --git a/setting/admin.py b/setting/admin.py deleted file mode 100644 index c71db0a..0000000 --- a/setting/admin.py +++ /dev/null @@ -1,117 +0,0 @@ -from setting.views import log_view -from django.contrib import admin -from import_export import resources -from import_export.admin import ImportExportModelAdmin - -from .models import Notification, PushoverSetting, SystemMailSetting, Log, SlackSetting, TelegramSetting - - -class SystemMailSettingResource(resources.ModelResource): - class Meta: - model = SystemMailSetting - skip_unchanged = True - report_skipped = True - - -@admin.register(SystemMailSetting) -class SystemMailSettingAdmin(ImportExportModelAdmin): - resource_class = SystemMailSettingResource - - list_display = [ - 'mail_server', 'mail_port', 'mail_username', 'mail_sender', - 'mail_password' - ] - - list_editable = ('mail_server', 'mail_port', 'mail_username', - 'mail_sender', 'mail_password') - - list_display_links = None - actions_on_top = True - - -class PushoverSettingResource(resources.ModelResource): - class Meta: - model = PushoverSetting - skip_unchanged = True - report_skipped = True - - -@admin.register(PushoverSetting) -class PushoverSettingAdmin(ImportExportModelAdmin): - resource_class = PushoverSettingResource - - list_display = ['api_token'] - list_editable = ('api_token', ) - - list_display_links = None - actions_on_top = True - - actions = ['custom_button'] - - def custom_button(self, request, queryset): - pass - - custom_button.short_description = '新建Pushover Application' - custom_button.type = 'info' - custom_button.action_type = 2 - custom_button.action_url = 'https://pushover.net/' - - -class NotificatioResource(resources.ModelResource): - class Meta: - model = Notification - import_id_fields = ('name', ) - exclude = ('id', ) - skip_unchanged = True - report_skipped = True - - -@admin.register(Notification) -class NotificationAdmin(ImportExportModelAdmin): - resource_class = NotificatioResource - - list_display = ['name', 'type', 'content'] - list_editable = ('name', 'type', 'content') - - list_display_links = None - actions_on_top = True - - -@admin.register(Log) -class FeedbackStatsAdmin(admin.ModelAdmin): - def changelist_view(self, request, extra_content=None): - return log_view(request) - - -class SlackSettingResource(resources.ModelResource): - class Meta: - model = PushoverSetting - skip_unchanged = True - report_skipped = True - - -@admin.register(SlackSetting) -class SlackSettingAdmin(admin.ModelAdmin): - resource_class = SlackSettingResource - - list_display = ['token'] - list_editable = ('token', ) - - list_display_links = None - - -class TelegramSettingResource(resources.ModelResource): - class Meta: - model = TelegramSetting - skip_unchanged = True - report_skipped = True - - -@admin.register(TelegramSetting) -class TelegramSettingAdmin(admin.ModelAdmin): - resource_class = TelegramSettingResource - - list_display = ['token'] - list_editable = ('token', ) - - list_display_links = None diff --git a/setting/apps.py b/setting/apps.py deleted file mode 100644 index 0c63129..0000000 --- a/setting/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class SettingConfig(AppConfig): - name = 'setting' - verbose_name = '系统管理' diff --git a/setting/management/commands/initadmin.py b/setting/management/commands/initadmin.py deleted file mode 100644 index a6a6f65..0000000 --- a/setting/management/commands/initadmin.py +++ /dev/null @@ -1,24 +0,0 @@ -from django.core.management.base import BaseCommand -from django.contrib.auth.models import User - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument('--username', required=False) - parser.add_argument('--password', required=False) - - def handle(self, *args, **options): - username = options['username'] - password = options['password'] - - if User.objects.count() == 0: - print(username, password) - admin = User.objects.create_superuser(username=username, - email='', - password=password) - admin.is_active = True - admin.is_superuser = True - admin.save() - else: - print( - 'Admin accounts can only be initialized if no Accounts exist') diff --git a/setting/migrations/0001_initial.py b/setting/migrations/0001_initial.py deleted file mode 100644 index 388de9c..0000000 --- a/setting/migrations/0001_initial.py +++ /dev/null @@ -1,53 +0,0 @@ -# Generated by Django 2.2 on 2020年04月18日 18:09 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Notification', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(default='默认名称', max_length=32, unique=True, verbose_name='通知方式名称')), - ('type', models.IntegerField(choices=[(0, '邮箱'), (1, '微信'), (2, 'pushover'), (3, 'Bark'), (4, '自定义通知')], default='邮箱', verbose_name='通知方式类型')), - ('content', models.CharField(max_length=100, verbose_name='通知方式')), - ], - options={ - 'verbose_name': '通知方式', - 'verbose_name_plural': '通知方式', - }, - ), - migrations.CreateModel( - name='PushoverSetting', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('api_token', models.CharField(max_length=100, verbose_name='Pushover API Token')), - ], - options={ - 'verbose_name': 'Pushover 设置', - 'verbose_name_plural': 'Pushover 设置', - }, - ), - migrations.CreateModel( - name='SystemMailSetting', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('mail_server', models.CharField(default='localhost', max_length=32, verbose_name='邮箱服务器')), - ('mail_port', models.IntegerField(default=25, verbose_name='端口')), - ('mail_username', models.CharField(default='默认用户名', max_length=64, verbose_name='用户名')), - ('mail_sender', models.CharField(default='默认用户名@mail.com', max_length=64, verbose_name='发件人')), - ('mail_password', models.CharField(default='默认密码', max_length=64, verbose_name='密码')), - ], - options={ - 'verbose_name': '系统邮箱', - 'verbose_name_plural': '系统邮箱', - }, - ), - ] diff --git a/setting/migrations/0002_auto_20210131_1925.py b/setting/migrations/0002_auto_20210131_1925.py deleted file mode 100644 index e7d726d..0000000 --- a/setting/migrations/0002_auto_20210131_1925.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 2.2.13 on 2021年01月31日 19:25 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('setting', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='Log', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ], - options={ - 'verbose_name': '日志查看', - 'verbose_name_plural': '日志查看', - }, - ), - migrations.AlterField( - model_name='notification', - name='content', - field=models.CharField(max_length=512, verbose_name='邮箱地址 / Server 酱 SCKEY / Pushover User Key / Bark key / 自定义网址'), - ), - ] diff --git a/setting/migrations/0003_auto_20210201_2104.py b/setting/migrations/0003_auto_20210201_2104.py deleted file mode 100644 index 066d9b8..0000000 --- a/setting/migrations/0003_auto_20210201_2104.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 2.2.13 on 2021年02月01日 21:04 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('setting', '0002_auto_20210131_1925'), - ] - - operations = [ - migrations.CreateModel( - name='SlackBotSetting', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('token', models.CharField(max_length=100, verbose_name='Slack OAuth Access Token')), - ], - options={ - 'verbose_name': 'Slack Bot 设置', - 'verbose_name_plural': 'Slack Bot 设置', - }, - ), - migrations.AlterField( - model_name='notification', - name='content', - field=models.CharField(max_length=512, verbose_name='邮箱地址 / Server 酱 SCKEY / Pushover User Key / Bark key / 自定义网址 / Slack channel'), - ), - ] diff --git a/setting/migrations/0004_auto_20210201_2117.py b/setting/migrations/0004_auto_20210201_2117.py deleted file mode 100644 index 3439d88..0000000 --- a/setting/migrations/0004_auto_20210201_2117.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.13 on 2021年02月01日 21:17 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('setting', '0003_auto_20210201_2104'), - ] - - operations = [ - migrations.AlterField( - model_name='notification', - name='type', - field=models.IntegerField(choices=[(0, '邮箱'), (1, '微信'), (2, 'pushover'), (3, 'Bark'), (4, '自定义通知'), (5, 'Slack bot')], default='邮箱', verbose_name='通知方式类型'), - ), - ] diff --git a/setting/migrations/0005_auto_20210201_2126.py b/setting/migrations/0005_auto_20210201_2126.py deleted file mode 100644 index 4ecd377..0000000 --- a/setting/migrations/0005_auto_20210201_2126.py +++ /dev/null @@ -1,26 +0,0 @@ -# Generated by Django 2.2.13 on 2021年02月01日 21:26 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('setting', '0004_auto_20210201_2117'), - ] - - operations = [ - migrations.RenameModel( - old_name='SlackBotSetting', - new_name='SlackSetting', - ), - migrations.AlterModelOptions( - name='slacksetting', - options={'verbose_name': 'Slack 设置', 'verbose_name_plural': 'Slack 设置'}, - ), - migrations.AlterField( - model_name='notification', - name='type', - field=models.IntegerField(choices=[(0, '邮箱'), (1, '微信'), (2, 'pushover'), (3, 'Bark'), (4, '自定义通知'), (5, 'Slack')], default='邮箱', verbose_name='通知方式类型'), - ), - ] diff --git a/setting/migrations/0006_auto_20210203_1749.py b/setting/migrations/0006_auto_20210203_1749.py deleted file mode 100644 index b24b493..0000000 --- a/setting/migrations/0006_auto_20210203_1749.py +++ /dev/null @@ -1,34 +0,0 @@ -# Generated by Django 2.2.13 on 2021年02月03日 17:49 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('setting', '0005_auto_20210201_2126'), - ] - - operations = [ - migrations.CreateModel( - name='TelegramSetting', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('token', models.CharField(max_length=100, verbose_name='Telegram Bot Token')), - ], - options={ - 'verbose_name': 'Telegram Bot 设置', - 'verbose_name_plural': 'Telegram Bot 设置', - }, - ), - migrations.AlterField( - model_name='notification', - name='content', - field=models.CharField(max_length=512, verbose_name='邮箱地址 / Server 酱 SCKEY / Pushover User Key / Bark key / 自定义网址 / Slack channel / Telegram chat_id'), - ), - migrations.AlterField( - model_name='notification', - name='type', - field=models.IntegerField(choices=[(0, '邮箱'), (1, '微信'), (2, 'pushover'), (3, 'Bark'), (4, '自定义通知'), (5, 'Slack'), (6, 'Telegram')], default='邮箱', verbose_name='通知方式类型'), - ), - ] diff --git a/setting/migrations/__init__.py b/setting/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setting/models.py b/setting/models.py deleted file mode 100644 index e9bb55e..0000000 --- a/setting/models.py +++ /dev/null @@ -1,99 +0,0 @@ -from django.db import models - - -class SystemMailSetting(models.Model): - mail_server = models.CharField(max_length=32, - null=False, - default='localhost', - verbose_name='邮箱服务器') - mail_port = models.IntegerField(null=False, default=25, verbose_name='端口') - mail_username = models.CharField(max_length=64, - null=False, - default='默认用户名', - verbose_name='用户名') - mail_sender = models.CharField(max_length=64, - null=False, - default='默认用户名@mail.com', - verbose_name='发件人') - mail_password = models.CharField(max_length=64, - null=False, - default='默认密码', - verbose_name='密码') - - class Meta: - verbose_name = "系统邮箱" - verbose_name_plural = "系统邮箱" - - def __str__(self): - return self.mail_server - - -class PushoverSetting(models.Model): - api_token = models.CharField(max_length=100, - null=False, - verbose_name='Pushover API Token') - - class Meta: - verbose_name = "Pushover 设置" - verbose_name_plural = "Pushover 设置" - - def __str__(self): - return 'Pushover ' + self.api_token - - -class Notification(models.Model): - type_choice = ((0, '邮箱'), (1, '微信'), (2, 'pushover'), (3, 'Bark'), - (4, '自定义通知'), (5, 'Slack'), (6, 'Telegram')) - name = models.CharField(max_length=32, - null=False, - verbose_name='通知方式名称', - unique=True, - default='默认名称') - type = models.IntegerField(null=False, - choices=type_choice, - default='邮箱', - verbose_name='通知方式类型') - content = models.CharField(max_length=512, - null=False, - verbose_name='邮箱地址 / Server 酱 SCKEY / \ - Pushover User Key / Bark key / 自定义网址 / Slack channel / Telegram chat_id' - ) - - class Meta: - verbose_name = "通知方式" - verbose_name_plural = "通知方式" - - def __str__(self): - return self.name - - -class Log(models.Model): - class Meta: - verbose_name = "日志查看" - verbose_name_plural = "日志查看" - - -class SlackSetting(models.Model): - token = models.CharField(max_length=100, - null=False, - verbose_name='Slack OAuth Access Token') - - class Meta: - verbose_name = "Slack 设置" - verbose_name_plural = "Slack 设置" - - def __str__(self): - return 'Slack ' + self.token - - -class TelegramSetting(models.Model): - token = models.CharField(max_length=100, - null=False, - verbose_name='Telegram Bot Token') - - class Meta: - verbose_name = "Telegram Bot 设置" - verbose_name_plural = "Telegram Bot 设置" - - def __str__(self): - return 'Telegram Bot ' + self.token diff --git a/setting/static/css/log.css b/setting/static/css/log.css deleted file mode 100644 index 63d44dc..0000000 --- a/setting/static/css/log.css +++ /dev/null @@ -1,7 +0,0 @@ -.log { - width: 100%; - height: 500px; - overflow: scroll; - overflow-x: hidden; - white-space: pre-line; -} \ No newline at end of file diff --git a/setting/static/js/log.js b/setting/static/js/log.js deleted file mode 100644 index 08e6a25..0000000 --- a/setting/static/js/log.js +++ /dev/null @@ -1,4 +0,0 @@ -window.onload = function () { - var hid = document.getElementById('msg_end'); - hid.scrollIntoView(false); -} diff --git a/setting/templates/log.html b/setting/templates/log.html deleted file mode 100644 index 4d0698e..0000000 --- a/setting/templates/log.html +++ /dev/null @@ -1,14 +0,0 @@ -{% extends "admin/base_site.html" %} -{% load static %} - - -{% block content %} - - - -
-

{{ content }}

- -
- -{% endblock %} \ No newline at end of file diff --git a/setting/tests.py b/setting/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/setting/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/setting/views.py b/setting/views.py deleted file mode 100644 index 21e8aba..0000000 --- a/setting/views.py +++ /dev/null @@ -1,16 +0,0 @@ -from django.shortcuts import render -import os - - -# Create your views here. -def log_view(request): - BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - log_path = os.path.join(BASE_DIR, 'static', 'log', 'log.txt') - - content = '日志文件不存在' - if os.path.exists(log_path): - with open(log_path, 'r') as f: - content = f.readlines()[-50:] - content = ''.join(content) - return render(request, 'log.html', {'content': content}) diff --git a/task/__init__.py b/task/__init__.py deleted file mode 100644 index 166eb39..0000000 --- a/task/__init__.py +++ /dev/null @@ -1 +0,0 @@ -default_app_config = 'task.apps.TaskConfig' diff --git a/task/admin.py b/task/admin.py deleted file mode 100644 index fd16dea..0000000 --- a/task/admin.py +++ /dev/null @@ -1,117 +0,0 @@ -import logging - -from django.contrib import admin, messages -from import_export import resources -from import_export.admin import ImportExportModelAdmin - -from task.models import Content, RSSTask, Task, TaskStatus -from task.utils.scheduler import remove_job - -logger = logging.getLogger('admin') - - -@admin.register(TaskStatus) -class TaskStatusAdmin(admin.ModelAdmin): - list_display = [ - 'task_name', 'last_run', 'short_last_status', 'task_status', - 'task_type' - ] - list_editable = ['task_status'] - list_per_page = 10 - list_display_links = None - - actions_on_top = True - - def has_add_permission(self, request): - return False - - def has_delete_permission(self, request, obj=None): - return False - - -class TaskResource(resources.ModelResource): - class Meta: - model = Task - import_id_fields = ('name', ) - exclude = ('id', ) - skip_unchanged = True - report_skipped = True - - -@admin.register(Task) -class TaskAdmin(ImportExportModelAdmin): - resource_class = TaskResource - - list_display = [ - 'id', 'name', 'url', 'frequency', 'selector', 'create_time', - 'is_chrome', 'regular_expression', 'rule', 'headers' - ] - list_editable = ('name', 'url', 'frequency', 'is_chrome', - 'regular_expression', 'rule', 'headers', 'selector') - filter_horizontal = ('notification', ) - - list_per_page = 10 - - def has_delete_permission(self, request, obj=None): - return False - - def redefine_delete_selected(self, request, obj): - for o in obj.all(): - id = o.id - remove_job(id) - - TaskStatus.objects.filter(task_id=id, task_type='html').delete() - Content.objects.filter(task_id=id, task_type='html').delete() - - o.delete() - logger.info('task_{}删除'.format(id)) - - messages.add_message(request, messages.SUCCESS, '删除成功') - - redefine_delete_selected.short_description = '删除' - redefine_delete_selected.icon = 'el-icon-delete' - redefine_delete_selected.style = 'color:white;background:red' - - actions = ['redefine_delete_selected'] - - -class RSSTaskResource(resources.ModelResource): - class Meta: - model = RSSTask - import_id_fields = ('name', ) - exclude = ('id', ) - skip_unchanged = True - report_skipped = True - - -@admin.register(RSSTask) -class RSSTaskAdmin(ImportExportModelAdmin): - resource_class = RSSTaskResource - - list_display = ['id', 'name', 'url', 'frequency', 'create_time'] - list_editable = ('name', 'url', 'frequency') - filter_horizontal = ('notification', ) - - list_per_page = 10 - - def has_delete_permission(self, request, obj=None): - return False - - def redefine_delete_selected(self, request, obj): - for o in obj.all(): - id = o.id - remove_job(id, 'rss') - - TaskStatus.objects.filter(task_id=id, task_type='rss').delete() - Content.objects.filter(task_id=id, task_type='rss').delete() - - o.delete() - logger.info('task_RSS{}删除'.format(id)) - - messages.add_message(request, messages.SUCCESS, '删除成功') - - redefine_delete_selected.short_description = '删除' - redefine_delete_selected.icon = 'el-icon-delete' - redefine_delete_selected.style = 'color:white;background:red' - - actions = ['redefine_delete_selected'] diff --git a/task/apps.py b/task/apps.py deleted file mode 100644 index 8f04920..0000000 --- a/task/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class TaskConfig(AppConfig): - name = 'task' - verbose_name = '任务管理' diff --git a/task/migrations/0001_initial.py b/task/migrations/0001_initial.py deleted file mode 100644 index 582ed57..0000000 --- a/task/migrations/0001_initial.py +++ /dev/null @@ -1,76 +0,0 @@ -# Generated by Django 2.2 on 2020年04月22日 17:08 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('setting', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='Content', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('task_id', models.IntegerField()), - ('content', models.CharField(max_length=512)), - ('task_type', models.CharField(default='html', max_length=32)), - ], - ), - migrations.CreateModel( - name='TaskStatus', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('task_id', models.IntegerField(verbose_name='任务ID')), - ('task_name', models.CharField(max_length=100, verbose_name='任务名称')), - ('last_run', models.DateTimeField(auto_now=True, verbose_name='上次运行时间')), - ('last_status', models.CharField(default='创建任务成功', max_length=100, verbose_name='上次运行结果')), - ('task_status', models.IntegerField(choices=[(0, 'run'), (1, 'stop')], default=0, verbose_name='任务状态')), - ('task_type', models.CharField(default='html', max_length=100, verbose_name='任务类型')), - ], - options={ - 'verbose_name': '任务状态', - 'verbose_name_plural': '任务状态', - }, - ), - migrations.CreateModel( - name='Task', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=100, verbose_name='任务名称')), - ('url', models.CharField(max_length=500, verbose_name='监控网址')), - ('selector_type', models.IntegerField(choices=[(0, 'Xpath'), (1, 'Css selector'), (2, 'JsonPath')], default='Xpath', verbose_name='元素选择器类型')), - ('selector', models.CharField(max_length=128, verbose_name='元素选择器')), - ('is_chrome', models.IntegerField(choices=[(0, 'no'), (1, 'yes')], default='no', verbose_name='是否使用无头浏览器')), - ('frequency', models.IntegerField(default=5, verbose_name='频率(分钟) ')), - ('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')), - ('regular_expression', models.CharField(blank=True, help_text='使用正则表达式进一步提取信息,可以留空', max_length=128, verbose_name='正则表达式')), - ('rule', models.CharField(blank=True, help_text='规则写法参考文档,留空则只简单监控内容变化', max_length=128, verbose_name='监控规则')), - ('headers', models.CharField(blank=True, help_text='自定义请求头,如可以设置cookie获取登录后才能查看的页面', max_length=1024, verbose_name='自定义请求头')), - ('notification', models.ManyToManyField(to='setting.Notification', verbose_name='通知方式')), - ], - options={ - 'verbose_name': '网页监控', - 'verbose_name_plural': '网页监控管理', - }, - ), - migrations.CreateModel( - name='RSSTask', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=32, verbose_name='任务名称')), - ('url', models.CharField(max_length=500, verbose_name='RSS地址')), - ('frequency', models.IntegerField(default=5, verbose_name='频率(分钟)')), - ('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')), - ('notification', models.ManyToManyField(to='setting.Notification', verbose_name='通知方式')), - ], - options={ - 'verbose_name': 'RSS监控', - 'verbose_name_plural': 'RSS监控管理', - }, - ), - ] diff --git a/task/migrations/0002_auto_20200807_1209.py b/task/migrations/0002_auto_20200807_1209.py deleted file mode 100644 index 962b2cb..0000000 --- a/task/migrations/0002_auto_20200807_1209.py +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Django 2.2.13 on 2020年08月07日 12:09 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('task', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='rsstask', - name='frequency', - field=models.FloatField(default=5, validators=[django.core.validators.MinValueValidator(0)], verbose_name='频率(分钟)'), - ), - migrations.AlterField( - model_name='rsstask', - name='url', - field=models.CharField(max_length=500, validators=[django.core.validators.URLValidator()], verbose_name='RSS地址'), - ), - migrations.AlterField( - model_name='task', - name='frequency', - field=models.FloatField(default=5, validators=[django.core.validators.MinValueValidator(0)], verbose_name='频率(分钟)'), - ), - migrations.AlterField( - model_name='task', - name='headers', - field=models.TextField(blank=True, help_text='自定义请求头,如可以设置cookie获取登录后才能查看的页面', verbose_name='自定义请求头'), - ), - migrations.AlterField( - model_name='task', - name='url', - field=models.CharField(max_length=500, validators=[django.core.validators.URLValidator()], verbose_name='监控网址'), - ), - ] diff --git a/task/migrations/0003_auto_20200814_1333.py b/task/migrations/0003_auto_20200814_1333.py deleted file mode 100644 index e9fa359..0000000 --- a/task/migrations/0003_auto_20200814_1333.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 2.2.13 on 2020年08月14日 13:33 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('task', '0002_auto_20200807_1209'), - ] - - operations = [ - migrations.AlterField( - model_name='task', - name='regular_expression', - field=models.CharField(blank=True, help_text='使用正则表达式进一步提取信息,可以留空', max_length=500, verbose_name='正则表达式'), - ), - migrations.AlterField( - model_name='task', - name='rule', - field=models.CharField(blank=True, help_text='规则写法参考文档,留空则只简单监控内容变化', max_length=500, verbose_name='监控规则'), - ), - migrations.AlterField( - model_name='task', - name='selector', - field=models.CharField(max_length=500, verbose_name='元素选择器'), - ), - ] diff --git a/task/migrations/0004_auto_20210131_1925.py b/task/migrations/0004_auto_20210131_1925.py deleted file mode 100644 index b55f3d2..0000000 --- a/task/migrations/0004_auto_20210131_1925.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.13 on 2021年01月31日 19:25 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('task', '0003_auto_20200814_1333'), - ] - - operations = [ - migrations.AlterField( - model_name='task', - name='selector', - field=models.TextField(verbose_name='元素选择器'), - ), - ] diff --git a/task/migrations/0005_task_template.py b/task/migrations/0005_task_template.py deleted file mode 100644 index 969a11c..0000000 --- a/task/migrations/0005_task_template.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.13 on 2021年01月31日 20:28 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('task', '0004_auto_20210131_1925'), - ] - - operations = [ - migrations.AddField( - model_name='task', - name='template', - field=models.TextField(blank=True, verbose_name='消息体模板'), - ), - ] diff --git a/task/migrations/0006_auto_20210201_1755.py b/task/migrations/0006_auto_20210201_1755.py deleted file mode 100644 index afb703c..0000000 --- a/task/migrations/0006_auto_20210201_1755.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 2.2.13 on 2021年02月01日 17:55 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('task', '0005_task_template'), - ] - - operations = [ - migrations.AlterField( - model_name='task', - name='selector', - field=models.TextField(help_text='一行一个元素选择器,每一行的格式为:选择器名称{选择器内容},例如:title{//*[@id="id3"]/h3/text()}。其中 url 为系统保留选择器名称,请不要使用且无法被覆盖', verbose_name='元素选择器'), - ), - migrations.AlterField( - model_name='task', - name='template', - field=models.TextField(blank=True, help_text='可为空,自定义发送的通知内容格式,按照选择器名称进行替换,具体示例见文档', verbose_name='消息体模板'), - ), - ] diff --git a/task/migrations/0007_auto_20210203_1817.py b/task/migrations/0007_auto_20210203_1817.py deleted file mode 100644 index 3d711b7..0000000 --- a/task/migrations/0007_auto_20210203_1817.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.13 on 2021年02月03日 18:17 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('task', '0006_auto_20210201_1755'), - ] - - operations = [ - migrations.AlterField( - model_name='task', - name='selector', - field=models.TextField(help_text='一行一个元素选择器,每一行的格式为:选择器名称{选择器内容}, 例如:title{//*[@id="id3"]/h3/text()}。其中 url 为系统保留选择器名称,请不要使用且无法被覆盖', verbose_name='元素选择器'), - ), - ] diff --git a/task/migrations/0008_auto_20210314_1924.py b/task/migrations/0008_auto_20210314_1924.py deleted file mode 100644 index 74e085a..0000000 --- a/task/migrations/0008_auto_20210314_1924.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 2.2.13 on 2021年03月14日 19:24 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('task', '0007_auto_20210203_1817'), - ] - - operations = [ - migrations.AlterField( - model_name='rsstask', - name='url', - field=models.CharField(max_length=1000, validators=[django.core.validators.URLValidator()], verbose_name='RSS地址'), - ), - migrations.AlterField( - model_name='task', - name='url', - field=models.CharField(max_length=1000, validators=[django.core.validators.URLValidator()], verbose_name='监控网址'), - ), - ] diff --git a/task/migrations/__init__.py b/task/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/task/models.py b/task/models.py deleted file mode 100644 index 6be507e..0000000 --- a/task/models.py +++ /dev/null @@ -1,265 +0,0 @@ -import logging -from datetime import datetime - -from django.core.validators import MinValueValidator, URLValidator -from django.db import models - -from setting.models import Notification - -logger = logging.getLogger('main') - -# def check_url(url): -# try: -# requests.get(url, timeout=10) -# except Exception as e: -# raise ValidationError({'url': e}) - -# def check_selector(selector_type, selector, url, is_chrome, headers): -# try: -# if is_chrome == 0: -# selector_handler = new_handler('request') -# else: -# selector_handler = new_handler('phantomjs') - -# if selector_type == 0: -# selector_handler.get_by_xpath(url, selector, headers) -# elif selector_type == 1: -# selector_handler.get_by_css(url, selector, headers) -# elif selector_type == 2: -# selector_handler.get_by_json(url, selector, headers) -# else: -# raise Exception('无效选择器') -# except Exception as e: -# raise ValidationError({'selector': e}) - - -class Content(models.Model): - task_id = models.IntegerField(null=False) - content = models.CharField(max_length=512, null=False) - task_type = models.CharField(max_length=32, null=False, default='html') - - -class TaskStatus(models.Model): - task_id = models.IntegerField(null=False, verbose_name='任务ID') - task_name = models.CharField(max_length=100, - null=False, - verbose_name='任务名称') - last_run = models.DateTimeField(auto_now=True, verbose_name='上次运行时间') - last_status = models.CharField(max_length=100, - null=False, - default='创建任务成功', - verbose_name='上次运行结果') - - status_choices = ((0, 'run'), (1, 'stop')) - - task_status = models.IntegerField(null=False, - default=0, - verbose_name='任务状态', - choices=status_choices) - task_type = models.CharField(max_length=100, - null=False, - default='html', - verbose_name='任务类型') - - class Meta: - verbose_name = "任务状态" - verbose_name_plural = "任务状态" - - def __str__(self): - return self.task_name - - def save(self, *args, **kwargs): - from task.utils.scheduler import add_job, remove_job - - super(TaskStatus, self).save(*args, **kwargs) - task_id = self.task_id - - if self.task_status == 0: - if self.last_status != '更新任务成功': - if self.task_type == 'html': - task = Task.objects.get(id=task_id) - add_job(task_id, task.frequency) - elif self.task_type == 'rss': - rss_task = RSSTask.objects.get(id=task_id) - add_job(task_id, rss_task.frequency, 'rss') - else: - if self.task_type == 'html': - remove_job(task_id) - elif self.task_type == 'rss': - remove_job(task_id, 'rss') - - def short_last_status(self): - if len(str(self.last_status))> 100: - return '{}......'.format(str(self.last_status)[:100]) - else: - return str(self.last_status) - - short_last_status.allow_tags = True - short_last_status.short_description = '上次运行结果' - - -class Task(models.Model): - name = models.CharField(max_length=100, verbose_name='任务名称', null=False) - url = models.CharField(max_length=1000, - verbose_name='监控网址', - null=False, - validators=[URLValidator()]) - - selector_choices = ( - (0, 'Xpath'), - (1, 'Css selector'), - (2, 'JsonPath'), - ) - - selector_type = models.IntegerField(verbose_name='元素选择器类型', - null=False, - default='Xpath', - choices=selector_choices) - selector = models.TextField(verbose_name='元素选择器', - blank=False, - help_text='一行一个元素选择器,每一行的格式为:选择器名称{选择器内容},\ - 例如:title{//*[@id="id3"]/h3/text()}。其中 url 为系统保留选择器名称,请不要使用且无法被覆盖') - template = models.TextField( - verbose_name='消息体模板', - blank=True, - help_text='可为空,自定义发送的通知内容格式,按照选择器名称进行替换,具体示例见文档') - is_chrome_choices = ((0, 'no'), (1, 'yes')) - is_chrome = models.IntegerField(null=False, - default='no', - verbose_name='是否使用无头浏览器', - choices=is_chrome_choices) - frequency = models.FloatField(null=False, - default=5, - verbose_name='频率(分钟)', - validators=[MinValueValidator(0)]) - create_time = models.DateTimeField(null=False, - auto_now_add=True, - verbose_name='创建时间') - - notification = models.ManyToManyField(Notification, - blank=False, - verbose_name='通知方式') - - regular_expression = models.CharField(max_length=500, - verbose_name='正则表达式', - blank=True, - help_text='使用正则表达式进一步提取信息,可以留空') - rule = models.CharField(max_length=500, - verbose_name='监控规则', - blank=True, - help_text='规则写法参考文档,留空则只简单监控内容变化') - headers = models.TextField(verbose_name='自定义请求头', - blank=True, - help_text='自定义请求头,如可以设置cookie获取登录后才能查看的页面') - - class Meta: - verbose_name = "网页监控" - verbose_name_plural = "网页监控管理" - - def __str__(self): - return self.name - - def save(self, *args, **kwargs): - from task.utils.scheduler import add_job - - # 新建 - if not self.pk: - super(Task, self).save(*args, **kwargs) - id = self.pk - - add_job(id, self.frequency) - - task_status = TaskStatus(task_name=self.name, task_id=id) - task_status.save() - else: - super(Task, self).save(*args, **kwargs) - id = self.pk - - task_status = TaskStatus.objects.get(task_id=id, task_type='html') - task_status.task_name = self.name - task_status.last_status = '更新任务成功' - task_status.last_run = datetime.now() - task_status.task_status = 0 - task_status.save() - - add_job(id, self.frequency) - logger.info('task_{}更新'.format(id)) - - def delete(self, *args, **kwargs): - from task.utils.scheduler import remove_job - - id = self.pk - remove_job(id) - - TaskStatus.objects.filter(task_id=id, task_type='html').delete() - Content.objects.filter(task_id=id, task_type='html').delete() - - logger.info('task_{}删除'.format(id)) - - super(Task, self).delete(*args, **kwargs) - - -class RSSTask(models.Model): - name = models.CharField(max_length=32, null=False, verbose_name='任务名称') - url = models.CharField(max_length=1000, - null=False, - verbose_name='RSS地址', - validators=[URLValidator()]) - frequency = models.FloatField(null=False, - default=5, - verbose_name='频率(分钟)', - validators=[MinValueValidator(0)]) - create_time = models.DateTimeField(null=False, - auto_now_add=True, - verbose_name='创建时间') - - notification = models.ManyToManyField(Notification, - blank=False, - verbose_name='通知方式') - - class Meta: - verbose_name = "RSS监控" - verbose_name_plural = "RSS监控管理" - - def __str__(self): - return self.name - - def save(self, *args, **kwargs): - from task.utils.scheduler import add_job - - # 新建 - if not self.pk: - super(RSSTask, self).save(*args, **kwargs) - id = self.pk - - add_job(id, self.frequency, 'rss') - task_status = TaskStatus(task_name=self.name, - task_id=id, - task_type='rss') - task_status.save() - else: - super(RSSTask, self).save(*args, **kwargs) - - id = self.pk - task_status = TaskStatus.objects.get(task_id=id, task_type='rss') - task_status.task_name = self.name - task_status.last_status = '更新任务成功' - task_status.last_run = datetime.now() - task_status.task_status = 0 - task_status.save() - - add_job(id, self.frequency, 'rss') - logger.info('task_RSS{}更新'.format(id)) - - def delete(self, *args, **kwargs): - from task.utils.scheduler import remove_job - - id = self.pk - remove_job(id, 'rss') - - TaskStatus.objects.filter(task_id=id, task_type='rss').delete() - Content.objects.filter(task_id=id, task_type='rss').delete() - - logger.info('task_RSS{}删除'.format(id)) - - super(RSSTask, self).delete(*args, **kwargs) diff --git a/task/tests.py b/task/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/task/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/task/utils/extract_info.py b/task/utils/extract_info.py deleted file mode 100644 index 0a49365..0000000 --- a/task/utils/extract_info.py +++ /dev/null @@ -1,96 +0,0 @@ -import logging -import re -from collections import OrderedDict - -import feedparser -from func_timeout import func_set_timeout -from task.utils.selector.selector_handler import new_handler - -logger = logging.getLogger('main') - - -def extract_by_re(conetnt, regular_expression): - m = re.search(regular_expression, conetnt) - - if m: - return m.group(1) - elif not m: - return "未检测到相关内容" - else: - logger.error('{} 无法使用正则提取'.format(regular_expression)) - raise Exception('无法使用正则提取') - - -def wrap_template_content(content_dict, content_template): - if content_template == '': - content_template = '\t'.join( - ['{' + k + '}' for k in content_dict.keys()]) - - for k, v in content_dict.items(): - content_template = content_template.replace('{' + k + '}', v) - - content = content_template - return content - - -def get_content(url, - is_chrome, - selector_type, - selector, - content_template, - regular_expression=None, - headers=None, - debug=False): - if is_chrome == 0: - selector_handler = new_handler('request', debug) - else: - selector_handler = new_handler('phantomjs', debug) - - # 兼容旧版本,默认转为{content} - selector_dict = OrderedDict() - if '{' not in selector: - selector_dict['content'] = selector - else: - selector_split_list = selector.split('\n') - for selector_split in selector_split_list: - selector_split = selector_split.strip() - key, value = selector_split.split('{') - value = value.split('}')[0] - selector_dict[key] = value - - if selector_type == 0: - content_dict = selector_handler.get_by_xpath(url, selector_dict, - headers) - elif selector_type == 1: - content_dict = selector_handler.get_by_css(url, selector_dict, headers) - elif selector_type == 2: - content_dict = selector_handler.get_by_json(url, selector_dict, - headers) - else: - logger.error('无效选择器') - raise Exception('无效选择器') - - # 添加或替换保留字段:{url} - if 'url' in content_dict: - content_dict['url'] = url - content = wrap_template_content(content_dict, content_template) - - if regular_expression: - content = extract_by_re(content, regular_expression) - return content - - -@func_set_timeout(10) -def get_rss_content(url): - feeds = feedparser.parse(url) - - if len(feeds.entries) == 0: - raise Exception('无内容') - - single_post = feeds.entries[0] - item = {} - item['title'] = single_post.title - item['link'] = single_post.link - item['guid'] = single_post.id - - return item diff --git a/task/utils/notification/__init__.py b/task/utils/notification/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/task/utils/notification/bark_notification.py b/task/utils/notification/bark_notification.py deleted file mode 100644 index 549c730..0000000 --- a/task/utils/notification/bark_notification.py +++ /dev/null @@ -1,36 +0,0 @@ -import json -import logging -import re -import requests - -from task.utils.notification.notification import Notification -import urllib.parse - -logger = logging.getLogger('main') - - -def getUrlQuery(content): - """ - Extract the first URL in the content with format of '?url=URL', return '' if none URL found. - """ - regex = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()]+|\(([^\s()]+|(\([^\s()]+\)))*\))+(?:\(([^\s()]+|(\([^\s()]+\)))*\)|[^\s`!()\[\]{};:'\".,?«»""‘’]))" - urls = re.findall(regex, content) - if len(urls): - url = [x[0] for x in urls][0] - url_query = f'?url={urllib.parse.quote_plus(url)}' - return url_query - return '' - - -class BarkNotification(Notification): - def send(self, to, header, content): - if to == '默认': - logger.error('没有设置Bark KEY,无法发送Bark通知') - raise Exception('没有设置Bark KEY,无法发送Bark通知') - url = 'https://api.day.app/{}/{}/{}{}'.format( - to, header, urllib.parse.quote_plus(content), getUrlQuery(content)) - r = requests.post(url) - - res = json.loads(r.text) - if res['code'] != 200: - raise Exception(res['message']) diff --git a/task/utils/notification/custom_notification.py b/task/utils/notification/custom_notification.py deleted file mode 100644 index 49de86b..0000000 --- a/task/utils/notification/custom_notification.py +++ /dev/null @@ -1,33 +0,0 @@ -import json -import logging - -import requests - -from task.utils.notification.notification import Notification -import urllib.parse - -logger = logging.getLogger('main') - - -class CustomNotification(Notification): - def send(self, to, header, content): - if to == '默认': - logger.error('没有设置通知网址,无法发送自定义通知') - raise Exception('没有设置通知网址,无法发送自定义通知') - loc = to.find("{data=") - if loc == -1: - url = to.replace('{header}', - urllib.parse.quote_plus(header)).replace( - '{content}', urllib.parse.quote_plus(content)) - r = requests.get(url) - res = json.loads(r.text) - logger.debug('自定义[GET]通知:{},结果:{}'.format(url, res)) - - else: - url = to[:loc] - data = to[loc + 6:to.rfind("}")].replace( - '{header}', - json.dumps(header)).replace('{content}', json.dumps(content)) - r = requests.post(url, json=json.loads(data)) - res = json.loads(r.text) - logger.debug('自定义[POST]通知:{},传输数据:{},结果:{}'.format(url, data, res)) diff --git a/task/utils/notification/notification_handler.py b/task/utils/notification/notification_handler.py deleted file mode 100644 index 4440c9f..0000000 --- a/task/utils/notification/notification_handler.py +++ /dev/null @@ -1,30 +0,0 @@ -from task.utils.notification.mail_notification import MailNotification -from task.utils.notification.wechat_notification import WechatNotification -from task.utils.notification.pushover_notification import PushoverNotification -from task.utils.notification.bark_notification import BarkNotification -from task.utils.notification.custom_notification import CustomNotification -from task.utils.notification.slack_notification import SlackNotification -from task.utils.notification.telegram_notification import TelegramNotification - -import logging -logger = logging.getLogger('main') - - -def new_handler(name): - if name == 'mail': - return MailNotification() - elif name == 'wechat': - return WechatNotification() - elif name == 'pushover': - return PushoverNotification() - elif name == 'bark': - return BarkNotification() - elif name == 'custom': - return CustomNotification() - elif name == 'slack': - return SlackNotification() - elif name == 'telegram': - return TelegramNotification() - else: - logger.error('通知方式错误') - raise Exception('通知方式错误') diff --git a/task/utils/notification/pushover_notification.py b/task/utils/notification/pushover_notification.py deleted file mode 100755 index 442954e..0000000 --- a/task/utils/notification/pushover_notification.py +++ /dev/null @@ -1,50 +0,0 @@ -import json -import logging -import traceback - -import requests -from requests.exceptions import RequestException -from setting.models import PushoverSetting -from task.utils.notification.notification import Notification - -logger = logging.getLogger('main') - - -class PushoverNotification(Notification): - def __init__(self): - try: - setting = PushoverSetting.objects.first() - except Exception: - logger.error('没有设置Pushover API Token,无法发送通知') - logger.error(traceback.format_exc()) - raise Exception('没有设置Pushover API Token,无法发送通知') - - self.token = setting.api_token - - def send(self, to, header, content): - if to == '默认': - logger.error('没有设置Prushover User Key,无法发送推送通知') - raise Exception('没有设置Prushover User Key,无法发送推送通知') - token = self.token - sendData = { - 'token': token, - 'user': to, - 'message': '【' + header + '】有更新!\n>>>新内容为:\n' + content, - } - pushoverApi = 'https://api.pushover.net/1/messages.json' - - try: - response = requests.post(pushoverApi, sendData, timeout=5) - except RequestException as e: - logger.error('请求错误', traceback.format_exc()) - raise Exception('Error: {}'.format(e)) - - res = json.loads(response.text) - - if res['status'] != 1: - raise Exception(res['errors']) - elif 'info' in res: - if 'no active devices to send to' in res['info']: - raise Exception('User key 对应的账户无激活设备,需要先行到官网购买 License') - else: - logger.debug(res['info']) diff --git a/task/utils/notification/slack_notification.py b/task/utils/notification/slack_notification.py deleted file mode 100644 index 45cd796..0000000 --- a/task/utils/notification/slack_notification.py +++ /dev/null @@ -1,33 +0,0 @@ -import logging -import traceback - -from setting.models import SlackSetting -from slack_sdk import WebClient -from slack_sdk.errors import SlackApiError -from task.utils.notification.notification import Notification - -logger = logging.getLogger('main') - - -class SlackNotification(Notification): - def __init__(self): - try: - setting = SlackSetting.objects.first() - except Exception: - logger.error('没有设置 Slack OAuth Access Token,无法发送通知') - logger.error(traceback.format_exc()) - raise Exception('没有设置 Slack OAuth Access Token,无法发送通知') - - self.token = setting.token - - def send(self, to, header, content): - if to == '默认': - logger.error('没有设置 channel 名称,无法发送 Slack 通知') - raise Exception('没有设置 channel 名称,无法发送 Slack 通知') - client = WebClient(token=self.token) - - try: - client.chat_postMessage(channel=to, - text='{}:{}'.format(header, content)) - except SlackApiError as e: - raise Exception(e.response['error']) diff --git a/task/utils/notification/telegram_notification.py b/task/utils/notification/telegram_notification.py deleted file mode 100644 index 9765450..0000000 --- a/task/utils/notification/telegram_notification.py +++ /dev/null @@ -1,34 +0,0 @@ -import logging -import traceback -import urllib.parse - -import requests -from setting.models import TelegramSetting -from task.utils.notification.notification import Notification - -logger = logging.getLogger('main') - - -class TelegramNotification(Notification): - def __init__(self): - try: - setting = TelegramSetting.objects.first() - except Exception: - logger.error('没有设置 Telegram bot token,无法发送通知') - logger.error(traceback.format_exc()) - raise Exception('没有设置 Telegram bot token,无法发送通知') - - self.token = setting.token - - def send(self, to, header, content): - if to == '默认': - logger.error('没有设置 chat_id,无法发送 Telegram 通知') - raise Exception('没有设置 chat_id,无法发送 Telegram 通知') - - r = requests.get( - 'https://api.telegram.org/bot{}/sendMessage?chat_id={}&text={}'. - format(self.token, to, - urllib.parse.quote_plus('{}: {}'.format(header, content)))) - result = r.json() - if not result['ok']: - raise Exception(result['description']) diff --git a/task/utils/scheduler.py b/task/utils/scheduler.py deleted file mode 100644 index f3e5ce4..0000000 --- a/task/utils/scheduler.py +++ /dev/null @@ -1,239 +0,0 @@ -import logging -import traceback -from datetime import datetime - -import markdown -from apscheduler.jobstores.base import JobLookupError -from func_timeout.exceptions import FunctionTimedOut - -from task.models import Content, RSSTask, Task, TaskStatus -from task.utils.extract_info import get_content, get_rss_content -from task.utils.notification.notification_handler import new_handler -from task.utils.rule import is_changed -from task.views import scheduler - -logger = logging.getLogger('main') - - -# 部分通知方式出错异常 -class PartNotificationError(Exception): - pass - - -def wraper_rss_msg(item): - title = item['title'] - link = item['link'] - - res = '''[{}]({})'''.format(title, link) - return res - - -def send_message(content, header, notifications): - if len(notifications) == 0: - raise Exception('通知方式为空') - - total = 0 - fail = 0 - - exception_content = '' - for notification in notifications: - total += 1 - - type = notification.type - notification_detail = notification.content - - try: - if type == 0: - handler = new_handler('mail') - content = markdown.markdown(content, - output_format='html5', - extensions=['extra']) - handler.send(notification_detail, header, content) - except Exception as e: - fail += 1 - exception_content += 'Mail Exception: {};'.format(repr(e)) - - try: - if type == 1: - handler = new_handler('wechat') - handler.send(notification_detail, header, content) - except Exception as e: - fail += 1 - exception_content += 'Wechat Exception: {};'.format(repr(e)) - - try: - if type == 2: - handler = new_handler('pushover') - handler.send(notification_detail, header, content) - except Exception as e: - fail += 1 - exception_content += 'Pushover Exception: {};'.format(repr(e)) - - try: - if type == 3: - handler = new_handler('bark') - handler.send(notification_detail, header, content) - except Exception as e: - fail += 1 - exception_content += 'Bark Exception: {};'.format(repr(e)) - - try: - if type == 4: - handler = new_handler('custom') - handler.send(notification_detail, header, content) - except Exception as e: - fail += 1 - exception_content += 'Custom Exception: {};'.format(repr(e)) - - try: - if type == 5: - handler = new_handler('slack') - handler.send(notification_detail, header, content) - except Exception as e: - fail += 1 - exception_content += 'Slack Exception: {};'.format(repr(e)) - - try: - if type == 6: - handler = new_handler('telegram') - handler.send(notification_detail, header, content) - except Exception as e: - fail += 1 - exception_content += 'Telegram Exception: {};'.format(repr(e)) - - if fail> 0: - if fail < total: - raise PartNotificationError('监测到变化,部分通知方式发送错误:' + - exception_content) - else: - raise Exception('监测到变化,但发送通知错误:' + exception_content) - - -def monitor(id, type): - status = '' - global_content = None - last = None - try: - if type == 'html': - task = Task.objects.get(pk=id) - name = task.name - url = task.url - selector_type = task.selector_type - selector = task.selector - is_chrome = task.is_chrome - content_template = task.template - - notifications = [i for i in task.notification.iterator()] - - regular_expression = task.regular_expression - rule = task.rule - headers = task.headers - - try: - last = Content.objects.get(task_id=id, task_type=type) - except Exception: - last = Content(task_id=id) - - last_content = last.content - content = get_content(url, is_chrome, selector_type, selector, - content_template, regular_expression, - headers) - global_content = content - status_code = is_changed(rule, content, last_content) - logger.info( - 'rule: {}, content: {}, last_content: {}, status_code: {}'. - format(rule, content, last_content, status_code)) - if status_code == 1: - status = '监测到变化,但未命中规则,最新值为{}'.format(content) - last.content = content - last.save() - elif status_code == 2: - status = '监测到变化,且命中规则,最新值为{}'.format(content) - send_message(content, name, notifications) - last.content = content - last.save() - elif status_code == 3: - status = '监测到变化,最新值为{}'.format(content) - send_message(content, name, notifications) - last.content = content - last.save() - elif status_code == 0: - status = '成功执行但未监测到变化,当前值为{}'.format(content) - elif type == 'rss': - rss_task = RSSTask.objects.get(id=id) - url = rss_task.url - name = rss_task.name - - notifications = [i for i in rss_task.notification.iterator()] - - try: - last = Content.objects.get(task_id=id, task_type=type) - except Exception: - last = Content(task_id=id, task_type='rss') - - last_guid = last.content - item = get_rss_content(url) - global_content = item['guid'] - if item['guid'] != last_guid: - content = wraper_rss_msg(item) - send_message(content, name, notifications) - last.content = item['guid'] - last.save() - status = '监测到变化,最新值:' + item['guid'] - else: - status = '成功执行但未监测到变化,当前值为{}'.format(last_guid) - - except FunctionTimedOut: - logger.error(traceback.format_exc()) - status = '解析RSS超时' - except PartNotificationError as e: - logger.error(traceback.format_exc()) - status = repr(e) - last.content = global_content - last.save() - except Exception as e: - logger.error(traceback.format_exc()) - status = repr(e) - - task_status = TaskStatus.objects.get(task_id=id, task_type=type) - task_status.last_run = datetime.now() - task_status.last_status = status - task_status.save() - - -def add_job(id, interval, type='html'): - task_id = '' - if type == 'html': - task_id = id - elif type == 'rss': - task_id = 'rss{}'.format(id) - try: - scheduler.remove_job(job_id='task_{}'.format(task_id)) - except Exception: - pass - scheduler.add_job(func=monitor, - args=( - id, - type, - ), - trigger='interval', - minutes=interval, - id='task_{}'.format(task_id), - replace_existing=True) - logger.info('添加定时任务task_{}'.format(task_id)) - - -def remove_job(id, type='html'): - task_id = '' - - if type == 'html': - task_id = id - elif type == 'rss': - task_id = 'rss{}'.format(id) - - try: - scheduler.remove_job('task_{}'.format(task_id)) - logger.info('删除定时任务task_{}'.format(task_id)) - except JobLookupError as e: - logger.info(e) - logger.info('task_{}不存在'.format(task_id)) diff --git a/task/utils/selector/request_selector.py b/task/utils/selector/request_selector.py deleted file mode 100644 index dbc452b..0000000 --- a/task/utils/selector/request_selector.py +++ /dev/null @@ -1,50 +0,0 @@ -import ast -from collections import OrderedDict - -import requests -from task.utils.selector.selector import SelectorABC as FatherSelector - - -class RequestsSelector(FatherSelector): - def __init__(self, debug=False): - self.debug = debug - - def get_html(self, url, headers): - if headers: - header_dict = ast.literal_eval(headers) - if type(header_dict) != dict: - raise Exception('必须是字典格式') - - r = requests.get(url, headers=header_dict, timeout=10) - else: - r = requests.get(url, timeout=10) - r.encoding = r.apparent_encoding - html = r.text - return html - - def get_by_xpath(self, url, selector_dict, headers=None): - html = self.get_html(url, headers) - - result = OrderedDict() - for key, xpath_ext in selector_dict.items(): - result[key] = self.xpath_parse(html, xpath_ext) - - return result - - def get_by_css(self, url, selector_dict, headers=None): - html = self.get_html(url, headers) - - result = OrderedDict() - for key, css_ext in selector_dict.items(): - result[key] = self.css_parse(html, css_ext) - - return result - - def get_by_json(self, url, selector_dict, headers=None): - html = self.get_html(url, headers) - - result = OrderedDict() - for key, json_ext in selector_dict.items(): - result[key] = self.json_parse(html, json_ext) - - return result diff --git a/task/utils/selector/selector.py b/task/utils/selector/selector.py deleted file mode 100644 index 22286da..0000000 --- a/task/utils/selector/selector.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# coding=UTF-8 -''' -@Author: LogicJake, Jacob -@Date: 2019-03-25 12:23:59 -@LastEditTime: 2020-03-01 14:54:14 -''' -import json -from abc import ABCMeta, abstractmethod - -import jsonpath -from scrapy.selector import Selector - - -class SelectorABC(): - __metaclass__ = ABCMeta - - def xpath_parse(self, html, xpath_ext): - if 'string()' in xpath_ext: - xpath_ext = xpath_ext.split('/') - xpath_ext = '/'.join(xpath_ext[:-1]) - res = Selector( - text=html).xpath(xpath_ext)[0].xpath('string(.)').extract() - else: - res = Selector(text=html).xpath(xpath_ext).extract() - - if len(res) != 0: - return res[0] - else: - raise Exception('无法获取文本信息') - - def css_parse(self, html, css_ext): - res = Selector(text=html).css(css_ext).extract() - - if len(res) != 0: - return res[0] - else: - raise Exception('无法获取文本信息') - - def json_parse(self, html, json_ext): - try: - resJson = json.loads(html) - except Exception: - raise Exception('Json转换错误') - res = json.dumps(jsonpath.jsonpath(resJson, json_ext), - ensure_ascii=False) - - if len(res) != 0: - return res - else: - raise Exception('无法获取文本信息') - - @abstractmethod - def get_by_xpath(self): - pass - - @abstractmethod - def get_by_css(self): - pass - - @abstractmethod - def get_by_json(self): - pass diff --git a/task/views.py b/task/views.py deleted file mode 100644 index 9906786..0000000 --- a/task/views.py +++ /dev/null @@ -1,22 +0,0 @@ -from apscheduler.schedulers.background import BackgroundScheduler -from django_apscheduler.jobstores import DjangoJobStore, register_events -import logging - -logger = logging.getLogger('main') - - -def ping(): - logger.info('pong!!') - - -# 定时器 -scheduler = BackgroundScheduler() -scheduler.configure(timezone='Asia/Shanghai') -scheduler.add_jobstore(DjangoJobStore(), 'default') -scheduler.add_job(func=ping, - trigger='interval', - minutes=1, - id='ping', - replace_existing=True) -register_events(scheduler) -scheduler.start() diff --git a/tests/test_extract_info.py b/tests/test_extract_info.py index ffd26c8..6d247ad 100644 --- a/tests/test_extract_info.py +++ b/tests/test_extract_info.py @@ -1,5 +1,5 @@ import unittest -from task.utils.extract_info import extract_by_re +from app.main.extract_info import extract_by_re class TestExtract(unittest.TestCase): @@ -13,6 +13,7 @@ def test_re2(self): regular_expression = r'([1-9]\d*)' content = '1391好贵' res = extract_by_re(content, regular_expression) + print(res) self.assertEqual(res, '1391') diff --git a/tests/test_rule.py b/tests/test_rule.py index 8989f04..82035f8 100644 --- a/tests/test_rule.py +++ b/tests/test_rule.py @@ -1,5 +1,12 @@ +#!/usr/bin/env python +# coding=UTF-8 +''' +@Author: LogicJake +@Date: 2019-03-31 10:45:15 +@LastEditTime: 2019-03-31 12:38:06 +''' import unittest -from task.utils.rule import is_changed +from app.main.rule import is_changed class TestRule(unittest.TestCase): @@ -36,19 +43,6 @@ def test_contains(self): res = is_changed(rule, content, last_content) self.assertEqual(res, 1) - def test_without(self): - rule = '-without 变化' - content = 'abcdas' - last_content = '不变化' - res = is_changed(rule, content, last_content) - self.assertEqual(res, 2) - - rule = '-without 变化' - content = '我发生变化了' - last_content = '不变化' - res = is_changed(rule, content, last_content) - self.assertEqual(res, 1) - def test_increase(self): rule = '-increase 3' content = '1888.1' @@ -156,49 +150,6 @@ def test_more(self): res = is_changed(rule, content, last_content) self.assertEqual(res, 0) - def test_multi_rules(self): - rule = '-contain 3;-contain 4' - content = '3' - last_content = '2' - res = is_changed(rule, content, last_content) - self.assertEqual(res, 2) - - rule = '-contain 3;-contain 4' - content = '4' - last_content = '2' - res = is_changed(rule, content, last_content) - self.assertEqual(res, 2) - - rule = '-contain 3;-contain 4' - content = '2' - last_content = '1' - res = is_changed(rule, content, last_content) - self.assertEqual(res, 1) - - rule = '-contain 3;-contain 4' - content = '3' - last_content = '3' - res = is_changed(rule, content, last_content) - self.assertEqual(res, 0) - - rule = '-contain 3;-contain 4' - content = '4' - last_content = '4' - res = is_changed(rule, content, last_content) - self.assertEqual(res, 0) - - rule = '-contain 2;-more 4' - content = '5' - last_content = '1' - res = is_changed(rule, content, last_content) - self.assertEqual(res, 2) - - rule = '-contain 2;-more 4' - content = '2' - last_content = '1' - res = is_changed(rule, content, last_content) - self.assertEqual(res, 2) - if __name__ == '__main__': unittest.main() diff --git a/webmonitor/__init__.py b/webmonitor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/webmonitor/asgi.py b/webmonitor/asgi.py deleted file mode 100644 index 0239df2..0000000 --- a/webmonitor/asgi.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -ASGI config for webmonitor project. - -It exposes the ASGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ -""" - -import os - -from django.core.asgi import get_asgi_application - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'webmonitor.settings') -application = get_asgi_application() diff --git a/webmonitor/settings.py b/webmonitor/settings.py deleted file mode 100644 index 884d22e..0000000 --- a/webmonitor/settings.py +++ /dev/null @@ -1,194 +0,0 @@ -""" -Django settings for webmonitor project. - -Generated by 'django-admin startproject' using Django 3.0.5. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/3.0/ref/settings/ -""" - -import os - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") - -SIMPLEUI_DEFAULT_THEME = 'admin.lte.css' - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '@c*hlp1prhh5up^i9c9&0w86&@2!d)fb*r$up1cf!hhnlyf_@&' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [ - '*', -] - -# Application definition - -INSTALLED_APPS = [ - 'import_export', 'setting.apps.SettingConfig', 'task.apps.TaskConfig', - 'simpleui', 'django_apscheduler', 'django.contrib.admin', - 'django.contrib.auth', 'django.contrib.contenttypes', - 'django.contrib.sessions', 'django.contrib.messages', - 'django.contrib.staticfiles' -] - -STATICFILES_DIRS = [ - os.path.join(BASE_DIR, 'static'), -] - -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -ROOT_URLCONF = 'webmonitor.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -WSGI_APPLICATION = 'webmonitor.wsgi.application' - -# Database -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases - -os.makedirs(os.path.join(BASE_DIR, 'db'), exist_ok=True) - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db', 'db.sqlite3'), - 'OPTIONS': { - 'timeout': 20, - } - } -} - -# Password validation -# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': - 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': - 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': - 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': - 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - -# Internationalization -# https://docs.djangoproject.com/en/3.0/topics/i18n/ - -LANGUAGE_CODE = 'zh-hans' - -TIME_ZONE = 'Asia/Shanghai' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = False - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.0/howto/static-files/ - -STATIC_URL = '/static/' - -# SIMPLEUI 配置 -SIMPLEUI_HOME_INFO = False -SIMPLEUI_CONFIG = { - 'system_keep': - True, - 'menus': [{ - 'name': '使用文档', - 'icon': 'fa fa-file', - 'url': 'https://www.logicjake.xyz/WebMonitor/#/how' - }], - 'menu_display': ['Simpleui', '任务管理', '系统管理', '使用文档'], -} - -SIMPLEUI_ICON = { - '系统管理': 'fas fa-cog', - '系统邮箱': 'fas fa-mail-bulk', - 'RSS监控管理': 'fas fa-rss', - '网页监控管理': 'far fa-file-code', - '任务状态': 'far fa-calendar-check', - '日志查看': 'fas fa-book-reader', -} - -SIMPLEUI_ANALYSIS = False -SIMPLEUI_STATIC_OFFLINE = True -SIMPLEUI_LOGO = 'https://www.logicjake.xyz/img/favicon.ico' - -# logging配置 -log_path = os.path.join(BASE_DIR, 'static', 'log') -log_file = os.path.join(log_path, 'log.txt') -if not os.path.exists(log_file): - os.makedirs(log_path, exist_ok=True) - -LOGGING = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'standard': { - 'format': - '%(asctime)s [%(threadName)s:%(thread)d] ' - '[%(module)s:%(funcName)s:%(lineno)d] [%(levelname)s]- %(message)s' - } - }, - 'filter': {}, - 'handlers': { - 'default': { - 'level': 'DEBUG', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': log_file, - 'maxBytes': 1024 * 1024 * 5, - 'backupCount': 5, - 'formatter': 'standard', - 'encoding': 'utf-8', - } - }, - 'loggers': { - 'main': { - 'handlers': ['default'], - 'level': 'INFO', - 'propagate': True - }, - } -} diff --git a/webmonitor/urls.py b/webmonitor/urls.py deleted file mode 100644 index ef86a09..0000000 --- a/webmonitor/urls.py +++ /dev/null @@ -1,24 +0,0 @@ -"""webmonitor URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/3.0/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" -from django.contrib import admin -from django.urls import path - -admin.site.site_title = 'I Am Watching You' -admin.site.site_header = 'I Am Watching You' - -urlpatterns = [ - path('', admin.site.urls), -] diff --git a/webmonitor/wsgi.py b/webmonitor/wsgi.py deleted file mode 100644 index 78d43fc..0000000 --- a/webmonitor/wsgi.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -WSGI config for webmonitor project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ -""" - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'webmonitor.settings') - -application = get_wsgi_application() diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 0000000..719230a --- /dev/null +++ b/wsgi.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# @Author: LogicJake +# @Date: 2019-02-15 19:53:12 +# @Last Modified time: 2019-03-12 18:31:47 +import os + +from app import create_app + +dotenv_path = os.path.join(os.path.dirname(__file__), '.env') +if os.path.exists(dotenv_path): + from dotenv import load_dotenv + load_dotenv(dotenv_path) + +env = os.getenv('FLASK_ENV') or 'production' +app = create_app(env)