diff --git a/.gitignore b/.gitignore index 7e21a43..407a684 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,6 @@ dmypy.json **__pycache__/ **.idea +**dist/ +**build +**__pycache__ diff --git a/001-Downloader/README.md b/001-Downloader/README.md index c3cd49b..f5c919b 100644 --- a/001-Downloader/README.md +++ b/001-Downloader/README.md @@ -1,9 +1,9 @@ # 资源下载器 -本项目主要通过网络上开源的项目聚合成了一个跨平台的下载工具,可批量下载抖音、快手和YouTube视音频资源。下载地址: +本项目主要通过网络上开源的项目聚合成了一个跨平台的下载工具,可批量下载抖音、快手视音频资源。下载地址: -MacOS:[Downloader1.0.1-mac](https://github.com/xhunmon/PythonIsTools/releases/download/v1.0.1/downloader1.0.1-mac) +MacOS:[Downloader1.0.3.app](https://github.com/xhunmon/PythonIsTools/releases/download/v1.0.3/Downloader1.0.3.app.zip) 下载后解压后使用 -Window:[downloader1.0.1-window.exe](https://github.com/xhunmon/PythonIsTools/releases/download/v1.0.1/downloader1.0.1-window.exe) +Window:[Downloader1.0.3.exe](https://github.com/xhunmon/PythonIsTools/releases/download/v1.0.3/Downloader1.0.3.exe.zip) 下载后解压后使用 效果如图: @@ -25,6 +25,8 @@ pyinstaller -F -i res/logo.ico main.py -w #3:再次进行打包,参考installer-mac.sh pyinstaller -F -i res/logo.ico main.spec -w ``` +打包脚本与配置已放在 `doc` 目录下,需要拷贝出根目录进行打包。 + 注意: pyinstaller打包工具的版本与python版本、python所需第三方库以及操作系统会存在各种问题,所以需要看日志查找问题。例如:打包后运用,发现导入pyppeteer报错,通过降低版本后能正常使用:pip install pyppeteer==0.2.2 diff --git a/001-Downloader/config.ini b/001-Downloader/config.ini index ca0245f..b0ce152 100644 --- a/001-Downloader/config.ini +++ b/001-Downloader/config.ini @@ -1,10 +1,10 @@ # 常用配置模块 [common] #软件使用截止日期 -expired_time=2021年12月15日 23:59:59 +expired_time=2025年12月15日 23:59:59 #app的版本名称 -version_name=1.0.1 +version_name=1.0.4 #app的版本号 -version_code=100 \ No newline at end of file +version_code=1040 \ No newline at end of file diff --git a/001-Downloader/doc/installer-window.sh b/001-Downloader/doc/installer-window.sh deleted file mode 100644 index d53c4c1..0000000 --- a/001-Downloader/doc/installer-window.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -pyinstaller -F -i res\\logo.ico -w main.spec main.py --p type_enum.py --p ui.py --p utils.py --p downloader.py --p douyin\\dy_download.py --p kuaishou\\ks_download.py --p pytube\\captions.py --p pytube\\cipher.py --p pytube\\cli.py --p pytube\\exceptions.py --p pytube\\extract.py --p pytube\\helpers.py --p pytube\\innertube.py --p pytube\\itags.py --p pytube\\metadata.py --p pytube\\monostate.py --p pytube\\parser.py --p pytube\\query.py --p pytube\\request.py --p pytube\\streams.py --p pytube\\version.py --p pytube\\__init__.py --p pytube\\__main__.py --p pytube\\contrib\\__init__.py --p pytube\\contrib\\channel.py --p pytube\\contrib\\playlist.py --p pytube\\contrib\\search.py \ No newline at end of file diff --git a/001-Downloader/main.spec b/001-Downloader/doc/mac-sh/main.spec similarity index 90% rename from 001-Downloader/main.spec rename to 001-Downloader/doc/mac-sh/main.spec index afd7f6d..6b54090 100644 --- a/001-Downloader/main.spec +++ b/001-Downloader/doc/mac-sh/main.spec @@ -38,3 +38,7 @@ exe = EXE(pyz, target_arch=None, codesign_identity=None, entitlements_file=None , icon='res/logo.ico') +app = BUNDLE(exe, + name='Downloader.app', + icon='res/logo.ico', + bundle_identifier=None) \ No newline at end of file diff --git a/001-Downloader/doc/mac-sh/pyinstaller.sh b/001-Downloader/doc/mac-sh/pyinstaller.sh new file mode 100644 index 0000000..ebc509e --- /dev/null +++ b/001-Downloader/doc/mac-sh/pyinstaller.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +pyinstaller -F -i res/logo.ico main.spec main.py -w \ +-p type_enum.py \ +-p ui.py \ +-p utils.py \ +-p downloader.py \ +-p douyin/dy_download.py \ +-p kuaishou/ks_download.py \ No newline at end of file diff --git a/001-Downloader/doc/main-window.spec b/001-Downloader/doc/win-sh/main.spec similarity index 87% rename from 001-Downloader/doc/main-window.spec rename to 001-Downloader/doc/win-sh/main.spec index 36e43cf..3f8b91e 100644 --- a/001-Downloader/doc/main-window.spec +++ b/001-Downloader/doc/win-sh/main.spec @@ -5,7 +5,7 @@ block_cipher = None a = Analysis(['main.py','type_enum.py','ui.py','utils.py','downloader.py','douyin\\dy_download.py'], - pathex=['D:\\develop\\Python\\project\\pyofafish\\crawling'], + pathex=['.'], binaries=[], datas=[('res\\logo.ico', 'images'),('config.ini', '.')], hiddenimports=[], @@ -38,3 +38,7 @@ exe = EXE(pyz, target_arch=None, codesign_identity=None, entitlements_file=None , icon='res\\logo.ico') +app = BUNDLE(exe, + name='Downloader.exe', + icon='res\\logo.ico', + bundle_identifier=None) diff --git a/001-Downloader/doc/win-sh/pyinstaller.sh b/001-Downloader/doc/win-sh/pyinstaller.sh new file mode 100644 index 0000000..d03605f --- /dev/null +++ b/001-Downloader/doc/win-sh/pyinstaller.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +pyinstaller -F -i res\\logo.ico -w main.spec main.py +-p type_enum.py +-p ui.py +-p utils.py +-p downloader.py +-p douyin\\dy_download.py +-p kuaishou\\ks_download.py \ No newline at end of file diff --git a/001-Downloader/douyin/dy_download.py b/001-Downloader/douyin/dy_download.py index 470dbba..f528828 100644 --- a/001-Downloader/douyin/dy_download.py +++ b/001-Downloader/douyin/dy_download.py @@ -11,10 +11,8 @@ import os import re import time -from urllib import parse import requests -import requests_html from downloader import Downloader @@ -32,7 +30,7 @@ def start(self, url, path): # 读取保存路径 self.save = path # 读取下载视频个数 - self.count = 35 + self.count = 10 # 读取下载是否下载音频 self.musicarg = True # 读取用户主页地址 @@ -50,6 +48,11 @@ def start(self, url, path): self.user = url else: self.single = url + # https://www.douyin.com/video/6979067378848042276?extra_params=%7B%22search_id%22%3A%22202109260757420101511740995D070AF5%22%2C%22search_result_id%22%3A%226979067378848042276%22%2C%22search_type%22%3A%22video%22%2C%22search_keyword%22%3A%22%E6%A8%A1%E7%89%B9%22%7D&previous_page=search_result + # try: + # self.single = re.findall(r'(http.+?)\?extra_params', url)[0] + # except: + # self.single = url if len(self.single)> 0: self.count = 1 @@ -59,13 +62,13 @@ def start(self, url, path): # 单条数据页面 def parse_single(self): - session = requests_html.HTMLSession() - html = session.get(self.single) - - result = re.findall(r'id="RENDER_DATA".+?json">(.+?)', html.text)[0] - urls = parse.unquote_plus(result) - jsonObj = json.loads(urls) - + url = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', self.single)[ + 0] + r = requests.get(url=url) + key = re.findall('video/(\d+)?', str(r.url))[0] + jx_url = f'https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={key}' # 官方接口 + js = json.loads(requests.get(url=jx_url, headers=self.headers).text) + detail = js['item_list'][0] # 作者信息 author_list = [] # 无水印视频链接 @@ -75,19 +78,12 @@ def parse_single(self): # 作者id nickname = [] max_cursor = 0 - - if ('C_12' in jsonObj): - detail = jsonObj['C_12']['aweme']['detail'] - elif ('C_14' in jsonObj): - detail = jsonObj['C_14']['aweme']['detail'] - else: - detail = jsonObj['C_0']['aweme']['detail'] author_list.append(str(detail['desc'])) - video_list.append(str("https:" + detail['video']['playAddr'][0]['src'])) - aweme_id.append(str(detail['awemeId'])) - nickname.append(str(detail['authorInfo']['nickname'])) + video_list.append(str(detail['video']['play_addr']['url_list'][0]).replace('playwm', 'play')) + aweme_id.append(str(detail['aweme_id'])) + nickname.append(str(detail['author']['nickname'])) Downloader.print_ui('开始下载单个视频' + video_list[0]) - self.videos_download(1, author_list, video_list, aweme_id, nickname, max_cursor) + self.videos_download(author_list, video_list, aweme_id, nickname, max_cursor) # 匹配粘贴的url地址 def Find(self, string): @@ -98,10 +94,11 @@ def Find(self, string): # 判断个人主页api链接 def judge_link(self): - user_url = self.user + user_url: str = self.user Downloader.print_ui('----为您下载多个视频----\r') - key = re.findall('/user/(.*?)\?', str(user_url))[0] + + key = re.findall('/user/(.*?)$', str(user_url))[0] if not key: key = user_url[28:83] Downloader.print_ui('----' + '用户的sec_id=' + key + '----\r') @@ -151,7 +148,8 @@ def next_data(self, max_cursor): return user_url = self.user # 获取用户sec_uid - key = re.findall('/user/(.*?)\?', str(user_url))[0] + # key = re.findall('/user/(.*?)\?', str(user_url))[0] + key = re.findall('/user/(.*?)$', str(user_url))[0] if not key: key = user_url[28:83] @@ -166,20 +164,21 @@ def next_data(self, max_cursor): self.end = True return index += 1 - Downloader.print_ui('----正在对' + max_cursor + '页进行第 %d 次尝试----\r' % index) - time.sleep(0.3) + # Downloader.print_ui('----正在对' + max_cursor + '页进行第 %d 次尝试----\r' % index) + Downloader.print_ui('----正在对{}页进行第 {} 次尝试----\r'.format(max_cursor, index)) + time.sleep(3) response = requests.get(url=api_naxt_post_url, headers=self.headers) html = json.loads(response.content.decode()) if self.end == False: # 下一页值 max_cursor = html['max_cursor'] result = html['aweme_list'] - Downloader.print_ui('----' + max_cursor + '页抓获数据成功----\r') + Downloader.print_ui('----{}页抓获数据成功----\r'.format(max_cursor)) # 处理下一页视频信息 self.video_info(result, max_cursor) else: - self.end == True - Downloader.print_ui('----' + max_cursor + '页抓获数据失败----\r') + self.end = True + Downloader.print_ui('----{}页抓获数据失败----\r'.format(max_cursor)) # sys.exit() # 处理视频信息 @@ -199,7 +198,7 @@ def video_info(self, result, max_cursor): # 封面大图 # dynamic_cover = [] - for i2 in range(self.count): + for i2 in range(len(result)): try: author_list.append(str(result[i2]['desc'])) video_list.append(str(result[i2]['video']['play_addr']['url_list'][0])) @@ -209,10 +208,11 @@ def video_info(self, result, max_cursor): except Exception as error: # Downloader.print_ui2(error) pass - self.videos_download(self.count, author_list, video_list, aweme_id, nickname, max_cursor) + self.videos_download(author_list, video_list, aweme_id, nickname, max_cursor) return self, author_list, video_list, aweme_id, nickname, max_cursor - def videos_download(self, count, author_list, video_list, aweme_id, nickname, max_cursor): + def videos_download(self, author_list, video_list, aweme_id, nickname, max_cursor): + count = len(author_list) Downloader.add_total_count(count) for i in range(count): if count == 1: @@ -225,41 +225,48 @@ def videos_download(self, count, author_list, video_list, aweme_id, nickname, ma except: pass Downloader.add_downloading_count() - try: - jx_url = f'https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={aweme_id[i]}' # 官方接口 - js = json.loads(requests.get(url=jx_url, headers=self.headers).text) - music_url = str(js['item_list'][0]['music']['play_url']['url_list'][0]) - music_title = str(js['item_list'][0]['music']['author']) - if self.musicarg == "yes": # 保留音频 - music = requests.get(music_url) # 保存音频 - start = time.time() # 下载开始时间 - size = 0 # 初始化已下载大小 - chunk_size = 1024 # 每次下载的数据大小 - content_size = int(music.headers['content-length']) # 下载文件总大小 - if music.status_code == 200: # 判断是否响应成功 - Downloader.print_ui('[ 音频 ]:' + author_list[i] + '[文件 大小]:{size:.2f} MB'.format( - size=content_size / chunk_size / 1024)) # 开始下载,显示下载文件大小 - # m_url = pre_save + music_title + '-[' + author_list[i] + '].mp3' - m_url = os.path.join(pre_save, - nickname[i] + "-" + music_title + '-[' + author_list[i] + '].mp3') - Downloader.print_ui("路径:" + m_url) - with open(m_url, 'wb') as file: # 显示进度条 - for data in music.iter_content(chunk_size=chunk_size): - file.write(data) - size += len(data) - Downloader.print_ui('\r' + music_title + '\n[下载进度]:%s%.2f%%' % ( - '>' * int(size * 50 / content_size), float(size / content_size * 100))) - end = time.time() # 下载结束时间 - Downloader.print_ui('\n' + music_title + '\n[下载完成]:耗时: %.2f秒\n' % (end - start)) # 输出下载用时时间 - Downloader.add_success_count() - except Exception as error: - # Downloader.print_ui2(error) - Downloader.print_ui('该页音频没有' + str(self.count) + '个,已为您跳过\r') - Downloader.add_failed_count() - break + # try: + # jx_url = f'https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={aweme_id[i]}' # 官方接口 + # js = json.loads(requests.get(url=jx_url, headers=self.headers).text) + # music_url = str(js['item_list'][0]['music']['play_url']['url_list'][0]) + # music_title = str(js['item_list'][0]['music']['author']) + # if self.musicarg == "yes": # 保留音频 + # music = requests.get(music_url) # 保存音频 + # start = time.time() # 下载开始时间 + # size = 0 # 初始化已下载大小 + # chunk_size = 1024 # 每次下载的数据大小 + # content_size = int(music.headers['content-length']) # 下载文件总大小 + # if music.status_code == 200: # 判断是否响应成功 + # Downloader.print_ui('[ 音频 ]:' + author_list[i] + '[文件 大小]:{size:.2f} MB'.format( + # size=content_size / chunk_size / 1024)) # 开始下载,显示下载文件大小 + # # m_url = pre_save + music_title + '-[' + author_list[i] + '].mp3' + # m_url = os.path.join(pre_save, + # nickname[i] + "-" + music_title + '-[' + author_list[i] + '].mp3') + # Downloader.print_ui("路径:" + m_url) + # with open(m_url, 'wb') as file: # 显示进度条 + # for data in music.iter_content(chunk_size=chunk_size): + # file.write(data) + # size += len(data) + # Downloader.print_ui('\r' + music_title + '\n[下载进度]:%s%.2f%%' % ( + # '>' * int(size * 50 / content_size), float(size / content_size * 100))) + # end = time.time() # 下载结束时间 + # Downloader.print_ui('\n' + music_title + '\n[下载完成]:耗时: %.2f秒\n' % (end - start)) # 输出下载用时时间 + # Downloader.add_success_count() + # except Exception as error: + # # Downloader.print_ui2(error) + # Downloader.print_ui('该页音频没有' + str(self.count) + '个\r') + # # Downloader.add_failed_count() + # # break try: - video = requests.get(video_list[i]) # 保存视频 + v_url = os.path.join(pre_save, nickname[i] + "-" + '[' + author_list[i] + '].mp4') + # 如果本地已经有了就跳过 + if os.path.exists(v_url): + Downloader.print_ui('{}-已存在!'.format(v_url)) + Downloader.add_success_count() + continue + + video = requests.get(video_list[i], headers=self.headers) # 保存视频 start = time.time() # 下载开始时间 size = 0 # 初始化已下载大小 chunk_size = 100 # 每次下载的数据大小 @@ -268,7 +275,7 @@ def videos_download(self, count, author_list, video_list, aweme_id, nickname, ma Downloader.print_ui( '[ 视频 ]:' + nickname[i] + '-' + author_list[i] + '[文件 大小]:{size:.2f} MB'.format( size=content_size / 1024 / 1024)) # 开始下载,显示下载文件大小 - v_url = os.path.join(pre_save, nickname[i] + "-" + '[' + author_list[i] + '].mp4') + # v_url = os.path.join(pre_save, nickname[i] + "-" + '[' + author_list[i] + '].mp4') # v_url = pre_save + '[' + author_list[i] + '].mp4' Downloader.print_ui("路径:" + v_url) with open(v_url, 'wb') as file: # 显示进度条 @@ -282,7 +289,7 @@ def videos_download(self, count, author_list, video_list, aweme_id, nickname, ma Downloader.add_success_count() except Exception as error: # Downloader.print_ui2(error) - Downloader.print_ui('该页视频没有' + str(self.count) + '个,已为您跳过\r') + Downloader.print_ui('该页视频没有' + str(count) + '个,已为您跳过\r') Downloader.add_failed_count() break self.next_data(max_cursor) diff --git a/001-Downloader/downloader.py b/001-Downloader/downloader.py index 9cc3aa9..1270700 100644 --- a/001-Downloader/downloader.py +++ b/001-Downloader/downloader.py @@ -40,11 +40,10 @@ def print_hint(): Downloader.print_ui( """ 使用说明: - 1、youtube下载需要先让电脑连接外网,地址如:https://www.youtube.com/watch?v=jKhP750VdXw - 2、快手下载用户批量视频如:https://www.kuaishou.com/profile/xxx - 3、快手下载单条视频如:https://www.kuaishou.com/short-video/xxx - 4、抖音下载用户批量视频如:https://www.douyin.com/user/xxx - 5、抖音下载单条视频如:https://www.douyin.com/video/xxx + 1、快手下载用户批量视频如:https://www.kuaishou.com/profile/xxx + 2、快手下载单条视频如:https://www.kuaishou.com/short-video/xxx + 3、抖音下载用户批量视频如:https://www.douyin.com/user/xxx + 4、抖音下载单条视频如:https://www.douyin.com/video/xxx """ ) diff --git a/001-Downloader/installer-mac.sh b/001-Downloader/installer-mac.sh deleted file mode 100644 index 04a0216..0000000 --- a/001-Downloader/installer-mac.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -pyinstaller -F -i res/logo.ico main.spec main.py -w \ --p type_enum.py \ --p ui.py \ --p utils.py \ --p downloader.py \ --p douyin/dy_download.py \ --p kuaishou/ks_download.py \ --p pytube/captions.py \ --p pytube/cipher.py \ --p pytube/cli.py \ --p pytube/exceptions.py \ --p pytube/extract.py \ --p pytube/helpers.py \ --p pytube/innertube.py \ --p pytube/itags.py \ --p pytube/metadata.py \ --p pytube/monostate.py \ --p pytube/parser.py \ --p pytube/query.py \ --p pytube/request.py \ --p pytube/streams.py \ --p pytube/version.py \ --p pytube/__init__.py \ --p pytube/__main__.py \ --p pytube/contrib/__init__.py \ --p pytube/contrib/channel.py \ --p pytube/contrib/playlist.py \ --p pytube/contrib/search.py \ No newline at end of file diff --git a/001-Downloader/main.py b/001-Downloader/main.py index f2d7090..ff73936 100644 --- a/001-Downloader/main.py +++ b/001-Downloader/main.py @@ -14,6 +14,7 @@ # 主模块执行 if __name__ == "__main__": path = os.path.dirname(os.path.realpath(sys.argv[0])) + # path = os.path.dirname('/Users/Qincji/Documents/zmt/') app = Ui() app.set_dir(path) # to do diff --git a/001-Downloader/pytube/__init__.py b/001-Downloader/pytube/__init__.py deleted file mode 100755 index 4eaa1b2..0000000 --- a/001-Downloader/pytube/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# flake8: noqa: F401 -# noreorder -""" -Pytube: a very serious Python library for downloading YouTube Videos. -""" -__title__ = "pytube" -__author__ = "Ronnie Ghose, Taylor Fox Dahlin, Nick Ficano" -__license__ = "The Unlicense (Unlicense)" -__js__ = None -__js_url__ = None - -from pytube.version import __version__ -from pytube.streams import Stream -from pytube.captions import Caption -from pytube.query import CaptionQuery, StreamQuery -from pytube.__main__ import YouTube -from pytube.contrib.playlist import Playlist -from pytube.contrib.channel import Channel -from pytube.contrib.search import Search diff --git a/001-Downloader/pytube/__main__.py b/001-Downloader/pytube/__main__.py deleted file mode 100755 index 1879a56..0000000 --- a/001-Downloader/pytube/__main__.py +++ /dev/null @@ -1,482 +0,0 @@ -""" -This module implements the core developer interface for pytube. - -The problem domain of the :class:`YouTube class focuses almost -exclusively on the developer interface. Pytube offloads the heavy lifting to -smaller peripheral modules and functions. - -""" -import logging -from typing import Any, Callable, Dict, List, Optional - -import pytube -import pytube.exceptions as exceptions -from pytube import extract, request -from pytube import Stream, StreamQuery -from pytube.helpers import install_proxy -from pytube.innertube import InnerTube -from pytube.metadata import YouTubeMetadata -from pytube.monostate import Monostate -from downloader import Downloader - -logger = logging.getLogger(__name__) - - -class YouTube(Downloader): - """Core developer interface for pytube.""" - - def __init__( - self, - url: str, - on_progress_callback: Optional[Callable[[Any, bytes, int], None]] = None, - on_complete_callback: Optional[Callable[[Any, Optional[str]], None]] = None, - proxies: Dict[str, str] = None, - use_oauth: bool = False, - allow_oauth_cache: bool = True - ): - Downloader.__init__(self) - """Construct a :class:`YouTube `. - - :param str url: - A valid YouTube watch URL. - :param func on_progress_callback: - (Optional) User defined callback function for stream download - progress events. - :param func on_complete_callback: - (Optional) User defined callback function for stream download - complete events. - - """ - self._js: Optional[str] = None # js fetched by js_url - self._js_url: Optional[str] = None # the url to the js, parsed from watch html - - self._vid_info: Optional[Dict] = None # content fetched from innertube/player - - self._watch_html: Optional[str] = None # the html of /watch?v= - self._embed_html: Optional[str] = None - self._player_config_args: Optional[Dict] = None # inline js in the html containing - self._age_restricted: Optional[bool] = None - - self._fmt_streams: Optional[List[Stream]] = None - - self._initial_data = None - self._metadata: Optional[YouTubeMetadata] = None - - # video_id part of /watch?v= - self.video_id = extract.video_id(url) - - self.watch_url = f"https://youtube.com/watch?v={self.video_id}" - self.embed_url = f"https://www.youtube.com/embed/{self.video_id}" - - # Shared between all instances of `Stream` (Borg pattern). - self.stream_monostate = Monostate( - on_progress=on_progress_callback, on_complete=on_complete_callback - ) - - if proxies: - install_proxy(proxies) - - self._author = None - self._title = None - self._publish_date = None - - self.use_oauth = use_oauth - self.allow_oauth_cache = allow_oauth_cache - - def __repr__(self): - return f'' - - def on_progress_callback(self, stream, chunk, bytes_remaining): - print("进度: %d" % (bytes_remaining)) - Downloader.print_ui(txt="%s-->进度: %d" % (self.title, bytes_remaining)) - - def on_complete_callback(self, any, file_path): - print("%s; " % self.title) - Downloader.print_ui(txt="%s 100%%" % self.title) - Downloader.add_success_count() - - def start(self, url, path): - self.register_on_complete_callback(self.on_complete_callback) - self.register_on_progress_callback(self.on_progress_callback) - Downloader.add_total_count() - Downloader.add_downloading_count() - Downloader.print_ui(txt="开始任务:%s" % url) - try: - # dl = self.streams.first() - # dl = self.streams.filter(progressive=True, file_extension='mp4').order_by('resolution').desc().first() - dl = self.streams.filter(progressive=True, file_extension='mp4').order_by('fps').asc().first() - Downloader.print_ui(txt="准备下载:%s" % self.title) - dl.download(output_path=path) - except Exception as e: - Downloader.print_ui(txt="%s 下载失败!" % self.title) - - @property - def watch_html(self): - if self._watch_html: - return self._watch_html - self._watch_html = request.get(url=self.watch_url) - return self._watch_html - - @property - def embed_html(self): - if self._embed_html: - return self._embed_html - self._embed_html = request.get(url=self.embed_url) - return self._embed_html - - @property - def age_restricted(self): - if self._age_restricted: - return self._age_restricted - self._age_restricted = extract.is_age_restricted(self.watch_html) - return self._age_restricted - - @property - def js_url(self): - if self._js_url: - return self._js_url - - if self.age_restricted: - self._js_url = extract.js_url(self.embed_html) - else: - self._js_url = extract.js_url(self.watch_html) - - return self._js_url - - @property - def js(self): - if self._js: - return self._js - - # If the js_url doesn't match the cached url, fetch the new js and update - # the cache; otherwise, load the cache. - if pytube.__js_url__ != self.js_url: - self._js = request.get(self.js_url) - pytube.__js__ = self._js - pytube.__js_url__ = self.js_url - else: - self._js = pytube.__js__ - - return self._js - - @property - def initial_data(self): - if self._initial_data: - return self._initial_data - self._initial_data = extract.initial_data(self.watch_html) - return self._initial_data - - @property - def streaming_data(self): - """Return streamingData from video info.""" - if 'streamingData' in self.vid_info: - return self.vid_info['streamingData'] - else: - self.bypass_age_gate() - return self.vid_info['streamingData'] - - @property - def fmt_streams(self): - """Returns a list of streams if they have been initialized. - - If the streams have not been initialized, finds all relevant - streams and initializes them. - """ - self.check_availability() - if self._fmt_streams: - return self._fmt_streams - - self._fmt_streams = [] - - stream_manifest = extract.apply_descrambler(self.streaming_data) - - # If the cached js doesn't work, try fetching a new js file - # https://github.com/pytube/pytube/issues/1054 - try: - extract.apply_signature(stream_manifest, self.vid_info, self.js) - except exceptions.ExtractError: - # To force an update to the js file, we clear the cache and retry - self._js = None - self._js_url = None - pytube.__js__ = None - pytube.__js_url__ = None - extract.apply_signature(stream_manifest, self.vid_info, self.js) - - # build instances of :class:`Stream ` - # Initialize stream objects - for stream in stream_manifest: - video = Stream( - stream=stream, - monostate=self.stream_monostate, - ) - self._fmt_streams.append(video) - - self.stream_monostate.title = self.title - self.stream_monostate.duration = self.length - - return self._fmt_streams - - def check_availability(self): - """Check whether the video is available. - - Raises different exceptions based on why the video is unavailable, - otherwise does nothing. - """ - status, messages = extract.playability_status(self.watch_html) - - for reason in messages: - if status == 'UNPLAYABLE': - if reason == ( - 'Join this channel to get access to members-only content ' - 'like this video, and other exclusive perks.' - ): - raise exceptions.MembersOnly(video_id=self.video_id) - elif reason == 'This live stream recording is not available.': - raise exceptions.RecordingUnavailable(video_id=self.video_id) - else: - raise exceptions.VideoUnavailable(video_id=self.video_id) - elif status == 'LOGIN_REQUIRED': - if reason == ( - 'This is a private video. ' - 'Please sign in to verify that you may see it.' - ): - raise exceptions.VideoPrivate(video_id=self.video_id) - elif status == 'ERROR': - if reason == 'Video unavailable': - raise exceptions.VideoUnavailable(video_id=self.video_id) - elif status == 'LIVE_STREAM': - raise exceptions.LiveStreamError(video_id=self.video_id) - - @property - def vid_info(self): - """Parse the raw vid info and return the parsed result. - - :rtype: Dict[Any, Any] - """ - if self._vid_info: - return self._vid_info - - innertube = InnerTube(use_oauth=self.use_oauth, allow_cache=self.allow_oauth_cache) - - innertube_response = innertube.player(self.video_id) - self._vid_info = innertube_response - return self._vid_info - - def bypass_age_gate(self): - """Attempt to update the vid_info by bypassing the age gate.""" - innertube = InnerTube( - client='ANDROID_EMBED', - use_oauth=self.use_oauth, - allow_cache=self.allow_oauth_cache - ) - innertube_response = innertube.player(self.video_id) - - playability_status = innertube_response['playabilityStatus'].get('status', None) - - # If we still can't access the video, raise an exception - # (tier 3 age restriction) - if playability_status == 'UNPLAYABLE': - raise exceptions.AgeRestrictedError(self.video_id) - - self._vid_info = innertube_response - - @property - def caption_tracks(self) -> List[pytube.Caption]: - """Get a list of :class:`Caption `. - - :rtype: List[Caption] - """ - raw_tracks = ( - self.vid_info.get("captions", {}) - .get("playerCaptionsTracklistRenderer", {}) - .get("captionTracks", []) - ) - return [pytube.Caption(track) for track in raw_tracks] - - @property - def captions(self) -> pytube.CaptionQuery: - """Interface to query caption tracks. - - :rtype: :class:`CaptionQuery `. - """ - return pytube.CaptionQuery(self.caption_tracks) - - @property - def streams(self) -> StreamQuery: - """Interface to query both adaptive (DASH) and progressive streams. - - :rtype: :class:`StreamQuery `. - """ - self.check_availability() - return StreamQuery(self.fmt_streams) - - @property - def thumbnail_url(self) -> str: - """Get the thumbnail url image. - - :rtype: str - """ - thumbnail_details = ( - self.vid_info.get("videoDetails", {}) - .get("thumbnail", {}) - .get("thumbnails") - ) - if thumbnail_details: - thumbnail_details = thumbnail_details[-1] # last item has max size - return thumbnail_details["url"] - - return f"https://img.youtube.com/vi/{self.video_id}/maxresdefault.jpg" - - @property - def publish_date(self): - """Get the publish date. - - :rtype: datetime - """ - if self._publish_date: - return self._publish_date - self._publish_date = extract.publish_date(self.watch_html) - return self._publish_date - - @publish_date.setter - def publish_date(self, value): - """Sets the publish date.""" - self._publish_date = value - - @property - def title(self) -> str: - """Get the video title. - - :rtype: str - """ - if self._title: - return self._title - - try: - self._title = self.vid_info['videoDetails']['title'] - except KeyError: - # Check_availability will raise the correct exception in most cases - # if it doesn't, ask for a report. - self.check_availability() - raise exceptions.PytubeError( - ( - f'Exception while accessing title of {self.watch_url}. ' - 'Please file a bug report at https://github.com/pytube/pytube' - ) - ) - - return self._title - - @title.setter - def title(self, value): - """Sets the title value.""" - self._title = value - - @property - def description(self) -> str: - """Get the video description. - - :rtype: str - """ - return self.vid_info.get("videoDetails", {}).get("shortDescription") - - @property - def rating(self) -> float: - """Get the video average rating. - - :rtype: float - - """ - return self.vid_info.get("videoDetails", {}).get("averageRating") - - @property - def length(self) -> int: - """Get the video length in seconds. - - :rtype: int - """ - return int(self.vid_info.get('videoDetails', {}).get('lengthSeconds')) - - @property - def views(self) -> int: - """Get the number of the times the video has been viewed. - - :rtype: int - """ - return int(self.vid_info.get("videoDetails", {}).get("viewCount")) - - @property - def author(self) -> str: - """Get the video author. - :rtype: str - """ - if self._author: - return self._author - self._author = self.vid_info.get("videoDetails", {}).get( - "author", "unknown" - ) - return self._author - - @author.setter - def author(self, value): - """Set the video author.""" - self._author = value - - @property - def keywords(self) -> List[str]: - """Get the video keywords. - - :rtype: List[str] - """ - return self.vid_info.get('videoDetails', {}).get('keywords', []) - - @property - def channel_id(self) -> str: - """Get the video poster's channel id. - - :rtype: str - """ - return self.vid_info.get('videoDetails', {}).get('channelId', None) - - @property - def channel_url(self) -> str: - """Construct the channel url for the video's poster from the channel id. - - :rtype: str - """ - return f'https://www.youtube.com/channel/{self.channel_id}' - - @property - def metadata(self) -> Optional[YouTubeMetadata]: - """Get the metadata for the video. - - :rtype: YouTubeMetadata - """ - if self._metadata: - return self._metadata - else: - self._metadata = extract.metadata(self.initial_data) - return self._metadata - - def register_on_progress_callback(self, func: Callable[[Any, bytes, int], None]): - """Register a download progress callback function post initialization. - - :param callable func: - A callback function that takes ``stream``, ``chunk``, - and ``bytes_remaining`` as parameters. - - :rtype: None - - """ - self.stream_monostate.on_progress = func - - def register_on_complete_callback(self, func: Callable[[Any, Optional[str]], None]): - """Register a download complete callback function post initialization. - - :param callable func: - A callback function that takes ``stream`` and ``file_path``. - - :rtype: None - - """ - self.stream_monostate.on_complete = func diff --git a/001-Downloader/pytube/captions.py b/001-Downloader/pytube/captions.py deleted file mode 100755 index ed55f9a..0000000 --- a/001-Downloader/pytube/captions.py +++ /dev/null @@ -1,154 +0,0 @@ -import math -import os -import time -import xml.etree.ElementTree as ElementTree -from html import unescape -from typing import Dict, Optional - -from pytube import request -from pytube.helpers import safe_filename, target_directory - - -class Caption: - """Container for caption tracks.""" - - def __init__(self, caption_track: Dict): - """Construct a :class:`Caption `. - - :param dict caption_track: - Caption track data extracted from ``watch_html``. - """ - self.url = caption_track.get("baseUrl") - - # Certain videos have runs instead of simpleText - # this handles that edge case - name_dict = caption_track['name'] - if 'simpleText' in name_dict: - self.name = name_dict['simpleText'] - else: - for el in name_dict['runs']: - if 'text' in el: - self.name = el['text'] - - # Use "vssId" instead of "languageCode", fix issue #779 - self.code = caption_track["vssId"] - # Remove preceding '.' for backwards compatibility, e.g.: - # English -> vssId: .en, languageCode: en - # English (auto-generated) -> vssId: a.en, languageCode: en - self.code = self.code.strip('.') - - @property - def xml_captions(self) -> str: - """Download the xml caption tracks.""" - return request.get(self.url) - - def generate_srt_captions(self) -> str: - """Generate "SubRip Subtitle" captions. - - Takes the xml captions from :meth:`~pytube.Caption.xml_captions` and - recompiles them into the "SubRip Subtitle" format. - """ - return self.xml_caption_to_srt(self.xml_captions) - - @staticmethod - def float_to_srt_time_format(d: float) -> str: - """Convert decimal durations into proper srt format. - - :rtype: str - :returns: - SubRip Subtitle (str) formatted time duration. - - float_to_srt_time_format(3.89) -> '00:00:03,890' - """ - fraction, whole = math.modf(d) - time_fmt = time.strftime("%H:%M:%S,", time.gmtime(whole)) - ms = f"{fraction:.3f}".replace("0.", "") - return time_fmt + ms - - def xml_caption_to_srt(self, xml_captions: str) -> str: - """Convert xml caption tracks to "SubRip Subtitle (srt)". - - :param str xml_captions: - XML formatted caption tracks. - """ - segments = [] - root = ElementTree.fromstring(xml_captions) - for i, child in enumerate(list(root)): - text = child.text or "" - caption = unescape(text.replace("\n", " ").replace(" ", " "),) - try: - duration = float(child.attrib["dur"]) - except KeyError: - duration = 0.0 - start = float(child.attrib["start"]) - end = start + duration - sequence_number = i + 1 # convert from 0-indexed to 1. - line = "{seq}\n{start} --> {end}\n{text}\n".format( - seq=sequence_number, - start=self.float_to_srt_time_format(start), - end=self.float_to_srt_time_format(end), - text=caption, - ) - segments.append(line) - return "\n".join(segments).strip() - - def download( - self, - title: str, - srt: bool = True, - output_path: Optional[str] = None, - filename_prefix: Optional[str] = None, - ) -> str: - """Write the media stream to disk. - - :param title: - Output filename (stem only) for writing media file. - If one is not specified, the default filename is used. - :type title: str - :param srt: - Set to True to download srt, false to download xml. Defaults to True. - :type srt bool - :param output_path: - (optional) Output path for writing media file. If one is not - specified, defaults to the current working directory. - :type output_path: str or None - :param filename_prefix: - (optional) A string that will be prepended to the filename. - For example a number in a playlist or the name of a series. - If one is not specified, nothing will be prepended - This is separate from filename so you can use the default - filename but still add a prefix. - :type filename_prefix: str or None - - :rtype: str - """ - if title.endswith(".srt") or title.endswith(".xml"): - filename = ".".join(title.split(".")[:-1]) - else: - filename = title - - if filename_prefix: - filename = f"{safe_filename(filename_prefix)}{filename}" - - filename = safe_filename(filename) - - filename += f" ({self.code})" - - if srt: - filename += ".srt" - else: - filename += ".xml" - - file_path = os.path.join(target_directory(output_path), filename) - - with open(file_path, "w", encoding="utf-8") as file_handle: - if srt: - file_handle.write(self.generate_srt_captions()) - else: - file_handle.write(self.xml_captions) - - return file_path - - def __repr__(self): - """Printable object representation.""" - return ''.format(s=self) diff --git a/001-Downloader/pytube/cipher.py b/001-Downloader/pytube/cipher.py deleted file mode 100755 index 70ac770..0000000 --- a/001-Downloader/pytube/cipher.py +++ /dev/null @@ -1,679 +0,0 @@ -""" -This module contains all logic necessary to decipher the signature. - -YouTube's strategy to restrict downloading videos is to send a ciphered version -of the signature to the client, along with the decryption algorithm obfuscated -in JavaScript. For the clients to play the videos, JavaScript must take the -ciphered version, cycle it through a series of "transform functions," and then -signs the media URL with the output. - -This module is responsible for (1) finding and extracting those "transform -functions" (2) maps them to Python equivalents and (3) taking the ciphered -signature and decoding it. - -""" -import logging -import re -from itertools import chain -from typing import Any, Callable, Dict, List, Optional, Tuple - -from pytube.exceptions import ExtractError, RegexMatchError -from pytube.helpers import cache, regex_search -from pytube.parser import find_object_from_startpoint, throttling_array_split - -logger = logging.getLogger(__name__) - - -class Cipher: - def __init__(self, js: str): - self.transform_plan: List[str] = get_transform_plan(js) - var_regex = re.compile(r"^\w+\W") - var_match = var_regex.search(self.transform_plan[0]) - if not var_match: - raise RegexMatchError( - caller="__init__", pattern=var_regex.pattern - ) - var = var_match.group(0)[:-1] - self.transform_map = get_transform_map(js, var) - self.js_func_patterns = [ - r"\w+\.(\w+)\(\w,(\d+)\)", - r"\w+\[(\"\w+\")\]\(\w,(\d+)\)" - ] - - self.throttling_plan = get_throttling_plan(js) - self.throttling_array = get_throttling_function_array(js) - - self.calculated_n = None - - def calculate_n(self, initial_n: list): - """Converts n to the correct value to prevent throttling.""" - if self.calculated_n: - return self.calculated_n - - # First, update all instances of 'b' with the list(initial_n) - for i in range(len(self.throttling_array)): - if self.throttling_array[i] == 'b': - self.throttling_array[i] = initial_n - - for step in self.throttling_plan: - curr_func = self.throttling_array[int(step[0])] - if not callable(curr_func): - logger.debug(f'{curr_func} is not callable.') - logger.debug(f'Throttling array:\n{self.throttling_array}\n') - raise ExtractError(f'{curr_func} is not callable.') - - first_arg = self.throttling_array[int(step[1])] - - if len(step) == 2: - curr_func(first_arg) - elif len(step) == 3: - second_arg = self.throttling_array[int(step[2])] - curr_func(first_arg, second_arg) - - self.calculated_n = ''.join(initial_n) - return self.calculated_n - - def get_signature(self, ciphered_signature: str) -> str: - """Decipher the signature. - - Taking the ciphered signature, applies the transform functions. - - :param str ciphered_signature: - The ciphered signature sent in the ``player_config``. - :rtype: str - :returns: - Decrypted signature required to download the media content. - """ - signature = list(ciphered_signature) - - for js_func in self.transform_plan: - name, argument = self.parse_function(js_func) # type: ignore - signature = self.transform_map[name](signature, argument) - logger.debug( - "applied transform function\n" - "output: %s\n" - "js_function: %s\n" - "argument: %d\n" - "function: %s", - "".join(signature), - name, - argument, - self.transform_map[name], - ) - - return "".join(signature) - - @cache - def parse_function(self, js_func: str) -> Tuple[str, int]: - """Parse the Javascript transform function. - - Break a JavaScript transform function down into a two element ``tuple`` - containing the function name and some integer-based argument. - - :param str js_func: - The JavaScript version of the transform function. - :rtype: tuple - :returns: - two element tuple containing the function name and an argument. - - **Example**: - - parse_function('DE.AJ(a,15)') - ('AJ', 15) - - """ - logger.debug("parsing transform function") - for pattern in self.js_func_patterns: - regex = re.compile(pattern) - parse_match = regex.search(js_func) - if parse_match: - fn_name, fn_arg = parse_match.groups() - return fn_name, int(fn_arg) - - raise RegexMatchError( - caller="parse_function", pattern="js_func_patterns" - ) - - -def get_initial_function_name(js: str) -> str: - """Extract the name of the function responsible for computing the signature. - :param str js: - The contents of the base.js asset file. - :rtype: str - :returns: - Function name from regex match - """ - - function_patterns = [ - r"\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[a-zA-Z0-9$]+)\(", # noqa: E501 - r"\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[a-zA-Z0-9$]+)\(", # noqa: E501 - r'(?:\b|[^a-zA-Z0-9$])(?P[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)', # noqa: E501 - r'(?P[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)', # noqa: E501 - r'(["\'])signature1円\s*,\s*(?P[a-zA-Z0-9$]+)\(', - r"\.sig\|\|(?P[a-zA-Z0-9$]+)\(", - r"yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P[a-zA-Z0-9$]+)\(", # noqa: E501 - r"\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P[a-zA-Z0-9$]+)\(", # noqa: E501 - r"\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P[a-zA-Z0-9$]+)\(", # noqa: E501 - r"\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P[a-zA-Z0-9$]+)\(", # noqa: E501 - r"\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P[a-zA-Z0-9$]+)\(", # noqa: E501 - r"\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P[a-zA-Z0-9$]+)\(", # noqa: E501 - ] - logger.debug("finding initial function name") - for pattern in function_patterns: - regex = re.compile(pattern) - function_match = regex.search(js) - if function_match: - logger.debug("finished regex search, matched: %s", pattern) - return function_match.group(1) - - raise RegexMatchError( - caller="get_initial_function_name", pattern="multiple" - ) - - -def get_transform_plan(js: str) -> List[str]: - """Extract the "transform plan". - - The "transform plan" is the functions that the ciphered signature is - cycled through to obtain the actual signature. - - :param str js: - The contents of the base.js asset file. - - **Example**: - - ['DE.AJ(a,15)', - 'DE.VR(a,3)', - 'DE.AJ(a,51)', - 'DE.VR(a,3)', - 'DE.kT(a,51)', - 'DE.kT(a,8)', - 'DE.VR(a,3)', - 'DE.kT(a,21)'] - """ - name = re.escape(get_initial_function_name(js)) - pattern = r"%s=function\(\w\){[a-z=\.\(\"\)]*;(.*);(?:.+)}" % name - logger.debug("getting transform plan") - return regex_search(pattern, js, group=1).split(";") - - -def get_transform_object(js: str, var: str) -> List[str]: - """Extract the "transform object". - - The "transform object" contains the function definitions referenced in the - "transform plan". The ``var`` argument is the obfuscated variable name - which contains these functions, for example, given the function call - ``DE.AJ(a,15)`` returned by the transform plan, "DE" would be the var. - - :param str js: - The contents of the base.js asset file. - :param str var: - The obfuscated variable name that stores an object with all functions - that descrambles the signature. - - **Example**: - ->>> get_transform_object(js, 'DE') - ['AJ:function(a){a.reverse()}', - 'VR:function(a,b){a.splice(0,b)}', - 'kT:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b]=c}'] - - """ - pattern = r"var %s={(.*?)};" % re.escape(var) - logger.debug("getting transform object") - regex = re.compile(pattern, flags=re.DOTALL) - transform_match = regex.search(js) - if not transform_match: - raise RegexMatchError(caller="get_transform_object", pattern=pattern) - - return transform_match.group(1).replace("\n", " ").split(", ") - - -def get_transform_map(js: str, var: str) -> Dict: - """Build a transform function lookup. - - Build a lookup table of obfuscated JavaScript function names to the - Python equivalents. - - :param str js: - The contents of the base.js asset file. - :param str var: - The obfuscated variable name that stores an object with all functions - that descrambles the signature. - - """ - transform_object = get_transform_object(js, var) - mapper = {} - for obj in transform_object: - # AJ:function(a){a.reverse()} => AJ, function(a){a.reverse()} - name, function = obj.split(":", 1) - fn = map_functions(function) - mapper[name] = fn - return mapper - - -def get_throttling_function_name(js: str) -> str: - """Extract the name of the function that computes the throttling parameter. - - :param str js: - The contents of the base.js asset file. - :rtype: str - :returns: - The name of the function used to compute the throttling parameter. - """ - function_patterns = [ - # https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-865985377 - # a.C&&(b=a.get("n"))&&(b=Dea(b),a.set("n",b))}}; - # In above case, `Dea` is the relevant function name - r'a\.C&&\(b=a\.get\("n"\)\)&&\(b=([^(]+)\(b\),a\.set\("n",b\)\)}};', - ] - logger.debug('Finding throttling function name') - for pattern in function_patterns: - regex = re.compile(pattern) - function_match = regex.search(js) - if function_match: - logger.debug("finished regex search, matched: %s", pattern) - return function_match.group(1) - - raise RegexMatchError( - caller="get_throttling_function_name", pattern="multiple" - ) - - -def get_throttling_function_code(js: str) -> str: - """Extract the raw code for the throttling function. - - :param str js: - The contents of the base.js asset file. - :rtype: str - :returns: - The name of the function used to compute the throttling parameter. - """ - # Begin by extracting the correct function name - name = re.escape(get_throttling_function_name(js)) - - # Identify where the function is defined - pattern_start = r"%s=function\(\w\)" % name - regex = re.compile(pattern_start) - match = regex.search(js) - - # Extract the code within curly braces for the function itself, and merge any split lines - code_lines_list = find_object_from_startpoint(js, match.span()[1]).split('\n') - joined_lines = "".join(code_lines_list) - - # Prepend function definition (e.g. `Dea=function(a)`) - return match.group(0) + joined_lines - - -def get_throttling_function_array(js: str) -> List[Any]: - """Extract the "c" array. - - :param str js: - The contents of the base.js asset file. - :returns: - The array of various integers, arrays, and functions. - """ - raw_code = get_throttling_function_code(js) - - array_start = r",c=\[" - array_regex = re.compile(array_start) - match = array_regex.search(raw_code) - - array_raw = find_object_from_startpoint(raw_code, match.span()[1] - 1) - str_array = throttling_array_split(array_raw) - - converted_array = [] - for el in str_array: - try: - converted_array.append(int(el)) - continue - except ValueError: - # Not an integer value. - pass - - if el == 'null': - converted_array.append(None) - continue - - if el.startswith('"') and el.endswith('"'): - # Convert e.g. '"abcdef"' to string without quotation marks, 'abcdef' - converted_array.append(el[1:-1]) - continue - - if el.startswith('function'): - mapper = ( - (r"{for\(\w=\(\w%\w\.length\+\w\.length\)%\w\.length;\w--;\)\w\.unshift\(\w.pop\(\)\)}", throttling_unshift), # noqa:E501 - (r"{\w\.reverse\(\)}", throttling_reverse), - (r"{\w\.push\(\w\)}", throttling_push), - (r";var\s\w=\w\[0\];\w\[0\]=\w\[\w\];\w\[\w\]=\w}", throttling_swap), - (r"case\s\d+", throttling_cipher_function), - (r"\w\.splice\(0,1,\w\.splice\(\w,1,\w\[0\]\)\[0\]\)", throttling_nested_splice), # noqa:E501 - (r";\w\.splice\(\w,1\)}", js_splice), - (r"\w\.splice\(-\w\)\.reverse\(\)\.forEach\(function\(\w\){\w\.unshift\(\w\)}\)", throttling_prepend), # noqa:E501 - (r"for\(var \w=\w\.length;\w;\)\w\.push\(\w\.splice\(--\w,1\)\[0\]\)}", throttling_reverse), # noqa:E501 - ) - - found = False - for pattern, fn in mapper: - if re.search(pattern, el): - converted_array.append(fn) - found = True - if found: - continue - - converted_array.append(el) - - # Replace null elements with array itself - for i in range(len(converted_array)): - if converted_array[i] is None: - converted_array[i] = converted_array - - return converted_array - - -def get_throttling_plan(js: str): - """Extract the "throttling plan". - - The "throttling plan" is a list of tuples used for calling functions - in the c array. The first element of the tuple is the index of the - function to call, and any remaining elements of the tuple are arguments - to pass to that function. - - :param str js: - The contents of the base.js asset file. - :returns: - The full function code for computing the throttlign parameter. - """ - raw_code = get_throttling_function_code(js) - - transform_start = r"try{" - plan_regex = re.compile(transform_start) - match = plan_regex.search(raw_code) - - transform_plan_raw = find_object_from_startpoint(raw_code, match.span()[1] - 1) - - # Steps are either c[x](c[y]) or c[x](c[y],c[z]) - step_start = r"c\[(\d+)\]\(c\[(\d+)\](,c(\[(\d+)\]))?\)" - step_regex = re.compile(step_start) - matches = step_regex.findall(transform_plan_raw) - transform_steps = [] - for match in matches: - if match[4] != '': - transform_steps.append((match[0],match[1],match[4])) - else: - transform_steps.append((match[0],match[1])) - - return transform_steps - - -def reverse(arr: List, _: Optional[Any]): - """Reverse elements in a list. - - This function is equivalent to: - - .. code-block:: javascript - - function(a, b) { a.reverse() } - - This method takes an unused ``b`` variable as their transform functions - universally sent two arguments. - - **Example**: - ->>> reverse([1, 2, 3, 4]) - [4, 3, 2, 1] - """ - return arr[::-1] - - -def splice(arr: List, b: int): - """Add/remove items to/from a list. - - This function is equivalent to: - - .. code-block:: javascript - - function(a, b) { a.splice(0, b) } - - **Example**: - ->>> splice([1, 2, 3, 4], 2) - [1, 2] - """ - return arr[b:] - - -def swap(arr: List, b: int): - """Swap positions at b modulus the list length. - - This function is equivalent to: - - .. code-block:: javascript - - function(a, b) { var c=a[0];a[0]=a[b%a.length];a[b]=c } - - **Example**: - ->>> swap([1, 2, 3, 4], 2) - [3, 2, 1, 4] - """ - r = b % len(arr) - return list(chain([arr[r]], arr[1:r], [arr[0]], arr[r + 1 :])) - - -def throttling_reverse(arr: list): - """Reverses the input list. - - Needs to do an in-place reversal so that the passed list gets changed. - To accomplish this, we create a reversed copy, and then change each - indvidual element. - """ - reverse_copy = arr.copy()[::-1] - for i in range(len(reverse_copy)): - arr[i] = reverse_copy[i] - - -def throttling_push(d: list, e: Any): - """Pushes an element onto a list.""" - d.append(e) - - -def throttling_mod_func(d: list, e: int): - """Perform the modular function from the throttling array functions. - - In the javascript, the modular operation is as follows: - e = (e % d.length + d.length) % d.length - - We simply translate this to python here. - """ - return (e % len(d) + len(d)) % len(d) - - -def throttling_unshift(d: list, e: int): - """Rotates the elements of the list to the right. - - In the javascript, the operation is as follows: - for(e=(e%d.length+d.length)%d.length;e--;)d.unshift(d.pop()) - """ - e = throttling_mod_func(d, e) - new_arr = d[-e:] + d[:-e] - d.clear() - for el in new_arr: - d.append(el) - - -def throttling_cipher_function(d: list, e: str): - """This ciphers d with e to generate a new list. - - In the javascript, the operation is as follows: - var h = [A-Za-z0-9-_], f = 96; // simplified from switch-case loop - d.forEach( - function(l,m,n){ - this.push( - n[m]=h[ - (h.indexOf(l)-h.indexOf(this[m])+m-32+f--)%h.length - ] - ) - }, - e.split("") - ) - """ - h = list('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_') - f = 96 - # by naming it "this" we can more closely reflect the js - this = list(e) - - # This is so we don't run into weirdness with enumerate while - # we change the input list - copied_list = d.copy() - - for m, l in enumerate(copied_list): - bracket_val = (h.index(l) - h.index(this[m]) + m - 32 + f) % len(h) - this.append( - h[bracket_val] - ) - d[m] = h[bracket_val] - f -= 1 - - -def throttling_nested_splice(d: list, e: int): - """Nested splice function in throttling js. - - In the javascript, the operation is as follows: - function(d,e){ - e=(e%d.length+d.length)%d.length; - d.splice( - 0, - 1, - d.splice( - e, - 1, - d[0] - )[0] - ) - } - - While testing, all this seemed to do is swap element 0 and e, - but the actual process is preserved in case there was an edge - case that was not considered. - """ - e = throttling_mod_func(d, e) - inner_splice = js_splice( - d, - e, - 1, - d[0] - ) - js_splice( - d, - 0, - 1, - inner_splice[0] - ) - - -def throttling_prepend(d: list, e: int): - """ - - In the javascript, the operation is as follows: - function(d,e){ - e=(e%d.length+d.length)%d.length; - d.splice(-e).reverse().forEach( - function(f){ - d.unshift(f) - } - ) - } - - Effectively, this moves the last e elements of d to the beginning. - """ - start_len = len(d) - # First, calculate e - e = throttling_mod_func(d, e) - - # Then do the prepending - new_arr = d[-e:] + d[:-e] - - # And update the input list - d.clear() - for el in new_arr: - d.append(el) - - end_len = len(d) - assert start_len == end_len - - -def throttling_swap(d: list, e: int): - """Swap positions of the 0'th and e'th elements in-place.""" - e = throttling_mod_func(d, e) - f = d[0] - d[0] = d[e] - d[e] = f - - -def js_splice(arr: list, start: int, delete_count=None, *items): - """Implementation of javascript's splice function. - - :param list arr: - Array to splice - :param int start: - Index at which to start changing the array - :param int delete_count: - Number of elements to delete from the array - :param *items: - Items to add to the array - - Reference: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice # noqa:E501 - """ - # Special conditions for start value - try: - if start> len(arr): - start = len(arr) - # If start is negative, count backwards from end - if start < 0: - start = len(arr) - start - except TypeError: - # Non-integer start values are treated as 0 in js - start = 0 - - # Special condition when delete_count is greater than remaining elements - if not delete_count or delete_count>= len(arr) - start: - delete_count = len(arr) - start # noqa: N806 - - deleted_elements = arr[start:start + delete_count] - - # Splice appropriately. - new_arr = arr[:start] + list(items) + arr[start + delete_count:] - - # Replace contents of input array - arr.clear() - for el in new_arr: - arr.append(el) - - return deleted_elements - - -def map_functions(js_func: str) -> Callable: - """For a given JavaScript transform function, return the Python equivalent. - - :param str js_func: - The JavaScript version of the transform function. - """ - mapper = ( - # function(a){a.reverse()} - (r"{\w\.reverse\(\)}", reverse), - # function(a,b){a.splice(0,b)} - (r"{\w\.splice\(0,\w\)}", splice), - # function(a,b){var c=a[0];a[0]=a[b%a.length];a[b]=c} - (r"{var\s\w=\w\[0\];\w\[0\]=\w\[\w\%\w.length\];\w\[\w\]=\w}", swap), - # function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c} - ( - r"{var\s\w=\w\[0\];\w\[0\]=\w\[\w\%\w.length\];\w\[\w\%\w.length\]=\w}", - swap, - ), - ) - - for pattern, fn in mapper: - if re.search(pattern, js_func): - return fn - raise RegexMatchError(caller="map_functions", pattern="multiple") diff --git a/001-Downloader/pytube/cli.py b/001-Downloader/pytube/cli.py deleted file mode 100755 index 7a98854..0000000 --- a/001-Downloader/pytube/cli.py +++ /dev/null @@ -1,560 +0,0 @@ -#!/usr/bin/env python3 -"""A simple command line application to download youtube videos.""" -import argparse -import gzip -import json -import logging -import os -import shutil -import sys -import datetime as dt -import subprocess # nosec -from typing import List, Optional - -import pytube.exceptions as exceptions -from pytube import __version__ -from pytube import CaptionQuery, Playlist, Stream, YouTube -from pytube.helpers import safe_filename, setup_logger - - -logger = logging.getLogger(__name__) - - -def main(): - """Command line application to download youtube videos.""" - # noinspection PyTypeChecker - parser = argparse.ArgumentParser(description=main.__doc__) - args = _parse_args(parser) - if args.verbose: - log_filename = None - if args.logfile: - log_filename = args.logfile - setup_logger(logging.DEBUG, log_filename=log_filename) - logger.debug(f'Pytube version: {__version__}') - - if not args.url or "youtu" not in args.url: - parser.print_help() - sys.exit(1) - - if "/playlist" in args.url: - print("Loading playlist...") - playlist = Playlist(args.url) - if not args.target: - args.target = safe_filename(playlist.title) - for youtube_video in playlist.videos: - try: - _perform_args_on_youtube(youtube_video, args) - except exceptions.PytubeError as e: - print(f"There was an error with video: {youtube_video}") - print(e) - else: - print("Loading video...") - youtube = YouTube(args.url) - _perform_args_on_youtube(youtube, args) - - -def _perform_args_on_youtube( - youtube: YouTube, args: argparse.Namespace -) -> None: - if len(sys.argv) == 2 : # no arguments parsed - download_highest_resolution_progressive( - youtube=youtube, resolution="highest", target=args.target - ) - if args.list_captions: - _print_available_captions(youtube.captions) - if args.list: - display_streams(youtube) - if args.build_playback_report: - build_playback_report(youtube) - if args.itag: - download_by_itag(youtube=youtube, itag=args.itag, target=args.target) - if args.caption_code: - download_caption( - youtube=youtube, lang_code=args.caption_code, target=args.target - ) - if args.resolution: - download_by_resolution( - youtube=youtube, resolution=args.resolution, target=args.target - ) - if args.audio: - download_audio( - youtube=youtube, filetype=args.audio, target=args.target - ) - if args.ffmpeg: - ffmpeg_process( - youtube=youtube, resolution=args.ffmpeg, target=args.target - ) - - -def _parse_args( - parser: argparse.ArgumentParser, args: Optional[List] = None -) -> argparse.Namespace: - parser.add_argument( - "url", help="The YouTube /watch or /playlist url", nargs="?" - ) - parser.add_argument( - "--version", action="version", version="%(prog)s " + __version__, - ) - parser.add_argument( - "--itag", type=int, help="The itag for the desired stream", - ) - parser.add_argument( - "-r", - "--resolution", - type=str, - help="The resolution for the desired stream", - ) - parser.add_argument( - "-l", - "--list", - action="store_true", - help=( - "The list option causes pytube cli to return a list of streams " - "available to download" - ), - ) - parser.add_argument( - "-v", - "--verbose", - action="store_true", - dest="verbose", - help="Set logger output to verbose output.", - ) - parser.add_argument( - "--logfile", - action="store", - help="logging debug and error messages into a log file", - ) - parser.add_argument( - "--build-playback-report", - action="store_true", - help="Save the html and js to disk", - ) - parser.add_argument( - "-c", - "--caption-code", - type=str, - help=( - "Download srt captions for given language code. " - "Prints available language codes if no argument given" - ), - ) - parser.add_argument( - '-lc', - '--list-captions', - action='store_true', - help=( - "List available caption codes for a video" - ) - ) - parser.add_argument( - "-t", - "--target", - help=( - "The output directory for the downloaded stream. " - "Default is current working directory" - ), - ) - parser.add_argument( - "-a", - "--audio", - const="mp4", - nargs="?", - help=( - "Download the audio for a given URL at the highest bitrate available" - "Defaults to mp4 format if none is specified" - ), - ) - parser.add_argument( - "-f", - "--ffmpeg", - const="best", - nargs="?", - help=( - "Downloads the audio and video stream for resolution provided" - "If no resolution is provided, downloads the best resolution" - "Runs the command line program ffmpeg to combine the audio and video" - ), - ) - - return parser.parse_args(args) - - -def build_playback_report(youtube: YouTube) -> None: - """Serialize the request data to json for offline debugging. - - :param YouTube youtube: - A YouTube object. - """ - ts = int(dt.datetime.utcnow().timestamp()) - fp = os.path.join(os.getcwd(), f"yt-video-{youtube.video_id}-{ts}.json.gz") - - js = youtube.js - watch_html = youtube.watch_html - vid_info = youtube.vid_info - - with gzip.open(fp, "wb") as fh: - fh.write( - json.dumps( - { - "url": youtube.watch_url, - "js": js, - "watch_html": watch_html, - "video_info": vid_info, - } - ).encode("utf8"), - ) - - -def display_progress_bar( - bytes_received: int, filesize: int, ch: str = "█", scale: float = 0.55 -) -> None: - """Display a simple, pretty progress bar. - - Example: - ~~~~~~~~ - PSY - GANGNAM STYLE(강남스타일) MV.mp4 - ↳ |███████████████████████████████████████| 100.0% - - :param int bytes_received: - The delta between the total file size (bytes) and bytes already - written to disk. - :param int filesize: - File size of the media stream in bytes. - :param str ch: - Character to use for presenting progress segment. - :param float scale: - Scale multiplier to reduce progress bar size. - - """ - columns = shutil.get_terminal_size().columns - max_width = int(columns * scale) - - filled = int(round(max_width * bytes_received / float(filesize))) - remaining = max_width - filled - progress_bar = ch * filled + " " * remaining - percent = round(100.0 * bytes_received / float(filesize), 1) - text = f" ↳ |{progress_bar}| {percent}%\r" - sys.stdout.write(text) - sys.stdout.flush() - - -# noinspection PyUnusedLocal -def on_progress( - stream: Stream, chunk: bytes, bytes_remaining: int -) -> None: # pylint: disable=W0613 - filesize = stream.filesize - bytes_received = filesize - bytes_remaining - display_progress_bar(bytes_received, filesize) - - -def _download( - stream: Stream, - target: Optional[str] = None, - filename: Optional[str] = None, -) -> None: - filesize_megabytes = stream.filesize // 1048576 - print(f"{filename or stream.default_filename} | {filesize_megabytes} MB") - file_path = stream.get_file_path(filename=filename, output_path=target) - if stream.exists_at_path(file_path): - print(f"Already downloaded at:\n{file_path}") - return - - stream.download(output_path=target, filename=filename) - sys.stdout.write("\n") - - -def _unique_name(base: str, subtype: str, media_type: str, target: str) -> str: - """ - Given a base name, the file format, and the target directory, will generate - a filename unique for that directory and file format. - :param str base: - The given base-name. - :param str subtype: - The filetype of the video which will be downloaded. - :param str media_type: - The media_type of the file, ie. "audio" or "video" - :param Path target: - Target directory for download. - """ - counter = 0 - while True: - file_name = f"{base}_{media_type}_{counter}" - file_path = os.path.join(target, f"{file_name}.{subtype}") - if not os.path.exists(file_path): - return file_name - counter += 1 - - -def ffmpeg_process( - youtube: YouTube, resolution: str, target: Optional[str] = None -) -> None: - """ - Decides the correct video stream to download, then calls _ffmpeg_downloader. - - :param YouTube youtube: - A valid YouTube object. - :param str resolution: - YouTube video resolution. - :param str target: - Target directory for download - """ - youtube.register_on_progress_callback(on_progress) - target = target or os.getcwd() - - if resolution == "best": - highest_quality_stream = ( - youtube.streams.filter(progressive=False) - .order_by("resolution") - .last() - ) - mp4_stream = ( - youtube.streams.filter(progressive=False, subtype="mp4") - .order_by("resolution") - .last() - ) - if highest_quality_stream.resolution == mp4_stream.resolution: - video_stream = mp4_stream - else: - video_stream = highest_quality_stream - else: - video_stream = youtube.streams.filter( - progressive=False, resolution=resolution, subtype="mp4" - ).first() - if not video_stream: - video_stream = youtube.streams.filter( - progressive=False, resolution=resolution - ).first() - if video_stream is None: - print(f"Could not find a stream with resolution: {resolution}") - print("Try one of these:") - display_streams(youtube) - sys.exit() - - audio_stream = youtube.streams.get_audio_only(video_stream.subtype) - if not audio_stream: - audio_stream = ( - youtube.streams.filter(only_audio=True).order_by("abr").last() - ) - if not audio_stream: - print("Could not find an audio only stream") - sys.exit() - _ffmpeg_downloader( - audio_stream=audio_stream, video_stream=video_stream, target=target - ) - - -def _ffmpeg_downloader( - audio_stream: Stream, video_stream: Stream, target: str -) -> None: - """ - Given a YouTube Stream object, finds the correct audio stream, downloads them both - giving them a unique name, them uses ffmpeg to create a new file with the audio - and video from the previously downloaded files. Then deletes the original adaptive - streams, leaving the combination. - - :param Stream audio_stream: - A valid Stream object representing the audio to download - :param Stream video_stream: - A valid Stream object representing the video to download - :param Path target: - A valid Path object - """ - video_unique_name = _unique_name( - safe_filename(video_stream.title), - video_stream.subtype, - "video", - target=target, - ) - audio_unique_name = _unique_name( - safe_filename(video_stream.title), - audio_stream.subtype, - "audio", - target=target, - ) - _download(stream=video_stream, target=target, filename=video_unique_name) - print("Loading audio...") - _download(stream=audio_stream, target=target, filename=audio_unique_name) - - video_path = os.path.join( - target, f"{video_unique_name}.{video_stream.subtype}" - ) - audio_path = os.path.join( - target, f"{audio_unique_name}.{audio_stream.subtype}" - ) - final_path = os.path.join( - target, f"{safe_filename(video_stream.title)}.{video_stream.subtype}" - ) - - subprocess.run( # nosec - [ - "ffmpeg", - "-i", - video_path, - "-i", - audio_path, - "-codec", - "copy", - final_path, - ] - ) - os.unlink(video_path) - os.unlink(audio_path) - - -def download_by_itag( - youtube: YouTube, itag: int, target: Optional[str] = None -) -> None: - """Start downloading a YouTube video. - - :param YouTube youtube: - A valid YouTube object. - :param int itag: - YouTube format identifier code. - :param str target: - Target directory for download - """ - stream = youtube.streams.get_by_itag(itag) - if stream is None: - print(f"Could not find a stream with itag: {itag}") - print("Try one of these:") - display_streams(youtube) - sys.exit() - - youtube.register_on_progress_callback(on_progress) - - try: - _download(stream, target=target) - except KeyboardInterrupt: - sys.exit() - - -def download_by_resolution( - youtube: YouTube, resolution: str, target: Optional[str] = None -) -> None: - """Start downloading a YouTube video. - - :param YouTube youtube: - A valid YouTube object. - :param str resolution: - YouTube video resolution. - :param str target: - Target directory for download - """ - # TODO(nficano): allow dash itags to be selected - stream = youtube.streams.get_by_resolution(resolution) - if stream is None: - print(f"Could not find a stream with resolution: {resolution}") - print("Try one of these:") - display_streams(youtube) - sys.exit() - - youtube.register_on_progress_callback(on_progress) - - try: - _download(stream, target=target) - except KeyboardInterrupt: - sys.exit() - - -def download_highest_resolution_progressive( - youtube: YouTube, resolution: str, target: Optional[str] = None -) -> None: - """Start downloading the highest resolution progressive stream. - - :param YouTube youtube: - A valid YouTube object. - :param str resolution: - YouTube video resolution. - :param str target: - Target directory for download - """ - youtube.register_on_progress_callback(on_progress) - try: - stream = youtube.streams.get_highest_resolution() - except exceptions.VideoUnavailable as err: - print(f"No video streams available: {err}") - else: - try: - _download(stream, target=target) - except KeyboardInterrupt: - sys.exit() - - -def display_streams(youtube: YouTube) -> None: - """Probe YouTube video and lists its available formats. - - :param YouTube youtube: - A valid YouTube watch URL. - - """ - for stream in youtube.streams: - print(stream) - - -def _print_available_captions(captions: CaptionQuery) -> None: - print( - f"Available caption codes are: {', '.join(c.code for c in captions)}" - ) - - -def download_caption( - youtube: YouTube, lang_code: Optional[str], target: Optional[str] = None -) -> None: - """Download a caption for the YouTube video. - - :param YouTube youtube: - A valid YouTube object. - :param str lang_code: - Language code desired for caption file. - Prints available codes if the value is None - or the desired code is not available. - :param str target: - Target directory for download - """ - try: - caption = youtube.captions[lang_code] - downloaded_path = caption.download( - title=youtube.title, output_path=target - ) - print(f"Saved caption file to: {downloaded_path}") - except KeyError: - print(f"Unable to find caption with code: {lang_code}") - _print_available_captions(youtube.captions) - - -def download_audio( - youtube: YouTube, filetype: str, target: Optional[str] = None -) -> None: - """ - Given a filetype, downloads the highest quality available audio stream for a - YouTube video. - - :param YouTube youtube: - A valid YouTube object. - :param str filetype: - Desired file format to download. - :param str target: - Target directory for download - """ - audio = ( - youtube.streams.filter(only_audio=True, subtype=filetype) - .order_by("abr") - .last() - ) - - if audio is None: - print("No audio only stream found. Try one of these:") - display_streams(youtube) - sys.exit() - - youtube.register_on_progress_callback(on_progress) - - try: - _download(audio, target=target) - except KeyboardInterrupt: - sys.exit() - - -if __name__ == "__main__": - main() diff --git a/001-Downloader/pytube/contrib/channel.py b/001-Downloader/pytube/contrib/channel.py deleted file mode 100755 index 147ff7e..0000000 --- a/001-Downloader/pytube/contrib/channel.py +++ /dev/null @@ -1,201 +0,0 @@ -# -*- coding: utf-8 -*- -"""Module for interacting with a user's youtube channel.""" -import json -import logging -from typing import Dict, List, Optional, Tuple - -from pytube import extract, Playlist, request -from pytube.helpers import uniqueify - -logger = logging.getLogger(__name__) - - -class Channel(Playlist): - def __init__(self, url: str, proxies: Optional[Dict[str, str]] = None): - """Construct a :class:`Channel `. - - :param str url: - A valid YouTube channel URL. - :param proxies: - (Optional) A dictionary of proxies to use for web requests. - """ - super().__init__(url, proxies) - - self.channel_uri = extract.channel_name(url) - - self.channel_url = ( - f"https://www.youtube.com{self.channel_uri}" - ) - - self.videos_url = self.channel_url + '/videos' - self.playlists_url = self.channel_url + '/playlists' - self.community_url = self.channel_url + '/community' - self.featured_channels_url = self.channel_url + '/channels' - self.about_url = self.channel_url + '/about' - - # Possible future additions - self._playlists_html = None - self._community_html = None - self._featured_channels_html = None - self._about_html = None - - @property - def channel_name(self): - """Get the name of the YouTube channel. - - :rtype: str - """ - return self.initial_data['metadata']['channelMetadataRenderer']['title'] - - @property - def channel_id(self): - """Get the ID of the YouTube channel. - - This will return the underlying ID, not the vanity URL. - - :rtype: str - """ - return self.initial_data['metadata']['channelMetadataRenderer']['externalId'] - - @property - def vanity_url(self): - """Get the vanity URL of the YouTube channel. - - Returns None if it doesn't exist. - - :rtype: str - """ - return self.initial_data['metadata']['channelMetadataRenderer'].get('vanityChannelUrl', None) # noqa:E501 - - @property - def html(self): - """Get the html for the /videos page. - - :rtype: str - """ - if self._html: - return self._html - self._html = request.get(self.videos_url) - return self._html - - @property - def playlists_html(self): - """Get the html for the /playlists page. - - Currently unused for any functionality. - - :rtype: str - """ - if self._playlists_html: - return self._playlists_html - else: - self._playlists_html = request.get(self.playlists_url) - return self._playlists_html - - @property - def community_html(self): - """Get the html for the /community page. - - Currently unused for any functionality. - - :rtype: str - """ - if self._community_html: - return self._community_html - else: - self._community_html = request.get(self.community_url) - return self._community_html - - @property - def featured_channels_html(self): - """Get the html for the /channels page. - - Currently unused for any functionality. - - :rtype: str - """ - if self._featured_channels_html: - return self._featured_channels_html - else: - self._featured_channels_html = request.get(self.featured_channels_url) - return self._featured_channels_html - - @property - def about_html(self): - """Get the html for the /about page. - - Currently unused for any functionality. - - :rtype: str - """ - if self._about_html: - return self._about_html - else: - self._about_html = request.get(self.about_url) - return self._about_html - - @staticmethod - def _extract_videos(raw_json: str) -> Tuple[List[str], Optional[str]]: - """Extracts videos from a raw json page - - :param str raw_json: Input json extracted from the page or the last - server response - :rtype: Tuple[List[str], Optional[str]] - :returns: Tuple containing a list of up to 100 video watch ids and - a continuation token, if more videos are available - """ - initial_data = json.loads(raw_json) - # this is the json tree structure, if the json was extracted from - # html - try: - videos = initial_data["contents"][ - "twoColumnBrowseResultsRenderer"][ - "tabs"][1]["tabRenderer"]["content"][ - "sectionListRenderer"]["contents"][0][ - "itemSectionRenderer"]["contents"][0][ - "gridRenderer"]["items"] - except (KeyError, IndexError, TypeError): - try: - # this is the json tree structure, if the json was directly sent - # by the server in a continuation response - important_content = initial_data[1]['response']['onResponseReceivedActions'][ - 0 - ]['appendContinuationItemsAction']['continuationItems'] - videos = important_content - except (KeyError, IndexError, TypeError): - try: - # this is the json tree structure, if the json was directly sent - # by the server in a continuation response - # no longer a list and no longer has the "response" key - important_content = initial_data['onResponseReceivedActions'][0][ - 'appendContinuationItemsAction']['continuationItems'] - videos = important_content - except (KeyError, IndexError, TypeError) as p: - logger.info(p) - return [], None - - try: - continuation = videos[-1]['continuationItemRenderer'][ - 'continuationEndpoint' - ]['continuationCommand']['token'] - videos = videos[:-1] - except (KeyError, IndexError): - # if there is an error, no continuation is available - continuation = None - - # remove duplicates - return ( - uniqueify( - list( - # only extract the video ids from the video data - map( - lambda x: ( - f"/watch?v=" - f"{x['gridVideoRenderer']['videoId']}" - ), - videos - ) - ), - ), - continuation, - ) diff --git a/001-Downloader/pytube/contrib/playlist.py b/001-Downloader/pytube/contrib/playlist.py deleted file mode 100755 index db9c718..0000000 --- a/001-Downloader/pytube/contrib/playlist.py +++ /dev/null @@ -1,411 +0,0 @@ -"""Module to download a complete playlist from a youtube channel.""" -import json -import logging -from collections.abc import Sequence -from datetime import date, datetime -from typing import Dict, Iterable, List, Optional, Tuple, Union - -from pytube import extract, request, YouTube -from pytube.helpers import cache, DeferredGeneratorList, install_proxy, uniqueify - -logger = logging.getLogger(__name__) - - -class Playlist(Sequence): - """Load a YouTube playlist with URL""" - - def __init__(self, url: str, proxies: Optional[Dict[str, str]] = None): - if proxies: - install_proxy(proxies) - - self._input_url = url - - # These need to be initialized as None for the properties. - self._html = None - self._ytcfg = None - self._initial_data = None - self._sidebar_info = None - - self._playlist_id = None - - @property - def playlist_id(self): - """Get the playlist id. - - :rtype: str - """ - if self._playlist_id: - return self._playlist_id - self._playlist_id = extract.playlist_id(self._input_url) - return self._playlist_id - - @property - def playlist_url(self): - """Get the base playlist url. - - :rtype: str - """ - return f"https://www.youtube.com/playlist?list={self.playlist_id}" - - @property - def html(self): - """Get the playlist page html. - - :rtype: str - """ - if self._html: - return self._html - self._html = request.get(self.playlist_url) - return self._html - - @property - def ytcfg(self): - """Extract the ytcfg from the playlist page html. - - :rtype: dict - """ - if self._ytcfg: - return self._ytcfg - self._ytcfg = extract.get_ytcfg(self.html) - return self._ytcfg - - @property - def initial_data(self): - """Extract the initial data from the playlist page html. - - :rtype: dict - """ - if self._initial_data: - return self._initial_data - else: - self._initial_data = extract.initial_data(self.html) - return self._initial_data - - @property - def sidebar_info(self): - """Extract the sidebar info from the playlist page html. - - :rtype: dict - """ - if self._sidebar_info: - return self._sidebar_info - else: - self._sidebar_info = self.initial_data['sidebar'][ - 'playlistSidebarRenderer']['items'] - return self._sidebar_info - - @property - def yt_api_key(self): - """Extract the INNERTUBE_API_KEY from the playlist ytcfg. - - :rtype: str - """ - return self.ytcfg['INNERTUBE_API_KEY'] - - def _paginate( - self, until_watch_id: Optional[str] = None - ) -> Iterable[List[str]]: - """Parse the video links from the page source, yields the /watch?v= - part from video link - - :param until_watch_id Optional[str]: YouTube Video watch id until - which the playlist should be read. - - :rtype: Iterable[List[str]] - :returns: Iterable of lists of YouTube watch ids - """ - videos_urls, continuation = self._extract_videos( - json.dumps(extract.initial_data(self.html)) - ) - if until_watch_id: - try: - trim_index = videos_urls.index(f"/watch?v={until_watch_id}") - yield videos_urls[:trim_index] - return - except ValueError: - pass - yield videos_urls - - # Extraction from a playlist only returns 100 videos at a time - # if self._extract_videos returns a continuation there are more - # than 100 songs inside a playlist, so we need to add further requests - # to gather all of them - if continuation: - load_more_url, headers, data = self._build_continuation_url(continuation) - else: - load_more_url, headers, data = None, None, None - - while load_more_url and headers and data: # there is an url found - logger.debug("load more url: %s", load_more_url) - # requesting the next page of videos with the url generated from the - # previous page, needs to be a post - req = request.post(load_more_url, extra_headers=headers, data=data) - # extract up to 100 songs from the page loaded - # returns another continuation if more videos are available - videos_urls, continuation = self._extract_videos(req) - if until_watch_id: - try: - trim_index = videos_urls.index(f"/watch?v={until_watch_id}") - yield videos_urls[:trim_index] - return - except ValueError: - pass - yield videos_urls - - if continuation: - load_more_url, headers, data = self._build_continuation_url( - continuation - ) - else: - load_more_url, headers, data = None, None, None - - def _build_continuation_url(self, continuation: str) -> Tuple[str, dict, dict]: - """Helper method to build the url and headers required to request - the next page of videos - - :param str continuation: Continuation extracted from the json response - of the last page - :rtype: Tuple[str, dict, dict] - :returns: Tuple of an url and required headers for the next http - request - """ - return ( - ( - # was changed to this format (and post requests) - # between 2021年03月02日 and 2021年03月03日 - "https://www.youtube.com/youtubei/v1/browse?key=" - f"{self.yt_api_key}" - ), - { - "X-YouTube-Client-Name": "1", - "X-YouTube-Client-Version": "2.20200720.00.02", - }, - # extra data required for post request - { - "continuation": continuation, - "context": { - "client": { - "clientName": "WEB", - "clientVersion": "2.20200720.00.02" - } - } - } - ) - - @staticmethod - def _extract_videos(raw_json: str) -> Tuple[List[str], Optional[str]]: - """Extracts videos from a raw json page - - :param str raw_json: Input json extracted from the page or the last - server response - :rtype: Tuple[List[str], Optional[str]] - :returns: Tuple containing a list of up to 100 video watch ids and - a continuation token, if more videos are available - """ - initial_data = json.loads(raw_json) - try: - # this is the json tree structure, if the json was extracted from - # html - section_contents = initial_data["contents"][ - "twoColumnBrowseResultsRenderer"][ - "tabs"][0]["tabRenderer"]["content"][ - "sectionListRenderer"]["contents"] - try: - # Playlist without submenus - important_content = section_contents[ - 0]["itemSectionRenderer"][ - "contents"][0]["playlistVideoListRenderer"] - except (KeyError, IndexError, TypeError): - # Playlist with submenus - important_content = section_contents[ - 1]["itemSectionRenderer"][ - "contents"][0]["playlistVideoListRenderer"] - videos = important_content["contents"] - except (KeyError, IndexError, TypeError): - try: - # this is the json tree structure, if the json was directly sent - # by the server in a continuation response - # no longer a list and no longer has the "response" key - important_content = initial_data['onResponseReceivedActions'][0][ - 'appendContinuationItemsAction']['continuationItems'] - videos = important_content - except (KeyError, IndexError, TypeError) as p: - logger.info(p) - return [], None - - try: - continuation = videos[-1]['continuationItemRenderer'][ - 'continuationEndpoint' - ]['continuationCommand']['token'] - videos = videos[:-1] - except (KeyError, IndexError): - # if there is an error, no continuation is available - continuation = None - - # remove duplicates - return ( - uniqueify( - list( - # only extract the video ids from the video data - map( - lambda x: ( - f"/watch?v=" - f"{x['playlistVideoRenderer']['videoId']}" - ), - videos - ) - ), - ), - continuation, - ) - - def trimmed(self, video_id: str) -> Iterable[str]: - """Retrieve a list of YouTube video URLs trimmed at the given video ID - - i.e. if the playlist has video IDs 1,2,3,4 calling trimmed(3) returns - [1,2] - :type video_id: str - video ID to trim the returned list of playlist URLs at - :rtype: List[str] - :returns: - List of video URLs from the playlist trimmed at the given ID - """ - for page in self._paginate(until_watch_id=video_id): - yield from (self._video_url(watch_path) for watch_path in page) - - def url_generator(self): - """Generator that yields video URLs. - - :Yields: Video URLs - """ - for page in self._paginate(): - for video in page: - yield self._video_url(video) - - @property # type: ignore - @cache - def video_urls(self) -> DeferredGeneratorList: - """Complete links of all the videos in playlist - - :rtype: List[str] - :returns: List of video URLs - """ - return DeferredGeneratorList(self.url_generator()) - - def videos_generator(self): - for url in self.video_urls: - yield YouTube(url) - - @property - def videos(self) -> Iterable[YouTube]: - """Yields YouTube objects of videos in this playlist - - :rtype: List[YouTube] - :returns: List of YouTube - """ - return DeferredGeneratorList(self.videos_generator()) - - def __getitem__(self, i: Union[slice, int]) -> Union[str, List[str]]: - return self.video_urls[i] - - def __len__(self) -> int: - return len(self.video_urls) - - def __repr__(self) -> str: - return f"{repr(self.video_urls)}" - - @property - @cache - def last_updated(self) -> Optional[date]: - """Extract the date that the playlist was last updated. - - :return: Date of last playlist update - :rtype: datetime.date - """ - last_updated_text = self.sidebar_info[0]['playlistSidebarPrimaryInfoRenderer'][ - 'stats'][2]['runs'][1]['text'] - date_components = last_updated_text.split() - month = date_components[0] - day = date_components[1].strip(',') - year = date_components[2] - return datetime.strptime( - f"{month} {day:0>2} {year}", "%b %d %Y" - ).date() - - @property - @cache - def title(self) -> Optional[str]: - """Extract playlist title - - :return: playlist title (name) - :rtype: Optional[str] - """ - return self.sidebar_info[0]['playlistSidebarPrimaryInfoRenderer'][ - 'title']['runs'][0]['text'] - - @property - def description(self) -> str: - return self.sidebar_info[0]['playlistSidebarPrimaryInfoRenderer'][ - 'description']['simpleText'] - - @property - def length(self): - """Extract the number of videos in the playlist. - - :return: Playlist video count - :rtype: int - """ - count_text = self.sidebar_info[0]['playlistSidebarPrimaryInfoRenderer'][ - 'stats'][0]['runs'][0]['text'] - count_text = count_text.replace(',','') - return int(count_text) - - @property - def views(self): - """Extract view count for playlist. - - :return: Playlist view count - :rtype: int - """ - # "1,234,567 views" - views_text = self.sidebar_info[0]['playlistSidebarPrimaryInfoRenderer'][ - 'stats'][1]['simpleText'] - # "1,234,567" - count_text = views_text.split()[0] - # "1234567" - count_text = count_text.replace(',', '') - return int(count_text) - - @property - def owner(self): - """Extract the owner of the playlist. - - :return: Playlist owner name. - :rtype: str - """ - return self.sidebar_info[1]['playlistSidebarSecondaryInfoRenderer'][ - 'videoOwner']['videoOwnerRenderer']['title']['runs'][0]['text'] - - @property - def owner_id(self): - """Extract the channel_id of the owner of the playlist. - - :return: Playlist owner's channel ID. - :rtype: str - """ - return self.sidebar_info[1]['playlistSidebarSecondaryInfoRenderer'][ - 'videoOwner']['videoOwnerRenderer']['title']['runs'][0][ - 'navigationEndpoint']['browseEndpoint']['browseId'] - - @property - def owner_url(self): - """Create the channel url of the owner of the playlist. - - :return: Playlist owner's channel url. - :rtype: str - """ - return f'https://www.youtube.com/channel/{self.owner_id}' - - @staticmethod - def _video_url(watch_path: str): - return f"https://www.youtube.com{watch_path}" diff --git a/001-Downloader/pytube/contrib/search.py b/001-Downloader/pytube/contrib/search.py deleted file mode 100755 index a10f00c..0000000 --- a/001-Downloader/pytube/contrib/search.py +++ /dev/null @@ -1,225 +0,0 @@ -"""Module for interacting with YouTube search.""" -# Native python imports -import logging - -# Local imports -from pytube import YouTube -from pytube.innertube import InnerTube - - -logger = logging.getLogger(__name__) - - -class Search: - def __init__(self, query): - """Initialize Search object. - - :param str query: - Search query provided by the user. - """ - self.query = query - self._innertube_client = InnerTube() - - # The first search, without a continuation, is structured differently - # and contains completion suggestions, so we must store this separately - self._initial_results = None - - self._results = None - self._completion_suggestions = None - - # Used for keeping track of query continuations so that new results - # are always returned when get_next_results() is called - self._current_continuation = None - - @property - def completion_suggestions(self): - """Return query autocompletion suggestions for the query. - - :rtype: list - :returns: - A list of autocomplete suggestions provided by YouTube for the query. - """ - if self._completion_suggestions: - return self._completion_suggestions - if self.results: - self._completion_suggestions = self._initial_results['refinements'] - return self._completion_suggestions - - @property - def results(self): - """Return search results. - - On first call, will generate and return the first set of results. - Additional results can be generated using ``.get_next_results()``. - - :rtype: list - :returns: - A list of YouTube objects. - """ - if self._results: - return self._results - - videos, continuation = self.fetch_and_parse() - self._results = videos - self._current_continuation = continuation - return self._results - - def get_next_results(self): - """Use the stored continuation string to fetch the next set of results. - - This method does not return the results, but instead updates the results property. - """ - if self._current_continuation: - videos, continuation = self.fetch_and_parse(self._current_continuation) - self._results.extend(videos) - self._current_continuation = continuation - else: - raise IndexError - - def fetch_and_parse(self, continuation=None): - """Fetch from the innertube API and parse the results. - - :param str continuation: - Continuation string for fetching results. - :rtype: tuple - :returns: - A tuple of a list of YouTube objects and a continuation string. - """ - # Begin by executing the query and identifying the relevant sections - # of the results - raw_results = self.fetch_query(continuation) - - # Initial result is handled by try block, continuations by except block - try: - sections = raw_results['contents']['twoColumnSearchResultsRenderer'][ - 'primaryContents']['sectionListRenderer']['contents'] - except KeyError: - sections = raw_results['onResponseReceivedCommands'][0][ - 'appendContinuationItemsAction']['continuationItems'] - item_renderer = None - continuation_renderer = None - for s in sections: - if 'itemSectionRenderer' in s: - item_renderer = s['itemSectionRenderer'] - if 'continuationItemRenderer' in s: - continuation_renderer = s['continuationItemRenderer'] - - # If the continuationItemRenderer doesn't exist, assume no further results - if continuation_renderer: - next_continuation = continuation_renderer['continuationEndpoint'][ - 'continuationCommand']['token'] - else: - next_continuation = None - - # If the itemSectionRenderer doesn't exist, assume no results. - if item_renderer: - videos = [] - raw_video_list = item_renderer['contents'] - for video_details in raw_video_list: - # Skip over ads - if video_details.get('searchPyvRenderer', {}).get('ads', None): - continue - - # Skip "recommended" type videos e.g. "people also watched" and "popular X" - # that break up the search results - if 'shelfRenderer' in video_details: - continue - - # Skip auto-generated "mix" playlist results - if 'radioRenderer' in video_details: - continue - - # Skip playlist results - if 'playlistRenderer' in video_details: - continue - - # Skip channel results - if 'channelRenderer' in video_details: - continue - - # Skip 'people also searched for' results - if 'horizontalCardListRenderer' in video_details: - continue - - # Can't seem to reproduce, probably related to typo fix suggestions - if 'didYouMeanRenderer' in video_details: - continue - - # Seems to be the renderer used for the image shown on a no results page - if 'backgroundPromoRenderer' in video_details: - continue - - if 'videoRenderer' not in video_details: - logger.warn('Unexpected renderer encountered.') - logger.warn(f'Renderer name: {video_details.keys()}') - logger.warn(f'Search term: {self.query}') - logger.warn( - 'Please open an issue at ' - 'https://github.com/pytube/pytube/issues ' - 'and provide this log output.' - ) - continue - - # Extract relevant video information from the details. - # Some of this can be used to pre-populate attributes of the - # YouTube object. - vid_renderer = video_details['videoRenderer'] - vid_id = vid_renderer['videoId'] - vid_url = f'https://www.youtube.com/watch?v={vid_id}' - vid_title = vid_renderer['title']['runs'][0]['text'] - vid_channel_name = vid_renderer['ownerText']['runs'][0]['text'] - vid_channel_uri = vid_renderer['ownerText']['runs'][0][ - 'navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'] - # Livestreams have "runs", non-livestreams have "simpleText", - # and scheduled releases do not have 'viewCountText' - if 'viewCountText' in vid_renderer: - if 'runs' in vid_renderer['viewCountText']: - vid_view_count_text = vid_renderer['viewCountText']['runs'][0]['text'] - else: - vid_view_count_text = vid_renderer['viewCountText']['simpleText'] - # Strip ' views' text, then remove commas - stripped_text = vid_view_count_text.split()[0].replace(',','') - if stripped_text == 'No': - vid_view_count = 0 - else: - vid_view_count = int(stripped_text) - else: - vid_view_count = 0 - if 'lengthText' in vid_renderer: - vid_length = vid_renderer['lengthText']['simpleText'] - else: - vid_length = None - - vid_metadata = { - 'id': vid_id, - 'url': vid_url, - 'title': vid_title, - 'channel_name': vid_channel_name, - 'channel_url': vid_channel_uri, - 'view_count': vid_view_count, - 'length': vid_length - } - - # Construct YouTube object from metadata and append to results - vid = YouTube(vid_metadata['url']) - vid.author = vid_metadata['channel_name'] - vid.title = vid_metadata['title'] - videos.append(vid) - else: - videos = None - - return videos, next_continuation - - def fetch_query(self, continuation=None): - """Fetch raw results from the innertube API. - - :param str continuation: - Continuation string for fetching results. - :rtype: dict - :returns: - The raw json object returned by the innertube API. - """ - query_results = self._innertube_client.search(self.query, continuation) - if not self._initial_results: - self._initial_results = query_results - return query_results # noqa:R504 diff --git a/001-Downloader/pytube/exceptions.py b/001-Downloader/pytube/exceptions.py deleted file mode 100755 index ec44d2a..0000000 --- a/001-Downloader/pytube/exceptions.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Library specific exception definitions.""" -from typing import Pattern, Union - - -class PytubeError(Exception): - """Base pytube exception that all others inherit. - - This is done to not pollute the built-in exceptions, which *could* result - in unintended errors being unexpectedly and incorrectly handled within - implementers code. - """ - - -class MaxRetriesExceeded(PytubeError): - """Maximum number of retries exceeded.""" - - -class HTMLParseError(PytubeError): - """HTML could not be parsed""" - - -class ExtractError(PytubeError): - """Data extraction based exception.""" - - -class RegexMatchError(ExtractError): - """Regex pattern did not return any matches.""" - - def __init__(self, caller: str, pattern: Union[str, Pattern]): - """ - :param str caller: - Calling function - :param str pattern: - Pattern that failed to match - """ - super().__init__(f"{caller}: could not find match for {pattern}") - self.caller = caller - self.pattern = pattern - - -class VideoUnavailable(PytubeError): - """Base video unavailable error.""" - def __init__(self, video_id: str): - """ - :param str video_id: - A YouTube video identifier. - """ - self.video_id = video_id - super().__init__(self.error_string) - - @property - def error_string(self): - return f'{self.video_id} is unavailable' - - -class AgeRestrictedError(VideoUnavailable): - """Video is age restricted, and cannot be accessed without OAuth.""" - def __init__(self, video_id: str): - """ - :param str video_id: - A YouTube video identifier. - """ - self.video_id = video_id - super().__init__(self.video_id) - - @property - def error_string(self): - return f"{self.video_id} is age restricted, and can't be accessed without logging in." - - -class LiveStreamError(VideoUnavailable): - """Video is a live stream.""" - def __init__(self, video_id: str): - """ - :param str video_id: - A YouTube video identifier. - """ - self.video_id = video_id - super().__init__(self.video_id) - - @property - def error_string(self): - return f'{self.video_id} is streaming live and cannot be loaded' - - -class VideoPrivate(VideoUnavailable): - def __init__(self, video_id: str): - """ - :param str video_id: - A YouTube video identifier. - """ - self.video_id = video_id - super().__init__(self.video_id) - - @property - def error_string(self): - return f'{self.video_id} is a private video' - - -class RecordingUnavailable(VideoUnavailable): - def __init__(self, video_id: str): - """ - :param str video_id: - A YouTube video identifier. - """ - self.video_id = video_id - super().__init__(self.video_id) - - @property - def error_string(self): - return f'{self.video_id} does not have a live stream recording available' - - -class MembersOnly(VideoUnavailable): - """Video is members-only. - - YouTube has special videos that are only viewable to users who have - subscribed to a content creator. - ref: https://support.google.com/youtube/answer/7544492?hl=en - """ - def __init__(self, video_id: str): - """ - :param str video_id: - A YouTube video identifier. - """ - self.video_id = video_id - super().__init__(self.video_id) - - @property - def error_string(self): - return f'{self.video_id} is a members-only video' - - -class VideoRegionBlocked(VideoUnavailable): - def __init__(self, video_id: str): - """ - :param str video_id: - A YouTube video identifier. - """ - self.video_id = video_id - super().__init__(self.video_id) - - @property - def error_string(self): - return f'{self.video_id} is not available in your region' diff --git a/001-Downloader/pytube/extract.py b/001-Downloader/pytube/extract.py deleted file mode 100755 index d083214..0000000 --- a/001-Downloader/pytube/extract.py +++ /dev/null @@ -1,579 +0,0 @@ -"""This module contains all non-cipher related data extraction logic.""" -import logging -import urllib.parse -import re -from collections import OrderedDict -from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple -from urllib.parse import parse_qs, quote, urlencode, urlparse - -from pytube.cipher import Cipher -from pytube.exceptions import HTMLParseError, LiveStreamError, RegexMatchError -from pytube.helpers import regex_search -from pytube.metadata import YouTubeMetadata -from pytube.parser import parse_for_object, parse_for_all_objects - - -logger = logging.getLogger(__name__) - - -def publish_date(watch_html: str): - """Extract publish date - :param str watch_html: - The html contents of the watch page. - :rtype: str - :returns: - Publish date of the video. - """ - try: - result = regex_search( - r"(?<=itemprop=\"datepublished\" content=\")\d{4}-\d{2}-\d{2}", - watch_html, group=0 - ) - except RegexMatchError: - return None - return datetime.strptime(result, '%Y-%m-%d') - - -def recording_available(watch_html): - """Check if live stream recording is available. - - :param str watch_html: - The html contents of the watch page. - :rtype: bool - :returns: - Whether or not the content is private. - """ - unavailable_strings = [ - 'This live stream recording is not available.' - ] - for string in unavailable_strings: - if string in watch_html: - return False - return True - - -def is_private(watch_html): - """Check if content is private. - - :param str watch_html: - The html contents of the watch page. - :rtype: bool - :returns: - Whether or not the content is private. - """ - private_strings = [ - "This is a private video. Please sign in to verify that you may see it.", - "\"simpleText\":\"Private video\"", - "This video is private." - ] - for string in private_strings: - if string in watch_html: - return True - return False - - -def is_age_restricted(watch_html: str) -> bool: - """Check if content is age restricted. - - :param str watch_html: - The html contents of the watch page. - :rtype: bool - :returns: - Whether or not the content is age restricted. - """ - try: - regex_search(r"og:restrictions:age", watch_html, group=0) - except RegexMatchError: - return False - return True - - -def playability_status(watch_html: str) -> (str, str): - """Return the playability status and status explanation of a video. - - For example, a video may have a status of LOGIN_REQUIRED, and an explanation - of "This is a private video. Please sign in to verify that you may see it." - - This explanation is what gets incorporated into the media player overlay. - - :param str watch_html: - The html contents of the watch page. - :rtype: bool - :returns: - Playability status and reason of the video. - """ - player_response = initial_player_response(watch_html) - status_dict = player_response.get('playabilityStatus', {}) - if 'liveStreamability' in status_dict: - return 'LIVE_STREAM', 'Video is a live stream.' - if 'status' in status_dict: - if 'reason' in status_dict: - return status_dict['status'], [status_dict['reason']] - if 'messages' in status_dict: - return status_dict['status'], status_dict['messages'] - return None, [None] - - -def video_id(url: str) -> str: - """Extract the ``video_id`` from a YouTube url. - - This function supports the following patterns: - - - :samp:`https://youtube.com/watch?v={video_id}` - - :samp:`https://youtube.com/embed/{video_id}` - - :samp:`https://youtu.be/{video_id}` - - :param str url: - A YouTube url containing a video id. - :rtype: str - :returns: - YouTube video id. - """ - return regex_search(r"(?:v=|\/)([0-9A-Za-z_-]{11}).*", url, group=1) - - -def playlist_id(url: str) -> str: - """Extract the ``playlist_id`` from a YouTube url. - - This function supports the following patterns: - - - :samp:`https://youtube.com/playlist?list={playlist_id}` - - :samp:`https://youtube.com/watch?v={video_id}&list={playlist_id}` - - :param str url: - A YouTube url containing a playlist id. - :rtype: str - :returns: - YouTube playlist id. - """ - parsed = urllib.parse.urlparse(url) - return parse_qs(parsed.query)['list'][0] - - -def channel_name(url: str) -> str: - """Extract the ``channel_name`` or ``channel_id`` from a YouTube url. - - This function supports the following patterns: - - - :samp:`https://youtube.com/c/{channel_name}/*` - - :samp:`https://youtube.com/channel/{channel_id}/* - - :samp:`https://youtube.com/u/{channel_name}/*` - - :samp:`https://youtube.com/user/{channel_id}/* - - :param str url: - A YouTube url containing a channel name. - :rtype: str - :returns: - YouTube channel name. - """ - patterns = [ - r"(?:\/(c)\/([%\d\w_\-]+)(\/.*)?)", - r"(?:\/(channel)\/([%\w\d_\-]+)(\/.*)?)", - r"(?:\/(u)\/([%\d\w_\-]+)(\/.*)?)", - r"(?:\/(user)\/([%\w\d_\-]+)(\/.*)?)" - ] - for pattern in patterns: - regex = re.compile(pattern) - function_match = regex.search(url) - if function_match: - logger.debug("finished regex search, matched: %s", pattern) - uri_style = function_match.group(1) - uri_identifier = function_match.group(2) - return f'/{uri_style}/{uri_identifier}' - - raise RegexMatchError( - caller="channel_name", pattern="patterns" - ) - - -def video_info_url(video_id: str, watch_url: str) -> str: - """Construct the video_info url. - - :param str video_id: - A YouTube video identifier. - :param str watch_url: - A YouTube watch url. - :rtype: str - :returns: - :samp:`https://youtube.com/get_video_info` with necessary GET - parameters. - """ - params = OrderedDict( - [ - ("video_id", video_id), - ("ps", "default"), - ("eurl", quote(watch_url)), - ("hl", "en_US"), - ("html5", "1"), - ("c", "TVHTML5"), - ("cver", "7.20201028"), - ] - ) - return _video_info_url(params) - - -def video_info_url_age_restricted(video_id: str, embed_html: str) -> str: - """Construct the video_info url. - - :param str video_id: - A YouTube video identifier. - :param str embed_html: - The html contents of the embed page (for age restricted videos). - :rtype: str - :returns: - :samp:`https://youtube.com/get_video_info` with necessary GET - parameters. - """ - try: - sts = regex_search(r'"sts"\s*:\s*(\d+)', embed_html, group=1) - except RegexMatchError: - sts = "" - # Here we use ``OrderedDict`` so that the output is consistent between - # Python 2.7+. - eurl = f"https://youtube.googleapis.com/v/{video_id}" - params = OrderedDict( - [ - ("video_id", video_id), - ("eurl", eurl), - ("sts", sts), - ("html5", "1"), - ("c", "TVHTML5"), - ("cver", "7.20201028"), - ] - ) - return _video_info_url(params) - - -def _video_info_url(params: OrderedDict) -> str: - return "https://www.youtube.com/get_video_info?" + urlencode(params) - - -def js_url(html: str) -> str: - """Get the base JavaScript url. - - Construct the base JavaScript url, which contains the decipher - "transforms". - - :param str html: - The html contents of the watch page. - """ - try: - base_js = get_ytplayer_config(html)['assets']['js'] - except (KeyError, RegexMatchError): - base_js = get_ytplayer_js(html) - return "https://youtube.com" + base_js - - -def mime_type_codec(mime_type_codec: str) -> Tuple[str, List[str]]: - """Parse the type data. - - Breaks up the data in the ``type`` key of the manifest, which contains the - mime type and codecs serialized together, and splits them into separate - elements. - - **Example**: - - mime_type_codec('audio/webm; codecs="opus"') -> ('audio/webm', ['opus']) - - :param str mime_type_codec: - String containing mime type and codecs. - :rtype: tuple - :returns: - The mime type and a list of codecs. - - """ - pattern = r"(\w+\/\w+)\;\scodecs=\"([a-zA-Z-0-9.,\s]*)\"" - regex = re.compile(pattern) - results = regex.search(mime_type_codec) - if not results: - raise RegexMatchError(caller="mime_type_codec", pattern=pattern) - mime_type, codecs = results.groups() - return mime_type, [c.strip() for c in codecs.split(",")] - - -def get_ytplayer_js(html: str) -> Any: - """Get the YouTube player base JavaScript path. - - :param str html - The html contents of the watch page. - :rtype: str - :returns: - Path to YouTube's base.js file. - """ - js_url_patterns = [ - r"(/s/player/[\w\d]+/[\w\d_/.]+/base\.js)" - ] - for pattern in js_url_patterns: - regex = re.compile(pattern) - function_match = regex.search(html) - if function_match: - logger.debug("finished regex search, matched: %s", pattern) - yt_player_js = function_match.group(1) - return yt_player_js - - raise RegexMatchError( - caller="get_ytplayer_js", pattern="js_url_patterns" - ) - - -def get_ytplayer_config(html: str) -> Any: - """Get the YouTube player configuration data from the watch html. - - Extract the ``ytplayer_config``, which is json data embedded within the - watch html and serves as the primary source of obtaining the stream - manifest data. - - :param str html: - The html contents of the watch page. - :rtype: str - :returns: - Substring of the html containing the encoded manifest data. - """ - logger.debug("finding initial function name") - config_patterns = [ - r"ytplayer\.config\s*=\s*", - r"ytInitialPlayerResponse\s*=\s*" - ] - for pattern in config_patterns: - # Try each pattern consecutively if they don't find a match - try: - return parse_for_object(html, pattern) - except HTMLParseError as e: - logger.debug(f'Pattern failed: {pattern}') - logger.debug(e) - continue - - # setConfig() needs to be handled a little differently. - # We want to parse the entire argument to setConfig() - # and use then load that as json to find PLAYER_CONFIG - # inside of it. - setconfig_patterns = [ - r"yt\.setConfig\(.*['\"]PLAYER_CONFIG['\"]:\s*" - ] - for pattern in setconfig_patterns: - # Try each pattern consecutively if they don't find a match - try: - return parse_for_object(html, pattern) - except HTMLParseError: - continue - - raise RegexMatchError( - caller="get_ytplayer_config", pattern="config_patterns, setconfig_patterns" - ) - - -def get_ytcfg(html: str) -> str: - """Get the entirety of the ytcfg object. - - This is built over multiple pieces, so we have to find all matches and - combine the dicts together. - - :param str html: - The html contents of the watch page. - :rtype: str - :returns: - Substring of the html containing the encoded manifest data. - """ - ytcfg = {} - ytcfg_patterns = [ - r"ytcfg\s=\s", - r"ytcfg\.set\(" - ] - for pattern in ytcfg_patterns: - # Try each pattern consecutively and try to build a cohesive object - try: - found_objects = parse_for_all_objects(html, pattern) - for obj in found_objects: - ytcfg.update(obj) - except HTMLParseError: - continue - - if len(ytcfg)> 0: - return ytcfg - - raise RegexMatchError( - caller="get_ytcfg", pattern="ytcfg_pattenrs" - ) - - -def apply_signature(stream_manifest: Dict, vid_info: Dict, js: str) -> None: - """Apply the decrypted signature to the stream manifest. - - :param dict stream_manifest: - Details of the media streams available. - :param str js: - The contents of the base.js asset file. - - """ - cipher = Cipher(js=js) - - for i, stream in enumerate(stream_manifest): - try: - url: str = stream["url"] - except KeyError: - live_stream = ( - vid_info.get("playabilityStatus", {},) - .get("liveStreamability") - ) - if live_stream: - raise LiveStreamError("UNKNOWN") - # 403 Forbidden fix. - if "signature" in url or ( - "s" not in stream and ("&sig=" in url or "&lsig=" in url) - ): - # For certain videos, YouTube will just provide them pre-signed, in - # which case there's no real magic to download them and we can skip - # the whole signature descrambling entirely. - logger.debug("signature found, skip decipher") - continue - - signature = cipher.get_signature(ciphered_signature=stream["s"]) - - logger.debug( - "finished descrambling signature for itag=%s", stream["itag"] - ) - parsed_url = urlparse(url) - - # Convert query params off url to dict - query_params = parse_qs(urlparse(url).query) - query_params = { - k: v[0] for k,v in query_params.items() - } - query_params['sig'] = signature - if 'ratebypass' not in query_params.keys(): - # Cipher n to get the updated value - - initial_n = list(query_params['n']) - new_n = cipher.calculate_n(initial_n) - query_params['n'] = new_n - - url = f'{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}?{urlencode(query_params)}' # noqa:E501 - - # 403 forbidden fix - stream_manifest[i]["url"] = url - - -def apply_descrambler(stream_data: Dict) -> None: - """Apply various in-place transforms to YouTube's media stream data. - - Creates a ``list`` of dictionaries by string splitting on commas, then - taking each list item, parsing it as a query string, converting it to a - ``dict`` and unquoting the value. - - :param dict stream_data: - Dictionary containing query string encoded values. - - **Example**: - ->>> d = {'foo': 'bar=1&var=test,em=5&t=url%20encoded'} ->>> apply_descrambler(d, 'foo') ->>> print(d) - {'foo': [{'bar': '1', 'var': 'test'}, {'em': '5', 't': 'url encoded'}]} - - """ - if 'url' in stream_data: - return None - - # Merge formats and adaptiveFormats into a single list - formats = [] - if 'formats' in stream_data.keys(): - formats.extend(stream_data['formats']) - if 'adaptiveFormats' in stream_data.keys(): - formats.extend(stream_data['adaptiveFormats']) - - # Extract url and s from signatureCiphers as necessary - for data in formats: - if 'url' not in data: - if 'signatureCipher' in data: - cipher_url = parse_qs(data['signatureCipher']) - data['url'] = cipher_url['url'][0] - data['s'] = cipher_url['s'][0] - data['is_otf'] = data.get('type') == 'FORMAT_STREAM_TYPE_OTF' - - logger.debug("applying descrambler") - return formats - - -def initial_data(watch_html: str) -> str: - """Extract the ytInitialData json from the watch_html page. - - This mostly contains metadata necessary for rendering the page on-load, - such as video information, copyright notices, etc. - - @param watch_html: Html of the watch page - @return: - """ - patterns = [ - r"window\[['\"]ytInitialData['\"]]\s*=\s*", - r"ytInitialData\s*=\s*" - ] - for pattern in patterns: - try: - return parse_for_object(watch_html, pattern) - except HTMLParseError: - pass - - raise RegexMatchError(caller='initial_data', pattern='initial_data_pattern') - - -def initial_player_response(watch_html: str) -> str: - """Extract the ytInitialPlayerResponse json from the watch_html page. - - This mostly contains metadata necessary for rendering the page on-load, - such as video information, copyright notices, etc. - - @param watch_html: Html of the watch page - @return: - """ - patterns = [ - r"window\[['\"]ytInitialPlayerResponse['\"]]\s*=\s*", - r"ytInitialPlayerResponse\s*=\s*" - ] - for pattern in patterns: - try: - return parse_for_object(watch_html, pattern) - except HTMLParseError: - pass - - raise RegexMatchError( - caller='initial_player_response', - pattern='initial_player_response_pattern' - ) - - -def metadata(initial_data) -> Optional[YouTubeMetadata]: - """Get the informational metadata for the video. - - e.g.: - [ - { - 'Song': '강남스타일(Gangnam Style)', - 'Artist': 'PSY', - 'Album': 'PSY SIX RULES Pt.1', - 'Licensed to YouTube by': 'YG Entertainment Inc. [...]' - } - ] - - :rtype: YouTubeMetadata - """ - try: - metadata_rows: List = initial_data["contents"]["twoColumnWatchNextResults"][ - "results"]["results"]["contents"][1]["videoSecondaryInfoRenderer"][ - "metadataRowContainer"]["metadataRowContainerRenderer"]["rows"] - except (KeyError, IndexError): - # If there's an exception accessing this data, it probably doesn't exist. - return YouTubeMetadata([]) - - # Rows appear to only have "metadataRowRenderer" or "metadataRowHeaderRenderer" - # and we only care about the former, so we filter the others - metadata_rows = filter( - lambda x: "metadataRowRenderer" in x.keys(), - metadata_rows - ) - - # We then access the metadataRowRenderer key in each element - # and build a metadata object from this new list - metadata_rows = [x["metadataRowRenderer"] for x in metadata_rows] - - return YouTubeMetadata(metadata_rows) diff --git a/001-Downloader/pytube/helpers.py b/001-Downloader/pytube/helpers.py deleted file mode 100755 index 4cf02eb..0000000 --- a/001-Downloader/pytube/helpers.py +++ /dev/null @@ -1,335 +0,0 @@ -"""Various helper functions implemented by pytube.""" -import functools -import gzip -import json -import logging -import os -import re -import warnings -from typing import Any, Callable, Dict, List, Optional, TypeVar -from urllib import request - -from pytube.exceptions import RegexMatchError - -logger = logging.getLogger(__name__) - - -class DeferredGeneratorList: - """A wrapper class for deferring list generation. - - Pytube has some continuation generators that create web calls, which means - that any time a full list is requested, all of those web calls must be - made at once, which could lead to slowdowns. This will allow individual - elements to be queried, so that slowdowns only happen as necessary. For - example, you can iterate over elements in the list without accessing them - all simultaneously. This should allow for speed improvements for playlist - and channel interactions. - """ - def __init__(self, generator): - """Construct a :class:`DeferredGeneratorList `. - - :param generator generator: - The deferrable generator to create a wrapper for. - :param func func: - (Optional) A function to call on the generator items to produce the list. - """ - self.gen = generator - self._elements = [] - - def __eq__(self, other): - """We want to mimic list behavior for comparison.""" - return list(self) == other - - def __getitem__(self, key) -> Any: - """Only generate items as they're asked for.""" - # We only allow querying with indexes. - if not isinstance(key, (int, slice)): - raise TypeError('Key must be either a slice or int.') - - # Convert int keys to slice - key_slice = key - if isinstance(key, int): - key_slice = slice(key, key + 1, 1) - - # Generate all elements up to the final item - while len(self._elements) < key_slice.stop: - try: - next_item = next(self.gen) - except StopIteration: - # If we can't find enough elements for the slice, raise an IndexError - raise IndexError - else: - self._elements.append(next_item) - - return self._elements[key] - - def __iter__(self): - """Custom iterator for dynamically generated list.""" - iter_index = 0 - while True: - try: - curr_item = self[iter_index] - except IndexError: - return - else: - yield curr_item - iter_index += 1 - - def __next__(self) -> Any: - """Fetch next element in iterator.""" - try: - curr_element = self[self.iter_index] - except IndexError: - raise StopIteration - self.iter_index += 1 - return curr_element # noqa:R504 - - def __len__(self) -> int: - """Return length of list of all items.""" - self.generate_all() - return len(self._elements) - - def __repr__(self) -> str: - """String representation of all items.""" - self.generate_all() - return str(self._elements) - - def __reversed__(self): - self.generate_all() - return self._elements[::-1] - - def generate_all(self): - """Generate all items.""" - while True: - try: - next_item = next(self.gen) - except StopIteration: - break - else: - self._elements.append(next_item) - - -def regex_search(pattern: str, string: str, group: int) -> str: - """Shortcut method to search a string for a given pattern. - - :param str pattern: - A regular expression pattern. - :param str string: - A target string to search. - :param int group: - Index of group to return. - :rtype: - str or tuple - :returns: - Substring pattern matches. - """ - regex = re.compile(pattern) - results = regex.search(string) - if not results: - raise RegexMatchError(caller="regex_search", pattern=pattern) - - logger.debug("matched regex search: %s", pattern) - - return results.group(group) - - -def safe_filename(s: str, max_length: int = 255) -> str: - """Sanitize a string making it safe to use as a filename. - - This function was based off the limitations outlined here: - https://en.wikipedia.org/wiki/Filename. - - :param str s: - A string to make safe for use as a file name. - :param int max_length: - The maximum filename character length. - :rtype: str - :returns: - A sanitized string. - """ - # Characters in range 0-31 (0x00-0x1F) are not allowed in ntfs filenames. - ntfs_characters = [chr(i) for i in range(0, 31)] - characters = [ - r'"', - r"\#", - r"\$", - r"\%", - r"'", - r"\*", - r",円", - r"\.", - r"\/", - r"\:", - r'"', - r"\;", - r"\<", - r"\>", - r"\?", - r"\\", - r"\^", - r"\|", - r"\~", - r"\\\\", - ] - pattern = "|".join(ntfs_characters + characters) - regex = re.compile(pattern, re.UNICODE) - filename = regex.sub("", s) - return filename[:max_length].rsplit(" ", 0)[0] - - -def setup_logger(level: int = logging.ERROR, log_filename: Optional[str] = None) -> None: - """Create a configured instance of logger. - - :param int level: - Describe the severity level of the logs to handle. - """ - fmt = "[%(asctime)s] %(levelname)s in %(module)s: %(message)s" - date_fmt = "%H:%M:%S" - formatter = logging.Formatter(fmt, datefmt=date_fmt) - - # https://github.com/pytube/pytube/issues/163 - logger = logging.getLogger("pytube") - logger.setLevel(level) - - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(formatter) - logger.addHandler(stream_handler) - - if log_filename is not None: - file_handler = logging.FileHandler(log_filename) - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - - -GenericType = TypeVar("GenericType") - - -def cache(func: Callable[..., GenericType]) -> GenericType: - """ mypy compatible annotation wrapper for lru_cache""" - return functools.lru_cache()(func) # type: ignore - - -def deprecated(reason: str) -> Callable: - """ - This is a decorator which can be used to mark functions - as deprecated. It will result in a warning being emitted - when the function is used. - """ - - def decorator(func1): - message = "Call to deprecated function {name} ({reason})." - - @functools.wraps(func1) - def new_func1(*args, **kwargs): - warnings.simplefilter("always", DeprecationWarning) - warnings.warn( - message.format(name=func1.__name__, reason=reason), - category=DeprecationWarning, - stacklevel=2, - ) - warnings.simplefilter("default", DeprecationWarning) - return func1(*args, **kwargs) - - return new_func1 - - return decorator - - -def target_directory(output_path: Optional[str] = None) -> str: - """ - Function for determining target directory of a download. - Returns an absolute path (if relative one given) or the current - path (if none given). Makes directory if it does not exist. - - :type output_path: str - :rtype: str - :returns: - An absolute directory path as a string. - """ - if output_path: - if not os.path.isabs(output_path): - output_path = os.path.join(os.getcwd(), output_path) - else: - output_path = os.getcwd() - os.makedirs(output_path, exist_ok=True) - return output_path - - -def install_proxy(proxy_handler: Dict[str, str]) -> None: - proxy_support = request.ProxyHandler(proxy_handler) - opener = request.build_opener(proxy_support) - request.install_opener(opener) - - -def uniqueify(duped_list: List) -> List: - """Remove duplicate items from a list, while maintaining list order. - - :param List duped_list - List to remove duplicates from - - :return List result - De-duplicated list - """ - seen: Dict[Any, bool] = {} - result = [] - for item in duped_list: - if item in seen: - continue - seen[item] = True - result.append(item) - return result - - -def generate_all_html_json_mocks(): - """Regenerate the video mock json files for all current test videos. - - This should automatically output to the test/mocks directory. - """ - test_vid_ids = [ - '2lAe1cqCOXo', - '5YceQ8YqYMc', - 'irauhITDrsE', - 'm8uHb5jIGN8', - 'QRS8MkLhQmM', - 'WXxV9g7lsFE' - ] - for vid_id in test_vid_ids: - create_mock_html_json(vid_id) - - -def create_mock_html_json(vid_id) -> Dict[str, Any]: - """Generate a json.gz file with sample html responses. - - :param str vid_id - YouTube video id - - :return dict data - Dict used to generate the json.gz file - """ - from pytube import YouTube - gzip_filename = 'yt-video-%s-html.json.gz' % vid_id - - # Get the pytube directory in order to navigate to /tests/mocks - pytube_dir_path = os.path.abspath( - os.path.join( - os.path.dirname(__file__), - os.path.pardir - ) - ) - pytube_mocks_path = os.path.join(pytube_dir_path, 'tests', 'mocks') - gzip_filepath = os.path.join(pytube_mocks_path, gzip_filename) - - yt = YouTube(f'https://www.youtube.com/watch?v={vid_id}') - html_data = { - 'url': yt.watch_url, - 'js': yt.js, - 'embed_html': yt.embed_html, - 'watch_html': yt.watch_html, - 'vid_info': yt.vid_info - } - - logger.info(f'Outputing json.gz file to {gzip_filepath}') - with gzip.open(gzip_filepath, 'wb') as f: - f.write(json.dumps(html_data).encode('utf-8')) - - return html_data diff --git a/001-Downloader/pytube/innertube.py b/001-Downloader/pytube/innertube.py deleted file mode 100755 index c5d940a..0000000 --- a/001-Downloader/pytube/innertube.py +++ /dev/null @@ -1,359 +0,0 @@ -"""This module is designed to interact with the innertube API. - -This module is NOT intended to be used directly by end users, as each of the -interfaces returns raw results. These should instead be parsed to extract -the useful information for the end user. -""" -# Native python imports -import json -import os -import pathlib -import time -from urllib import parse - -# Local imports -from pytube import request - -# YouTube on TV client secrets -_client_id = '861556708454-d6dlm3lh05idd8npek18k6be8ba3oc68.apps.googleusercontent.com' -_client_secret = 'SboVhoG9s0rNafixCSGGKXAT' - -# Extracted API keys -- unclear what these are linked to. -_api_keys = [ - 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8', - 'AIzaSyCtkvNIR1HCEwzsqK6JuE6KqpyjusIRI30', - 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w', - 'AIzaSyC8UYZpvA2eknNex0Pjid0_eTLJoDu6los', - 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw', - 'AIzaSyDHQ9ipnphqTzDqZsbtd8_Ru4_kiKVQe2k' -] - -_default_clients = { - 'WEB': { - 'context': { - 'client': { - 'clientName': 'WEB', - 'clientVersion': '2.20200720.00.02' - } - }, - 'api_key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8' - }, - 'ANDROID': { - 'context': { - 'client': { - 'clientName': 'ANDROID', - 'clientVersion': '16.20' - } - }, - 'api_key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8' - }, - 'WEB_EMBED': { - 'context': { - 'client': { - 'clientName': 'WEB', - 'clientVersion': '2.20210721.00.00', - 'clientScreen': 'EMBED' - } - }, - 'api_key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8' - }, - 'ANDROID_EMBED': { - 'context': { - 'client': { - 'clientName': 'ANDROID', - 'clientVersion': '16.20', - 'clientScreen': 'EMBED' - } - }, - 'api_key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8' - } -} -_token_timeout = 1800 -_cache_dir = pathlib.Path(__file__).parent.resolve() / '__cache__' -_token_file = os.path.join(_cache_dir, 'tokens.json') - - -class InnerTube: - """Object for interacting with the innertube API.""" - def __init__(self, client='ANDROID', use_oauth=False, allow_cache=True): - """Initialize an InnerTube object. - - :param str client: - Client to use for the object. - Default to web because it returns the most playback types. - :param bool use_oauth: - Whether or not to authenticate to YouTube. - :param bool allow_cache: - Allows caching of oauth tokens on the machine. - """ - self.context = _default_clients[client]['context'] - self.api_key = _default_clients[client]['api_key'] - self.access_token = None - self.refresh_token = None - self.use_oauth = use_oauth - self.allow_cache = allow_cache - - # Stored as epoch time - self.expires = None - - # Try to load from file if specified - if self.use_oauth and self.allow_cache: - # Try to load from file if possible - if os.path.exists(_token_file): - with open(_token_file) as f: - data = json.load(f) - self.access_token = data['access_token'] - self.refresh_token = data['refresh_token'] - self.expires = data['expires'] - self.refresh_bearer_token() - - def cache_tokens(self): - """Cache tokens to file if allowed.""" - if not self.allow_cache: - return - - data = { - 'access_token': self.access_token, - 'refresh_token': self.refresh_token, - 'expires': self.expires - } - if not os.path.exists(_cache_dir): - os.mkdir(_cache_dir) - with open(_token_file, 'w') as f: - json.dump(data, f) - - def refresh_bearer_token(self, force=False): - """Refreshes the OAuth token if necessary. - - :param bool force: - Force-refresh the bearer token. - """ - if not self.use_oauth: - return - # Skip refresh if it's not necessary and not forced - if self.expires> time.time() and not force: - return - - # Subtracting 30 seconds is arbitrary to avoid potential time discrepencies - start_time = int(time.time() - 30) - data = { - 'client_id': _client_id, - 'client_secret': _client_secret, - 'grant_type': 'refresh_token', - 'refresh_token': self.refresh_token - } - response = request._execute_request( - 'https://oauth2.googleapis.com/token', - 'POST', - headers={ - 'Content-Type': 'application/json' - }, - data=data - ) - response_data = json.loads(response.read()) - - self.access_token = response_data['access_token'] - self.expires = start_time + response_data['expires_in'] - self.cache_tokens() - - def fetch_bearer_token(self): - """Fetch an OAuth token.""" - # Subtracting 30 seconds is arbitrary to avoid potential time discrepencies - start_time = int(time.time() - 30) - data = { - 'client_id': _client_id, - 'scope': 'https://www.googleapis.com/auth/youtube' - } - response = request._execute_request( - 'https://oauth2.googleapis.com/device/code', - 'POST', - headers={ - 'Content-Type': 'application/json' - }, - data=data - ) - response_data = json.loads(response.read()) - verification_url = response_data['verification_url'] - user_code = response_data['user_code'] - print(f'Please open {verification_url} and input code {user_code}') - input('Press enter when you have completed this step.') - - data = { - 'client_id': _client_id, - 'client_secret': _client_secret, - 'device_code': response_data['device_code'], - 'grant_type': 'urn:ietf:params:oauth:grant-type:device_code' - } - response = request._execute_request( - 'https://oauth2.googleapis.com/token', - 'POST', - headers={ - 'Content-Type': 'application/json' - }, - data=data - ) - response_data = json.loads(response.read()) - - self.access_token = response_data['access_token'] - self.refresh_token = response_data['refresh_token'] - self.expires = start_time + response_data['expires_in'] - self.cache_tokens() - - @property - def base_url(self): - """Return the base url endpoint for the innertube API.""" - return 'https://www.youtube.com/youtubei/v1' - - @property - def base_data(self): - """Return the base json data to transmit to the innertube API.""" - return { - 'context': self.context - } - - @property - def base_params(self): - """Return the base query parameters to transmit to the innertube API.""" - return { - 'key': self.api_key, - 'contentCheckOk': True, - 'racyCheckOk': True - } - - def _call_api(self, endpoint, query, data): - """Make a request to a given endpoint with the provided query parameters and data.""" - # Remove the API key if oauth is being used. - if self.use_oauth: - del query['key'] - - endpoint_url = f'{endpoint}?{parse.urlencode(query)}' - headers = { - 'Content-Type': 'application/json', - } - # Add the bearer token if applicable - if self.use_oauth: - if self.access_token: - self.refresh_bearer_token() - headers['Authorization'] = f'Bearer {self.access_token}' - else: - self.fetch_bearer_token() - headers['Authorization'] = f'Bearer {self.access_token}' - - response = request._execute_request( - endpoint_url, - 'POST', - headers=headers, - data=data - ) - return json.loads(response.read()) - - def browse(self): - """Make a request to the browse endpoint. - - TODO: Figure out how we can use this - """ - # endpoint = f'{self.base_url}/browse' # noqa:E800 - ... - # return self._call_api(endpoint, query, self.base_data) # noqa:E800 - - def config(self): - """Make a request to the config endpoint. - - TODO: Figure out how we can use this - """ - # endpoint = f'{self.base_url}/config' # noqa:E800 - ... - # return self._call_api(endpoint, query, self.base_data) # noqa:E800 - - def guide(self): - """Make a request to the guide endpoint. - - TODO: Figure out how we can use this - """ - # endpoint = f'{self.base_url}/guide' # noqa:E800 - ... - # return self._call_api(endpoint, query, self.base_data) # noqa:E800 - - def next(self): - """Make a request to the next endpoint. - - TODO: Figure out how we can use this - """ - # endpoint = f'{self.base_url}/next' # noqa:E800 - ... - # return self._call_api(endpoint, query, self.base_data) # noqa:E800 - - def player(self, video_id): - """Make a request to the player endpoint. - - :param str video_id: - The video id to get player info for. - :rtype: dict - :returns: - Raw player info results. - """ - endpoint = f'{self.base_url}/player' - query = { - 'videoId': video_id, - } - query.update(self.base_params) - return self._call_api(endpoint, query, self.base_data) - - def search(self, search_query, continuation=None): - """Make a request to the search endpoint. - - :param str search_query: - The query to search. - :rtype: dict - :returns: - Raw search query results. - """ - endpoint = f'{self.base_url}/search' - query = { - 'query': search_query - } - query.update(self.base_params) - data = {} - if continuation: - data['continuation'] = continuation - data.update(self.base_data) - return self._call_api(endpoint, query, data) - - def verify_age(self, video_id): - """Make a request to the age_verify endpoint. - - Notable examples of the types of video this verification step is for: - * https://www.youtube.com/watch?v=QLdAhwSBZ3w - * https://www.youtube.com/watch?v=hc0ZDaAZQT0 - - :param str video_id: - The video id to get player info for. - :rtype: dict - :returns: - Returns information that includes a URL for bypassing certain restrictions. - """ - endpoint = f'{self.base_url}/verify_age' - data = { - 'nextEndpoint': { - 'urlEndpoint': { - 'url': f'/watch?v={video_id}' - } - }, - 'setControvercy': True - } - data.update(self.base_data) - result = self._call_api(endpoint, self.base_params, data) - return result - - def get_transcript(self, video_id): - """Make a request to the get_transcript endpoint. - - This is likely related to captioning for videos, but is currently untested. - """ - endpoint = f'{self.base_url}/get_transcript' - query = { - 'videoId': video_id, - } - query.update(self.base_params) - result = self._call_api(endpoint, query, self.base_data) - return result diff --git a/001-Downloader/pytube/itags.py b/001-Downloader/pytube/itags.py deleted file mode 100755 index 2f23cae..0000000 --- a/001-Downloader/pytube/itags.py +++ /dev/null @@ -1,144 +0,0 @@ -"""This module contains a lookup table of YouTube's itag values.""" -from typing import Dict - -PROGRESSIVE_VIDEO = { - 5: ("240p", "64kbps"), - 6: ("270p", "64kbps"), - 13: ("144p", None), - 17: ("144p", "24kbps"), - 18: ("360p", "96kbps"), - 22: ("720p", "192kbps"), - 34: ("360p", "128kbps"), - 35: ("480p", "128kbps"), - 36: ("240p", None), - 37: ("1080p", "192kbps"), - 38: ("3072p", "192kbps"), - 43: ("360p", "128kbps"), - 44: ("480p", "128kbps"), - 45: ("720p", "192kbps"), - 46: ("1080p", "192kbps"), - 59: ("480p", "128kbps"), - 78: ("480p", "128kbps"), - 82: ("360p", "128kbps"), - 83: ("480p", "128kbps"), - 84: ("720p", "192kbps"), - 85: ("1080p", "192kbps"), - 91: ("144p", "48kbps"), - 92: ("240p", "48kbps"), - 93: ("360p", "128kbps"), - 94: ("480p", "128kbps"), - 95: ("720p", "256kbps"), - 96: ("1080p", "256kbps"), - 100: ("360p", "128kbps"), - 101: ("480p", "192kbps"), - 102: ("720p", "192kbps"), - 132: ("240p", "48kbps"), - 151: ("720p", "24kbps"), - 300: ("720p", "128kbps"), - 301: ("1080p", "128kbps"), -} - -DASH_VIDEO = { - # DASH Video - 133: ("240p", None), # MP4 - 134: ("360p", None), # MP4 - 135: ("480p", None), # MP4 - 136: ("720p", None), # MP4 - 137: ("1080p", None), # MP4 - 138: ("2160p", None), # MP4 - 160: ("144p", None), # MP4 - 167: ("360p", None), # WEBM - 168: ("480p", None), # WEBM - 169: ("720p", None), # WEBM - 170: ("1080p", None), # WEBM - 212: ("480p", None), # MP4 - 218: ("480p", None), # WEBM - 219: ("480p", None), # WEBM - 242: ("240p", None), # WEBM - 243: ("360p", None), # WEBM - 244: ("480p", None), # WEBM - 245: ("480p", None), # WEBM - 246: ("480p", None), # WEBM - 247: ("720p", None), # WEBM - 248: ("1080p", None), # WEBM - 264: ("1440p", None), # MP4 - 266: ("2160p", None), # MP4 - 271: ("1440p", None), # WEBM - 272: ("4320p", None), # WEBM - 278: ("144p", None), # WEBM - 298: ("720p", None), # MP4 - 299: ("1080p", None), # MP4 - 302: ("720p", None), # WEBM - 303: ("1080p", None), # WEBM - 308: ("1440p", None), # WEBM - 313: ("2160p", None), # WEBM - 315: ("2160p", None), # WEBM - 330: ("144p", None), # WEBM - 331: ("240p", None), # WEBM - 332: ("360p", None), # WEBM - 333: ("480p", None), # WEBM - 334: ("720p", None), # WEBM - 335: ("1080p", None), # WEBM - 336: ("1440p", None), # WEBM - 337: ("2160p", None), # WEBM - 394: ("144p", None), # MP4 - 395: ("240p", None), # MP4 - 396: ("360p", None), # MP4 - 397: ("480p", None), # MP4 - 398: ("720p", None), # MP4 - 399: ("1080p", None), # MP4 - 400: ("1440p", None), # MP4 - 401: ("2160p", None), # MP4 - 402: ("4320p", None), # MP4 - 571: ("4320p", None), # MP4 -} - -DASH_AUDIO = { - # DASH Audio - 139: (None, "48kbps"), # MP4 - 140: (None, "128kbps"), # MP4 - 141: (None, "256kbps"), # MP4 - 171: (None, "128kbps"), # WEBM - 172: (None, "256kbps"), # WEBM - 249: (None, "50kbps"), # WEBM - 250: (None, "70kbps"), # WEBM - 251: (None, "160kbps"), # WEBM - 256: (None, "192kbps"), # MP4 - 258: (None, "384kbps"), # MP4 - 325: (None, None), # MP4 - 328: (None, None), # MP4 -} - -ITAGS = { - **PROGRESSIVE_VIDEO, - **DASH_VIDEO, - **DASH_AUDIO, -} - -HDR = [330, 331, 332, 333, 334, 335, 336, 337] -_3D = [82, 83, 84, 85, 100, 101, 102] -LIVE = [91, 92, 93, 94, 95, 96, 132, 151] - - -def get_format_profile(itag: int) -> Dict: - """Get additional format information for a given itag. - - :param str itag: - YouTube format identifier code. - """ - itag = int(itag) - if itag in ITAGS: - res, bitrate = ITAGS[itag] - else: - res, bitrate = None, None - return { - "resolution": res, - "abr": bitrate, - "is_live": itag in LIVE, - "is_3d": itag in _3D, - "is_hdr": itag in HDR, - "is_dash": ( - itag in DASH_AUDIO - or itag in DASH_VIDEO - ), - } diff --git a/001-Downloader/pytube/metadata.py b/001-Downloader/pytube/metadata.py deleted file mode 100755 index be12c63..0000000 --- a/001-Downloader/pytube/metadata.py +++ /dev/null @@ -1,48 +0,0 @@ -"""This module contains the YouTubeMetadata class.""" -import json -from typing import Dict, List, Optional - - -class YouTubeMetadata: - def __init__(self, metadata: List): - self._raw_metadata: List = metadata - self._metadata = [{}] - - for el in metadata: - # We only add metadata to the dict if it has a simpleText title. - if 'title' in el and 'simpleText' in el['title']: - metadata_title = el['title']['simpleText'] - else: - continue - - contents = el['contents'][0] - if 'simpleText' in contents: - self._metadata[-1][metadata_title] = contents['simpleText'] - elif 'runs' in contents: - self._metadata[-1][metadata_title] = contents['runs'][0]['text'] - - # Upon reaching a dividing line, create a new grouping - if el.get('hasDividerLine', False): - self._metadata.append({}) - - # If we happen to create an empty dict at the end, drop it - if self._metadata[-1] == {}: - self._metadata = self._metadata[:-1] - - def __getitem__(self, key): - return self._metadata[key] - - def __iter__(self): - for el in self._metadata: - yield el - - def __str__(self): - return json.dumps(self._metadata) - - @property - def raw_metadata(self) -> Optional[Dict]: - return self._raw_metadata - - @property - def metadata(self): - return self._metadata diff --git a/001-Downloader/pytube/monostate.py b/001-Downloader/pytube/monostate.py deleted file mode 100755 index 7968af5..0000000 --- a/001-Downloader/pytube/monostate.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Any, Callable, Optional - - -class Monostate: - def __init__( - self, - on_progress: Optional[Callable[[Any, bytes, int], None]], - on_complete: Optional[Callable[[Any, Optional[str]], None]], - title: Optional[str] = None, - duration: Optional[int] = None, - ): - self.on_progress = on_progress - self.on_complete = on_complete - self.title = title - self.duration = duration diff --git a/001-Downloader/pytube/parser.py b/001-Downloader/pytube/parser.py deleted file mode 100755 index 8edea35..0000000 --- a/001-Downloader/pytube/parser.py +++ /dev/null @@ -1,178 +0,0 @@ -import ast -import json -import re -from pytube.exceptions import HTMLParseError - - -def parse_for_all_objects(html, preceding_regex): - """Parses input html to find all matches for the input starting point. - - :param str html: - HTML to be parsed for an object. - :param str preceding_regex: - Regex to find the string preceding the object. - :rtype list: - :returns: - A list of dicts created from parsing the objects. - """ - result = [] - regex = re.compile(preceding_regex) - match_iter = regex.finditer(html) - for match in match_iter: - if match: - start_index = match.end() - try: - obj = parse_for_object_from_startpoint(html, start_index) - except HTMLParseError: - # Some of the instances might fail because set is technically - # a method of the ytcfg object. We'll skip these since they - # don't seem relevant at the moment. - continue - else: - result.append(obj) - - if len(result) == 0: - raise HTMLParseError(f'No matches for regex {preceding_regex}') - - return result - - -def parse_for_object(html, preceding_regex): - """Parses input html to find the end of a JavaScript object. - - :param str html: - HTML to be parsed for an object. - :param str preceding_regex: - Regex to find the string preceding the object. - :rtype dict: - :returns: - A dict created from parsing the object. - """ - regex = re.compile(preceding_regex) - result = regex.search(html) - if not result: - raise HTMLParseError(f'No matches for regex {preceding_regex}') - - start_index = result.end() - return parse_for_object_from_startpoint(html, start_index) - - -def find_object_from_startpoint(html, start_point): - """Parses input html to find the end of a JavaScript object. - - :param str html: - HTML to be parsed for an object. - :param int start_point: - Index of where the object starts. - :rtype dict: - :returns: - A dict created from parsing the object. - """ - html = html[start_point:] - if html[0] not in ['{','[']: - raise HTMLParseError(f'Invalid start point. Start of HTML:\n{html[:20]}') - - # First letter MUST be a open brace, so we put that in the stack, - # and skip the first character. - stack = [html[0]] - i = 1 - - context_closers = { - '{': '}', - '[': ']', - '"': '"' - } - - while i < len(html): - if len(stack) == 0: - break - curr_char = html[i] - curr_context = stack[-1] - - # If we've reached a context closer, we can remove an element off the stack - if curr_char == context_closers[curr_context]: - stack.pop() - i += 1 - continue - - # Strings require special context handling because they can contain - # context openers *and* closers - if curr_context == '"': - # If there's a backslash in a string, we skip a character - if curr_char == '\\': - i += 2 - continue - else: - # Non-string contexts are when we need to look for context openers. - if curr_char in context_closers.keys(): - stack.append(curr_char) - - i += 1 - - full_obj = html[:i] - return full_obj # noqa: R504 - - -def parse_for_object_from_startpoint(html, start_point): - """JSONifies an object parsed from HTML. - - :param str html: - HTML to be parsed for an object. - :param int start_point: - Index of where the object starts. - :rtype dict: - :returns: - A dict created from parsing the object. - """ - full_obj = find_object_from_startpoint(html, start_point) - try: - return json.loads(full_obj) - except json.decoder.JSONDecodeError: - try: - return ast.literal_eval(full_obj) - except (ValueError, SyntaxError): - raise HTMLParseError('Could not parse object.') - - -def throttling_array_split(js_array): - """Parses the throttling array into a python list of strings. - - Expects input to begin with `[` and close with `]`. - - :param str js_array: - The javascript array, as a string. - :rtype: list: - :returns: - A list of strings representing splits on `,` in the throttling array. - """ - results = [] - curr_substring = js_array[1:] - - comma_regex = re.compile(r",") - func_regex = re.compile(r"function\([^)]+\)") - - while len(curr_substring)> 0: - if curr_substring.startswith('function'): - # Handle functions separately. These can contain commas - match = func_regex.search(curr_substring) - match_start, match_end = match.span() - - function_text = find_object_from_startpoint(curr_substring, match.span()[1]) - full_function_def = curr_substring[:match_end + len(function_text)] - results.append(full_function_def) - curr_substring = curr_substring[len(full_function_def) + 1:] - else: - match = comma_regex.search(curr_substring) - - # Try-catch to capture end of array - try: - match_start, match_end = match.span() - except AttributeError: - match_start = len(curr_substring) - 1 - match_end = match_start + 1 - - curr_el = curr_substring[:match_start] - results.append(curr_el) - curr_substring = curr_substring[match_end:] - - return results diff --git a/001-Downloader/pytube/query.py b/001-Downloader/pytube/query.py deleted file mode 100755 index d4878ba..0000000 --- a/001-Downloader/pytube/query.py +++ /dev/null @@ -1,421 +0,0 @@ -"""This module provides a query interface for media streams and captions.""" -from collections.abc import Mapping, Sequence -from typing import Callable, List, Optional, Union - -from pytube import Caption, Stream -from pytube.helpers import deprecated - - -class StreamQuery(Sequence): - """Interface for querying the available media streams.""" - - def __init__(self, fmt_streams): - """Construct a :class:`StreamQuery `. - - param list fmt_streams: - list of :class:`Stream ` instances. - """ - self.fmt_streams = fmt_streams - self.itag_index = {int(s.itag): s for s in fmt_streams} - - def filter( - self, - fps=None, - res=None, - resolution=None, - mime_type=None, - type=None, - subtype=None, - file_extension=None, - abr=None, - bitrate=None, - video_codec=None, - audio_codec=None, - only_audio=None, - only_video=None, - progressive=None, - adaptive=None, - is_dash=None, - custom_filter_functions=None, - ): - """Apply the given filtering criterion. - - :param fps: - (optional) The frames per second. - :type fps: - int or None - - :param resolution: - (optional) Alias to ``res``. - :type res: - str or None - - :param res: - (optional) The video resolution. - :type resolution: - str or None - - :param mime_type: - (optional) Two-part identifier for file formats and format contents - composed of a "type", a "subtype". - :type mime_type: - str or None - - :param type: - (optional) Type part of the ``mime_type`` (e.g.: audio, video). - :type type: - str or None - - :param subtype: - (optional) Sub-type part of the ``mime_type`` (e.g.: mp4, mov). - :type subtype: - str or None - - :param file_extension: - (optional) Alias to ``sub_type``. - :type file_extension: - str or None - - :param abr: - (optional) Average bitrate (ABR) refers to the average amount of - data transferred per unit of time (e.g.: 64kbps, 192kbps). - :type abr: - str or None - - :param bitrate: - (optional) Alias to ``abr``. - :type bitrate: - str or None - - :param video_codec: - (optional) Video compression format. - :type video_codec: - str or None - - :param audio_codec: - (optional) Audio compression format. - :type audio_codec: - str or None - - :param bool progressive: - Excludes adaptive streams (one file contains both audio and video - tracks). - - :param bool adaptive: - Excludes progressive streams (audio and video are on separate - tracks). - - :param bool is_dash: - Include/exclude dash streams. - - :param bool only_audio: - Excludes streams with video tracks. - - :param bool only_video: - Excludes streams with audio tracks. - - :param custom_filter_functions: - (optional) Interface for defining complex filters without - subclassing. - :type custom_filter_functions: - list or None - - """ - filters = [] - if res or resolution: - filters.append(lambda s: s.resolution == (res or resolution)) - - if fps: - filters.append(lambda s: s.fps == fps) - - if mime_type: - filters.append(lambda s: s.mime_type == mime_type) - - if type: - filters.append(lambda s: s.type == type) - - if subtype or file_extension: - filters.append(lambda s: s.subtype == (subtype or file_extension)) - - if abr or bitrate: - filters.append(lambda s: s.abr == (abr or bitrate)) - - if video_codec: - filters.append(lambda s: s.video_codec == video_codec) - - if audio_codec: - filters.append(lambda s: s.audio_codec == audio_codec) - - if only_audio: - filters.append( - lambda s: ( - s.includes_audio_track and not s.includes_video_track - ), - ) - - if only_video: - filters.append( - lambda s: ( - s.includes_video_track and not s.includes_audio_track - ), - ) - - if progressive: - filters.append(lambda s: s.is_progressive) - - if adaptive: - filters.append(lambda s: s.is_adaptive) - - if custom_filter_functions: - filters.extend(custom_filter_functions) - - if is_dash is not None: - filters.append(lambda s: s.is_dash == is_dash) - - return self._filter(filters) - - def _filter(self, filters: List[Callable]) -> "StreamQuery": - fmt_streams = self.fmt_streams - for filter_lambda in filters: - fmt_streams = filter(filter_lambda, fmt_streams) - return StreamQuery(list(fmt_streams)) - - def order_by(self, attribute_name: str) -> "StreamQuery": - """Apply a sort order. Filters out stream the do not have the attribute. - - :param str attribute_name: - The name of the attribute to sort by. - """ - has_attribute = [ - s - for s in self.fmt_streams - if getattr(s, attribute_name) is not None - ] - # Check that the attributes have string values. - if has_attribute and isinstance( - getattr(has_attribute[0], attribute_name), str - ): - # Try to return a StreamQuery sorted by the integer representations - # of the values. - try: - return StreamQuery( - sorted( - has_attribute, - key=lambda s: int( - "".join( - filter(str.isdigit, getattr(s, attribute_name)) - ) - ), # type: ignore # noqa: E501 - ) - ) - except ValueError: - pass - - return StreamQuery( - sorted(has_attribute, key=lambda s: getattr(s, attribute_name)) - ) - - def desc(self) -> "StreamQuery": - """Sort streams in descending order. - - :rtype: :class:`StreamQuery ` - - """ - return StreamQuery(self.fmt_streams[::-1]) - - def asc(self) -> "StreamQuery": - """Sort streams in ascending order. - - :rtype: :class:`StreamQuery ` - - """ - return self - - def get_by_itag(self, itag: int) -> Optional[Stream]: - """Get the corresponding :class:`Stream ` for a given itag. - - :param int itag: - YouTube format identifier code. - :rtype: :class:`Stream ` or None - :returns: - The :class:`Stream ` matching the given itag or None if - not found. - - """ - return self.itag_index.get(int(itag)) - - def get_by_resolution(self, resolution: str) -> Optional[Stream]: - """Get the corresponding :class:`Stream ` for a given resolution. - - Stream must be a progressive mp4. - - :param str resolution: - Video resolution i.e. "720p", "480p", "360p", "240p", "144p" - :rtype: :class:`Stream ` or None - :returns: - The :class:`Stream ` matching the given itag or None if - not found. - - """ - return self.filter( - progressive=True, subtype="mp4", resolution=resolution - ).first() - - def get_lowest_resolution(self) -> Optional[Stream]: - """Get lowest resolution stream that is a progressive mp4. - - :rtype: :class:`Stream ` or None - :returns: - The :class:`Stream ` matching the given itag or None if - not found. - - """ - return ( - self.filter(progressive=True, subtype="mp4") - .order_by("resolution") - .first() - ) - - def get_highest_resolution(self) -> Optional[Stream]: - """Get highest resolution stream that is a progressive video. - - :rtype: :class:`Stream ` or None - :returns: - The :class:`Stream ` matching the given itag or None if - not found. - - """ - return self.filter(progressive=True).order_by("resolution").last() - - def get_audio_only(self, subtype: str = "mp4") -> Optional[Stream]: - """Get highest bitrate audio stream for given codec (defaults to mp4) - - :param str subtype: - Audio subtype, defaults to mp4 - :rtype: :class:`Stream ` or None - :returns: - The :class:`Stream ` matching the given itag or None if - not found. - """ - return ( - self.filter(only_audio=True, subtype=subtype) - .order_by("abr") - .last() - ) - - def otf(self, is_otf: bool = False) -> "StreamQuery": - """Filter stream by OTF, useful if some streams have 404 URLs - - :param bool is_otf: Set to False to retrieve only non-OTF streams - :rtype: :class:`StreamQuery ` - :returns: A StreamQuery object with otf filtered streams - """ - return self._filter([lambda s: s.is_otf == is_otf]) - - def first(self) -> Optional[Stream]: - """Get the first :class:`Stream ` in the results. - - :rtype: :class:`Stream ` or None - :returns: - the first result of this query or None if the result doesn't - contain any streams. - - """ - try: - return self.fmt_streams[0] - except IndexError: - return None - - def last(self): - """Get the last :class:`Stream ` in the results. - - :rtype: :class:`Stream ` or None - :returns: - Return the last result of this query or None if the result - doesn't contain any streams. - - """ - try: - return self.fmt_streams[-1] - except IndexError: - pass - - @deprecated("Get the size of this list directly using len()") - def count(self, value: Optional[str] = None) -> int: # pragma: no cover - """Get the count of items in the list. - - :rtype: int - """ - if value: - return self.fmt_streams.count(value) - - return len(self) - - @deprecated("This object can be treated as a list, all() is useless") - def all(self) -> List[Stream]: # pragma: no cover - """Get all the results represented by this query as a list. - - :rtype: list - - """ - return self.fmt_streams - - def __getitem__(self, i: Union[slice, int]): - return self.fmt_streams[i] - - def __len__(self) -> int: - return len(self.fmt_streams) - - def __repr__(self) -> str: - return f"{self.fmt_streams}" - - -class CaptionQuery(Mapping): - """Interface for querying the available captions.""" - - def __init__(self, captions: List[Caption]): - """Construct a :class:`Caption `. - - param list captions: - list of :class:`Caption ` instances. - - """ - self.lang_code_index = {c.code: c for c in captions} - - @deprecated( - "This object can be treated as a dictionary, i.e. captions['en']" - ) - def get_by_language_code( - self, lang_code: str - ) -> Optional[Caption]: # pragma: no cover - """Get the :class:`Caption ` for a given ``lang_code``. - - :param str lang_code: - The code that identifies the caption language. - :rtype: :class:`Caption ` or None - :returns: - The :class:`Caption ` matching the given ``lang_code`` or - None if it does not exist. - """ - return self.lang_code_index.get(lang_code) - - @deprecated("This object can be treated as a dictionary") - def all(self) -> List[Caption]: # pragma: no cover - """Get all the results represented by this query as a list. - - :rtype: list - - """ - return list(self.lang_code_index.values()) - - def __getitem__(self, i: str): - return self.lang_code_index[i] - - def __len__(self) -> int: - return len(self.lang_code_index) - - def __iter__(self): - return iter(self.lang_code_index.values()) - - def __repr__(self) -> str: - return f"{self.lang_code_index}" diff --git a/001-Downloader/pytube/request.py b/001-Downloader/pytube/request.py deleted file mode 100755 index b31b760..0000000 --- a/001-Downloader/pytube/request.py +++ /dev/null @@ -1,268 +0,0 @@ -"""Implements a simple wrapper around urlopen.""" -import http.client -import json -import logging -import re -import socket -from functools import lru_cache -from urllib import parse -from urllib.error import URLError -from urllib.request import Request, urlopen - -from pytube.exceptions import RegexMatchError, MaxRetriesExceeded -from pytube.helpers import regex_search - -import ssl -ssl._create_default_https_context = ssl._create_unverified_context - -logger = logging.getLogger(__name__) -default_range_size = 9437184 # 9MB - - -def _execute_request( - url, - method=None, - headers=None, - data=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT -): - base_headers = {"User-Agent": "Mozilla/5.0", "accept-language": "en-US,en"} - if headers: - base_headers.update(headers) - if data: - # encode data for request - if not isinstance(data, bytes): - data = bytes(json.dumps(data), encoding="utf-8") - if url.lower().startswith("http"): - request = Request(url, headers=base_headers, method=method, data=data) - else: - raise ValueError("Invalid URL") - return urlopen(request, timeout=timeout) # nosec - - -def get(url, extra_headers=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - """Send an http GET request. - - :param str url: - The URL to perform the GET request for. - :param dict extra_headers: - Extra headers to add to the request - :rtype: str - :returns: - UTF-8 encoded string of response - """ - if extra_headers is None: - extra_headers = {} - response = _execute_request(url, headers=extra_headers, timeout=timeout) - return response.read().decode("utf-8") - - -def post(url, extra_headers=None, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - """Send an http POST request. - - :param str url: - The URL to perform the POST request for. - :param dict extra_headers: - Extra headers to add to the request - :param dict data: - The data to send on the POST request - :rtype: str - :returns: - UTF-8 encoded string of response - """ - # could technically be implemented in get, - # but to avoid confusion implemented like this - if extra_headers is None: - extra_headers = {} - if data is None: - data = {} - # required because the youtube servers are strict on content type - # raises HTTPError [400]: Bad Request otherwise - extra_headers.update({"Content-Type": "application/json"}) - response = _execute_request( - url, - headers=extra_headers, - data=data, - timeout=timeout - ) - return response.read().decode("utf-8") - - -def seq_stream( - url, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - max_retries=0 -): - """Read the response in sequence. - :param str url: The URL to perform the GET request for. - :rtype: Iterable[bytes] - """ - # YouTube expects a request sequence number as part of the parameters. - split_url = parse.urlsplit(url) - base_url = '%s://%s/%s?' % (split_url.scheme, split_url.netloc, split_url.path) - - querys = dict(parse.parse_qsl(split_url.query)) - - # The 0th sequential request provides the file headers, which tell us - # information about how the file is segmented. - querys['sq'] = 0 - url = base_url + parse.urlencode(querys) - - segment_data = b'' - for chunk in stream(url, timeout=timeout, max_retries=max_retries): - yield chunk - segment_data += chunk - - # We can then parse the header to find the number of segments - stream_info = segment_data.split(b'\r\n') - segment_count_pattern = re.compile(b'Segment-Count: (\\d+)') - for line in stream_info: - match = segment_count_pattern.search(line) - if match: - segment_count = int(match.group(1).decode('utf-8')) - - # We request these segments sequentially to build the file. - seq_num = 1 - while seq_num <= segment_count: - # Create sequential request URL - querys['sq'] = seq_num - url = base_url + parse.urlencode(querys) - - yield from stream(url, timeout=timeout, max_retries=max_retries) - seq_num += 1 - return # pylint: disable=R1711 - - -def stream( - url, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - max_retries=0 -): - """Read the response in chunks. - :param str url: The URL to perform the GET request for. - :rtype: Iterable[bytes] - """ - file_size: int = default_range_size # fake filesize to start - downloaded = 0 - while downloaded < file_size: - stop_pos = min(downloaded + default_range_size, file_size) - 1 - range_header = f"bytes={downloaded}-{stop_pos}" - tries = 0 - - # Attempt to make the request multiple times as necessary. - while True: - # If the max retries is exceeded, raise an exception - if tries>= 1 + max_retries: - raise MaxRetriesExceeded() - - # Try to execute the request, ignoring socket timeouts - try: - response = _execute_request( - url, - method="GET", - headers={"Range": range_header}, - timeout=timeout - ) - except URLError as e: - # We only want to skip over timeout errors, and - # raise any other URLError exceptions - if isinstance(e.reason, socket.timeout): - pass - else: - raise - except http.client.IncompleteRead: - # Allow retries on IncompleteRead errors for unreliable connections - pass - else: - # On a successful request, break from loop - break - tries += 1 - - if file_size == default_range_size: - try: - content_range = response.info()["Content-Range"] - file_size = int(content_range.split("/")[1]) - except (KeyError, IndexError, ValueError) as e: - logger.error(e) - while True: - chunk = response.read() - if not chunk: - break - downloaded += len(chunk) - yield chunk - return # pylint: disable=R1711 - - -@lru_cache() -def filesize(url): - """Fetch size in bytes of file at given URL - - :param str url: The URL to get the size of - :returns: int: size in bytes of remote file - """ - return int(head(url)["content-length"]) - - -@lru_cache() -def seq_filesize(url): - """Fetch size in bytes of file at given URL from sequential requests - - :param str url: The URL to get the size of - :returns: int: size in bytes of remote file - """ - total_filesize = 0 - # YouTube expects a request sequence number as part of the parameters. - split_url = parse.urlsplit(url) - base_url = '%s://%s/%s?' % (split_url.scheme, split_url.netloc, split_url.path) - querys = dict(parse.parse_qsl(split_url.query)) - - # The 0th sequential request provides the file headers, which tell us - # information about how the file is segmented. - querys['sq'] = 0 - url = base_url + parse.urlencode(querys) - response = _execute_request( - url, method="GET" - ) - - response_value = response.read() - # The file header must be added to the total filesize - total_filesize += len(response_value) - - # We can then parse the header to find the number of segments - segment_count = 0 - stream_info = response_value.split(b'\r\n') - segment_regex = b'Segment-Count: (\\d+)' - for line in stream_info: - # One of the lines should contain the segment count, but we don't know - # which, so we need to iterate through the lines to find it - try: - segment_count = int(regex_search(segment_regex, line, 1)) - except RegexMatchError: - pass - - if segment_count == 0: - raise RegexMatchError('seq_filesize', segment_regex) - - # We make HEAD requests to the segments sequentially to find the total filesize. - seq_num = 1 - while seq_num <= segment_count: - # Create sequential request URL - querys['sq'] = seq_num - url = base_url + parse.urlencode(querys) - - total_filesize += int(head(url)['content-length']) - seq_num += 1 - return total_filesize - - -def head(url): - """Fetch headers returned http GET request. - - :param str url: - The URL to perform the GET request for. - :rtype: dict - :returns: - dictionary of lowercase headers - """ - response_headers = _execute_request(url, method="HEAD").info() - return {k.lower(): v for k, v in response_headers.items()} diff --git a/001-Downloader/pytube/streams.py b/001-Downloader/pytube/streams.py deleted file mode 100755 index 05ec6c1..0000000 --- a/001-Downloader/pytube/streams.py +++ /dev/null @@ -1,374 +0,0 @@ -""" -This module contains a container for stream manifest data. - -A container object for the media stream (video only / audio only / video+audio -combined). This was referred to as ``Video`` in the legacy pytube version, but -has been renamed to accommodate DASH (which serves the audio and video -separately). -""" -import logging -import os -from datetime import datetime -from typing import BinaryIO, Dict, Optional, Tuple -from urllib.error import HTTPError -from urllib.parse import parse_qs - -from pytube import extract, request -from pytube.helpers import safe_filename, target_directory -from pytube.itags import get_format_profile -from pytube.monostate import Monostate - -logger = logging.getLogger(__name__) - - -class Stream: - """Container for stream manifest data.""" - - def __init__( - self, stream: Dict, monostate: Monostate - ): - """Construct a :class:`Stream `. - - :param dict stream: - The unscrambled data extracted from YouTube. - :param dict monostate: - Dictionary of data shared across all instances of - :class:`Stream `. - """ - # A dictionary shared between all instances of :class:`Stream ` - # (Borg pattern). - self._monostate = monostate - - self.url = stream["url"] # signed download url - self.itag = int( - stream["itag"] - ) # stream format id (youtube nomenclature) - - # set type and codec info - - # 'video/webm; codecs="vp8, vorbis"' -> 'video/webm', ['vp8', 'vorbis'] - self.mime_type, self.codecs = extract.mime_type_codec(stream["mimeType"]) - - # 'video/webm' -> 'video', 'webm' - self.type, self.subtype = self.mime_type.split("/") - - # ['vp8', 'vorbis'] -> video_codec: vp8, audio_codec: vorbis. DASH - # streams return NoneType for audio/video depending. - self.video_codec, self.audio_codec = self.parse_codecs() - - self.is_otf: bool = stream["is_otf"] - self.bitrate: Optional[int] = stream["bitrate"] - - # filesize in bytes - self._filesize: Optional[int] = int(stream.get('contentLength', 0)) - - # Additional information about the stream format, such as resolution, - # frame rate, and whether the stream is live (HLS) or 3D. - itag_profile = get_format_profile(self.itag) - self.is_dash = itag_profile["is_dash"] - self.abr = itag_profile["abr"] # average bitrate (audio streams only) - if 'fps' in stream: - self.fps = stream['fps'] # Video streams only - self.resolution = itag_profile[ - "resolution" - ] # resolution (e.g.: "480p") - self.is_3d = itag_profile["is_3d"] - self.is_hdr = itag_profile["is_hdr"] - self.is_live = itag_profile["is_live"] - - @property - def is_adaptive(self) -> bool: - """Whether the stream is DASH. - - :rtype: bool - """ - # if codecs has two elements (e.g.: ['vp8', 'vorbis']): 2 % 2 = 0 - # if codecs has one element (e.g.: ['vp8']) 1 % 2 = 1 - return bool(len(self.codecs) % 2) - - @property - def is_progressive(self) -> bool: - """Whether the stream is progressive. - - :rtype: bool - """ - return not self.is_adaptive - - @property - def includes_audio_track(self) -> bool: - """Whether the stream only contains audio. - - :rtype: bool - """ - return self.is_progressive or self.type == "audio" - - @property - def includes_video_track(self) -> bool: - """Whether the stream only contains video. - - :rtype: bool - """ - return self.is_progressive or self.type == "video" - - def parse_codecs(self) -> Tuple[Optional[str], Optional[str]]: - """Get the video/audio codecs from list of codecs. - - Parse a variable length sized list of codecs and returns a - constant two element tuple, with the video codec as the first element - and audio as the second. Returns None if one is not available - (adaptive only). - - :rtype: tuple - :returns: - A two element tuple with audio and video codecs. - - """ - video = None - audio = None - if not self.is_adaptive: - video, audio = self.codecs - elif self.includes_video_track: - video = self.codecs[0] - elif self.includes_audio_track: - audio = self.codecs[0] - return video, audio - - @property - def filesize(self) -> int: - """File size of the media stream in bytes. - - :rtype: int - :returns: - Filesize (in bytes) of the stream. - """ - if self._filesize == 0: - try: - self._filesize = request.filesize(self.url) - except HTTPError as e: - if e.code != 404: - raise - self._filesize = request.seq_filesize(self.url) - return self._filesize - - @property - def title(self) -> str: - """Get title of video - - :rtype: str - :returns: - Youtube video title - """ - return self._monostate.title or "Unknown YouTube Video Title" - - @property - def filesize_approx(self) -> int: - """Get approximate filesize of the video - - Falls back to HTTP call if there is not sufficient information to approximate - - :rtype: int - :returns: size of video in bytes - """ - if self._monostate.duration and self.bitrate: - bits_in_byte = 8 - return int( - (self._monostate.duration * self.bitrate) / bits_in_byte - ) - - return self.filesize - - @property - def expiration(self) -> datetime: - expire = parse_qs(self.url.split("?")[1])["expire"][0] - return datetime.utcfromtimestamp(int(expire)) - - @property - def default_filename(self) -> str: - """Generate filename based on the video title. - - :rtype: str - :returns: - An os file system compatible filename. - """ - filename = safe_filename(self.title) - return f"{filename}.{self.subtype}" - - def download( - self, - output_path: Optional[str] = None, - filename: Optional[str] = None, - filename_prefix: Optional[str] = None, - skip_existing: bool = True, - timeout: Optional[int] = None, - max_retries: Optional[int] = 0 - ) -> str: - """Write the media stream to disk. - - :param output_path: - (optional) Output path for writing media file. If one is not - specified, defaults to the current working directory. - :type output_path: str or None - :param filename: - (optional) Output filename (stem only) for writing media file. - If one is not specified, the default filename is used. - :type filename: str or None - :param filename_prefix: - (optional) A string that will be prepended to the filename. - For example a number in a playlist or the name of a series. - If one is not specified, nothing will be prepended - This is separate from filename so you can use the default - filename but still add a prefix. - :type filename_prefix: str or None - :param skip_existing: - (optional) Skip existing files, defaults to True - :type skip_existing: bool - :param timeout: - (optional) Request timeout length in seconds. Uses system default. - :type timeout: int - :param max_retries: - (optional) Number of retries to attempt after socket timeout. Defaults to 0. - :type max_retries: int - :returns: - Path to the saved video - :rtype: str - - """ - file_path = self.get_file_path( - filename=filename, - output_path=output_path, - filename_prefix=filename_prefix, - ) - - if skip_existing and self.exists_at_path(file_path): - logger.debug(f'file {file_path} already exists, skipping') - self.on_complete(file_path) - return file_path - - bytes_remaining = self.filesize - logger.debug(f'downloading ({self.filesize} total bytes) file to {file_path}') - - with open(file_path, "wb") as fh: - try: - for chunk in request.stream( - self.url, - timeout=timeout, - max_retries=max_retries - ): - # reduce the (bytes) remainder by the length of the chunk. - bytes_remaining -= len(chunk) - # send to the on_progress callback. - self.on_progress(chunk, fh, bytes_remaining) - except HTTPError as e: - if e.code != 404: - raise - # Some adaptive streams need to be requested with sequence numbers - for chunk in request.seq_stream( - self.url, - timeout=timeout, - max_retries=max_retries - ): - # reduce the (bytes) remainder by the length of the chunk. - bytes_remaining -= len(chunk) - # send to the on_progress callback. - self.on_progress(chunk, fh, bytes_remaining) - self.on_complete(file_path) - return file_path - - def get_file_path( - self, - filename: Optional[str] = None, - output_path: Optional[str] = None, - filename_prefix: Optional[str] = None, - ) -> str: - if not filename: - filename = self.default_filename - if filename_prefix: - filename = f"{filename_prefix}{filename}" - return os.path.join(target_directory(output_path), filename) - - def exists_at_path(self, file_path: str) -> bool: - return ( - os.path.isfile(file_path) - and os.path.getsize(file_path) == self.filesize - ) - - def stream_to_buffer(self, buffer: BinaryIO) -> None: - """Write the media stream to buffer - - :rtype: io.BytesIO buffer - """ - bytes_remaining = self.filesize - logger.info( - "downloading (%s total bytes) file to buffer", self.filesize, - ) - - for chunk in request.stream(self.url): - # reduce the (bytes) remainder by the length of the chunk. - bytes_remaining -= len(chunk) - # send to the on_progress callback. - self.on_progress(chunk, buffer, bytes_remaining) - self.on_complete(None) - - def on_progress( - self, chunk: bytes, file_handler: BinaryIO, bytes_remaining: int - ): - """On progress callback function. - - This function writes the binary data to the file, then checks if an - additional callback is defined in the monostate. This is exposed to - allow things like displaying a progress bar. - - :param bytes chunk: - Segment of media file binary data, not yet written to disk. - :param file_handler: - The file handle where the media is being written to. - :type file_handler: - :py:class:`io.BufferedWriter` - :param int bytes_remaining: - The delta between the total file size in bytes and amount already - downloaded. - - :rtype: None - - """ - file_handler.write(chunk) - logger.debug("download remaining: %s", bytes_remaining) - if self._monostate.on_progress: - self._monostate.on_progress(self, chunk, bytes_remaining) - - def on_complete(self, file_path: Optional[str]): - """On download complete handler function. - - :param file_path: - The file handle where the media is being written to. - :type file_path: str - - :rtype: None - - """ - logger.debug("download finished") - on_complete = self._monostate.on_complete - if on_complete: - logger.debug("calling on_complete callback %s", on_complete) - on_complete(self, file_path) - - def __repr__(self) -> str: - """Printable object representation. - - :rtype: str - :returns: - A string representation of a :class:`Stream ` object. - """ - parts = ['itag="{s.itag}"', 'mime_type="{s.mime_type}"'] - if self.includes_video_track: - parts.extend(['res="{s.resolution}"', 'fps="{s.fps}fps"']) - if not self.is_adaptive: - parts.extend( - ['vcodec="{s.video_codec}"', 'acodec="{s.audio_codec}"',] - ) - else: - parts.extend(['vcodec="{s.video_codec}"']) - else: - parts.extend(['abr="{s.abr}"', 'acodec="{s.audio_codec}"']) - parts.extend(['progressive="{s.is_progressive}"', 'type="{s.type}"']) - return f"" diff --git a/001-Downloader/pytube/version.py b/001-Downloader/pytube/version.py deleted file mode 100755 index 4168adc..0000000 --- a/001-Downloader/pytube/version.py +++ /dev/null @@ -1,4 +0,0 @@ -__version__ = "11.0.0" - -if __name__ == "__main__": - print(__version__) diff --git a/001-Downloader/test/bilibili_video_download_v1.py b/001-Downloader/test/bilibili_video_download_v1.py new file mode 100644 index 0000000..086caa2 --- /dev/null +++ b/001-Downloader/test/bilibili_video_download_v1.py @@ -0,0 +1,243 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# time: 2019年04月17日--08:12 +__author__ = 'Henry' + +''' +项目: B站视频下载 + +版本1: 加密API版,不需要加入cookie,直接即可下载1080p视频 + +20190422 - 增加多P视频单独下载其中一集的功能 +''' + +import requests, time, hashlib, urllib.request, re, json +from moviepy.editor import * +import os, sys + + +# 访问API地址 +def get_play_list(start_url, cid, quality): + entropy = 'rbMCKn@KuamXWlPMoJGsKcbiJKUfkPF_8dABscJntvqhRSETg' + appkey, sec = ''.join([chr(ord(i) + 2) for i in entropy[::-1]]).split(':') + params = 'appkey=%s&cid=%s&otype=json&qn=%s&quality=%s&type=' % (appkey, cid, quality, quality) + chksum = hashlib.md5(bytes(params + sec, 'utf8')).hexdigest() + url_api = 'https://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum) + headers = { + 'Referer': start_url, # 注意加上referer + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36' + } + # print(url_api) + html = requests.get(url_api, headers=headers).json() + # print(json.dumps(html)) + video_list = [] + for i in html['durl']: + video_list.append(i['url']) + # print(video_list) + return video_list + + +# 下载视频 +''' + urllib.urlretrieve 的回调函数: +def callbackfunc(blocknum, blocksize, totalsize): + @blocknum: 已经下载的数据块 + @blocksize: 数据块的大小 + @totalsize: 远程文件的大小 +''' + + +def Schedule_cmd(blocknum, blocksize, totalsize): + speed = (blocknum * blocksize) / (time.time() - start_time) + # speed_str = " Speed: %.2f" % speed + speed_str = " Speed: %s" % format_size(speed) + recv_size = blocknum * blocksize + + # 设置下载进度条 + f = sys.stdout + pervent = recv_size / totalsize + percent_str = "%.2f%%" % (pervent * 100) + n = round(pervent * 50) + s = ('#' * n).ljust(50, '-') + f.write(percent_str.ljust(8, ' ') + '[' + s + ']' + speed_str) + f.flush() + # time.sleep(0.1) + f.write('\r') + + +def Schedule(blocknum, blocksize, totalsize): + speed = (blocknum * blocksize) / (time.time() - start_time) + # speed_str = " Speed: %.2f" % speed + speed_str = " Speed: %s" % format_size(speed) + recv_size = blocknum * blocksize + + # 设置下载进度条 + f = sys.stdout + pervent = recv_size / totalsize + percent_str = "%.2f%%" % (pervent * 100) + n = round(pervent * 50) + s = ('#' * n).ljust(50, '-') + print(percent_str.ljust(6, ' ') + '-' + speed_str) + f.flush() + time.sleep(2) + # print('\r') + + +# 字节bytes转化K\M\G +def format_size(bytes): + try: + bytes = float(bytes) + kb = bytes / 1024 + except: + print("传入的字节格式不对") + return "Error" + if kb>= 1024: + M = kb / 1024 + if M>= 1024: + G = M / 1024 + return "%.3fG" % (G) + else: + return "%.3fM" % (M) + else: + return "%.3fK" % (kb) + + +# 下载视频 +def down_video(video_list, title, start_url, page): + num = 1 + print('[正在下载P{}段视频,请稍等...]:'.format(page) + title) + currentVideoPath = os.path.join(sys.path[0], 'bilibili_video', title) # 当前目录作为下载目录 + for i in video_list: + opener = urllib.request.build_opener() + # 请求头 + opener.addheaders = [ + # ('Host', 'upos-hz-mirrorks3.acgvideo.com'), #注意修改host,不用也行 + ('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:56.0) Gecko/20100101 Firefox/56.0'), + ('Accept', '*/*'), + ('Accept-Language', 'en-US,en;q=0.5'), + ('Accept-Encoding', 'gzip, deflate, br'), + ('Range', 'bytes=0-'), # Range 的值要为 bytes=0- 才能下载完整视频 + ('Referer', start_url), # 注意修改referer,必须要加的! + ('Origin', 'https://www.bilibili.com'), + ('Connection', 'keep-alive'), + ] + urllib.request.install_opener(opener) + # 创建文件夹存放下载的视频 + if not os.path.exists(currentVideoPath): + os.makedirs(currentVideoPath) + # 开始下载 + if len(video_list)> 1: + urllib.request.urlretrieve(url=i, filename=os.path.join(currentVideoPath, r'{}-{}.mp4'.format(title, num)), + reporthook=Schedule_cmd) # 写成mp4也行 title + '-' + num + '.flv' + else: + urllib.request.urlretrieve(url=i, filename=os.path.join(currentVideoPath, r'{}.mp4'.format(title)), + reporthook=Schedule_cmd) # 写成mp4也行 title + '-' + num + '.flv' + num += 1 + + +# 合并视频 +def combine_video(video_list, title): + currentVideoPath = os.path.join(sys.path[0], 'bilibili_video', title) # 当前目录作为下载目录 + if not os.path.exists(currentVideoPath): + os.makedirs(currentVideoPath) + if len(video_list)>= 2: + # 视频大于一段才要合并 + print('[下载完成,正在合并视频...]:' + title) + # 定义一个数组 + L = [] + # 访问 video 文件夹 (假设视频都放在这里面) + root_dir = currentVideoPath + # 遍历所有文件 + for file in sorted(os.listdir(root_dir), key=lambda x: int(x[x.rindex("-") + 1:x.rindex(".")])): + # 如果后缀名为 .mp4/.flv + if os.path.splitext(file)[1] == '.flv': + # 拼接成完整路径 + filePath = os.path.join(root_dir, file) + # 载入视频 + video = VideoFileClip(filePath) + # 添加到数组 + L.append(video) + # 拼接视频 + final_clip = concatenate_videoclips(L) + # 生成目标视频文件 + final_clip.to_videofile(os.path.join(root_dir, r'{}.mp4'.format(title)), fps=24, remove_temp=False) + print('[视频合并完成]' + title) + + else: + # 视频只有一段则直接打印下载完成 + print('[视频合并完成]:' + title) + + +def getAid(Bid): + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36' + } + url = "https://api.bilibili.com/x/web-interface/view?bvid=" + Bid + print(url) + r = requests.get(url, headers=headers) + j = json.loads(r.text) + # print(j["data"]["aid"]) + print(j) + return j["data"]["aid"] + + +if __name__ == '__main__': + # 用户输入av号或者视频链接地址 + print('*' * 30 + 'B站视频下载小助手' + '*' * 30) + start = input('请输入您要下载的B站av号、bv号或者视频链接地址:') + if 'http' in start: + if 'video/BV' in start: + bv = re.findall(r'video/(.*?)\?', start)[0] + start = str(getAid(bv)) + print(start) + if start.isdigit() == True: # 如果输入的是av号 + # 获取cid的api, 传入aid即可 + start_url = 'https://api.bilibili.com/x/web-interface/view?aid=' + start + else: + # https://www.bilibili.com/video/av46958874/?spm_id_from=333.334.b_63686965665f7265636f6d6d656e64.16 + start_url = 'https://api.bilibili.com/x/web-interface/view?aid=' + re.search(r'/av(\d+)/*', start).group(1) + # https://www.bilibili.com/video/BV1jL4y1e7Uz?t=7.2 + # start_url = 'https://api.bilibili.com/x/web-interface/view?aid=' + re.findall(r'video/(.*?)\?', start)[0] + print(start_url) + # 视频质量 + # + # + # + quality = input('请输入您要下载视频的清晰度(1080p:80;720p:64;480p:32;360p:16)(填写80或64或32或16):') + # 获取视频的cid,title + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36' + } + html = requests.get(start_url, headers=headers).json() + data = html['data'] + video_title = data["title"].replace(" ", "_") + cid_list = [] + if '?p=' in start: + # 单独下载分P视频中的一集 + p = re.search(r'\?p=(\d+)', start).group(1) + cid_list.append(data['pages'][int(p) - 1]) + else: + # 如果p不存在就是全集下载 + cid_list = data['pages'] + # print(cid_list) + for item in cid_list: + cid = str(item['cid']) + title = item['part'] + if not title: + title = video_title + title = re.sub(r'[\/\\:*?"|]', '', title) # 替换为空的 + print('[下载视频的cid]:' + cid) + print('[下载视频的标题]:' + title) + page = str(item['page']) + start_url = start_url + "/?p=" + page + video_list = get_play_list(start_url, cid, quality) + start_time = time.time() + down_video(video_list, title, start_url, page) + combine_video(video_list, title) + + # 如果是windows系统,下载完成后打开下载目录 + currentVideoPath = os.path.join(sys.path[0], 'bilibili_video') # 当前目录作为下载目录 + if (sys.platform.startswith('win')): + os.startfile(currentVideoPath) + +# 分P视频下载测试: https://www.bilibili.com/video/av19516333/ diff --git a/001-Downloader/test/ff_video.py b/001-Downloader/test/ff_video.py new file mode 100644 index 0000000..45850b7 --- /dev/null +++ b/001-Downloader/test/ff_video.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: ffmpeg去掉最后一帧,改变md5 +@Date :2022年02月17日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import os + + +def cute_video(folder): + files = next(os.walk(folder))[2] # 获取文件 + for file in files: + file_path = os.path.join(folder, file) + shotname, extension = os.path.splitext(file) + if len(shotname) == 0 or len(extension) == 0: + continue + out_file = os.path.join(folder, 'out-{}{}'.format(shotname, extension)) + # 获取时间。输入自己系统安装的ffmpeg,注意斜杠 + time = os.popen( + r"/usr/local/ffmpeg/bin/ffmpeg -i {} 2>&1 | grep 'Duration' | cut -d ' ' -f 4 | sed s/,//".format( + file_path)).read().replace('\n', '').replace(' ', '') + if '.' in time: + match_time = time.split('.')[0] + else: + match_time = time + print(match_time) + ts = match_time.split(':') + sec = int(ts[0]) * 60 * 60 + int(ts[1]) * 60 + int(ts[2]) + # 从0分0秒100毫秒开始截切(目的就是去头去尾) + os.popen(r"/usr/local/ffmpeg/bin/ffmpeg -ss 0:00.100 -i {} -t {} -c:v copy -c:a copy {}".format(file_path, sec, + out_file)) + + +# 主模块执行 +if __name__ == "__main__": + # path = os.path.dirname('/Users/Qincji/Downloads/ffmpeg/') + path = os.path.dirname('需要处理的目录') # 目录下的所有视频 + cute_video(path) diff --git a/001-Downloader/pytube/contrib/__init__.py b/001-Downloader/test/urls.txt old mode 100755 new mode 100644 similarity index 100% rename from 001-Downloader/pytube/contrib/__init__.py rename to 001-Downloader/test/urls.txt diff --git a/001-Downloader/test/xhs_download.py b/001-Downloader/test/xhs_download.py new file mode 100644 index 0000000..554c740 --- /dev/null +++ b/001-Downloader/test/xhs_download.py @@ -0,0 +1,67 @@ +import os +import random +import time + +import requests +from my_fake_useragent import UserAgent + +ua = UserAgent(family='chrome') +pre_save = os.path.join(os.path.curdir, '0216') + +''' + +''' + + +def download_url(url, index): + try: + headers = { + 'Accept': '*/*', + 'Accept-Encoding': 'identity;q=1, *;q=0', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Cookie': 'xhsTrackerId=6970aca9-a496-4f50-cf98-118929f063bf; timestamp2=2022021544322a4e45f1e1dec93beb82; timestamp2.sig=jk1cFo-zHueSZUpZRvlqyJwTFoA1y8ch9t76Bfy28_Q; solar.beaker.session.id=1644906492328060192125; xhsTracker=url=index&searchengine=google', + 'Host': 'v.xiaohongshu.com', + 'Pragma': 'no-cache', + 'Referer': url, + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36' + } + video = requests.get(url, headers=headers) # 保存视频 + start = time.time() # 下载开始时间 + size = 0 # 初始化已下载大小 + chunk_size = 100 # 每次下载的数据大小 + content_size = int(video.headers['content-length']) # 下载文件总大小 + print(video.status_code) + if video.status_code == 200: # 判断是否响应成功 + print(str(index) + '[文件 大小]:{size:.2f} MB'.format(size=content_size / 1024 / 1024)) # 开始下载,显示下载文件大小 + v_url = os.path.join(pre_save, '{}.mp4'.format(index)) + # v_url = pre_save + '[' + author_list[i] + '].mp4' + with open(v_url, 'wb') as file: # 显示进度条 + for data in video.iter_content(chunk_size=chunk_size): + file.write(data) + size += len(data) + # print('\r' + i + '\n[下载进度]:%s%.2f%%' % ( + # '>' * int(size * 50 / content_size), float(size / content_size * 100))) + end = time.time() # 下载结束时间 + print('\n' + str(index) + '\n[下载完成]:耗时: %.2f秒\n' % (end - start)) # 输出下载用时时间 + except Exception as error: + # Downloader.print_ui2(error) + print(error) + print('该页视频没有' + str(index) + ',已为您跳过\r') + + +if __name__ == '__main__': + ls = [] + if not os.path.exists(pre_save): + os.makedirs(pre_save) + with open('../xhs/urls.txt', 'r') as f: + for line in f: + if 'http' in line: + ls.append(line.replace('\n', '').replace(' ', '')) + size = len(ls) + for i in range(0, size): + url = ls[i] + print('{}-{}'.format(i, url)) + download_url(url, i) + time.sleep(random.randint(5, 10)) diff --git a/001-Downloader/ui.py b/001-Downloader/ui.py index 2ba4967..55d2a2b 100644 --- a/001-Downloader/ui.py +++ b/001-Downloader/ui.py @@ -12,7 +12,6 @@ from douyin.dy_download import DouYin from downloader import Downloader from kuaishou.ks_download import KuaiShou -from pytube import YouTube from type_enum import PrintType from utils import * @@ -31,7 +30,8 @@ def __init__(self, master=None): self.createWidgets() def window_init(self): - self.master.title('欢迎使用-自媒体资源下载器' + Config.instance().get_version_name() + ',本程序仅用于学习交流!如有疑问请联系:xhunmon@gmail.com') + self.master.title( + '欢迎使用-自媒体资源下载器' + Config.instance().get_version_name() + ',本程序仅用于学习交流!如有疑问请联系:xhunmon@gmail.com') self.master.bg = bg_color width, height = self.master.maxsize() # self.master.geometry("{}x{}".format(width, height)) @@ -150,10 +150,7 @@ def start_download(self): url = self.urlEntry.get() path = self.dirEntry.get() domain = get_domain(url) - if "youtube" in domain: - # downloader: Downloader = YouTube(url).streams.first().download() - downloader: Downloader = YouTube(url) - elif "kwaicdn" in domain or "kuaishou" in domain: + if "kwaicdn" in domain or "kuaishou" in domain: downloader: KuaiShou = KuaiShou() # downloader.set_cookie() else: diff --git a/002-V2rayPool/README.md b/002-V2rayPool/README.md index a9ac2da..850d008 100644 --- a/002-V2rayPool/README.md +++ b/002-V2rayPool/README.md @@ -28,32 +28,27 @@ 1. 下载[v2ray-core-v4.31.0](https://github.com/v2fly/v2ray-core/releases/download/v4.31.0/v2ray-macos-64.zip) 2. 配置解压目录的路径: ```python -Config.set_v2ray_core_path('xxx/Downloads/v2ray-macos-64') +Config.set_v2ray_core_path('xxx/v2ray-macos-64') ``` 3. 查看是否能正常启动: ```python -client.Creator().v2ray_start('ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@37.120.144.211:43893#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%2041') +client.Creator().v2ray_start('xxx') +#如果需要开启全局代理 +client.Creator().v2ray_start('xxx',True) ``` -## 2021年10月12日检测可用测试节点: +## 2022年1月11日检测可用测试节点(注意去掉后面","开始的内容): ```shell -#,64.44.42.61,美国纽约布法罗 -ss://YWVzLTI1Ni1nY206YVlOZUtETXpZUVl3NEtiVWJKQThXc3pxQDY0LjQ0LjQyLjYwOjMxOTQ0#US -#,82.102.26.94,意大利米兰 -ss://YWVzLTI1Ni1nY206RmFURzR6QUxacnU3Mmd4amdTSFE3SmRo@82.102.26.93:42185#github.com/freefq%20-%20%E8%91%A1%E8%90%84%E7%89%99%20%209 -#,91.90.123.156,罗马尼亚布加勒斯特 -ss://YWVzLTI1Ni1nY206bjh3NFN0bmJWRDlkbVhZbjRBanQ4N0VB@91.90.123.155:31572#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%2011 -#,89.238.130.228,英国英格兰曼彻斯特 -ss://YWVzLTI1Ni1nY206c3V1Y1NlVkxtdDZQUUtBUDc3TnRHdzl4@89.238.130.227:49339#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%E6%9B%BC%E5%BD%BB%E6%96%AF%E7%89%B9M247%E7%BD%91%E7%BB%9C%2012 -#,2.59.214.204,俄罗斯莫斯科 -trojan://360ecd87-72e8-4706-b252-79d0d0cfe6aa@t7.ssrsub.com:8443#github.com/freefq%20-%20%E4%B9%8C%E5%85%8B%E5%85%B0%20%2026 -#,198.200.51.189,美国加利福尼亚圣何塞 -vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogIkBTU1JTVUItVjA1LeS7mOi0ueaOqOiNkDpzdW8ueXQvc3Nyc3ViIiwNCiAgImFkZCI6ICIxOTguMjAwLjUxLjE4OSIsDQogICJwb3J0IjogIjUzOTMwIiwNCiAgImlkIjogIjAxYzYxODY2LTQ1ODYtNGM4My04MmQxLTA1ZWUwOGNiZmE5YSIsDQogICJhaWQiOiAiMCIsDQogICJzY3kiOiAiYXV0byIsDQogICJuZXQiOiAidGNwIiwNCiAgInR5cGUiOiAibm9uZSIsDQogICJob3N0IjogInQubWUvdnBuaGF0IiwNCiAgInBhdGgiOiAidC5tZS92cG5wb29sIiwNCiAgInRscyI6ICIiLA0KICAic25pIjogIiINCn0= -#,192.74.254.112,美国加利福尼亚圣何塞 -vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogIkBTU1JTVUItVjE0LeS7mOi0ueaOqOiNkDpzdW8ueXQvc3Nyc3ViIiwNCiAgImFkZCI6ICIxOTIuNzQuMjU0LjExMiIsDQogICJwb3J0IjogIjU4NzE5IiwNCiAgImlkIjogImFlMTA0OGViLWE5NjItNDhmNi1iMTVmLTAxM2Q4M2QwYjZjNyIsDQogICJhaWQiOiAiNjQiLA0KICAic2N5IjogImF1dG8iLA0KICAibmV0IjogInRjcCIsDQogICJ0eXBlIjogIm5vbmUiLA0KICAiaG9zdCI6ICIxOTIuNzQuMjU0LjExMiIsDQogICJwYXRoIjogIiIsDQogICJ0bHMiOiAiIiwNCiAgInNuaSI6ICIiDQp9 +ss://YWVzLTI1Ni1nY206MWY2YWNhM2NlYmQyMWE0Y2Q1YTgwNzE4ZWQxNmI3NGNAMTIwLjIzMi4yMTQuMzY6NTAwMg#%F0%9F%87%B8%F0%9F%87%ACSingapore,8.25.96.100,美国 Level3 +ss://YWVzLTI1Ni1nY206Y2RCSURWNDJEQ3duZklO@139.99.62.207:8119#github.com/freefq%20-%20%E6%96%B0%E5%8A%A0%E5%9D%A1OVH%201,139.99.62.207,新加坡 OVH +ss://YWVzLTI1Ni1nY206UmV4bkJnVTdFVjVBRHhH@167.88.61.60:7002#github.com/freefq%20-%20%E7%91%9E%E5%85%B8%20%203,167.88.61.60,美国加利福尼亚圣克拉拉 +ss://YWVzLTI1Ni1nY206WTZSOXBBdHZ4eHptR0M@38.143.66.71:3389#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%E5%8D%8E%E7%9B%9B%E9%A1%BFCogent%E9%80%9A%E4%BF%A1%E5%85%AC%E5%8F%B8%204,38.143.66.71,美国华盛顿西雅图 Cogent +trojan://e6c36d58-6070-4b55-a437-146e6b53ec57@t1.ssrsub.com:8443#github.com/freefq%20-%20%E5%8A%A0%E6%8B%BF%E5%A4%A7%20%2012,142.47.89.64,加拿大安大略 +ss://YWVzLTI1Ni1nY206ZTRGQ1dyZ3BramkzUVk@172.99.190.87:9101#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%20%2013,172.99.190.87,美国乔治亚亚特兰大 +ss://YWVzLTI1Ni1nY206UENubkg2U1FTbmZvUzI3@46.29.218.6:8091#github.com/freefq%20-%20%E6%8C%AA%E5%A8%81%20%2019,46.29.218.6,挪威 ``` -注意:本程序虽可跨平台,但因博主能力有限,无法在更多系统上去尝试和改进,望谅解! +注意:本程序虽可跨平台,但因博主能力有限,只在macos系统操作过,无法在更多系统上去尝试和改进,望谅解! ------- diff --git a/002-V2rayPool/base/net_proxy.py b/002-V2rayPool/base/net_proxy.py index 91c2148..74326e4 100644 --- a/002-V2rayPool/base/net_proxy.py +++ b/002-V2rayPool/base/net_proxy.py @@ -42,6 +42,10 @@ def get_header(self, headers, key): else: return '' + def update_agent(self): + self.USER_AGENT = self._ua.random() # 随机生成的agent + self._headers = {"user-agent": self.USER_AGENT, 'Connection': 'close'} + def get_urls(self) -> []: """需子类实现""" pass diff --git a/002-V2rayPool/core/client.py b/002-V2rayPool/core/client.py index c81e0e3..207fa78 100755 --- a/002-V2rayPool/core/client.py +++ b/002-V2rayPool/core/client.py @@ -9,6 +9,7 @@ from core.conf import Config from core.group import Vmess, Vless, Socks, SS, Mtproto, Trojan, Group, Dyport from core.utils import ProtocolType +import core.utils as util class ClientWriter: @@ -251,25 +252,27 @@ def __kill_threading(self): a = os.popen("kill %d" % int(pid)).read() except Exception as e: pass + util.sys_proxy_off() - def __child_thread(self, url: str): + def __child_thread(self, url: str, isSysOn=False): self.generateAndWrite(url) # 执行就可,不需要知道结果 - cur_dir = os.path.dirname(os.path.abspath(__file__)) if Config.get_v2ray_core_path() is None: raise Exception('请先调用#Config.set_v2ray_core_path 设置路径') v2ray_path = os.path.join(Config.get_v2ray_core_path(), 'v2ray') - config_path = os.path.join(cur_dir, 'config.json') + config_path = os.path.join(Config.get_v2ray_core_path(), 'config.json') os.popen("%s -config %s>/dev/null 2>&1" % (v2ray_path, config_path)) print("%s -config %s>/dev/null 2>&1" % (v2ray_path, config_path)) + if isSysOn: + util.sys_v2ray_on() - def v2ray_start(self, url: str): + def v2ray_start(self, url: str, isSysOn=False): self.__kill_threading() - self.__child_thread(url) + self.__child_thread(url, isSysOn) - def v2ray_start_with_log(self, url: str): + def v2ray_start_with_log(self, url: str, isSysOn=False): try: - self.v2ray_start(url) + self.v2ray_start(url, isSysOn) except Exception as e: print(e) print("启动异常:%s" % url) diff --git a/002-V2rayPool/core/conf.py b/002-V2rayPool/core/conf.py index 7a744dc..2e09091 100755 --- a/002-V2rayPool/core/conf.py +++ b/002-V2rayPool/core/conf.py @@ -7,11 +7,12 @@ class Config: __v2ray_core_path = None + __v2ray_node_path = None def __init__(self): self.config = configparser.ConfigParser() parent_dir = os.path.dirname(os.path.abspath(__file__)) - self.config_path = os.path.join(parent_dir, 'config.json') + self.config_path = os.path.join(Config.__v2ray_core_path, 'config.json') self.json_path = os.path.join(parent_dir, 'json_template') # self.config.read(self.config_path) @@ -35,3 +36,13 @@ def set_v2ray_core_path(dir: str): def get_v2ray_core_path(): """获取当前v2ray_core程序的目录""" return Config.__v2ray_core_path + + @staticmethod + def set_v2ray_node_path(dir: str): + """设置当前v2ray保存节点的目录""" + Config.__v2ray_node_path = dir + + @staticmethod + def get_v2ray_node_path(): + """获取当前v2ray保存节点的目录""" + return Config.__v2ray_node_path diff --git a/002-V2rayPool/core/json_template/client.json b/002-V2rayPool/core/json_template/client.json index 204b210..1865d51 100755 --- a/002-V2rayPool/core/json_template/client.json +++ b/002-V2rayPool/core/json_template/client.json @@ -16,6 +16,14 @@ "clients": null }, "streamSettings": null + }, + { + "listen": "127.0.0.1", + "protocol": "http", + "settings": { + "timeout": 360 + }, + "port": "1087" } ], "outbounds": [ diff --git a/002-V2rayPool/core/json_template/client_socks.json b/002-V2rayPool/core/json_template/client_socks.json index ae7fdfc..c952c47 100755 --- a/002-V2rayPool/core/json_template/client_socks.json +++ b/002-V2rayPool/core/json_template/client_socks.json @@ -16,6 +16,14 @@ "clients": null }, "streamSettings": null + }, + { + "listen": "127.0.0.1", + "protocol": "http", + "settings": { + "timeout": 360 + }, + "port": "1087" } ], "outbounds": diff --git a/002-V2rayPool/core/json_template/client_ss.json b/002-V2rayPool/core/json_template/client_ss.json index 621e0a0..dd76925 100755 --- a/002-V2rayPool/core/json_template/client_ss.json +++ b/002-V2rayPool/core/json_template/client_ss.json @@ -16,6 +16,14 @@ "clients": null }, "streamSettings": null + }, + { + "listen": "127.0.0.1", + "protocol": "http", + "settings": { + "timeout": 360 + }, + "port": "1087" } ], "outbounds": [ diff --git a/002-V2rayPool/core/json_template/client_trojan.json b/002-V2rayPool/core/json_template/client_trojan.json index cc44fad..2694c58 100755 --- a/002-V2rayPool/core/json_template/client_trojan.json +++ b/002-V2rayPool/core/json_template/client_trojan.json @@ -16,6 +16,14 @@ "clients": null }, "streamSettings": null + }, + { + "listen": "127.0.0.1", + "protocol": "http", + "settings": { + "timeout": 360 + }, + "port": "1087" } ], "outbounds": [ diff --git a/002-V2rayPool/core/utils.py b/002-V2rayPool/core/utils.py index 8636db5..37f0c0f 100755 --- a/002-V2rayPool/core/utils.py +++ b/002-V2rayPool/core/utils.py @@ -189,38 +189,43 @@ def readchar(prompt=""): return ch.strip() +def kill_all_v2ray(): + pids = os.popen("ps aux |grep v2ray |awk '{print 2ドル}'").read().split('\n') + for pid in pids: + try: + import subprocess + # subprocess.check_output("kill %d" % int(pid)) + a = os.popen("kill %d" % int(pid)).read() + except Exception as e: + pass + sys_proxy_off() + + # netstat -nlp | grep :1080 | awk '{print 7ドル}' | awk -F\" / \" '{ print 1ドル }' def kill_process_by_port(port): try: - # kill -9 `ps -ef |grep act1 |awk 'NR==1{print 3ドル}'` - # pids = os.popen("pgrep -f v2ray|xargs -n1 kill -9").read().split('\n') pids = os.popen("pgrep -f v2ray|xargs kill -9").read().split('\n') print(pids) - # a = os.kill(int(pid), 0) - # a = os.popen("kill %d" % int(pid)).read() - # print('已杀死port为%d,pid为%s的进程, 返回值是:%s' % (port, pid, a)) except: pass - pid_all = [] - # ps aux | grep 'v2ray' | awk '{print 2ドル} - # pids = os.popen("lsof -i:%d | awk '{print 2ドル}'" % (port)).read().split('\n') - - - # pids = os.popen("ps aux |grep v2ray |awk '{print 2ドル}'").read().split('\n') - # # pids = os.popen("ps aux | grep 'v2ray' | awk '{print 2ドル}'").read().split('\n') - # print(pids) - # for pid in pids: - # temp = pid.strip() - # if len(temp)> 1 and temp != 'PID' and temp != '0' and temp not in pid_all: - # pid_all.append(temp) - # for pid in pid_all: - # # a = os.kill(int(pid), 0) - # # a = os.popen("kill -9 %d" % int(pid)).read() - # try: - # process = subprocess.Popen("kill %d" % int(pid), shell=True) - # os.killpg(os.getpgid(process.pid), signal.SIGTERM) - # # a = os.kill(int(pid), 0) - # # a = os.popen("kill %d" % int(pid)).read() - # print('已杀死port为%d,pid为%s的进程, 返回值是:%s' % (port, pid, a)) - # except: - # pass + + +def sys_proxy_on(proxy, port): + ''''控制macOS系统代理''' + os.system('networksetup -setwebproxy wi-fi %s %d' % (proxy, port)) # http + os.system('networksetup -setsecurewebproxy wi-fi %s %d' % (proxy, port)) # https + os.system('networksetup -setsocksfirewallproxy wi-fi %s %d' % (proxy, port)) # socks + + +def sys_v2ray_on(): + # proxy_on("127.0.0.1", 1080) + '''端口要对应起v2ray开启的,具体要看写入config.json文件中inbounds节点部分''' + os.system('networksetup -setwebproxy wi-fi 127.0.0.1 1087') + os.system('networksetup -setsecurewebproxy wi-fi 127.0.0.1 1087') + os.system('networksetup -setsocksfirewallproxy wi-fi 127.0.0.1 1080') + + +def sys_proxy_off(): + os.system('networksetup -setwebproxystate wi-fi off') + os.system('networksetup -setsecurewebproxystate wi-fi off') + os.system('networksetup -setsocksfirewallproxystate wi-fi off') diff --git a/002-V2rayPool/db/_db-checked.txt b/002-V2rayPool/db/_db-checked.txt deleted file mode 100644 index 4dc114b..0000000 --- a/002-V2rayPool/db/_db-checked.txt +++ /dev/null @@ -1,32 +0,0 @@ -vmess://eyJ2IjogIjIiLCAicHMiOiAiZ2l0aHViLmNvbS9mcmVlZnEgLSBcdTViODlcdTVmYmRcdTc3MDFcdTgwNTRcdTkwMWEgMSIsICJhZGQiOiAiNDIuMTU3LjguMTYyIiwgInBvcnQiOiAiNDYwMDYiLCAiaWQiOiAiMzk1OTQ3N2UtNTVjNC00NTNmLWJjODAtM2IxM2U2NDg5MWFjIiwgImFpZCI6ICI2NCIsICJzY3kiOiAiYXV0byIsICJuZXQiOiAidGNwIiwgInR5cGUiOiAibm9uZSIsICJob3N0IjogIiIsICJwYXRoIjogIi8iLCAidGxzIjogIiIsICJzbmkiOiAiIn0=,42.157.8.162,中国安徽合肥 联通 -vmess://eyJ2IjogIjIiLCAicHMiOiAiZ2l0aHViLmNvbS9mcmVlZnEgLSBcdTdmOGVcdTU2ZmQgIDMiLCAiYWRkIjogIjE0NC4xNzIuMTE4LjQ3IiwgInBvcnQiOiAiODg4OCIsICJpZCI6ICI5ZTlmMGY5Yi1iY2FhLTQ0MjEtOTI0MS0xNDUzNDNlMWE0NjUiLCAiYWlkIjogIjIzMyIsICJzY3kiOiAiYXV0byIsICJuZXQiOiAidGNwIiwgInR5cGUiOiAibm9uZSIsICJob3N0IjogIiIsICJwYXRoIjogIiIsICJ0bHMiOiAiIiwgInNuaSI6ICIifQ==,144.172.118.47,美国佛罗里达杰克逊维尔 -ss://YWVzLTI1Ni1nY206c3V1Y1NlVkxtdDZQUUtBUDc3TnRHdzl4@89.238.130.227:49339#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%E6%9B%BC%E5%BD%BB%E6%96%AF%E7%89%B9M247%E7%BD%91%E7%BB%9C%204,89.238.130.228,英国英格兰曼彻斯特 -ss://YWVzLTI1Ni1nY206Q1VuZFNabllzUEtjdTZLajhUSFZNQkhE@193.29.106.61:39772#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%207,193.29.106.62,罗马尼亚 -vmess://eyJ2IjogIjIiLCAicHMiOiAiZ2l0aHViLmNvbS9mcmVlZnEgLSBcdTdmOGVcdTU2ZmRcdTUyYTBcdTUyMjlcdTc5OGZcdTVjM2NcdTRlOWFcdTVkZGVcdTU3MjNcdTRmNTVcdTU4NWVQRUcgVEVDSFx1NjU3MFx1NjM2ZVx1NGUyZFx1NWZjMyA4IiwgImFkZCI6ICIxOTIuNzQuMjU0LjExMiIsICJwb3J0IjogIjU4NzE5IiwgImlkIjogImFlMTA0OGViLWE5NjItNDhmNi1iMTVmLTAxM2Q4M2QwYjZjNyIsICJhaWQiOiAiNjQiLCAibmV0IjogInRjcCIsICJ0eXBlIjogIm5vbmUiLCAiaG9zdCI6ICIiLCAicGF0aCI6ICIiLCAidGxzIjogIm5vbmUifQ==,192.74.254.112,美国加利福尼亚圣何塞 -ss://YWVzLTI1Ni1nY206Q1VuZFNabllzUEtjdTZLajhUSFZNQkhE@193.29.106.29:39772#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%2012,193.29.106.30,罗马尼亚 -ss://YWVzLTI1Ni1nY206V0N1ejd5cmZaU0NRUVhTTnJ0R1B6MkhU@89.46.223.239:50168#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%2013,89.46.223.240,英国英格兰伦敦 -ss://YWVzLTI1Ni1nY206YVlOZUtETXpZUVl3NEtiVWJKQThXc3px@66.115.177.141:31944#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%20%2014,66.115.177.142,美国乔治亚亚特兰大 -trojan://ed4b6594-ea16-40f6-a935-f3985c433973@t2.ssrsub.one:8443#github.com/freefq%20-%20%E4%BF%84%E7%BD%97%E6%96%AF%20%2015,213.59.118.168,美国加利福尼亚洛杉矶 -ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@193.29.107.237:43893#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%2016,193.29.107.238,罗马尼亚 -ss://YWVzLTI1Ni1nY206Q1VuZFNabllzUEtjdTZLajhUSFZNQkhE@45.12.221.179:39772#github.com/freefq%20-%20%E6%AC%A7%E7%9B%9F%20%2017,45.12.221.180,丹麦 -trojan://ed4b6594-ea16-40f6-a935-f3985c433973@t8.ssrsub.one:8443#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%20%2019,152.70.119.197,美国 -ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@185.188.61.119:43893#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%20%2022,185.188.61.120,英国 -ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@89.37.95.23:43893#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%2026,89.37.95.24,罗马尼亚 -ss://YWVzLTI1Ni1nY206S3F1djVVaHZaWE5NZW1BUXk4RHhaN3Fu@185.38.150.41:38620#github.com/freefq%20-%20%E6%AC%A7%E6%B4%B2%20%2027,185.38.150.49,英国英格兰布里斯托尔 -trojan://ed4b6594-ea16-40f6-a935-f3985c433973@t6.ssrsub.one:8443#github.com/freefq%20-%20%E4%BF%84%E7%BD%97%E6%96%AF%20%2030,31.184.204.97,俄罗斯圣彼得堡 -ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@37.120.144.211:43893#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%2031,37.120.144.212,英国英格兰伦敦 -trojan://ed4b6594-ea16-40f6-a935-f3985c433973@t1.ssrsub.one:8443#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%E5%86%85%E5%8D%8E%E8%BE%BE%E5%B7%9E%E6%8B%89%E6%96%AF%E7%BB%B4%E5%8A%A0%E6%96%AFBuyVM%E6%95%B0%E6%8D%AE%E4%B8%AD%E5%BF%83%2033,209.141.62.129,美国内华达拉斯维加斯 -ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@89.238.130.253:43893#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%E6%9B%BC%E5%BD%BB%E6%96%AF%E7%89%B9M247%E7%BD%91%E7%BB%9C%2037,89.238.130.254,英国英格兰曼彻斯特 -ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@84.17.41.83:43893#github.com/freefq%20-%20%E9%A6%99%E6%B8%AFCDN77%E8%8A%82%E7%82%B9%2038,84.17.41.84,美国华盛顿西雅图 -ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@84.17.35.83:43893#github.com/freefq%20-%20%E9%A6%99%E6%B8%AFCDN77%E8%8A%82%E7%82%B9%2039,84.17.35.84,美国纽约纽约市 -trojan://ed4b6594-ea16-40f6-a935-f3985c433973@t3.ssrsub.one:8443#github.com/freefq%20-%20%E4%BF%84%E7%BD%97%E6%96%AF%E6%96%B0%E8%A5%BF%E4%BC%AF%E5%88%A9%E4%BA%9AJustHost%2040,45.89.229.223,俄罗斯新西伯利亚 -ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@84.17.53.86:43893#github.com/freefq%20-%20%E9%A6%99%E6%B8%AFCDN77%E8%8A%82%E7%82%B9%2042,84.17.53.87,英国 -ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@193.29.107.163:43893#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%2045,193.29.107.164,罗马尼亚 -ss://YWVzLTI1Ni1nY206NGVqSjhuNWRkTHVZRFVIR1hKcmUydWZK@86.106.136.85:48938#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%2046,86.106.136.86,罗马尼亚 -vmess://eyJ2IjogIjIiLCAicHMiOiAiZ2l0aHViLmNvbS9mcmVlZnEgLSBcdTY1ZTVcdTY3MmNcdTRlMWNcdTRlYWNHb29nbGVcdTRlOTFcdThiYTFcdTdiOTdcdTY1NzBcdTYzNmVcdTRlMmRcdTVmYzMgNDgiLCAiYWRkIjogIjM1LjIwMC4yMDIuMTc4IiwgInBvcnQiOiAiMzEyNDgiLCAiaWQiOiAiYTBkYWI1NmMtYjI3MC00NmRiLWJjYmEtNTQxOWNkMTc2NDg1IiwgImFpZCI6ICI2NCIsICJuZXQiOiAidGNwIiwgInR5cGUiOiAibm9uZSIsICJob3N0IjogIiIsICJwYXRoIjogIiIsICJ0bHMiOiAibm9uZSJ9,35.200.202.178,印度马哈拉施特拉孟买 谷歌云 -trojan://ed4b6594-ea16-40f6-a935-f3985c433973@t4.ssrsub.one:8443#github.com/freefq%20-%20%E4%BF%84%E7%BD%97%E6%96%AF%20%2049,194.87.103.61,俄罗斯莫斯科 -ss://YWVzLTI1Ni1nY206OG42cHdBY3JydjJwajZ0RlkycDNUYlE2@185.153.151.85:33992#github.com/freefq%20-%20%E5%8D%A2%E6%A3%AE%E5%A0%A1%20%2051,185.153.151.86,卢森堡 -trojan://ed4b6594-ea16-40f6-a935-f3985c433973@t7.ssrsub.one:8443#github.com/freefq%20-%20%E4%B9%8C%E5%85%8B%E5%85%B0%20%2052,2.59.214.204,俄罗斯莫斯科 -vmess://eyJ2IjogIjIiLCAicHMiOiAiZ2l0aHViLmNvbS9mcmVlZnEgLSBcdTY1ZTVcdTY3MmNcdTRlMWNcdTRlYWNBbWF6b25cdTY1NzBcdTYzNmVcdTRlMmRcdTVmYzMgNTQiLCAiYWRkIjogImFmMDEudXdvcmsubW9iaSIsICJwb3J0IjogIjgwIiwgImlkIjogIjAwMDAwMDAwLTAwMDAtMDAwMC0wMDAwLTAwMDAwMDAwMDAwMSIsICJhaWQiOiAiNjQiLCAibmV0IjogInRjcCIsICJ0eXBlIjogIm5vbmUiLCAiaG9zdCI6ICIiLCAicGF0aCI6ICIiLCAidGxzIjogIm5vbmUifQ==

,8.37.43.232,日本东京 Level3 -vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogIkBTU1JTVUItVjI3LeS7mOi0ueaOqOiNkDpzdW8ueXQvc3Nyc3ViIiwNCiAgImFkZCI6ICIxMzIuMjI2LjE2OS45MyIsDQogICJwb3J0IjogIjMzMjU0IiwNCiAgImlkIjogIjA0NTMxNTRiLWRiNTEtNGE2Ni04ZjdjLTA4ODc4NzhjYjlhMyIsDQogICJhaWQiOiAiMCIsDQogICJzY3kiOiAiYXV0byIsDQogICJuZXQiOiAidGNwIiwNCiAgInR5cGUiOiAibm9uZSIsDQogICJob3N0IjogIjEzMi4yMjYuMTY5LjkzIiwNCiAgInBhdGgiOiAiIiwNCiAgInRscyI6ICIiLA0KICAic25pIjogIiINCn0=,132.226.169.93,荷兰阿姆斯特丹 -vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogImh0dHBzOi8vZ2l0LmlvL3Y5OTk5IOiOq+aWr+enkWciLA0KICAiYWRkIjogIjE4NS4yMi4xNTMuMTg3IiwNCiAgInBvcnQiOiAiNDM1NDYiLA0KICAiaWQiOiAiMjViOGI4YTYtMTdiOC0xMWVjLWJiNTAtMGFiM2VhMTM0ZDA2IiwNCiAgImFpZCI6ICIwIiwNCiAgIm5ldCI6ICJ0Y3AiLA0KICAidHlwZSI6ICJub25lIiwNCiAgImhvc3QiOiAiIiwNCiAgInBhdGgiOiAiIiwNCiAgInRscyI6ICIiLA0KICAic25pIjogIiINCn0=,185.22.153.187,俄罗斯莫斯科 diff --git a/002-V2rayPool/db/db_main.py b/002-V2rayPool/db/db_main.py index c41f4e0..61b9e6e 100644 --- a/002-V2rayPool/db/db_main.py +++ b/002-V2rayPool/db/db_main.py @@ -12,217 +12,214 @@ from db.local import DbLocal, DbEnable from db.net import * -dbLocal = DbLocal() -check = PYCheck() -dbEnable = DbEnable().get() - - -def __add_urls_de_dup(all_urls: [], new_urls: []) -> []: - """合并数组,并且去重,去空""" - if not new_urls: # 为 [] 或者 None - return all_urls - for url in new_urls: - temp = url.strip().replace('\n', '') - if temp in all_urls or len(temp) < 20: - continue - all_urls.append(temp) - - -def start_random_v2ray_by_local(): - """从本地随机启动一个可用的proxy""" - urls = load_enable_urls_by_local() - for url in urls: - if client.Creator().v2ray_start_with_log(random.choice(urls)) is False: - time.sleep(1) - continue - time.sleep(2) - ips = PYCheck().get_curren_ip() - if not ips: - print('无效地址:%s' % url) - continue - print('代理开启成功') - time.sleep(1) - return True - return False +class DBManage(object): + def init(self): + self.dbLocal = DbLocal() + self.check = PYCheck() + self.dbEnable = DbEnable().get() + + def __add_urls_de_dup(self, all_urls: [], new_urls: []) -> []: + """合并数组,并且去重,去空""" + if not new_urls: # 为 [] 或者 None + return all_urls + for url in new_urls: + temp = url.strip().replace('\n', '') + if temp in all_urls or len(temp) < 20: + continue + all_urls.append(temp) -def load_urls_and_save_auto(): - """首先通过不需要代理的网页获取节点,当代理有可用时,开启代理,获取需要代理获取的网页""" - dbLocal.clear_local() - all_urls = load_urls_by_not_proxy() - proxy_url = None - for url in all_urls: - if client.Creator().v2ray_start_with_log(url) is False: + def start_random_v2ray_by_local(self, isSysOn=False): + """从本地随机启动一个可用的proxy""" + urls = self.load_enable_urls_by_local() + for url in urls: + if client.Creator().v2ray_start_with_log(random.choice(urls), isSysOn) is False: + time.sleep(1) + continue + time.sleep(2) + ips = PYCheck().get_curren_ip() + if not ips: + print('无效地址:%s' % url) + continue + print('代理开启成功') time.sleep(1) - continue - time.sleep(2) - ips = PYCheck().get_curren_ip() - if not ips: - print('无效地址:%s' % url) - continue - proxy_url = url - break - if proxy_url is None: - raise Exception("无代理可用,退出!") - print("获得可用代理地址:%s" % proxy_url) - proxy_urls = load_urls_by_net_with_proxy(proxy_url=proxy_url) - all_urls = all_urls + proxy_urls - check_and_save(all_urls, append=False) - - -def load_urls_by_not_proxy(save_local=True): - all_urls = [] - # 1. 先把不需要代理的先请求下来 - __add_urls_de_dup(all_urls, PNSsfree().get_urls()) - print("获取https://view.ssfree.ru/后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PNFreevpnX().get_urls()) - print("获取https://freevpn-x.com/后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PNGithubIwxf().get_urls()) - print("获取https://github.com/iwxf/free-v2ray/blob/master/README.md 后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PNFreeV2ray().get_urls()) - print("获取https://view.freev2ray.org/ 后数目:%d" % len(all_urls)) - if save_local: # 保存到本地 - dbLocal.save_urls(all_urls) - return all_urls - - -def load_urls_by_net_with_proxy(proxy_url=None, save_local=True): - all_urls = [] - if save_local: # 保存到本地 - dbLocal.save_urls(all_urls) - if not proxy_url: - proxy_url = 'ss://YWVzLTI1Ni1nY206NGVqSjhuNWRkTHVZRFVIR1hKcmUydWZK@212.102.40.68:48938#github.com/freefq%20-%20%E6%84%8F%E5%A4%A7%E5%88%A9%20%201' - creator = client.Creator() - creator.v2ray_start(proxy_url) - time.sleep(2) - __add_urls_de_dup(all_urls, PYIvmess().get_urls()) - print("获取https://t.me/s/ivmess 后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PYFlyingboat().get_urls()) - print("获取https://t.me/s/flyingboat 后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PYFreevpnnet().get_urls()) - print("获取https://www.freevpnnet.com/ 后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PYMerlinblog().get_urls()) - print("获取https://merlinblog.xyz/wiki/freess.html 后数目:%d" % len(all_urls)) - # __add_urls_de_dup(all_urls, PYFreeFq().get_urls()) - __add_urls_de_dup(all_urls, PYFreeFq().download_urls(f_day=1)) # 前2天的地址 - print("获取从https://freefq.com/ 后数目:%d" % len(all_urls)) - if save_local: # 保存到本地 - dbLocal.save_urls(all_urls) - return all_urls + return True + return False + def load_urls_and_save_auto(self): + """首先通过不需要代理的网页获取节点,当代理有可用时,开启代理,获取需要代理获取的网页""" + self.dbLocal.clear_local() + all_urls = self.load_urls_by_not_proxy() + proxy_url = None + for url in all_urls: + if client.Creator().v2ray_start_with_log(url) is False: + time.sleep(1) + continue + time.sleep(2) + ips = PYCheck().get_curren_ip() + if not ips: + print('无效地址:%s' % url) + continue + proxy_url = url + break + if proxy_url is None: + raise Exception("无代理可用,退出!") + print("获得可用代理地址:%s" % proxy_url) + proxy_urls = self.load_urls_by_net_with_proxy(proxy_url=proxy_url) + all_urls = all_urls + proxy_urls + self.check_and_save(all_urls, append=False) + + def load_urls_by_not_proxy(self, save_local=True): + all_urls = [] + # 1. 先把不需要代理的先请求下来 + self.__add_urls_de_dup(all_urls, PNTWGithubV2ray().get_urls()) + print("获取https://hub.xn--gzu630h.xn--kpry57d/freefq/free后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PNSsfree().get_urls()) + print("获取https://view.ssfree.ru/后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PNFreevpnX().get_urls()) + print("获取https://freevpn-x.com/后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PNGithubIwxf().get_urls()) + print("获取https://github.com/iwxf/free-v2ray/blob/master/README.md 后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PNFreeV2ray().get_urls()) + print("获取https://view.freev2ray.org/ 后数目:%d" % len(all_urls)) + if save_local: # 保存到本地 + self.dbLocal.save_urls(all_urls) + return all_urls -def load_urls_by_net(proxy_url=None, save_local=True, need_proxy=True): - """ - 通过网络获取最新的节点,但是需要代理 - :param proxy_url: 代理url,默认的如果失效了则回去失败 - :param save_local: 是否保存到本地 - :param need_proxy: 如果程序本身就在外网跑,就不需要开启代理获取了 - :return: - """ - all_urls = [] - # 1. 先把不需要代理的先请求下来 - __add_urls_de_dup(all_urls, PNSsfree().get_urls()) - print("获取https://view.ssfree.ru/后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PNFreevpnX().get_urls()) - print("获取https://freevpn-x.com/后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PNGithubIwxf().get_urls()) - print("获取https://github.com/iwxf/free-v2ray/blob/master/README.md 后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PNFreeV2ray().get_urls()) - print("获取https://view.freev2ray.org/ 后数目:%d" % len(all_urls)) - print("准备开启代理获取...") - if need_proxy: + def load_urls_by_net_with_proxy(self, proxy_url=None, save_local=True): + all_urls = [] + if save_local: # 保存到本地 + self.dbLocal.save_urls(all_urls) if not proxy_url: proxy_url = 'ss://YWVzLTI1Ni1nY206NGVqSjhuNWRkTHVZRFVIR1hKcmUydWZK@212.102.40.68:48938#github.com/freefq%20-%20%E6%84%8F%E5%A4%A7%E5%88%A9%20%201' - # 需要代理 creator = client.Creator() creator.v2ray_start(proxy_url) - time.sleep(2) - __add_urls_de_dup(all_urls, PYIvmess().get_urls()) - print("获取https://t.me/s/ivmess 后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PYFlyingboat().get_urls()) - print("获取https://t.me/s/flyingboat 后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PYFreevpnnet().get_urls()) - print("获取https://www.freevpnnet.com/ 后数目:%d" % len(all_urls)) - __add_urls_de_dup(all_urls, PYMerlinblog().get_urls()) - print("获取https://merlinblog.xyz/wiki/freess.html 后数目:%d" % len(all_urls)) - # __add_urls_de_dup(all_urls, PYFreeFq().get_urls()) - __add_urls_de_dup(all_urls, PYFreeFq().download_urls(f_day=1)) # 前2天的地址 - print("获取从https://freefq.com/ 后数目:%d" % len(all_urls)) - if save_local: # 保存到本地 - dbLocal.save_urls(all_urls, append=False) - return all_urls - - -def load_unchecked_urls_by_local(): - """获取本地未校验过的url""" - dbLocal.get_urls(False) - urls = dbLocal.get_checked_urls() - return urls - + time.sleep(2) + self.__add_urls_de_dup(all_urls, PYIvmess().get_urls()) + print("获取https://t.me/s/ivmess 后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PYFlyingboat().get_urls()) + print("获取https://t.me/s/flyingboat 后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PYFreevpnnet().get_urls()) + print("获取https://www.freevpnnet.com/ 后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PYMerlinblog().get_urls()) + print("获取https://merlinblog.xyz/wiki/freess.html 后数目:%d" % len(all_urls)) + # __add_urls_de_dup(all_urls, PYFreeFq().get_urls()) + self.__add_urls_de_dup(all_urls, PYFreeFq().download_urls(f_day=1)) # 前2天的地址 + print("获取从https://freefq.com/ 后数目:%d" % len(all_urls)) + if save_local: # 保存到本地 + self.dbLocal.save_urls(all_urls) + return all_urls -def load_enable_urls_by_local(): - """获取已检测过的url""" - return dbEnable.get_urls() + def load_urls_by_net(self, proxy_url=None, save_local=True, need_proxy=True): + """ + 通过网络获取最新的节点,但是需要代理 + :param proxy_url: 代理url,默认的如果失效了则回去失败 + :param save_local: 是否保存到本地 + :param need_proxy: 如果程序本身就在外网跑,就不需要开启代理获取了 + :return: + """ + all_urls = [] + # 1. 先把不需要代理的先请求下来 + self.__add_urls_de_dup(all_urls, PNTWGithubV2ray().get_urls()) + print("获取https://hub.xn--gzu630h.xn--kpry57d/freefq/free后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PNSsfree().get_urls()) + print("获取https://view.ssfree.ru/后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PNFreevpnX().get_urls()) + print("获取https://freevpn-x.com/后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PNGithubIwxf().get_urls()) + print("获取https://github.com/iwxf/free-v2ray/blob/master/README.md 后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PNFreeV2ray().get_urls()) + print("获取https://view.freev2ray.org/ 后数目:%d" % len(all_urls)) + print("准备开启代理获取...") + if need_proxy: + if not proxy_url: + proxy_url = 'ss://YWVzLTI1Ni1nY206NGVqSjhuNWRkTHVZRFVIR1hKcmUydWZK@212.102.40.68:48938#github.com/freefq%20-%20%E6%84%8F%E5%A4%A7%E5%88%A9%20%201' + # 需要代理 + creator = client.Creator() + creator.v2ray_start(proxy_url) + time.sleep(2) + self.__add_urls_de_dup(all_urls, PYIvmess().get_urls()) + print("获取https://t.me/s/ivmess 后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PYFlyingboat().get_urls()) + print("获取https://t.me/s/flyingboat 后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PYFreevpnnet().get_urls()) + print("获取https://www.freevpnnet.com/ 后数目:%d" % len(all_urls)) + self.__add_urls_de_dup(all_urls, PYMerlinblog().get_urls()) + print("获取https://merlinblog.xyz/wiki/freess.html 后数目:%d" % len(all_urls)) + # __add_urls_de_dup(all_urls, PYFreeFq().get_urls()) + self.__add_urls_de_dup(all_urls, PYFreeFq().download_urls(f_day=1)) # 前2天的地址 + print("获取从https://freefq.com/ 后数目:%d" % len(all_urls)) + if save_local: # 保存到本地 + self.dbLocal.save_urls(all_urls, append=False) + return all_urls + def load_unchecked_urls_by_local(self): + """获取本地未校验过的url""" + self.dbLocal.get_urls(False) + urls = self.dbLocal.get_checked_urls() + return urls -def check_url_single(url: str): - client.Creator().v2ray_start(url) - time.sleep(2) - ips = PYCheck().get_curren_ip() - if not ips: - print('地址无效!') - return False - print('检查地址结果:%s' % url) - print(ips) - return True + def load_enable_urls_by_local(self): + """获取已检测过的url""" + return self.dbEnable.get_urls() + def check_url_single(self, url: str): + client.Creator().v2ray_start(url) + time.sleep(2) + ips = PYCheck().get_curren_ip() + if not ips: + print('地址无效!') + return False + print('检查地址结果:%s' % url) + print(ips) + return True -def check_and_save(urls: [], append=True): - """检测url是否可用,并且保存到本地""" - if not append: - dbEnable.clear_local() - all_infos = dbEnable.get_infos() - new_infos = [] - size = len(urls) - for i in range(size): - try: - if i % 30 == 0: # 每三十个更新一次 - dbLocal.save_urls(urls=urls, append=False) # 更新剩下的 - url = urls.pop() - in_all = False - for item in all_infos: - if url in item: - in_all = True - break - if in_all: - print('取出地址已存在:%s' % url) - continue - if client.Creator().v2ray_start_with_log(url) is False: - time.sleep(1) - continue - time.sleep(2) - ips = PYCheck().get_curren_ip() - if not ips: - print('地址无效!') - continue - ip, add = str(ips[0]), ips[1] - hase_item = False # 已存在 - for item in all_infos: - if ip in item: - hase_item = True - break - if hase_item: - print('ip=%s已存在!' % ip) - continue - info = r'%s,%s,%s' % (url.strip(), ip.strip(), add.strip().replace('\n', '')) - print('%s!总共:%d |待检测:%d |可用:%d' % ('地址有效', size, len(urls), len(all_infos))) - new_infos.append(info) - all_infos.append(info) - dbEnable.save_urls(new_infos) # 写入已通过的 - new_infos.clear() - except Exception as e: - print(e) - # 最后 - print('%s!总共:%d |待检测:%d |可用:%d' % ('全部检测完毕!', size, len(urls), len(all_infos))) - dbEnable.save_urls(new_infos) - dbLocal.clear_local() + def check_and_save(self, urls: [], append=True): + """检测url是否可用,并且保存到本地""" + if not append: + self.dbEnable.clear_local() + all_infos = self.dbEnable.get_infos() + new_infos = [] + size = len(urls) + for i in range(size): + try: + if i % 30 == 0: # 每三十个更新一次 + self.dbLocal.save_urls(urls=urls, append=False) # 更新剩下的 + url = urls.pop() + in_all = False + for item in all_infos: + if url in item: + in_all = True + break + if in_all: + print('取出地址已存在:%s' % url) + continue + if client.Creator().v2ray_start_with_log(url) is False: + time.sleep(1) + continue + time.sleep(2) + ips = PYCheck().get_curren_ip() + if not ips: + print('地址无效!') + continue + ip, add = str(ips[0]), ips[1] + hase_item = False # 已存在 + for item in all_infos: + if ip in item: + hase_item = True + break + if hase_item: + print('ip=%s已存在!' % ip) + continue + info = r'%s,%s,%s' % (url.strip(), ip.strip(), add.strip().replace('\n', '')) + print('%s!总共:%d |待检测:%d |可用:%d' % ('地址有效', size, len(urls), len(all_infos))) + new_infos.append(info) + all_infos.append(info) + self.dbEnable.save_urls(new_infos) # 写入已通过的 + new_infos.clear() + except Exception as e: + print(e) + # 最后 + print('%s!总共:%d |待检测:%d |可用:%d' % ('全部检测完毕!', size, len(urls), len(all_infos))) + self.dbEnable.save_urls(new_infos) + self.dbLocal.clear_local() diff --git a/002-V2rayPool/db/local.py b/002-V2rayPool/db/local.py index 9c230dd..fed407b 100644 --- a/002-V2rayPool/db/local.py +++ b/002-V2rayPool/db/local.py @@ -11,22 +11,24 @@ from concurrent.futures import ThreadPoolExecutor from threading import Lock +from core.conf import Config + class DbLocal(object): """ 加载本地文件的url,并进行检测是否合法 """ - def __init__(self, save_path=None, get_path=None): + def __init__(self): + path = Config.get_v2ray_node_path() parent_dir = os.path.dirname(os.path.abspath(__file__)) - if not save_path: - self.__save_path = os.path.join(parent_dir, '_db-uncheck.txt') - else: - self.__save_path = save_path - if not get_path: - self.__get_path = os.path.join(parent_dir, '_db-uncheck.txt') - else: - self.get_path = get_path + if path: + parent_dir = path + if not os.path.exists(parent_dir): + os.mkdir(parent_dir) + + self.__save_path = os.path.join(parent_dir, '_db-uncheck.txt') + self.__get_path = os.path.join(parent_dir, '_db-uncheck.txt') if not os.path.isfile(self.__save_path): open(self.__save_path, mode='w', encoding="utf-8").write('') if not os.path.isfile(self.__get_path): @@ -149,19 +151,21 @@ class DbEnable(object): """ _instance_lock = Lock() - def __init__(self, path=None): + def __init__(self): self.__urls = [] self.__index = 0 self.__END = '.back' self.__mutex = Lock() self.__default_url = 'ss://YWVzLTI1Ni1nY206WWd1c0gyTVdBOFBXYzNwMlZEc1I3QVZ2@81.19.223.189:31764#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%20%208' self.__config_path = '' + path = Config.get_v2ray_node_path() # 获取本地文件实例? parent_dir = os.path.dirname(os.path.abspath(__file__)) - if not path: - self.__path = os.path.join(parent_dir, '_db-checked.txt') - else: - self.__path = path + if path: + parent_dir = path + if not os.path.exists(parent_dir): + os.mkdir(parent_dir) + self.__path = os.path.join(parent_dir, '_db-checked.txt') if not os.path.isfile(self.__path): open(self.__path, mode='w', encoding="utf-8").write('') @@ -181,7 +185,7 @@ def init(self, config_path, def_url=None): """ if def_url is not None: self.__default_url = def_url - if len(config_path) < len('config.json'): + if len(config_path) < len('(参考用)config.json'): return False if not os.path.isfile(config_path): return False diff --git a/002-V2rayPool/db/net.py b/002-V2rayPool/db/net.py index 2170586..2d21fbe 100644 --- a/002-V2rayPool/db/net.py +++ b/002-V2rayPool/db/net.py @@ -6,11 +6,13 @@ @Author :xhunmon @Mail :xhunmon@gmail.com """ +import json from base.net_proxy import Net import re import time +import chardet def re_vmess_ss_trojan(pattern, html) -> []: @@ -39,17 +41,23 @@ def re_vmess_ss_trojan(pattern, html) -> []: class PYCheck(Net): - def get_curren_ip(self): + def get_curren_ip(self, url='https://ip.cn/api/index?ip=&type=0'): """获取内容""" try: - r = self.request_zh('https://2021.ip138.com') - if r.status_code != 200: - return None - r.encoding = r.apparent_encoding - results = re.findall(r'\[(.+?).+?:(.+?)\n

', r.text, re.DOTALL) - if not results: - return None - return results[0] + r = self.request_zh(url) + if r.status_code == 200: + charset = chardet.detect(r.content) + content = r.content.decode(charset['encoding']) + r.encoding = r.apparent_encoding + # {"rs":1,"code":0,"address":"德国 Hessen ","ip":"51.38.122.98","isDomain":0} + results = json.loads(content) + if not results: + return None + return [results['ip'], results['address']] + elif r.status_code == 301 or r.status_code == 302 or r.status_code == 303: + location = r.headers['Location'] + time.sleep(1) + return self.get_curren_ip(location) except Exception as e: print(e) return None @@ -72,6 +80,23 @@ def get_urls(self) -> []: return None +class PNTWGithubV2ray(Net): + """ + # https://hub.xn--gzu630h.xn--kpry57d/freefq/free + """ + + def get_urls(self) -> []: + try: + r = self.request(r'https://hub.xn--gzu630h.xn--kpry57d/freefq/free') + if r.status_code != 200: + return None + r.encoding = r.apparent_encoding + return re_vmess_ss_trojan(r'"%s"', r.text) + except Exception as e: + print(e) + return None + + class PNSsfree(Net): """ https://view.ssfree.ru/ diff --git a/002-V2rayPool/core/config.json "b/002-V2rayPool/doc/(345円217円202円350円200円203円347円224円250円)config.json" similarity index 59% rename from 002-V2rayPool/core/config.json rename to "002-V2rayPool/doc/(345円217円202円350円200円203円347円224円250円)config.json" index 9db50d8..1487683 100644 --- a/002-V2rayPool/core/config.json +++ "b/002-V2rayPool/doc/(345円217円202円350円200円203円347円224円250円)config.json" @@ -16,38 +16,29 @@ "clients": null }, "streamSettings": null + }, + { + "listen": "127.0.0.1", + "protocol": "http", + "settings": { + "timeout": 360 + }, + "port": "1087" } ], "outbounds": [ { - "protocol": "vmess", + "protocol": "shadowsocks", "settings": { - "vnext": [ + "servers": [ { - "address": "api.ssfree.ru", - "port": 443, - "users": [ - { - "id": "24a39774-1bbe-11ec-a4ee-000017022008", - "alterId": 64, - "security": "aes-128-gcm" - } - ] + "address": "167.88.61.60", + "method": "aes-256-gcm", + "ota": false, + "password": "RexnBgU7EV5ADxG", + "port": 7002 } ] - }, - "streamSettings": { - "security": "", - "tlsSettings": {}, - "wsSettings": {}, - "httpSettings": {}, - "network": "tcp", - "kcpSettings": {}, - "tcpSettings": {}, - "quicSettings": {} - }, - "mux": { - "enabled": true } }, { @@ -58,13 +49,6 @@ "tag": "direct" } ], - "dns": { - "servers": [ - "8.8.8.8", - "8.8.4.4", - "localhost" - ] - }, "routing": { "domainStrategy": "IPIfNonMatch", "rules": [ diff --git a/002-V2rayPool/main.py b/002-V2rayPool/main.py deleted file mode 100644 index b52a5bc..0000000 --- a/002-V2rayPool/main.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- -""" -@Description: 主要入口 -@Date :2021年08月25日 -@Author :xhunmon -@Mail :xhunmon@gmail.com -""" -from core.conf import Config -from db.db_main import * - -EXIT_NUM = 100 - -if __name__ == '__main__': - Config.set_v2ray_core_path('xxx/Downloads/v2ray-macos-64') - url = 'ss://YWVzLTI1Ni1nY206WXlDQmVEZFlYNGNhZEhwQ2trbWRKTHE4@37.120.144.211:43893#github.com/freefq%20-%20%E7%BD%97%E9%A9%AC%E5%B0%BC%E4%BA%9A%20%2041' - if check_url_single(url): - urls = load_urls_by_net(proxy_url=url) - check_and_save(urls, append=False) - # print(urls) - # urls = load_unchecked_urls_by_local() - # urls = load_enable_urls_by_local() - # load_urls_and_save_auto() diff --git a/002-V2rayPool/test_main.py b/002-V2rayPool/test_main.py new file mode 100644 index 0000000..fc96891 --- /dev/null +++ b/002-V2rayPool/test_main.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 主要入口 +@Date :2021年08月25日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +from core import utils +from core.conf import Config +from db.db_main import * + +EXIT_NUM = 100 + +if __name__ == '__main__': + utils.kill_all_v2ray() + Config.set_v2ray_core_path('/Users/Qincji/Desktop/develop/soft/intalled/v2ray-macos-64') # v2ray内核存放路径 + Config.set_v2ray_node_path('/Users/Qincji/Desktop/develop/py/project/PythonIsTools/002-V2rayPool') # 保存获取到节点的路径 + proxy_url = 'vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogIkBTU1JTVUItVjUyLeS7mOi0ueaOqOiNkDpzdW8ueXQvc3Nyc3ViIiwNCiAgImFkZCI6ICIxMTIuMzMuMzIuMTM2IiwNCiAgInBvcnQiOiAiMTAwMDMiLA0KICAiaWQiOiAiNjVjYWM1NmQtNDE1NS00M2M4LWJhZTAtZjM2OGNiMjFmNzcxIiwNCiAgImFpZCI6ICIxIiwNCiAgInNjeSI6ICJhdXRvIiwNCiAgIm5ldCI6ICJ0Y3AiLA0KICAidHlwZSI6ICJub25lIiwNCiAgImhvc3QiOiAiMTEyLjMzLjMyLjEzNiIsDQogICJwYXRoIjogIiIsDQogICJ0bHMiOiAiIiwNCiAgInNuaSI6ICIiDQp9' + dbm = DBManage() + dbm.init() # 必须初始化 + if dbm.check_url_single(proxy_url): + urls = dbm.load_urls_by_net(proxy_url=proxy_url) + dbm.check_and_save(urls, append=False) + # print(urls) + # urls = dbm.load_unchecked_urls_by_local() + # dbm.check_and_save(urls, append=False) + # urls = dbm.load_enable_urls_by_local() + # dbm.load_urls_and_save_auto() + utils.kill_all_v2ray() diff --git a/003-Keywords/.gitignore b/003-Keywords/.gitignore new file mode 100644 index 0000000..b6e4761 --- /dev/null +++ b/003-Keywords/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/003-Keywords/Necklace/Necklace.jpg b/003-Keywords/Necklace/Necklace.jpg new file mode 100644 index 0000000..3c6f070 Binary files /dev/null and b/003-Keywords/Necklace/Necklace.jpg differ diff --git a/003-Keywords/Necklace/Necklace.xlsx b/003-Keywords/Necklace/Necklace.xlsx new file mode 100644 index 0000000..249db9e Binary files /dev/null and b/003-Keywords/Necklace/Necklace.xlsx differ diff --git a/003-Keywords/README.md b/003-Keywords/README.md new file mode 100644 index 0000000..0601159 --- /dev/null +++ b/003-Keywords/README.md @@ -0,0 +1,20 @@ +# 通过google trends查找相关关键词,并且生成趋势 + +## 效果 + +例子:通过`women ring`关键词查找出有1千个相关关键词:[women ring.csv](women-ring/women ring.csv) + +![](women-ring/2.jpg) + +以及其生成关键词趋势,如:[swarovski rings.jpg](women-ring/swarovski rings.jpg) + +![](women-ring/1.jpg) + +![](women-ring/3.jpg) + +## 实现 +1. 使用[pytrends](https://github.com/GeneralMills/pytrends) 开源库。 +2. 使用[002-V2rayPool](../002-V2rayPool) 代理(可选择第三方代理)。 +3. 实现入口请参照:[main.py](main.py) + +> 注:项目架构是使用[Scrapy](https://www.osgeo.cn/scrapy/intro/overview.html) 实现的,可实现amazon关键词查询等。 \ No newline at end of file diff --git a/002-V2rayPool/db/_db-uncheck.txt b/003-Keywords/__init__.py similarity index 100% rename from 002-V2rayPool/db/_db-uncheck.txt rename to 003-Keywords/__init__.py diff --git a/003-Keywords/amazon/items.py b/003-Keywords/amazon/items.py new file mode 100644 index 0000000..6d9650d --- /dev/null +++ b/003-Keywords/amazon/items.py @@ -0,0 +1,12 @@ +# Define here the models for your scraped items +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class AmazonItem(scrapy.Item): + # define the fields for your item here like: + # name = scrapy.Field() + pass diff --git a/003-Keywords/amazon/middlewares.py b/003-Keywords/amazon/middlewares.py new file mode 100644 index 0000000..7b40bc4 --- /dev/null +++ b/003-Keywords/amazon/middlewares.py @@ -0,0 +1,132 @@ +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html +import requests +from scrapy import signals + +# useful for handling different item types with a single interface +from scrapy.http import TextResponse + + +class AmazonSpiderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, or item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request or item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class AmazonDownloaderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class AmazonProxyMiddleware(object): + def process_request(self, request, spider): + print('执行AmazonProxyMiddleware......') + # Set the location of the proxy + proxy_url = "socks5://127.0.0.1:1080" + request.meta['proxy'] = proxy_url + # 考虑socks代理,使用requests库进行请求 + if proxy_url.startswith('socks'): + url = request.url + method = request.method + headers = {key: request.headers[key] for key in request.headers} + body = request.body + cookies = request.cookies + timeout = request.meta.get('download_timeout', 10) + proxies = {'http': proxy_url, + 'https': proxy_url} + + resp = requests.request(method, url, + data=body, + headers=headers, + cookies=cookies, + verify=False, timeout=timeout, proxies=proxies) + resp.headers['content-encoding'] = None + response = TextResponse(url=url, headers=resp.headers, body=resp.content, + request=request, encoding=resp.encoding) + return response + return None \ No newline at end of file diff --git a/003-Keywords/amazon/pipelines.py b/003-Keywords/amazon/pipelines.py new file mode 100644 index 0000000..ace0ff9 --- /dev/null +++ b/003-Keywords/amazon/pipelines.py @@ -0,0 +1,13 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html + + +# useful for handling different item types with a single interface +from itemadapter import ItemAdapter + + +class AmazonPipeline: + def process_item(self, item, spider): + return item diff --git a/003-Keywords/amazon/settings.py b/003-Keywords/amazon/settings.py new file mode 100644 index 0000000..0dd7495 --- /dev/null +++ b/003-Keywords/amazon/settings.py @@ -0,0 +1,89 @@ +# Scrapy settings for amazon project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = 'amazon' + +SPIDER_MODULES = ['amazon.spiders'] +NEWSPIDER_MODULE = 'amazon.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = 'amazon (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = False + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +DOWNLOAD_DELAY = 2 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'amazon.middlewares.AmazonSpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# DOWNLOADER_MIDDLEWARES = { +# # 'amazon.middlewares.AmazonDownloaderMiddleware': 543, +# 'amazon.middlewares.AmazonProxyMiddleware': 543 +# } + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +#ITEM_PIPELINES = { +# 'amazon.pipelines.AmazonPipeline': 300, +#} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/003-Keywords/amazon/spiders/__init__.py b/003-Keywords/amazon/spiders/__init__.py new file mode 100644 index 0000000..ebd689a --- /dev/null +++ b/003-Keywords/amazon/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/003-Keywords/amazon/spiders/alibaba.py b/003-Keywords/amazon/spiders/alibaba.py new file mode 100644 index 0000000..79b4818 --- /dev/null +++ b/003-Keywords/amazon/spiders/alibaba.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 亚马逊相关关键词获取 +@Date :2021年09月24日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import re +from urllib.parse import quote_plus + +import scrapy +from my_fake_useragent import UserAgent + + +class AlibabaSpider(scrapy.Spider): + name = 'alibaba' + allowed_domains = ['alibaba.com'] + results = [] + keywords = [] + headers = { + 'Host': 'www.alibaba.com', + 'User-Agent': UserAgent().random() + } + + def start_requests(self): + print('alibaba start_requests') + """ + start_requests做为程序的入口,可以重写,自定义第一批请求 + + """ + start_urls = ['https://www.alibaba.com/trade/search?fsb=y&IndexArea=product_en&CatId=&SearchText={k}'.format( + k=quote_plus(k)) for k in self.keywords] + # , meta={'proxy': 'socks5h://127.0.0.1:1080/'} + for url in start_urls: + yield scrapy.Request(url, headers=self.headers, + callback=self.parse) + + def parse(self, response): + print('alibaba parse') + with open('alibaba.html', mode='w') as f: + f.write(response.text) \ No newline at end of file diff --git a/003-Keywords/amazon/spiders/amazon.py b/003-Keywords/amazon/spiders/amazon.py new file mode 100644 index 0000000..72b4abc --- /dev/null +++ b/003-Keywords/amazon/spiders/amazon.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 亚马逊相关关键词获取 +@Date :2021年09月24日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import re +from urllib.parse import quote_plus + +import scrapy +from my_fake_useragent import UserAgent + + +class AmazonSpider(scrapy.Spider): + name = 'amazon' + allowed_domains = ['amazon.com'] + results = [] + keywords = [] + headers = { + 'Host': 'www.amazon.com', + 'User-Agent': UserAgent().random() + } + + def start_requests(self): + print('amazon start_requests') + """ + start_requests做为程序的入口,可以重写,自定义第一批请求 + """ + start_urls = ['https://www.amazon.com/s?k={k}'.format(k=quote_plus(k)) for k in self.keywords] + # , meta={'proxy': 'socks5h://127.0.0.1:1080/'} + for url in start_urls: + yield scrapy.Request(url, headers=self.headers, + callback=self.parse) + + def parse(self, response): + print('amazon parse') + temps = re.findall(r'(.+?)', response.text, + re.DOTALL) + deal = [x.replace('\n', '').strip() for x in temps] + print(deal) + self.results += deal diff --git a/003-Keywords/amazon/spiders/checkip.py b/003-Keywords/amazon/spiders/checkip.py new file mode 100644 index 0000000..17bdde9 --- /dev/null +++ b/003-Keywords/amazon/spiders/checkip.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 检查当前代理是否起作用 +@Date :2021年09月24日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import re + +import scrapy +from my_fake_useragent import UserAgent + + +class CheckIpSpider(scrapy.Spider): + name = 'ip138' + allowed_domains = ['ip138.com'] + ips = None + headers = { + 'User-Agent': UserAgent().random() + } + + def start_requests(self): + print('CheckIpSpider start_requests') + """ + start_requests做为程序的入口,可以重写,自定义第一批请求 + """ + start_urls = ['https://2021.ip138.com'] + # , meta={'proxy': 'socks5h://127.0.0.1:1080/'} + for url in start_urls: + yield scrapy.Request(url, headers=self.headers, + callback=self.parse) + + def parse(self, response): + print('CheckIpSpider parse') + results = re.findall(r'\[(.+?).+?:(.+?)\n

', response.text, re.DOTALL) + print(results) + self.ips = results diff --git a/003-Keywords/amazon/spiders/spiders.py b/003-Keywords/amazon/spiders/spiders.py new file mode 100644 index 0000000..5053ce1 --- /dev/null +++ b/003-Keywords/amazon/spiders/spiders.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 爬虫页面集合 +@Date :2021年09月26日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import re +from urllib.parse import quote_plus + +import scrapy +from my_fake_useragent import UserAgent + + +class CheckIpSpider(scrapy.Spider): + """ + 检查当前代理ip信息 + """ + name = 'ip138' + allowed_domains = ['ip138.com'] + ips = None + headers = { + 'User-Agent': UserAgent().random() + } + + def start_requests(self): + print('CheckIpSpider start_requests') + """ + start_requests做为程序的入口,可以重写,自定义第一批请求 + """ + start_urls = ['https://2021.ip138.com'] + # , meta={'proxy': 'socks5h://127.0.0.1:1080/'} + for url in start_urls: + yield scrapy.Request(url, headers=self.headers, + callback=self.parse) + + def parse(self, response): + print('CheckIpSpider parse') + results = re.findall(r'\[(.+?).+?:(.+?)\n

', response.text, re.DOTALL) + print(results) + self.ips = results + + +class AmazonSpider(scrapy.Spider): + """ + https://www.amazon.com/s?k=?? + 亚马逊页面获取搜索词相关: + """ + name = 'amazon' + allowed_domains = ['amazon.com'] + results = [] + keywords = [] + headers = { + 'Host': 'www.amazon.com', + 'User-Agent': UserAgent().random() + } + + def start_requests(self): + print('amazon start_requests') + """ + start_requests做为程序的入口,可以重写,自定义第一批请求 + """ + start_urls = ['https://www.amazon.com/s?k={k}'.format(k=quote_plus(k)) for k in self.keywords] + # , meta={'proxy': 'socks5h://127.0.0.1:1080/'} + for url in start_urls: + yield scrapy.Request(url, headers=self.headers, + callback=self.parse) + + def parse(self, response): + print('amazon parse') + temps = re.findall(r'(.+?)', response.text, + re.DOTALL) + deal = [x.replace('\n', '').strip() for x in temps] + print(deal) + self.results += deal + + +class EtsySpider(scrapy.Spider): + """需要连接外网 + https://www.etsy.com/market/ + """ + name = 'etsy' + allowed_domains = ['etsy.com'] + results = [] + keywords = [] + headers = { + 'User-Agent': UserAgent().random() + } + + def start_requests(self): + print('etsy start_requests') + """ + start_requests做为程序的入口,可以重写,自定义第一批请求 + """ + start_urls = ['https://www.etsy.com/market/{k}'.format(k=quote_plus(k)) for k in self.keywords] + # , meta={'proxy': 'socks5h://127.0.0.1:1080/'} + for url in start_urls: + yield scrapy.Request(url, headers=self.headers, + callback=self.parse) + + def parse(self, response): + print('etsy parse') + with open('etsy.html', mode='w') as f: + f.write(response.text) + f.close() + temps = re.findall(r'(.+?)', response.text, + re.DOTALL) + deal = [x.replace('\n', '').strip() for x in temps] + print(deal) + self.results += deal + + +class LakesideSpider(scrapy.Spider): + """一个商城网站 + https://www.lakeside.com/browse/Clothing-Accessories + """ + name = 'lakeside' + allowed_domains = ['lakeside.com'] + results = [] + keywords = [] + headers = { + 'User-Agent': UserAgent().random() + } + + def start_requests(self): + print('lakeside start_requests') + """ + start_requests做为程序的入口,可以重写,自定义第一批请求 + """ + start_urls = ['https://www.lakeside.com/browse/{k}'.format(k=quote_plus(k)) for k in self.keywords] + # , meta={'proxy': 'socks5h://127.0.0.1:1080/'} + for url in start_urls: + yield scrapy.Request(url, headers=self.headers, + callback=self.parse) + + def parse(self, response): + print('lakeside parse') + with open('lakeside.html', mode='w') as f: + f.write(response.text) + f.close() + # temps = re.findall(r'(.+?)', response.text, + # re.DOTALL) + # deal = [x.replace('\n', '').strip() for x in temps] + # print(deal) + # self.results += deal + + +class WordtrackerSpider(scrapy.Spider): + """ + https://www.wordtracker.com/search?query=food%20bags + """ + name = 'wordtracker' + allowed_domains = ['wordtracker.com'] + results = [] + keywords = [] + headers = { + 'User-Agent': UserAgent().random() + } + + def start_requests(self): + print('wordtracker start_requests') + """ + start_requests做为程序的入口,可以重写,自定义第一批请求 + """ + start_urls = ['https://www.wordtracker.com/search?query={k}'.format(k=quote_plus(k)) for k in self.keywords] + # , meta={'proxy': 'socks5h://127.0.0.1:1080/'} + for url in start_urls: + yield scrapy.Request(url, headers=self.headers, + callback=self.parse) + + def parse(self, response): + print('wordtracker parse') + with open('wordtracker.html', mode='w') as f: + f.write(response.text) + f.close() + temps = re.findall(r'(.+?)', response.text, + re.DOTALL) + deal = [x.replace('\n', '').strip() for x in temps] + print(deal) + self.results += deal diff --git a/003-Keywords/google.py b/003-Keywords/google.py new file mode 100644 index 0000000..9cdf56a --- /dev/null +++ b/003-Keywords/google.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: google相关获取 +@Date :2021年10月08日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import xlsxwriter +import os.path + +import matplotlib.pyplot as plt + +from mypytrends.request import TrendReq + + +class GoogleTrend(object): + def __init__(self): + self.data = {} + self.max_column = 0 + + def search(self, keyword, path, hl='en-US', proxies=False, retries=2, timeframe='2019-10-01 2022年01月01日'): + if not os.path.exists(path): + os.makedirs(path) + csv_file = os.path.join(path, "%s.xlsx" % keyword) + workbook = xlsxwriter.Workbook(csv_file) + # 设置整个工作薄的格式 + workbook.formats[0].set_align('vcenter') # 单元格垂直居中 + # workbook.formats[0].set_text_wrap() # 自动换行 + + worksheet = workbook.add_worksheet('sheet1') + i_row, i_column = 0, 0 + first_row = ['keyword', 'no', 'top keyword', 'top range', 'rising keyword', 'rising range'] + self.max_column = len(first_row) + for i in range(self.max_column): + worksheet.write(i_row, i, first_row[i]) + i_row += 1 + tr = self.__get_req(hl=hl, proxies=proxies, retries=retries) + i_row = self.__search_trends(i_row, worksheet, path, keyword, timeframe, tr) + first_data = self.__search_related_queries(keyword, timeframe, tr) + tops = first_data[keyword]['top'] + risings = first_data[keyword]['rising'] + top_size = len(tops) + rising_size = len(risings) + max_len = top_size if top_size> rising_size else rising_size + for i in range(max_len): + top_key, top_range, rising_key, rising_range = None, None, None, None + if i < top_size: + top_key = tops[i]['keyword'] + top_range = tops[i]['range'] + if i < rising_size: + rising_key = risings[i]['keyword'] + rising_range = risings[i]['range'] + max_datas = [keyword, i, top_key, top_range, rising_key, rising_range] + for j in range(len(max_datas)): + worksheet.write(i_row, j, max_datas[j]) + i_row += 1 + # self.__save_line(csv_file, [keyword, i, top_key, top_range, rising_key, rising_range]) + # self.__save_line(csv_file, ['', '']) # 换行 + i_row += 1 + for top in tops: + top_key = top['keyword'] + try: + i_row = self.__sub_search(top_key, i_row, worksheet, path, csv_file, timeframe, tr) + # self.__save_line(csv_file, ['', '']) # 换行 + i_row += 1 + except: + pass + for rising in risings: + rising_key = rising['keyword'] + try: + i_row = self.__sub_search(rising_key, i_row, worksheet, path, csv_file, timeframe, tr) + # self.__save_line(csv_file, ['', '']) # 换行 + i_row += 1 + except: + pass + workbook.close() + + def __sub_search(self, i_row, worksheet, keyword, path, csv_file, timeframe, tr): + i_row = self.__search_trends(i_row, worksheet, path, keyword, timeframe, tr) + first_data = self.__search_related_queries(keyword, timeframe, tr) + tops = first_data[keyword]['top'] + risings = first_data[keyword]['rising'] + top_size = len(tops) + rising_size = len(risings) + max_len = top_size if top_size> rising_size else rising_size + for i in range(max_len): + top_key, top_range, rising_key, rising_range = None, None, None, None + if i < top_size: + top_key = tops[i]['keyword'] + top_range = tops[i]['range'] + try: + i_row = self.__search_trends(i_row, worksheet, path, top_key, timeframe, tr) + except: + pass + if i < rising_size: + rising_key = risings[i]['keyword'] + rising_range = risings[i]['range'] + try: + i_row = self.__search_trends(i_row, worksheet, path, rising_key, timeframe, tr) + except: + pass + # self.__save_line(csv_file, [keyword, i, top_key, top_range, rising_key, rising_range]) + max_datas = [keyword, i, top_key, top_range, rising_key, rising_range] + for j in range(len(max_datas)): + worksheet.write(i_row, j, max_datas[j]) + i_row += 1 + + return i_row + + def __search_related_queries(self, keyword, timeframe, tr: TrendReq) -> {}: + tr.build_payload([keyword, ], cat=0, timeframe=timeframe, geo='', gprop='') + related = tr.related_queries() + r_value = [related[key] for key in related][0] + r_top = r_value['top'] + r_rising = r_value['rising'] + tops = [] + risings = [] + print('---------top--------') + for index, row in r_top.iterrows(): + print(index, row["query"], row["value"]) + tops.append({"keyword": row["query"], "range": row["value"]}) + print('---------rising--------') + for index, row in r_rising.iterrows(): + print(index, row["query"], row["value"]) + risings.append({"keyword": row["query"], "range": row["value"]}) + return {keyword: {"top": tops, "rising": risings}} + + def __search_trends(self, i_row, worksheet, path, keyword, timeframe, tr: TrendReq): + tr.build_payload([keyword, ], cat=0, timeframe=timeframe, geo='', gprop='') + trends = tr.interest_over_time() + x_data = [] + y_data = [] + year = '' + month = '' + temp_value = 0 + count = 0 + for time, row in trends.iterrows(): + value = row[keyword] + date = str(time) + t = date.split(' ')[0] if ' ' in date else date + y = t.split('-')[0] + m = t.split('-')[1] + if month != m and month != '': + y_data.append(int(temp_value / count)) + x_data.append(year[2:] + "-" + month) + year = '' + month = '' + count = 0 + temp_value = 0 + continue + year = y + month = m + count += 1 + temp_value += value + y_data.append(int(temp_value / count)) + x_data.append(year[2:] + "-" + month) + print(y_data) + print(x_data) + print('%s : %d - %d' % (keyword, len(y_data), len(x_data))) + # self.__draw_graph(x_data, y_data, 'pci.jpg', keyword) + img_path = os.path.join(path, "%s.jpg" % keyword) + self.__draw_histogram(x_data, y_data, img_path, keyword) + worksheet.insert_image(i_row - 1, self.max_column, img_path, + {'x_scale': 0.2, 'y_scale': 0.2, 'object_position': 1}) + i_row += 1 + return i_row + + def __draw_histogram(self, x: [], y: [], path, title, x_name='date', y_name='trends'): + plt.figure(dpi=60) + plt.ylim(0, 100) + plt.style.use('ggplot') + plt.bar(x, y, label=title) + # 显示图例(使绘制生效) + plt.legend() + # 横坐标名称 + plt.xlabel(x_name) + # 纵坐标名称 + plt.ylabel(y_name) + # 横坐标显示倒立 + plt.xticks(rotation=90) + # 保存图片到本地 + plt.savefig(path) + # 显示图片 + # plt.show() + + def __draw_graph(self, x: [], y: [], path, title, x_name='date', y_name='trends'): + plt.figure() + '''绘制第一条数据线 + 1、节点为圆圈 + 2、线颜色为红色 + 3、标签名字为y1-data + ''' + plt.ylim(0, 100) + plt.plot(x, y, marker='o', color='r', label=title) + # 显示图例(使绘制生效) + plt.legend() + # 横坐标名称 + plt.xlabel(x_name) + # 纵坐标名称 + plt.ylabel(y_name) + # 横坐标显示倒立 + plt.xticks(rotation=90) + # 保存图片到本地 + plt.savefig(path) + # 显示图片 + # plt.show() + + def __save_line(self, path, line, mode='a'): + with open(path, mode=mode, encoding='utf-8', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(line) + csvfile.close() + + def __get_req(self, hl='en-US', proxies=False, retries=2) -> TrendReq: + if proxies: + return TrendReq(hl=hl, tz=360, timeout=(10, 35), proxies=['socks5h://127.0.0.1:1080', ], retries=retries, + backoff_factor=0.1, requests_args={'verify': False}) + else: + return TrendReq(hl=hl, tz=360, timeout=(10, 35), retries=retries, backoff_factor=0.1, + requests_args={'verify': False}) diff --git a/003-Keywords/main.py b/003-Keywords/main.py new file mode 100644 index 0000000..c96a38a --- /dev/null +++ b/003-Keywords/main.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 关键词获取 +@Date :2021年09月22日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +# from amazon import run_api +import os +import time + +import xlwt + +import run_api +import v2ray_util as v2util +from google import GoogleTrend +from openpyxl import load_workbook + + +def Write_Img(): + import xlsxwriter + book = xlsxwriter.Workbook('test_source.xlsx') + sheet = book.get_worksheet_by_name('Sheet1') + # sheet = book.add_worksheet('demo') + # sheet.insert_image(0, 5, 'Necklace/Necklace.jpg', {'x_scale': 0.2, 'y_scale': 0.2, 'object_position': 1}) + print(sheet) + book.close() + + +def read_xlsl(): + import pandas as pd + df = pd.read_excel('test_source.xlsx', sheet_name='Sheet1') + data = df.values + print(data) + print('\n') + df = pd.read_excel('test_souce1.xlsx', sheet_name='2021xuqiu') + data = df.values + print(data) + + + +if __name__ == "__main__": + # v2util.restart_v2ray() + # 获取amazon中相关词,代理需要看:middlewares.py, + # results = run_api.crawl_amazon(['plastic packaging', ]) + # 把通过google trends查出关键词相关词,以及其词的趋势图,如: + # GoogleTrend().search('Necklace', 'Necklace', proxies=True, timeframe='2021-01-01 2022年01月01日') + # v2util.kill_all_v2ray() + # Write_Img() + read_xlsl() diff --git a/003-Keywords/mypytrends/__init__.py b/003-Keywords/mypytrends/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/003-Keywords/mypytrends/dailydata.py b/003-Keywords/mypytrends/dailydata.py new file mode 100755 index 0000000..eece481 --- /dev/null +++ b/003-Keywords/mypytrends/dailydata.py @@ -0,0 +1,127 @@ +from datetime import date, timedelta +from functools import partial +from time import sleep +from calendar import monthrange + +import pandas as pd + +from mypytrends.exceptions import ResponseError +from mypytrends.request import TrendReq + + +def get_last_date_of_month(year: int, month: int) -> date: + """Given a year and a month returns an instance of the date class + containing the last day of the corresponding month. + + Source: https://stackoverflow.com/questions/42950/get-last-day-of-the-month-in-python + """ + return date(year, month, monthrange(year, month)[1]) + + +def convert_dates_to_timeframe(start: date, stop: date) -> str: + """Given two dates, returns a stringified version of the interval between + the two dates which is used to retrieve data for a specific time frame + from Google Trends. + """ + return f"{start.strftime('%Y-%m-%d')} {stop.strftime('%Y-%m-%d')}" + + +def _fetch_data(pytrends, build_payload, timeframe: str) -> pd.DataFrame: + """Attempts to fecth data and retries in case of a ResponseError.""" + attempts, fetched = 0, False + while not fetched: + try: + build_payload(timeframe=timeframe) + except ResponseError as err: + print(err) + print(f'Trying again in {60 + 5 * attempts} seconds.') + sleep(60 + 5 * attempts) + attempts += 1 + if attempts> 3: + print('Failed after 3 attemps, abort fetching.') + break + else: + fetched = True + return pytrends.interest_over_time() + + +def get_daily_data(word: str, + start_year: int, + start_mon: int, + stop_year: int, + stop_mon: int, + geo: str = 'US', + verbose: bool = True, + wait_time: float = 5.0) -> pd.DataFrame: + """Given a word, fetches daily search volume data from Google Trends and + returns results in a pandas DataFrame. + + Details: Due to the way Google Trends scales and returns data, special + care needs to be taken to make the daily data comparable over different + months. To do that, we download daily data on a month by month basis, + and also monthly data. The monthly data is downloaded in one go, so that + the monthly values are comparable amongst themselves and can be used to + scale the daily data. The daily data is scaled by multiplying the daily + value by the monthly search volume divided by 100. + For a more detailed explanation see http://bit.ly/trendsscaling + + Args: + word (str): Word to fetch daily data for. + start_year (int): the start year + start_mon (int): start 1st day of the month + stop_year (int): the end year + stop_mon (int): end at the last day of the month + geo (str): geolocation + verbose (bool): If True, then prints the word and current time frame + we are fecthing the data for. + + Returns: + complete (pd.DataFrame): Contains 4 columns. + The column named after the word argument contains the daily search + volume already scaled and comparable through time. + The column f'{word}_unscaled' is the original daily data fetched + month by month, and it is not comparable across different months + (but is comparable within a month). + The column f'{word}_monthly' contains the original monthly data + fetched at once. The values in this column have been backfilled + so that there are no NaN present. + The column 'scale' contains the scale used to obtain the scaled + daily data. + """ + + # Set up start and stop dates + start_date = date(start_year, start_mon, 1) + stop_date = get_last_date_of_month(stop_year, stop_mon) + + # Start pytrends for US region + pytrends = TrendReq(hl='en-US', tz=360) + # Initialize build_payload with the word we need data for + build_payload = partial(pytrends.build_payload, + kw_list=[word], cat=0, geo=geo, gprop='') + + # Obtain monthly data for all months in years [start_year, stop_year] + monthly = _fetch_data(pytrends, build_payload, + convert_dates_to_timeframe(start_date, stop_date)) + + # Get daily data, month by month + results = {} + # if a timeout or too many requests error occur we need to adjust wait time + current = start_date + while current < stop_date: + last_date_of_month = get_last_date_of_month(current.year, current.month) + timeframe = convert_dates_to_timeframe(current, last_date_of_month) + if verbose: + print(f'{word}:{timeframe}') + results[current] = _fetch_data(pytrends, build_payload, timeframe) + current = last_date_of_month + timedelta(days=1) + sleep(wait_time) # don't go too fast or Google will send 429s + + daily = pd.concat(results.values()).drop(columns=['isPartial']) + complete = daily.join(monthly, lsuffix='_unscaled', rsuffix='_monthly') + + # Scale daily data by monthly weights so the data is comparable + complete[f'{word}_monthly'].ffill(inplace=True) # fill NaN values + complete['scale'] = complete[f'{word}_monthly'] / 100 + complete[word] = complete[f'{word}_unscaled'] * complete.scale + + return complete diff --git a/003-Keywords/mypytrends/exceptions.py b/003-Keywords/mypytrends/exceptions.py new file mode 100755 index 0000000..0a24083 --- /dev/null +++ b/003-Keywords/mypytrends/exceptions.py @@ -0,0 +1,8 @@ +class ResponseError(Exception): + """Something was wrong with the response from Google""" + + def __init__(self, message, response): + super(Exception, self).__init__(message) + + # pass response so it can be handled upstream + self.response = response diff --git a/003-Keywords/mypytrends/request.py b/003-Keywords/mypytrends/request.py new file mode 100755 index 0000000..7f922f1 --- /dev/null +++ b/003-Keywords/mypytrends/request.py @@ -0,0 +1,560 @@ +import json +import sys +import time +from datetime import datetime, timedelta + +import pandas as pd +import requests + +from pandas.io.json._normalize import nested_to_record +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + +from mypytrends import exceptions + +from urllib.parse import quote + + +class TrendReq(object): + """ + Google Trends API + """ + GET_METHOD = 'get' + POST_METHOD = 'post' + GENERAL_URL = 'https://trends.google.com/trends/api/explore' + INTEREST_OVER_TIME_URL = 'https://trends.google.com/trends/api/widgetdata/multiline' + INTEREST_BY_REGION_URL = 'https://trends.google.com/trends/api/widgetdata/comparedgeo' + RELATED_QUERIES_URL = 'https://trends.google.com/trends/api/widgetdata/relatedsearches' + TRENDING_SEARCHES_URL = 'https://trends.google.com/trends/hottrends/visualize/internal/data' + TOP_CHARTS_URL = 'https://trends.google.com/trends/api/topcharts' + SUGGESTIONS_URL = 'https://trends.google.com/trends/api/autocomplete/' + CATEGORIES_URL = 'https://trends.google.com/trends/api/explore/pickers/category' + TODAY_SEARCHES_URL = 'https://trends.google.com/trends/api/dailytrends' + ERROR_CODES = (500, 502, 504, 429) + + def __init__(self, hl='en-US', tz=360, geo='', timeout=(2, 5), proxies='', + retries=0, backoff_factor=0, requests_args=None): + """ + Initialize default values for params + """ + # google rate limit + self.google_rl = 'You have reached your quota limit. Please try again later.' + self.results = None + # set user defined options used globally + self.tz = tz + self.hl = hl + self.geo = geo + self.kw_list = list() + self.timeout = timeout + self.proxies = proxies # add a proxy option + self.retries = retries + self.backoff_factor = backoff_factor + self.proxy_index = 0 + self.requests_args = requests_args or {} + self.cookies = self.GetGoogleCookie() + # intialize widget payloads + self.token_payload = dict() + self.interest_over_time_widget = dict() + self.interest_by_region_widget = dict() + self.related_topics_widget_list = list() + self.related_queries_widget_list = list() + + def GetGoogleCookie(self): + """ + Gets google cookie (used for each and every proxy; once on init otherwise) + Removes proxy from the list on proxy error + """ + while True: + if "proxies" in self.requests_args: + try: + return dict(filter(lambda i: i[0] == 'NID', requests.get( + 'https://trends.google.com/?geo={geo}'.format( + geo=self.hl[-2:]), + timeout=self.timeout, + **self.requests_args + ).cookies.items())) + except: + continue + else: + if len(self.proxies)> 0: + proxy = {'https': self.proxies[self.proxy_index]} + else: + proxy = '' + try: + return dict(filter(lambda i: i[0] == 'NID', requests.get( + 'https://trends.google.com/?geo={geo}'.format( + geo=self.hl[-2:]), + timeout=self.timeout, + proxies=proxy, + **self.requests_args + ).cookies.items())) + except requests.exceptions.ProxyError: + print('Proxy error. Changing IP') + if len(self.proxies)> 1: + self.proxies.remove(self.proxies[self.proxy_index]) + else: + print('No more proxies available. Bye!') + raise + continue + + def GetNewProxy(self): + """ + Increment proxy INDEX; zero on overflow + """ + if self.proxy_index < (len(self.proxies) - 1): + self.proxy_index += 1 + else: + self.proxy_index = 0 + + def _get_data(self, url, method=GET_METHOD, trim_chars=0, **kwargs): + """Send a request to Google and return the JSON response as a Python object + :param url: the url to which the request will be sent + :param method: the HTTP method ('get' or 'post') + :param trim_chars: how many characters should be trimmed off the beginning of the content of the response + before this is passed to the JSON parser + :param kwargs: any extra key arguments passed to the request builder (usually query parameters or data) + :return: + """ + s = requests.session() + # Retries mechanism. Activated when one of statements>0 (best used for proxy) + if self.retries> 0 or self.backoff_factor> 0: + retry = Retry(total=self.retries, read=self.retries, + connect=self.retries, + backoff_factor=self.backoff_factor, + status_forcelist=TrendReq.ERROR_CODES, + method_whitelist=frozenset(['GET', 'POST'])) + s.mount('https://', HTTPAdapter(max_retries=retry)) + + s.headers.update({'accept-language': self.hl}) + if len(self.proxies)> 0: + self.cookies = self.GetGoogleCookie() + s.proxies.update({'https': self.proxies[self.proxy_index]}) + if method == TrendReq.POST_METHOD: + response = s.post(url, timeout=self.timeout, + cookies=self.cookies, **kwargs, + **self.requests_args) # DO NOT USE retries or backoff_factor here + else: + response = s.get(url, timeout=self.timeout, cookies=self.cookies, + **kwargs, **self.requests_args) # DO NOT USE retries or backoff_factor here + # check if the response contains json and throw an exception otherwise + # Google mostly sends 'application/json' in the Content-Type header, + # but occasionally it sends 'application/javascript + # and sometimes even 'text/javascript + if response.status_code == 200 and 'application/json' in \ + response.headers['Content-Type'] or \ + 'application/javascript' in response.headers['Content-Type'] or \ + 'text/javascript' in response.headers['Content-Type']: + # trim initial characters + # some responses start with garbage characters, like ")]}'," + # these have to be cleaned before being passed to the json parser + content = response.text[trim_chars:] + # parse json + self.GetNewProxy() + return json.loads(content) + else: + # error + raise exceptions.ResponseError( + 'The request failed: Google returned a ' + 'response with code {0}.'.format(response.status_code), + response=response) + + def build_payload(self, kw_list, cat=0, timeframe='today 5-y', geo='', + gprop=''): + """Create the payload for related queries, interest over time and interest by region""" + if gprop not in ['', 'images', 'news', 'youtube', 'froogle']: + raise ValueError('gprop must be empty (to indicate web), images, news, youtube, or froogle') + self.kw_list = kw_list + self.geo = geo or self.geo + self.token_payload = { + 'hl': self.hl, + 'tz': self.tz, + 'req': {'comparisonItem': [], 'category': cat, 'property': gprop} + } + + # build out json for each keyword + for kw in self.kw_list: + keyword_payload = {'keyword': kw, 'time': timeframe, + 'geo': self.geo} + self.token_payload['req']['comparisonItem'].append(keyword_payload) + # requests will mangle this if it is not a string + self.token_payload['req'] = json.dumps(self.token_payload['req']) + # get tokens + self._tokens() + return + + def _tokens(self): + """Makes request to Google to get API tokens for interest over time, interest by region and related queries""" + # make the request and parse the returned json + widget_dicts = self._get_data( + url=TrendReq.GENERAL_URL, + method=TrendReq.GET_METHOD, + params=self.token_payload, + trim_chars=4, + )['widgets'] + # order of the json matters... + first_region_token = True + # clear self.related_queries_widget_list and self.related_topics_widget_list + # of old keywords'widgets + self.related_queries_widget_list[:] = [] + self.related_topics_widget_list[:] = [] + # assign requests + for widget in widget_dicts: + if widget['id'] == 'TIMESERIES': + self.interest_over_time_widget = widget + if widget['id'] == 'GEO_MAP' and first_region_token: + self.interest_by_region_widget = widget + first_region_token = False + # response for each term, put into a list + if 'RELATED_TOPICS' in widget['id']: + self.related_topics_widget_list.append(widget) + if 'RELATED_QUERIES' in widget['id']: + self.related_queries_widget_list.append(widget) + return + + def interest_over_time(self): + """Request data from Google's Interest Over Time section and return a dataframe""" + + over_time_payload = { + # convert to string as requests will mangle + 'req': json.dumps(self.interest_over_time_widget['request']), + 'token': self.interest_over_time_widget['token'], + 'tz': self.tz + } + + # make the request and parse the returned json + req_json = self._get_data( + url=TrendReq.INTEREST_OVER_TIME_URL, + method=TrendReq.GET_METHOD, + trim_chars=5, + params=over_time_payload, + ) + + df = pd.DataFrame(req_json['default']['timelineData']) + if (df.empty): + return df + + df['date'] = pd.to_datetime(df['time'].astype(dtype='float64'), + unit='s') + df = df.set_index(['date']).sort_index() + # split list columns into seperate ones, remove brackets and split on comma + result_df = df['value'].apply(lambda x: pd.Series( + str(x).replace('[', '').replace(']', '').split(','))) + # rename each column with its search term, relying on order that google provides... + for idx, kw in enumerate(self.kw_list): + # there is currently a bug with assigning columns that may be + # parsed as a date in pandas: use explicit insert column method + result_df.insert(len(result_df.columns), kw, + result_df[idx].astype('int')) + del result_df[idx] + + if 'isPartial' in df: + # make other dataframe from isPartial key data + # split list columns into seperate ones, remove brackets and split on comma + df = df.fillna(False) + result_df2 = df['isPartial'].apply(lambda x: pd.Series( + str(x).replace('[', '').replace(']', '').split(','))) + result_df2.columns = ['isPartial'] + # Change to a bool type. + result_df2.isPartial = result_df2.isPartial == 'True' + # concatenate the two dataframes + final = pd.concat([result_df, result_df2], axis=1) + else: + final = result_df + final['isPartial'] = False + + return final + + def interest_by_region(self, resolution='COUNTRY', inc_low_vol=False, + inc_geo_code=False): + """Request data from Google's Interest by Region section and return a dataframe""" + + # make the request + region_payload = dict() + if self.geo == '': + self.interest_by_region_widget['request'][ + 'resolution'] = resolution + elif self.geo == 'US' and resolution in ['DMA', 'CITY', 'REGION']: + self.interest_by_region_widget['request'][ + 'resolution'] = resolution + + self.interest_by_region_widget['request'][ + 'includeLowSearchVolumeGeos'] = inc_low_vol + + # convert to string as requests will mangle + region_payload['req'] = json.dumps( + self.interest_by_region_widget['request']) + region_payload['token'] = self.interest_by_region_widget['token'] + region_payload['tz'] = self.tz + + # parse returned json + req_json = self._get_data( + url=TrendReq.INTEREST_BY_REGION_URL, + method=TrendReq.GET_METHOD, + trim_chars=5, + params=region_payload, + ) + df = pd.DataFrame(req_json['default']['geoMapData']) + if (df.empty): + return df + + # rename the column with the search keyword + df = df[['geoName', 'geoCode', 'value']].set_index( + ['geoName']).sort_index() + # split list columns into separate ones, remove brackets and split on comma + result_df = df['value'].apply(lambda x: pd.Series( + str(x).replace('[', '').replace(']', '').split(','))) + if inc_geo_code: + result_df['geoCode'] = df['geoCode'] + + # rename each column with its search term + for idx, kw in enumerate(self.kw_list): + result_df[kw] = result_df[idx].astype('int') + del result_df[idx] + + return result_df + + def related_topics(self): + """Request data from Google's Related Topics section and return a dictionary of dataframes + + If no top and/or rising related topics are found, the value for the key "top" and/or "rising" will be None + """ + + # make the request + related_payload = dict() + result_dict = dict() + for request_json in self.related_topics_widget_list: + # ensure we know which keyword we are looking at rather than relying on order + kw = request_json['request']['restriction'][ + 'complexKeywordsRestriction']['keyword'][0]['value'] + # convert to string as requests will mangle + related_payload['req'] = json.dumps(request_json['request']) + related_payload['token'] = request_json['token'] + related_payload['tz'] = self.tz + + # parse the returned json + req_json = self._get_data( + url=TrendReq.RELATED_QUERIES_URL, + method=TrendReq.GET_METHOD, + trim_chars=5, + params=related_payload, + ) + + # top topics + try: + top_list = req_json['default']['rankedList'][0][ + 'rankedKeyword'] + df_top = pd.DataFrame( + [nested_to_record(d, sep='_') for d in top_list]) + except KeyError: + # in case no top topics are found, the lines above will throw a KeyError + df_top = None + + # rising topics + try: + rising_list = req_json['default']['rankedList'][1][ + 'rankedKeyword'] + df_rising = pd.DataFrame( + [nested_to_record(d, sep='_') for d in rising_list]) + except KeyError: + # in case no rising topics are found, the lines above will throw a KeyError + df_rising = None + + result_dict[kw] = {'rising': df_rising, 'top': df_top} + return result_dict + + def related_queries(self): + """Request data from Google's Related Queries section and return a dictionary of dataframes + + If no top and/or rising related queries are found, the value for the key "top" and/or "rising" will be None + """ + + # make the request + related_payload = dict() + result_dict = dict() + for request_json in self.related_queries_widget_list: + # ensure we know which keyword we are looking at rather than relying on order + kw = request_json['request']['restriction'][ + 'complexKeywordsRestriction']['keyword'][0]['value'] + # convert to string as requests will mangle + related_payload['req'] = json.dumps(request_json['request']) + related_payload['token'] = request_json['token'] + related_payload['tz'] = self.tz + + # parse the returned json + req_json = self._get_data( + url=TrendReq.RELATED_QUERIES_URL, + method=TrendReq.GET_METHOD, + trim_chars=5, + params=related_payload, + ) + + # top queries + try: + top_df = pd.DataFrame( + req_json['default']['rankedList'][0]['rankedKeyword']) + top_df = top_df[['query', 'value']] + except KeyError: + # in case no top queries are found, the lines above will throw a KeyError + top_df = None + + # rising queries + try: + rising_df = pd.DataFrame( + req_json['default']['rankedList'][1]['rankedKeyword']) + rising_df = rising_df[['query', 'value']] + except KeyError: + # in case no rising queries are found, the lines above will throw a KeyError + rising_df = None + + result_dict[kw] = {'top': top_df, 'rising': rising_df} + return result_dict + + def trending_searches(self, pn='united_states'): + """Request data from Google's Hot Searches section and return a dataframe""" + + # make the request + # forms become obsolete due to the new TRENDING_SEARCHES_URL + # forms = {'ajax': 1, 'pn': pn, 'htd': '', 'htv': 'l'} + req_json = self._get_data( + url=TrendReq.TRENDING_SEARCHES_URL, + method=TrendReq.GET_METHOD + )[pn] + result_df = pd.DataFrame(req_json) + return result_df + + def today_searches(self, pn='US'): + """Request data from Google Daily Trends section and returns a dataframe""" + forms = {'ns': 15, 'geo': pn, 'tz': '-180', 'hl': 'en-US'} + req_json = self._get_data( + url=TrendReq.TODAY_SEARCHES_URL, + method=TrendReq.GET_METHOD, + trim_chars=5, + params=forms, + **self.requests_args + )['default']['trendingSearchesDays'][0]['trendingSearches'] + result_df = pd.DataFrame() + # parse the returned json + sub_df = pd.DataFrame() + for trend in req_json: + sub_df = sub_df.append(trend['title'], ignore_index=True) + result_df = pd.concat([result_df, sub_df]) + return result_df.iloc[:, -1] + + def top_charts(self, date, hl='en-US', tz=300, geo='GLOBAL'): + """Request data from Google's Top Charts section and return a dataframe""" + + try: + date = int(date) + except: + raise ValueError( + 'The date must be a year with format YYYY. See https://github.com/GeneralMills/pytrends/issues/355') + + # create the payload + chart_payload = {'hl': hl, 'tz': tz, 'date': date, 'geo': geo, + 'isMobile': False} + + # make the request and parse the returned json + req_json = self._get_data( + url=TrendReq.TOP_CHARTS_URL, + method=TrendReq.GET_METHOD, + trim_chars=5, + params=chart_payload, + **self.requests_args + ) + try: + df = pd.DataFrame(req_json['topCharts'][0]['listItems']) + except IndexError: + df = None + return df + + def suggestions(self, keyword): + """Request data from Google's Keyword Suggestion dropdown and return a dictionary""" + + # make the request + kw_param = quote(keyword) + parameters = {'hl': self.hl} + + req_json = self._get_data( + url=TrendReq.SUGGESTIONS_URL + kw_param, + params=parameters, + method=TrendReq.GET_METHOD, + trim_chars=5, + **self.requests_args + )['default']['topics'] + return req_json + + def categories(self): + """Request available categories data from Google's API and return a dictionary""" + + params = {'hl': self.hl} + + req_json = self._get_data( + url=TrendReq.CATEGORIES_URL, + params=params, + method=TrendReq.GET_METHOD, + trim_chars=5, + **self.requests_args + ) + return req_json + + def get_historical_interest(self, keywords, year_start=2018, month_start=1, + day_start=1, hour_start=0, year_end=2018, + month_end=2, day_end=1, hour_end=0, cat=0, + geo='', gprop='', sleep=0): + """Gets historical hourly data for interest by chunking requests to 1 week at a time (which is what Google allows)""" + + # construct datetime objects - raises ValueError if invalid parameters + initial_start_date = start_date = datetime(year_start, month_start, + day_start, hour_start) + end_date = datetime(year_end, month_end, day_end, hour_end) + + # the timeframe has to be in 1 week intervals or Google will reject it + delta = timedelta(days=7) + + df = pd.DataFrame() + + date_iterator = start_date + date_iterator += delta + + while True: + # format date to comply with API call + + start_date_str = start_date.strftime('%Y-%m-%dT%H') + date_iterator_str = date_iterator.strftime('%Y-%m-%dT%H') + + tf = start_date_str + ' ' + date_iterator_str + + try: + self.build_payload(keywords, cat, tf, geo, gprop) + week_df = self.interest_over_time() + df = df.append(week_df) + except Exception as e: + print(e) + pass + + start_date += delta + date_iterator += delta + + if (date_iterator> end_date): + # Run for 7 more days to get remaining data that would have been truncated if we stopped now + # This is needed because google requires 7 days yet we may end up with a week result less than a full week + start_date_str = start_date.strftime('%Y-%m-%dT%H') + date_iterator_str = date_iterator.strftime('%Y-%m-%dT%H') + + tf = start_date_str + ' ' + date_iterator_str + + try: + self.build_payload(keywords, cat, tf, geo, gprop) + week_df = self.interest_over_time() + df = df.append(week_df) + except Exception as e: + print(e) + pass + break + + # just in case you are rate-limited by Google. Recommended is 60 if you are. + if sleep> 0: + time.sleep(sleep) + + # Return the dataframe with results from our timeframe + return df.loc[initial_start_date:end_date] diff --git a/003-Keywords/mypytrends/test_trendReq.py b/003-Keywords/mypytrends/test_trendReq.py new file mode 100755 index 0000000..7b0149d --- /dev/null +++ b/003-Keywords/mypytrends/test_trendReq.py @@ -0,0 +1,99 @@ +from unittest import TestCase +import pandas.api.types as ptypes + +from mypytrends.request import TrendReq + + +class TestTrendReq(TestCase): + + def test__get_data(self): + """Should use same values as in the documentation""" + pytrend = TrendReq() + self.assertEqual(pytrend.hl, 'en-US') + self.assertEqual(pytrend.tz, 360) + self.assertEqual(pytrend.geo, '') + self.assertTrue(pytrend.cookies['NID']) + + def test_build_payload(self): + """Should return the widgets to get data""" + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel']) + self.assertIsNotNone(pytrend.token_payload) + + def test__tokens(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel']) + self.assertIsNotNone(pytrend.related_queries_widget_list) + + def test_interest_over_time(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel']) + self.assertIsNotNone(pytrend.interest_over_time()) + + def test_interest_over_time_images(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel'], gprop='images') + self.assertIsNotNone(pytrend.interest_over_time()) + + def test_interest_over_time_news(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel'], gprop='news') + self.assertIsNotNone(pytrend.interest_over_time()) + + def test_interest_over_time_youtube(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel'], gprop='youtube') + self.assertIsNotNone(pytrend.interest_over_time()) + + def test_interest_over_time_froogle(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel'], gprop='froogle') + self.assertIsNotNone(pytrend.interest_over_time()) + + def test_interest_over_time_bad_gprop(self): + pytrend = TrendReq() + with self.assertRaises(ValueError): + pytrend.build_payload(kw_list=['pizza', 'bagel'], gprop=' ') + + def test_interest_by_region(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel']) + self.assertIsNotNone(pytrend.interest_by_region()) + + def test_related_topics(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel']) + self.assertIsNotNone(pytrend.related_topics()) + + def test_related_queries(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel']) + self.assertIsNotNone(pytrend.related_queries()) + + def test_trending_searches(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel']) + self.assertIsNotNone(pytrend.trending_searches()) + + def test_top_charts(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel']) + self.assertIsNotNone(pytrend.top_charts(date=2019)) + + def test_suggestions(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel']) + self.assertIsNotNone(pytrend.suggestions(keyword='pizza')) + + def test_ispartial_dtype(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel']) + df = pytrend.interest_over_time() + assert ptypes.is_bool_dtype(df.isPartial) + + def test_ispartial_dtype_timeframe_all(self): + pytrend = TrendReq() + pytrend.build_payload(kw_list=['pizza', 'bagel'], + timeframe='all') + df = pytrend.interest_over_time() + assert ptypes.is_bool_dtype(df.isPartial) diff --git a/003-Keywords/run_api.py b/003-Keywords/run_api.py new file mode 100644 index 0000000..9f77bb5 --- /dev/null +++ b/003-Keywords/run_api.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 通过框架自带命令 启动命令脚本 +@Date :2021年09月20日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +from scrapy.crawler import CrawlerProcess +from scrapy.utils.log import configure_logging +from scrapy.utils.project import get_project_settings + +from amazon.spiders.alibaba import AlibabaSpider +from amazon.spiders.amazon import AmazonSpider +from amazon.spiders.checkip import CheckIpSpider + +settings = get_project_settings() +configure_logging(settings) +crawler = CrawlerProcess(settings) + + +def check_ip(): + spider = CheckIpSpider + crawler.crawl(spider) + crawler.start() + return spider.ips + + +def crawl_amazon(keywords: []): + spider = AmazonSpider + spider.keywords = keywords + crawler.crawl(spider) + crawler.start() + return spider.results + + +def crawl_alibaba(keywords: []): + spider = AlibabaSpider + spider.keywords = keywords + crawler.crawl(spider) + crawler.start() diff --git a/003-Keywords/scrapy.cfg b/003-Keywords/scrapy.cfg new file mode 100644 index 0000000..999e0f0 --- /dev/null +++ b/003-Keywords/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = amazon.settings + +[deploy] +#url = http://localhost:6800/ +project = amazon diff --git a/003-Keywords/test_souce1.xlsx b/003-Keywords/test_souce1.xlsx new file mode 100755 index 0000000..e0b8dd3 Binary files /dev/null and b/003-Keywords/test_souce1.xlsx differ diff --git a/003-Keywords/test_source.xlsx b/003-Keywords/test_source.xlsx new file mode 100755 index 0000000..9678be4 Binary files /dev/null and b/003-Keywords/test_source.xlsx differ diff --git a/003-Keywords/v2ray_pool/__init__.py b/003-Keywords/v2ray_pool/__init__.py new file mode 100644 index 0000000..7a4ce9d --- /dev/null +++ b/003-Keywords/v2ray_pool/__init__.py @@ -0,0 +1,13 @@ +# 运行时路径。并非__init__.py的路径 +import os +import sys + +BASE_DIR = "../002-V2rayPool" +if os.path.exists(BASE_DIR): + sys.path.append(BASE_DIR) + +from core import utils +from core.conf import Config +from core.client import Creator +from db.db_main import DBManage +from base.net_proxy import Net \ No newline at end of file diff --git a/003-Keywords/v2ray_pool/_db-checked.txt b/003-Keywords/v2ray_pool/_db-checked.txt new file mode 100644 index 0000000..7c33833 --- /dev/null +++ b/003-Keywords/v2ray_pool/_db-checked.txt @@ -0,0 +1,33 @@ +ss://YWVzLTI1Ni1nY206ZzVNZUQ2RnQzQ1dsSklkQDE0Mi4yMDIuNDguMTA4OjUwMDM#%E7%BE%8E%E5%9B%BD-2.48MB/s,142.202.48.108,加拿大 魁北克 魁北克 +ss://YWVzLTI1Ni1nY206ZmFCQW9ENTRrODdVSkc3QDM4LjY4LjEzNC4xOTE6MjM3NQ#%E7%BE%8E%E5%9B%BD-1.81MB/s(Youtube:%E4%B8%8D%E8%89%AF%E6%9E%97),38.68.134.191,美国 新泽西 科进 +ss://YWVzLTI1Ni1nY206ZzVNZUQ2RnQzQ1dsSklkQDE2OS4xOTcuMTQzLjE1Nzo1MDA0#%E7%BE%8E%E5%9B%BD-988.6KB/s(Youtube:%E4%B8%8D%E8%89%AF%E6%9E%97),169.197.143.157,美国 佐治亚 亚特兰大 +ss://YWVzLTI1Ni1nY206ZmFCQW9ENTRrODdVSkc3QDM4LjEyMS40My43MToyMzc2#%E7%BE%8E%E5%9B%BD-2.82MB/s,38.121.43.71,美国 加利福尼亚 科进 +ss://YWVzLTI1Ni1nY206Rm9PaUdsa0FBOXlQRUdQ@167.88.61.204:7307#github.com/freefq%20-%20%E7%91%9E%E5%85%B8%20%203,167.88.61.204,瑞典 斯德哥尔摩 +ss://YWVzLTI1Ni1nY206ZmFCQW9ENTRrODdVSkc3@38.68.134.202:2376#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%E5%8D%8E%E7%9B%9B%E9%A1%BFCogent%E9%80%9A%E4%BF%A1%E5%85%AC%E5%8F%B8%205,38.68.134.202,美国 新泽西 科进 +ss://YWVzLTI1Ni1nY206ZzVNZUQ2RnQzQ1dsSklk@38.68.134.23:5003#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%E5%8D%8E%E7%9B%9B%E9%A1%BFCogent%E9%80%9A%E4%BF%A1%E5%85%AC%E5%8F%B8%206,38.68.134.23,美国 新泽西 科进 +ss://YWVzLTI1Ni1nY206ZmFCQW9ENTRrODdVSkc3@38.91.101.11:2375#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%E5%8D%8E%E7%9B%9B%E9%A1%BFCogent%E9%80%9A%E4%BF%A1%E5%85%AC%E5%8F%B8%207,38.91.101.11,美国 纽约 纽约 科进 +ss://YWVzLTI1Ni1nY206Rm9PaUdsa0FBOXlQRUdQ@142.202.48.52:7306#github.com/freefq%20-%20%E5%8A%A0%E6%8B%BF%E5%A4%A7%20%208,142.202.48.52,加拿大 魁北克 魁北克 +ss://YWVzLTI1Ni1nY206Rm9PaUdsa0FBOXlQRUdQ@142.202.48.34:7307#github.com/freefq%20-%20%E5%8A%A0%E6%8B%BF%E5%A4%A7%20%2010,142.202.48.34,加拿大 魁北克 魁北克 +ss://YWVzLTI1Ni1nY206ZmFCQW9ENTRrODdVSkc3@38.75.136.45:2376#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%E5%8D%8E%E7%9B%9B%E9%A1%BFCogent%E9%80%9A%E4%BF%A1%E5%85%AC%E5%8F%B8%2011,38.75.136.45,美国 科进 +ss://YWVzLTI1Ni1nY206WTZSOXBBdHZ4eHptR0M@38.143.66.71:3389#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%E5%8D%8E%E7%9B%9B%E9%A1%BFCogent%E9%80%9A%E4%BF%A1%E5%85%AC%E5%8F%B8%2012,38.143.66.71,美国 科进 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t1.ssrsub.com:8443#github.com/freefq%20-%20%E5%8A%A0%E6%8B%BF%E5%A4%A7%20%2014,142.47.89.64,加拿大 安大略 多伦多 +ss://YWVzLTI1Ni1nY206UENubkg2U1FTbmZvUzI3@145.239.1.137:8091#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%20%2015,51.38.122.98,德国 Hessen +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t2.ssrsub.com:8443#github.com/freefq%20-%20%E4%BF%84%E7%BD%97%E6%96%AF%20%2020,195.133.53.209,俄罗斯 莫斯科 莫斯科 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t7.ssrsub.com:8443#github.com/freefq%20-%20%E4%BF%84%E7%BD%97%E6%96%AF%E8%8E%AB%E6%96%AF%E7%A7%91Relcom%E7%BD%91%E7%BB%9C%2022,194.87.238.109,俄罗斯 莫斯科 莫斯科 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t6.ssrsub.com:8443#github.com/freefq%20-%20%E4%BF%84%E7%BD%97%E6%96%AF%20%2023,195.133.48.9,俄罗斯 莫斯科 莫斯科 +ss://YWVzLTI1Ni1nY206Y2RCSURWNDJEQ3duZklO@167.88.61.60:8118#github.com/freefq%20-%20%E7%91%9E%E5%85%B8%20%2025,167.88.61.60,瑞典 斯德哥尔摩 +ss://YWVzLTI1Ni1nY206ZmFCQW9ENTRrODdVSkc3@169.197.142.39:2375#github.com/freefq%20-%20%E5%8C%97%E7%BE%8E%E5%9C%B0%E5%8C%BA%20%2026,169.197.142.39,美国 佐治亚 亚特兰大 +ss://YWVzLTI1Ni1nY206ZTRGQ1dyZ3BramkzUVk@172.99.190.87:9101#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%20%2030,172.99.190.87,美国 康涅狄格 +ss://Y2hhY2hhMjAtaWV0Zi1wb2x5MTMwNTpzRjQzWHQyZ09OcWNnRlg1NjM@141.95.0.26:826#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%20%2031,54.38.217.138,美国 新泽西 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t3.ssrsub.com:8443#github.com/freefq%20-%20%E5%8A%A0%E6%8B%BF%E5%A4%A7%E5%AE%89%E5%A4%A7%E7%95%A5%E7%9C%81%E5%9F%BA%E5%A5%87%E7%BA%B3DataCity%E6%95%B0%E6%8D%AE%E4%B8%AD%E5%BF%83%2032,45.62.247.153,加拿大 安大略 基奇纳 +vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogIkBTU1JTVUItVjI4LeS7mOi0ueaOqOiNkDpzdW8ueXQvc3Nyc3ViIiwNCiAgImFkZCI6ICI0Mi4xNTcuOC4xNjIiLA0KICAicG9ydCI6ICI1MDAwMiIsDQogICJpZCI6ICI0MTgwNDhhZi1hMjkzLTRiOTktOWIwYy05OGNhMzU4MGRkMjQiLA0KICAiYWlkIjogIjY0IiwNCiAgInNjeSI6ICJhdXRvIiwNCiAgIm5ldCI6ICJ0Y3AiLA0KICAidHlwZSI6ICJub25lIiwNCiAgImhvc3QiOiAiNDIuMTU3LjguMTYyIiwNCiAgInBhdGgiOiAiIiwNCiAgInRscyI6ICIiLA0KICAic25pIjogIiINCn0=,42.157.8.162,中国 安徽省 合肥市 联通 +vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogImh0dHBzOi8vZ2l0aHViLmNvbS9BbHZpbjk5OTkvbmV3LXBhYy93aWtpIOS/hOe9l+aWr2cyIiwNCiAgImFkZCI6ICIxOTUuMTMzLjUzLjg4IiwNCiAgInBvcnQiOiAiMjU5NjQiLA0KICAiaWQiOiAiYmE5M2U1NmMtNzdmOS0xMWVjLWFmNDMtZDJlOGI0YzNhNzVhIiwNCiAgImFpZCI6ICIwIiwNCiAgInNjeSI6ICJhdXRvIiwNCiAgIm5ldCI6ICJ0Y3AiLA0KICAidHlwZSI6ICJub25lIiwNCiAgImhvc3QiOiAiIiwNCiAgInBhdGgiOiAiIiwNCiAgInRscyI6ICIiLA0KICAic25pIjogIiIsDQogICJhbHBuIjogIiINCn0=,195.133.53.88,俄罗斯 莫斯科 莫斯科 +vmess://eyJ2IjoiMiIsInBzIjoi57+75aKZ5YWaZmFucWlhbmdkYW5nLmNvbUAwMTIwX0NOXzE4IiwiYWRkIjoiMTEyLjMzLjMyLjEzNiIsInBvcnQiOiIxMDAwMyIsImlkIjoiNjVjYWM1NmQtNDE1NS00M2M4LWJhZTAtZjM2OGNiMjFmNzcxIiwiYWlkIjoiMCIsInNjeSI6ImF1dG8iLCJuZXQiOiJ0Y3AiLCJ0eXBlIjoibm9uZSIsImhvc3QiOiJhaWNvbzZkdS5jb20iLCJwYXRoIjoiL3dzIiwidGxzIjoiIiwic25pIjoiIn0=,114.43.130.6,中国 台湾省 中华电信 +vmess://eyJ2IjoiMiIsInBzIjoi57+75aKZ5YWaZmFucWlhbmdkYW5nLmNvbUAwMTIwX0NOXzI1IiwiYWRkIjoiMTEyLjMzLjMyLjEzNiIsInBvcnQiOiIxMDAwNCIsImlkIjoiNjVjYWM1NmQtNDE1NS00M2M4LWJhZTAtZjM2OGNiMjFmNzcxIiwiYWlkIjoiMCIsInNjeSI6ImF1dG8iLCJuZXQiOiJ0Y3AiLCJ0eXBlIjoibm9uZSIsImhvc3QiOiIxMTIuMzMuMzIuMTM2IiwicGF0aCI6Ii9zL2QwYTg2OTIuZm0uYXBwbGUuY29tOjMyNjY3IiwidGxzIjoiIiwic25pIjoiIn0=,172.70.214.120,美国 加利福尼亚 旧金山 +vmess://eyJ2IjoiMiIsInBzIjoi57+75aKZ5YWaZmFucWlhbmdkYW5nLmNvbUAwMTIwX0NOXzI2IiwiYWRkIjoic2hjdTAxLmlwbGMxODguY29tIiwicG9ydCI6IjEwMDA0IiwiaWQiOiI2NWNhYzU2ZC00MTU1LTQzYzgtYmFlMC1mMzY4Y2IyMWY3NzEiLCJhaWQiOiIwIiwic2N5IjoiYXV0byIsIm5ldCI6InRjcCIsInR5cGUiOiJub25lIiwiaG9zdCI6InNoY3UwMS5pcGxjMTg4LmNvbSIsInBhdGgiOiIvd3MiLCJ0bHMiOiIiLCJzbmkiOiIifQ==,210.71.214.218,中国 台湾省 中华电信 +vmess://eyJ2IjoiMiIsInBzIjoi57+75aKZ5YWaZmFucWlhbmdkYW5nLmNvbUAwMTIwX0NOXzI4IiwiYWRkIjoiNDIuMTU3LjguNTIiLCJwb3J0IjoiNDg3MjciLCJpZCI6IjU3YWE1YWMzLWQxZDAtNGUyZi1iMzJlLTY0ODhkNWE3Y2I0NSIsImFpZCI6IjY0Iiwic2N5IjoiYXV0byIsIm5ldCI6InRjcCIsInR5cGUiOiJub25lIiwiaG9zdCI6IiIsInBhdGgiOiIiLCJ0bHMiOiIiLCJzbmkiOiIifQ==,42.157.8.52,中国 安徽省 合肥市 联通 +vmess://eyJ2IjoiMiIsInBzIjoi57+75aKZ5YWaZmFucWlhbmdkYW5nLmNvbUAwMTIwX0NOXzMzIiwiYWRkIjoiMTEyLjMzLjMyLjEzNiIsInBvcnQiOiIxMDAwNCIsImlkIjoiNjVjYWM1NmQtNDE1NS00M2M4LWJhZTAtZjM2OGNiMjFmNzcxIiwiYWlkIjoiMCIsInNjeSI6ImF1dG8iLCJuZXQiOiJ0Y3AiLCJ0eXBlIjoibm9uZSIsImhvc3QiOiIxNDYuNTYuMTc3LjM0IiwicGF0aCI6Ii92MnJheSIsInRscyI6IiIsInNuaSI6IiJ9,172.70.211.7,美国 加利福尼亚 旧金山 +vmess://eyJ2IjoiMiIsInBzIjoi57+75aKZ5YWaZmFucWlhbmdkYW5nLmNvbUAwMTIwX0NOXzM0IiwiYWRkIjoiMTEyLjMzLjMyLjEzNiIsInBvcnQiOiIxMDAwNCIsImlkIjoiNjVjYWM1NmQtNDE1NS00M2M4LWJhZTAtZjM2OGNiMjFmNzcxIiwiYWlkIjoiMCIsInNjeSI6ImF1dG8iLCJuZXQiOiJ0Y3AiLCJ0eXBlIjoibm9uZSIsImhvc3QiOiIxMTIuMzMuMzIuMTM2IiwicGF0aCI6Ii93cyIsInRscyI6IiIsInNuaSI6IiJ9,172.70.210.228,美国 加利福尼亚 旧金山 +vmess://eyJ2IjoiMiIsInBzIjoi57+75aKZ5YWaZmFucWlhbmdkYW5nLmNvbUAwMTIwX0NOXzM1IiwiYWRkIjoiNDIuMTkzLjQ4LjY0IiwicG9ydCI6IjUwMDAyIiwiaWQiOiI0MTgwNDhhZi1hMjkzLTRiOTktOWIwYy05OGNhMzU4MGRkMjQiLCJhaWQiOiI2NCIsInNjeSI6ImF1dG8iLCJuZXQiOiJ0Y3AiLCJ0eXBlIjoibm9uZSIsImhvc3QiOiI0Mi4xOTMuNDguNjQiLCJwYXRoIjoiIiwidGxzIjoiIiwic25pIjoiIn0=,42.193.48.64,中国 上海 上海市 电信 +vmess://eyJ2IjoiMiIsInBzIjoi57+75aKZ5YWaZmFucWlhbmdkYW5nLmNvbUAwMTIwX0NOXzM3IiwiYWRkIjoiMTEyLjMzLjMyLjEzNiIsInBvcnQiOiIxMDAwNCIsImlkIjoiNjVjYWM1NmQtNDE1NS00M2M4LWJhZTAtZjM2OGNiMjFmNzcxIiwiYWlkIjoiMCIsInNjeSI6ImF1dG8iLCJuZXQiOiJ0Y3AiLCJ0eXBlIjoibm9uZSIsImhvc3QiOiIxMTIuMzMuMzIuMTM2IiwicGF0aCI6Ii8iLCJ0bHMiOiIiLCJzbmkiOiIifQ==,172.70.206.182,美国 加利福尼亚 旧金山 +vmess://eyJ2IjoiMiIsInBzIjoi57+75aKZ5YWaZmFucWlhbmdkYW5nLmNvbUAwMTIwX0NOXzM5IiwiYWRkIjoiMTEyLjMzLjMyLjEzNiIsInBvcnQiOiIxMDAwNCIsImlkIjoiNjVjYWM1NmQtNDE1NS00M2M4LWJhZTAtZjM2OGNiMjFmNzcxIiwiYWlkIjoiMCIsInNjeSI6ImF1dG8iLCJuZXQiOiJ0Y3AiLCJ0eXBlIjoibm9uZSIsImhvc3QiOiIxMTIuMzMuMzIuMTM2IiwicGF0aCI6Ii9zLzUzMTg0NDIuZm0uYXBwbGUuY29tOjU0MDgwIiwidGxzIjoiIiwic25pIjoiIn0=,172.69.33.158,美国 加利福尼亚 \ No newline at end of file diff --git a/003-Keywords/v2ray_pool/_db-uncheck.txt b/003-Keywords/v2ray_pool/_db-uncheck.txt new file mode 100644 index 0000000..c853100 --- /dev/null +++ b/003-Keywords/v2ray_pool/_db-uncheck.txt @@ -0,0 +1,19 @@ +vmess://eyJ2IjogIjIiLCAicHMiOiAiZ2l0aHViLmNvbS9mcmVlZnEgLSBcdTViODlcdTVmYmRcdTc3MDFcdTgwNTRcdTkwMWEgMSIsICJhZGQiOiAiNDIuMTU3LjguMTYyIiwgInBvcnQiOiAiNTAwMDIiLCAiaWQiOiAiNDE4MDQ4YWYtYTI5My00Yjk5LTliMGMtOThjYTM1ODBkZDI0IiwgImFpZCI6ICI2NCIsICJzY3kiOiAiYXV0byIsICJuZXQiOiAidGNwIiwgInR5cGUiOiAibm9uZSIsICJob3N0IjogIiIsICJwYXRoIjogIiIsICJ0bHMiOiAiIiwgInNuaSI6ICIifQ==,42.157.8.162,中国 安徽省 合肥市 联通 +ss://YWVzLTI1Ni1nY206ZzVNZUQ2RnQzQ1dsSklk@38.86.135.27:5003#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%E5%8D%8E%E7%9B%9B%E9%A1%BFCogent%E9%80%9A%E4%BF%A1%E5%85%AC%E5%8F%B8%204,38.86.135.27,美国 科进 +ss://YWVzLTI1Ni1nY206ZmFCQW9ENTRrODdVSkc3@169.197.142.39:2375#github.com/freefq%20-%20%E5%8C%97%E7%BE%8E%E5%9C%B0%E5%8C%BA%20%205,169.197.142.39,美国 佐治亚 亚特兰大 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t4.ssrsub.com:8443#github.com/freefq%20-%20%E5%8A%A0%E6%8B%BF%E5%A4%A7%E5%AE%89%E5%A4%A7%E7%95%A5%E7%9C%81%E5%9F%BA%E5%A5%87%E7%BA%B3DataCity%E6%95%B0%E6%8D%AE%E4%B8%AD%E5%BF%83%206,45.62.250.251,加拿大 安大略 基奇纳 +ss://YWVzLTI1Ni1nY206ZmFCQW9ENTRrODdVSkc3@46.29.218.6:2376#github.com/freefq%20-%20%E6%8C%AA%E5%A8%81%20%207,46.29.218.6,挪威 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t3.ssrsub.com:8443#github.com/freefq%20-%20%E5%8A%A0%E6%8B%BF%E5%A4%A7%E5%AE%89%E5%A4%A7%E7%95%A5%E7%9C%81%E5%9F%BA%E5%A5%87%E7%BA%B3DataCity%E6%95%B0%E6%8D%AE%E4%B8%AD%E5%BF%83%2011,45.62.247.153,加拿大 安大略 基奇纳 +ss://YWVzLTI1Ni1nY206ZTRGQ1dyZ3BramkzUVk@172.99.190.87:9101#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%20%2012,172.99.190.87,美国 康涅狄格 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t6.ssrsub.com:8443#github.com/freefq%20-%20%E4%BF%84%E7%BD%97%E6%96%AF%20%2013,195.133.48.9,俄罗斯 莫斯科 莫斯科 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t7.ssrsub.com:8443#github.com/freefq%20-%20%E4%BF%84%E7%BD%97%E6%96%AF%E8%8E%AB%E6%96%AF%E7%A7%91Relcom%E7%BD%91%E7%BB%9C%2022,194.87.238.109,俄罗斯 莫斯科 莫斯科 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t1.ssrsub.com:8443#github.com/freefq%20-%20%E5%8A%A0%E6%8B%BF%E5%A4%A7%20%2023,142.47.89.64,加拿大 安大略 多伦多 +ss://YWVzLTI1Ni1nY206cEtFVzhKUEJ5VFZUTHRN@134.195.196.12:443#github.com/freefq%20-%20%E5%8C%97%E7%BE%8E%E5%9C%B0%E5%8C%BA%20%2024,134.195.196.12,美国 +ss://YWVzLTI1Ni1nY206cEtFVzhKUEJ5VFZUTHRN@38.143.66.71:443#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%E5%8D%8E%E7%9B%9B%E9%A1%BFCogent%E9%80%9A%E4%BF%A1%E5%85%AC%E5%8F%B8%2026,38.143.66.71,美国 科进 +ss://YWVzLTI1Ni1nY206UENubkg2U1FTbmZvUzI3@145.239.1.137:8091#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%20%2029,51.38.122.98,德国 Hessen +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t8.ssrsub.com:8443#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%20%2030,152.69.200.26,美国 加利福尼亚 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t2.ssrsub.com:8443#github.com/freefq%20-%20%E4%BF%84%E7%BD%97%E6%96%AF%20%2032,195.133.53.209,俄罗斯 莫斯科 莫斯科 +ss://YWVzLTI1Ni1nY206WTZSOXBBdHZ4eHptR0M@167.88.61.60:5601#github.com/freefq%20-%20%E7%91%9E%E5%85%B8%20%2034,167.88.61.60,瑞典 斯德哥尔摩 +ss://Y2hhY2hhMjAtaWV0Zi1wb2x5MTMwNTpzRjQzWHQyZ09OcWNnRlg1NjM@141.95.0.26:826#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%20%2037,54.38.217.138,美国 新泽西 +trojan://64c3ab43-dc1b-401c-9437-9adf7bcf4a28@t9.ssrsub.com:8443#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%20%2038,150.230.215.123,美国 加利福尼亚 +vmess://eyJ2IjogIjIiLCAicHMiOiAiZ2l0aHViLmNvbS9mcmVlZnEgLSBcdTRlMGFcdTZkNzdcdTVlMDJcdTVmOTBcdTZjNDdcdTUzM2FcdTgwNTRcdTkwMWFcdTZmMTVcdTZjYjNcdTZjZmVcdTY1NzBcdTYzNmVcdTRlMmRcdTVmYzMgNDAiLCAiYWRkIjogInNoY3UwMS5pcGxjMTg4LmNvbSIsICJwb3J0IjogIjEwMDA0IiwgImlkIjogIjY1Y2FjNTZkLTQxNTUtNDNjOC1iYWUwLWYzNjhjYjIxZjc3MSIsICJhaWQiOiAiMSIsICJzY3kiOiAiYXV0byIsICJuZXQiOiAidGNwIiwgInR5cGUiOiAibm9uZSIsICJob3N0IjogInNoY3UwMS5pcGxjMTg4LmNvbSIsICJwYXRoIjogIiIsICJ0bHMiOiAiIiwgInNuaSI6ICIifQ==,61.216.19.199,中国 台湾省 中华电信 diff --git a/003-Keywords/v2ray_util.py b/003-Keywords/v2ray_util.py new file mode 100644 index 0000000..a7308b7 --- /dev/null +++ b/003-Keywords/v2ray_util.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 管理v2ray_pool的工具 +@Date :2022年1月14日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" + +import time + +from v2ray_pool import utils, Config, DBManage + + +def search_node(): + # 如果有系统全局代理,可不需要开启v2ray_core代理,GoogleTrend(proxies=False) + utils.kill_all_v2ray() + Config.set_v2ray_core_path('/Users/Qincji/Desktop/develop/soft/intalled/v2ray-macos-64') # v2ray内核存放路径 + Config.set_v2ray_node_path( + '/Users/Qincji/Desktop/develop/py/project/PythonIsTools/005-PaidSource/v2ray_pool') # 保存获取到节点的路径 + proxy_url = 'ss://YWVzLTI1Ni1nY206UENubkg2U1FTbmZvUzI3@145.239.1.137:8091#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%20%207' + dbm = DBManage() + dbm.init() # 必须初始化 + # if dbm.check_url_single(proxy_url): + # urls = dbm.load_urls_by_net(proxy_url=proxy_url) + # dbm.check_and_save(urls, append=False) + dbm.load_urls_and_save_auto() + # urls = dbm.load_unchecked_urls_by_local() + # dbm.check_and_save(urls, append=False) + utils.kill_all_v2ray() + + +def restart_v2ray(isSysOn=False): + utils.kill_all_v2ray() + Config.set_v2ray_core_path('/Users/Qincji/Desktop/develop/soft/intalled/v2ray-macos-64') # v2ray内核存放路径 + Config.set_v2ray_node_path( + '/Users/Qincji/Desktop/develop/py/project/PythonIsTools/005-PaidSource/v2ray_pool') # 保存获取到节点的路径 + dbm = DBManage() + dbm.init() # 必须初始化 + while 1: + if dbm.start_random_v2ray_by_local(isSysOn=isSysOn): + break + else: + print("启动失败,进行重试!") + time.sleep(1) + + +def kill_all_v2ray(): + utils.kill_all_v2ray() diff --git a/003-Keywords/women-ring/1.jpg b/003-Keywords/women-ring/1.jpg new file mode 100644 index 0000000..a4ac524 Binary files /dev/null and b/003-Keywords/women-ring/1.jpg differ diff --git a/003-Keywords/women-ring/2.jpg b/003-Keywords/women-ring/2.jpg new file mode 100644 index 0000000..fd9d3f9 Binary files /dev/null and b/003-Keywords/women-ring/2.jpg differ diff --git a/003-Keywords/women-ring/3.jpg b/003-Keywords/women-ring/3.jpg new file mode 100644 index 0000000..d2c9f28 Binary files /dev/null and b/003-Keywords/women-ring/3.jpg differ diff --git a/003-Keywords/women-ring/swarovski rings.jpg b/003-Keywords/women-ring/swarovski rings.jpg new file mode 100644 index 0000000..92dc408 Binary files /dev/null and b/003-Keywords/women-ring/swarovski rings.jpg differ diff --git a/003-Keywords/women-ring/women ring.csv b/003-Keywords/women-ring/women ring.csv new file mode 100644 index 0000000..5967da3 --- /dev/null +++ b/003-Keywords/women-ring/women ring.csv @@ -0,0 +1,885 @@ +keyword,no,top keyword,top range,rising keyword,rising range +women ring,0,ring for women,100,are men and women ring sizes the same,550 +women ring,1,gold women ring,40,emerald rings for women,200 +women ring,2,gold ring,40,ruby ring for women,200 +women ring,3,ring for women gold,31,gucci rings women,200 +women ring,4,women rings,31,cartier ring,190 +women ring,5,rings,30,emerald ring for women,170 +women ring,6,rings for women,24,thumb ring women,170 +women ring,7,wedding ring women,20,cartier,170 +women ring,8,wedding ring,20,macys,170 +women ring,9,diamond ring,17,pinky rings for women,160 +women ring,10,diamond,17,moonstone ring,160 +women ring,11,diamond ring women,16,gold price today,150 +women ring,12,women ring size,16,versace ring women,140 +women ring,13,ring size,16,gold rate today,130 +women ring,14,engagement,14,how to measure ring size,130 +women ring,15,engagement ring women,14,cartier love ring,120 +women ring,16,diamond ring for women,14,kay jewelers,110 +women ring,17,engagement ring,13,thumb ring for women,110 +women ring,18,wedding ring for women,13,opal ring for women,110 +women ring,19,engagement ring for women,12,swarovski,110 +women ring,20,silver ring women,11,diamond ring for women price,100 +women ring,21,silver ring,11,louis vuitton,100 +women ring,22,women ring design,11,the bling ring,100 +women ring,23,ring design,11,how to measure ring size women,100 +women ring,24,women ring finger,10,eternity ring for women,90 +, +ring for women,0,gold ring women,100,gold ring for women under 5000,3950 +ring for women,1,ring for women gold,98,fendi ring,450 +ring for women,2,gold ring,94,dior ring,350 +ring for women,3,rings,70,silver rate today,300 +ring for women,4,rings for women,69,opal rings for women,250 +ring for women,5,diamond,49,ruby ring for women,250 +ring for women,6,diamond ring for women,46,pinky rings for women,200 +ring for women,7,diamond ring,44,emerald ring for women,200 +ring for women,8,engagement ring,41,ruby rings for women,190 +ring for women,9,wedding ring for women,41,eternity ring for women,150 +ring for women,10,engagement,39,solitaire ring for women,150 +ring for women,11,wedding ring,38,gold pendant,120 +ring for women,12,engagement ring for women,36,diamond ring price for women,120 +ring for women,13,ring size for women,28,pearl ring,120 +ring for women,14,ring design,26,gold ring for women under 10000,110 +ring for women,15,silver ring,26,thumb ring for women,110 +ring for women,16,engagement rings,23,pearl ring for women,100 +ring for women,17,silver ring for women,23,gucci ring,100 +ring for women,18,ring design for women,23,custom rings for women,100 +ring for women,19,engagement rings for women,23,kay jewelers,90 +ring for women,20,ring finger for women,18,amethyst ring for women,90 +ring for women,21,gold rings for women,18,emerald rings for women,80 +ring for women,22,gold ring design,18,rose gold rings for women,80 +ring for women,23,wedding rings,17,opal ring for women,70 +ring for women,24,ring finger,17,types of rings for women,70 +, +gold women ring,0,ring for women gold,100,gold ring for women under 10000,36200 +gold women ring,1,ring for women,98,gold price today,350 +gold women ring,2,gold rings,24,gold ring for women under 5000,250 +gold women ring,3,gold ring design,23,white gold engagement ring for women,130 +gold women ring,4,gold rings women,23,ring design for men,110 +gold women ring,5,women gold ring design,22,silver ring for women,80 +gold women ring,6,women ring design,22,ring design for women,80 +gold women ring,7,ring design,22,gold rate today,70 +gold women ring,8,gold ring design for women,21,gold ring design for women,60 +gold women ring,9,ring design for women,20,ring for women gold,50 +gold women ring,10,gold rings for women,20,ring for women,50 +gold women ring,11,rings for women,19,gold ring design,50 +gold women ring,12,gold ring women price,19,gold ring women price,50 +gold women ring,13,gold ring price,19,gold ring for girls,50 +gold women ring,14,gold price,18,women ring design,50 +gold women ring,15,gold ring for women price,15,white gold ring for women,50 +gold women ring,16,diamond ring,12,women gold ring design,40 +gold women ring,17,diamond ring for women,11,gold rate,40 +gold women ring,18,ring designs for women,9,gold chain for women,40 +gold women ring,19,gold ring designs for women,8,, +gold women ring,20,gold engagement ring for women,7,, +gold women ring,21,gold rate,7,, +gold women ring,22,engagement ring for women,7,, +gold women ring,23,gold ring for men,7,, +gold women ring,24,silver ring for women,6,, +, +gold ring,0,diamond,100,nose ring designs in gold for female,750 +gold ring,1,gold diamond ring,99,simple gold ring design for female,200 +gold ring,2,diamond ring,97,how to measure ring size,170 +gold ring,3,white gold ring,93,new gold ring design for female,150 +gold ring,4,white gold,90,ear ring design in gold,150 +gold ring,5,rose gold ring,81,2 gram gold ring price,130 +gold ring,6,rose gold,79,boys gold ring design,130 +gold ring,7,gold rings,77,3 gram gold ring,120 +gold ring,8,gold ring price,77,today gold price,120 +gold ring,9,gold price,77,2 gram gold ring,110 +gold ring,10,ring design gold,77,gold ring price in bd,110 +gold ring,11,ring design,76,nose ring gold design,100 +gold ring,12,rings,74,gold engagement ring designs for female,90 +gold ring,13,engagement ring gold,57,today gold rate,80 +gold ring,14,engagement ring,56,bulgari ring gold,80 +gold ring,15,gold ring men,47,white gold ring for women,80 +gold ring,16,gold wedding ring,47,ear ring design gold,80 +gold ring,17,wedding ring,46,3 gram gold ring price,70 +gold ring,18,mens gold ring,40,ring designs for girls gold,70 +gold ring,19,gold band ring,34,white gold nose ring,70 +gold ring,20,ring for men gold,32,gold dome ring,70 +gold ring,21,women gold ring,31,ring design for women,60 +gold ring,22,14k gold ring,30,simple gold ring design,60 +gold ring,23,yellow gold ring,30,couple gold ring design,60 +gold ring,24,gold nose ring,27,ring light,60 +, +ring for women gold,0,rings for women,100,gold ring for women under 5000,400 +ring for women gold,1,gold rings for women,96,gold rate today,200 +ring for women gold,2,gold ring design for women,90,gold ring for women under 10000,200 +ring for women gold,3,gold ring design,87,gold ring for girls,170 +ring for women gold,4,ring design for women,82,white gold ring for women,60 +ring for women gold,5,gold ring price for women,65,engagement ring for women,50 +ring for women gold,6,gold price,65,gold ring design for women,50 +ring for women gold,7,diamond ring,58,tanishq gold ring for women,50 +ring for women gold,8,diamond ring for women,48,ring design for women,50 +ring for women gold,9,engagement ring for women,41,gold ring design,50 +ring for women gold,10,ring designs for women,38,, +ring for women gold,11,gold ring designs for women,37,, +ring for women gold,12,white gold ring for women,33,, +ring for women gold,13,wedding ring for women,30,, +ring for women gold,14,gold rate,29,, +ring for women gold,15,white gold ring,29,, +ring for women gold,16,gold ring for men,28,, +ring for women gold,17,gold engagement rings for women,26,, +ring for women gold,18,engagement rings for women,24,, +ring for women gold,19,tanishq gold ring for women,24,, +ring for women gold,20,tanishq,24,, +ring for women gold,21,gold rate today,23,, +ring for women gold,22,silver ring for women,22,, +ring for women gold,23,gold earrings for women,22,, +ring for women gold,24,gold chain,21,, +, +women rings,0,rings for women,100,diamond engagement rings for women with price,1750 +women rings,1,engagement,25,turquoise rings for women,1300 +women rings,2,engagement rings,25,adjustable rings for women,250 +women rings,3,women engagement rings,25,gucci ring,250 +women rings,4,wedding rings,24,personalized rings for women,200 +women rings,5,wedding rings women,24,fendi rings women,200 +women rings,6,engagement rings for women,24,eternity ring,200 +women rings,7,ring,20,unique engagement rings for women,180 +women rings,8,wedding rings for women,20,opal rings for women,170 +women rings,9,gold rings women,20,emerald rings for women,160 +women rings,10,gold rings,19,class rings for women,130 +women rings,11,rings for women gold,16,chocolate diamond rings for women,130 +women rings,12,ring for women,16,types of rings for women,130 +women rings,13,diamond,13,rubber rings for women,130 +women rings,14,women diamond rings,12,celtic rings for women,130 +women rings,15,diamond rings for women,12,gucci ring women,110 +women rings,16,diamond rings,12,silicone rings for women,110 +women rings,17,silver rings,7,solitaire rings for women,110 +women rings,18,silver rings women,7,silicone wedding rings for women,110 +women rings,19,silver rings for women,6,simple engagement rings for women,100 +women rings,20,engagement ring,5,gucci rings women,100 +women rings,21,gold ring,5,tiffany rings for women,90 +women rings,22,men rings,5,nose rings for women,90 +women rings,23,jewelry,5,tiffany and co,90 +women rings,24,pandora,5,eternity ring for women,90 +, +rings,0,lord of the rings,100,rolex rings ipo,34750 +rings,1,the lord of the rings,98,rolex rings share price,11350 +rings,2,engagement rings,65,floating rings weeping woods,9650 +rings,3,ring,55,rolex rings ipo gmp,7050 +rings,4,wedding rings,39,rolex rings ipo allotment status,5950 +rings,5,gold rings,29,floating rings at weeping woods,5400 +rings,6,diamond rings,25,rolex rings ipo allotment date,5350 +rings,7,pandora rings,16,shang-chi and the legend of the ten rings,5000 +rings,8,pandora,16,shang chi and the legend of the ten rings,2450 +rings,9,silver rings,15,shang chi,2200 +rings,10,mens rings,14,ten rings,1250 +rings,11,onion rings,13,where to watch lord of the rings,550 +rings,12,men rings,12,lord of the rings 4k,350 +rings,13,promise rings,12,clay rings,350 +rings,14,rings for women,10,air fryer onion rings,300 +rings,15,diamond engagement rings,9,mejuri rings,250 +rings,16,rings movie,8,onion rings in air fryer,200 +rings,17,nose rings,8,akatsuki rings,200 +rings,18,amazon rings,8,chunky rings,180 +rings,19,rings for men,8,glamira rings,170 +rings,20,ten rings,6,harry styles rings,160 +rings,21,gold engagement rings,6,matching rings,120 +rings,22,the hobbit,5,engagement rings for women,110 +rings,23,cheap rings,5,pura vida rings,100 +rings,24,lord of the rings cast,5,peach rings,90 +, +rings for women,0,engagement,100,amethyst rings for women,12700 +rings for women,1,engagement rings,100,gold thumb rings for women,4800 +rings for women,2,engagement rings for women,99,emerald rings for women,350 +rings for women,3,wedding rings for women,78,key rings for women,300 +rings for women,4,wedding rings,76,sapphire rings for women,300 +rings for women,5,rings for women gold,70,chocolate diamond rings for women,250 +rings for women,6,gold rings,67,turquoise rings for women,250 +rings for women,7,ring,62,unique engagement rings for women,250 +rings for women,8,ring for women,60,nose rings for women,200 +rings for women,9,diamond rings,45,cartier rings for women,200 +rings for women,10,diamond rings for women,43,louis vuitton,190 +rings for women,11,diamond,42,engagement rings for women near me,170 +rings for women,12,rings for women silver,27,real gold rings for women,160 +rings for women,13,silver rings,26,sapphire rings,160 +rings for women,14,engagement ring for women,19,opal rings for women,150 +rings for women,15,engagement ring,18,engagement rings for men and women,150 +rings for women,16,gold ring for women,17,eternity rings for women,150 +rings for women,17,gold ring,16,anklets for women,150 +rings for women,18,jewelry,16,diamond engagement rings for women with price,140 +rings for women,19,pandora,16,james avery,130 +rings for women,20,promise rings for women,15,ruby rings for women,120 +rings for women,21,pandora rings for women,15,ruby ring for women,90 +rings for women,22,pandora rings,14,adjustable rings for women,90 +rings for women,23,promise rings,14,gemstone rings for women,90 +rings for women,24,diamond ring,14,unique rings for women,90 +, +wedding ring women,0,wedding ring for women,100,wedding sets for women,90 +wedding ring women,1,wedding rings,54,wedding ring sets,70 +wedding ring women,2,rings for women,44,wedding ring sets for women,60 +wedding ring women,3,wedding rings for women,43,engagement rings for women,50 +wedding ring women,4,wedding bands,25,wedding dresses,40 +wedding ring women,5,wedding ring sets,23,, +wedding ring women,6,diamond ring for women,20,, +wedding ring women,7,wedding sets for women,19,, +wedding ring women,8,engagement rings,19,, +wedding ring women,9,wedding bands for women,19,, +wedding ring women,10,engagement rings for women,18,, +wedding ring women,11,wedding ring sets for women,18,, +wedding ring women,12,wedding band for women,17,, +wedding ring women,13,women wedding ring set,12,, +wedding ring women,14,wedding ring finger women,10,, +wedding ring women,15,wedding ring finger,9,, +wedding ring women,16,wedding ring set for women,8,, +wedding ring women,17,wedding ring finger for women,5,, +wedding ring women,18,wedding dresses,4,, +wedding ring women,19,men and women wedding ring sets,2,, +wedding ring women,20,kay jewelers,1,, +, +wedding ring,0,wedding rings,100,wedding ring sheathing,69850 +wedding ring,1,rings,96,vanessa bryant wedding ring,10700 +wedding ring,2,engagement ring,93,wedding ring sheating,2200 +wedding ring,3,ring for wedding,79,alex lovell no wedding ring,2150 +wedding ring,4,wedding band,76,andy murray wedding ring,300 +wedding ring,5,gold wedding ring,58,hailey bieber wedding ring,300 +wedding ring,6,the wedding ring,57,ariana grande wedding ring,250 +wedding ring,7,diamond ring,56,lotr wedding ring,250 +wedding ring,8,diamond wedding ring,55,kobe bryant wedding ring,190 +wedding ring,9,wedding ring hand,50,couples wedding ring sets,150 +wedding ring,10,ring finger,42,how many people should i invite to my wedding,130 +wedding ring,11,wedding ring finger,42,finance wedding ring,100 +wedding ring,12,mens wedding ring,36,which hand does wedding ring go on,100 +wedding ring,13,mens ring,36,wedding ring tattoo ideas,100 +wedding ring,14,wedding ring sets,35,his and her wedding ring sets,90 +wedding ring,15,engagement rings,32,what hand does a wedding ring go on for a man,90 +wedding ring,16,wedding ring bands,30,which hand does your wedding ring go on,90 +wedding ring,17,wedding bands,30,outlander wedding ring,80 +wedding ring,18,wedding ring set,27,do you wear your engagement ring on your wedding day,80 +wedding ring,19,engagement ring and wedding ring,25,which finger does a wedding ring go on,80 +wedding ring,20,men wedding ring,25,which hand does a wedding ring go on,80 +wedding ring,21,wedding dress,24,opal engagement ring,80 +wedding ring,22,wedding ring men,24,which hand do you wear a wedding ring on,80 +wedding ring,23,what hand wedding ring,21,wedding showers,70 +wedding ring,24,black wedding ring,17,what finger for wedding ring,70 +, +diamond ring,0,diamond engagement ring,100,lab grown diamonds,300 +diamond ring,1,engagement ring,96,ban maxxis diamond ring 14,300 +diamond ring,2,gold diamond ring,85,diamond hoop nose ring,130 +diamond ring,3,rings,70,diamond ring price in bangladesh,120 +diamond ring,4,diamond rings,67,salt and pepper diamond,110 +diamond ring,5,diamond price,51,levian chocolate diamond ring,110 +diamond ring,6,diamond ring price,49,diamond ring design for female,110 +diamond ring,7,wedding ring,38,salt and pepper diamond ring,100 +diamond ring,8,diamond wedding ring,37,diamond ring price in uae,100 +diamond ring,9,diamond engagement rings,32,10 carat diamond ring price,90 +diamond ring,10,engagement rings,31,diamond ring price in pakistan,70 +diamond ring,11,diamond band ring,30,radiant cut diamond ring,70 +diamond ring,12,black diamond,25,heart shape diamond ring,70 +diamond ring,13,black diamond ring,25,radiant diamond ring,70 +diamond ring,14,white gold diamond ring,22,3k diamond ring,70 +diamond ring,15,emerald diamond ring,22,diamond ring for women,60 +diamond ring,16,solitaire diamond ring,20,4 ct diamond ring,50 +diamond ring,17,princess diamond ring,20,best way to clean diamond ring,40 +diamond ring,18,solitaire ring,20,diamond ring for girls,40 +diamond ring,19,2 carat diamond ring,19,2 carat emerald cut diamond ring,40 +diamond ring,20,1 carat diamond,19,, +diamond ring,21,1 carat diamond ring,18,, +diamond ring,22,diamond sapphire ring,17,, +diamond ring,23,princess cut ring,17,, +diamond ring,24,oval diamond ring,17,, +, +diamond,0,diamond ring,100,diamond casino heist,22950 +diamond,1,black diamond,78,pokemon brilliant diamond,12700 +diamond,2,free diamond,73,diamond league 2021,10600 +diamond,3,diamond free fire,60,free fire hack diamond 2020,9400 +diamond,4,free fire,58,diamond jeje,7850 +diamond,5,free fire free diamond,57,double diamond top up,7150 +diamond,6,diamond painting,50,lil uzi vert diamond,6750 +diamond,7,blue diamond,42,lil uzi vert,6000 +diamond,8,diamond rings,41,lil uzi diamond,5050 +diamond,9,diamond price,38,blaq diamond love letter,4100 +diamond,10,diamond earrings,32,free fire redeem code,3100 +diamond,11,diamonds,29,diamond pang,2900 +diamond,12,diamond cut,28,avenues of the diamond,2700 +diamond,13,neil diamond,26,dunia games free fire 70 diamond,1950 +diamond,14,diamond white,26,diamond princess cruise ship,1850 +diamond,15,diamond necklace,26,diamond and pearl remake,900 +diamond,16,white diamond,25,free fire diamond hack generator,900 +diamond,17,dustin diamond,22,pokemon diamond and pearl remake,900 +diamond,18,diamond princess,21,dustin diamond,900 +diamond,19,diamond ff,20,diamond painting wereld,700 +diamond,20,kinemaster,20,ff diamond hack,700 +diamond,21,kinemaster diamond,20,kinemaster diamond mod apk,700 +diamond,22,pokemon diamond,19,kinemaster diamond apk download,650 +diamond,23,diamond platnumz,18,generator diamond online free fire,650 +diamond,24,minecraft diamond,16,diamond princess cruise,600 +, +diamond ring women,0,diamond ring for women,100,eternity ring for women,46600 +diamond ring women,1,diamond rings,41,platinum ring for women,70 +diamond ring women,2,diamond rings for women,36,, +diamond ring women,3,rings for women,36,, +diamond ring women,4,gold ring for women,23,, +diamond ring women,5,gold diamond ring for women,21,, +diamond ring women,6,engagement rings,17,, +diamond ring women,7,diamond ring price,16,, +diamond ring women,8,engagement rings for women,16,, +diamond ring women,9,diamond ring for women price,13,, +diamond ring women,10,wedding rings,13,, +diamond ring women,11,wedding rings for women,9,, +diamond ring women,12,gold rings for women,8,, +diamond ring women,13,platinum ring for women,7,, +diamond ring women,14,diamond ring for men,6,, +diamond ring women,15,tanishq,6,, +diamond ring women,16,tanishq diamond ring for women,5,, +diamond ring women,17,black diamond ring,5,, +diamond ring women,18,wedding bands for women,4,, +diamond ring women,19,black diamond ring for women,4,, +diamond ring women,20,diamond bands for women,3,, +diamond ring women,21,white gold rings for women,2,, +diamond ring women,22,eternity ring for women,1,, +, +women ring size,0,ring size for women,100,etsy,41000 +women ring size,1,average women ring size,53,average women ring size,130 +women ring size,2,average ring size for women,43,how to measure ring size women,100 +women ring size,3,ring size chart,38,gold rings for women,100 +women ring size,4,women ring size chart,37,average ring size for women,90 +women ring size,5,rings for women,26,engagement rings for women,80 +women ring size,6,ring size chart for women,26,how to measure ring size,80 +women ring size,7,how to measure ring size,23,rings for women,60 +women ring size,8,how to measure ring size women,22,, +women ring size,9,how to measure ring size for women,6,, +women ring size,10,ring sizes for women,5,, +women ring size,11,wedding rings for women,5,, +women ring size,12,engagement rings for women,4,, +women ring size,13,how to find ring size,3,, +women ring size,14,gold rings for women,3,, +women ring size,15,etsy,2,, +, +ring size,0,ring size chart,100,what is the screen size of new fire-boltt ring smartwatch?,3700 +ring size,1,measure ring size,60,how to figure out ring size at home,600 +ring size,2,ring size mm,57,how to know the size of your ring finger,500 +ring size,3,how to measure ring size,48,measuring ring size at home,500 +ring size,4,uk ring size,45,how can you measure your ring size,400 +ring size,5,rings,44,how to know the size of your ring,350 +ring size,6,us ring size,39,fendi ring,350 +ring size,7,size of ring,37,charmed aroma,250 +ring size,8,ring size in mm,26,royal essence,250 +ring size,9,pandora ring,25,ring size chart nz,250 +ring size,10,pandora ring size,25,2.25 inches ring size,200 +ring size,11,find ring size,24,how to.measure ring size,190 +ring size,12,pandora,24,2.5 inches to mm,180 +ring size,13,ring finger size,24,how do you measure your ring size,170 +ring size,14,how to size a ring,23,how to find ring size at home,170 +ring size,15,cm to ring size,22,size 8 ring in cm,140 +ring size,16,ring size in cm,19,how to measure ring size women,140 +ring size,17,mens ring size,18,how to know what size ring you are,140 +ring size,18,average ring size,17,how do i measure my ring size,130 +ring size,19,how to find ring size,16,how to know your ring size female,120 +ring size,20,engagement ring,16,52 ring size in letters,110 +ring size,21,men ring size,16,how to find your ring size at home,110 +ring size,22,engagement ring size,16,how to work out ring size uk,110 +ring size,23,ring sizes,15,us ring size to eu,100 +ring size,24,cm to mm,14,how do i figure out my ring size,100 +, +engagement,0,ring engagement,100,bakhtawar bhutto engagement,4800 +engagement,1,rings,99,himanshi khurana engagement,4600 +engagement,2,engagement rings,96,ankita lokhande engagement,4300 +engagement,3,diamond engagement ring,14,ankita lokhande,4150 +engagement,4,diamond rings,13,engagement awc.odisha.gov.in,2950 +engagement,5,diamond engagement rings,13,emma stone engagement ring,2700 +engagement,6,employee engagement,11,hardik pandya engagement,1850 +engagement,7,wedding ring,10,youth engagement for global action,1750 +engagement,8,gold engagement rings,8,gwen stefani engagement ring,1450 +engagement,9,engagement party,8,katrina kaif engagement,1250 +engagement,10,engagement meaning,7,lily collins engagement ring,1200 +engagement,11,engagement dress,7,aaron rodgers engagement,1000 +engagement,12,instagram engagement,7,demi lovato engagement,800 +engagement,13,wedding rings,6,instagram engagement rate calculator,300 +engagement,14,engagement photos,6,engagement makeup look,300 +engagement,15,rules of engagement,6,mia khalifa engagement,250 +engagement,16,engagement wishes,5,engagement artinya,250 +engagement,17,community engagement,5,happy engagement wishes,200 +engagement,18,engagement rate,5,engagement cake design,190 +engagement,19,engagement quotes,4,engagement rate calculator,180 +engagement,20,engagement gifts,4,instagram engagement calculator,160 +engagement,21,customer engagement,4,engagement anniversary wishes to husband,160 +engagement,22,forfait sans engagement,4,engagement rings for couples,140 +engagement,23,tiffany,4,engagement adalah,140 +engagement,24,engagement rings for women,4,engagement anniversary wishes,140 +, +engagement ring women,0,engagement ring for women,100,white gold engagement ring for women,60 +engagement ring women,1,women engagement rings,71,, +engagement ring women,2,engagement rings,65,, +engagement ring women,3,rings for women,59,, +engagement ring women,4,engagement rings for women,55,, +engagement ring women,5,diamond ring for women,26,, +engagement ring women,6,wedding rings,22,, +engagement ring women,7,diamond rings for women,18,, +engagement ring women,8,diamond engagement rings for women,17,, +engagement ring women,9,wedding rings for women,16,, +engagement ring women,10,engagement ring finger,16,, +engagement ring women,11,gold engagement rings for women,13,, +engagement ring women,12,engagement ring finger for women,12,, +engagement ring women,13,wedding bands,8,, +engagement ring women,14,wedding bands for women,8,, +engagement ring women,15,engagement ring sets for women,5,, +engagement ring women,16,white gold engagement ring for women,5,, +engagement ring women,17,wedding ring sets for women,4,, +engagement ring women,18,kay jewelers,2,, +, +diamond ring for women,0,diamond rings,100,eternity ring for women,550 +diamond ring for women,1,diamond rings for women,98,, +diamond ring for women,2,rings for women,98,, +diamond ring for women,3,gold diamond ring for women,74,, +diamond ring for women,4,diamond ring price for women,46,, +diamond ring for women,5,diamond ring price,46,, +diamond ring for women,6,diamond engagement rings for women,46,, +diamond ring for women,7,engagement rings for women,40,, +diamond ring for women,8,engagement rings,39,, +diamond ring for women,9,wedding rings for women,25,, +diamond ring for women,10,tanishq diamond ring for women,21,, +diamond ring for women,11,tanishq,19,, +diamond ring for women,12,platinum diamond ring for women,15,, +diamond ring for women,13,platinum ring for women,12,, +diamond ring for women,14,black diamond ring for women,12,, +diamond ring for women,15,diamond ring for men,12,, +diamond ring for women,16,diamond earrings for women,10,, +diamond ring for women,17,eternity ring for women,10,, +diamond ring for women,18,wedding bands for women,9,, +, +engagement ring,0,engagement rings,100,patrick mahomes engagement ring,11650 +engagement ring,1,rings,98,heather rae young engagement ring,11100 +engagement ring,2,diamond engagement ring,87,shailene woodley engagement ring,5700 +engagement ring,3,diamond ring,82,gwen stefani,2700 +engagement ring,4,wedding ring,59,katie thurston engagement ring,1850 +engagement ring,5,engagement ring gold,44,gwen stefani engagement ring,1550 +engagement ring,6,diamond engagement rings,29,emma stone engagement ring,1350 +engagement ring,7,halo engagement ring,19,demi lovato engagement ring,600 +engagement ring,8,wedding rings,18,hidden halo engagement ring,550 +engagement ring,9,wedding ring and engagement ring,18,engagement ring booking,450 +engagement ring,10,engagement ring finger,18,1000 dollar engagement ring,300 +engagement ring,11,engagement ring hand,17,gold engagement ring designs for female,110 +engagement ring,12,tiffany,16,engagement ring platter,110 +engagement ring,13,tiffany engagement ring,15,how much are you supposed to spend on an engagement ring,110 +engagement ring,14,oval engagement ring,14,engagement ring designs for female,100 +engagement ring,15,rose engagement ring,14,three stone oval engagement ring,100 +engagement ring,16,sapphire engagement ring,13,which finger is for engagement ring,90 +engagement ring,17,engagement ring set,13,engagement and wedding ring set,80 +engagement ring,18,solitaire engagement ring,13,princess diana engagement ring,70 +engagement ring,19,pear engagement ring,13,nikki bella engagement ring,70 +engagement ring,20,engagement ring price,12,oval solitaire engagement ring,70 +engagement ring,21,emerald engagement ring,12,engagement ring cost rule,70 +engagement ring,22,rose gold engagement ring,12,do you wear your engagement ring on your wedding day,60 +engagement ring,23,white gold engagement ring,11,how much is an engagement ring,60 +engagement ring,24,engagement ring size,11,what finger does the engagement ring go on,60 +, +wedding ring for women,0,wedding rings for women,100,wedding ring set for women,110 +wedding ring for women,1,wedding rings,98,wedding ring sets for women,80 +wedding ring for women,2,rings for women,93,wedding ring sets,60 +wedding ring for women,3,wedding sets for women,48,wedding sets for women,60 +wedding ring for women,4,wedding ring sets,46,, +wedding ring for women,5,wedding ring sets for women,45,, +wedding ring for women,6,wedding bands for women,40,, +wedding ring for women,7,wedding band for women,39,, +wedding ring for women,8,engagement rings for women,39,, +wedding ring for women,9,wedding ring set for women,20,, +wedding ring for women,10,wedding ring for men,19,, +wedding ring for women,11,wedding ring finger for women,15,, +wedding ring for women,12,ring finger for women,14,, +wedding ring for women,13,gold wedding bands for women,8,, +wedding ring for women,14,best wedding rings for women,5,, +, +engagement ring for women,0,rings for women,100,unique engagement rings for women,300 +engagement ring for women,1,engagement rings,97,wedding ring sets for women,70 +engagement ring for women,2,engagement rings for women,92,diamond rings for women,60 +engagement ring for women,3,diamond ring for women,43,diamond engagement rings for women,50 +engagement ring for women,4,diamond rings for women,34,engagement rings for women,40 +engagement ring for women,5,diamond engagement rings for women,33,engagement rings,40 +engagement ring for women,6,wedding rings for women,31,, +engagement ring for women,7,gold engagement rings for women,26,, +engagement ring for women,8,engagement ring finger for women,18,, +engagement ring for women,9,engagement ring finger,18,, +engagement ring for women,10,wedding bands for women,10,, +engagement ring for women,11,white gold engagement ring for women,6,, +engagement ring for women,12,unique engagement rings for women,4,, +engagement ring for women,13,wedding ring sets for women,4,, +, +silver ring,0,sterling silver,100,re8 silver ring,9950 +silver ring,1,sterling silver ring,95,dior ring silver,900 +silver ring,2,silver rings,68,dior ring,900 +silver ring,3,rings,68,silver ring design for girl,400 +silver ring,4,gold ring,61,fendi ring,350 +silver ring,5,men silver ring,35,covetous silver serpent ring ds3,300 +silver ring,6,men ring,34,sterling silver ring blanks,250 +silver ring,7,mens silver ring,32,gold price today,250 +silver ring,8,silver ring price,30,boy ring design silver,250 +silver ring,9,diamond ring,30,silver price today,200 +silver ring,10,diamond silver ring,30,evil eye ring silver,180 +silver ring,11,silver price,29,sterling silver thumb ring,180 +silver ring,12,diamond,29,boys ring design,170 +silver ring,13,ring for men,24,gucci silver heart ring,170 +silver ring,14,silver 925 ring,23,silver wishbone ring,160 +silver ring,15,silver ring for men,23,sterling silver turtle ring,140 +silver ring,16,ring for men silver,23,snake ring,140 +silver ring,17,wedding ring silver,22,silver ring designs for men,130 +silver ring,18,925 silver,22,silver ring designs for girls,130 +silver ring,19,black silver ring,21,silver serpent ring ds3,120 +silver ring,20,ring design,21,today gold rate,120 +silver ring,21,silver ring design,20,vivienne westwood,120 +silver ring,22,silver band ring,18,leg ring silver,110 +silver ring,23,sterling silver rings,18,etsy uk,110 +silver ring,24,pandora ring,18,wrap around ring silver,110 +, +women ring design,0,ring design for women,100,gold rate today,300 +women ring design,1,women gold ring design,71,, +women ring design,2,gold ring,70,, +women ring design,3,gold ring design,66,, +women ring design,4,gold ring for women design,64,, +women ring design,5,gold ring for women,62,, +women ring design,6,silver ring for women,6,, +women ring design,7,ring design for men,5,, +women ring design,8,gold ring design for men,4,, +women ring design,9,gold rate today,3,, +, +ring design,0,gold,100,gold ring design 2020,70700 +ring design,1,ring gold,98,latest ring design 2020,32500 +ring design,2,gold ring design,97,"gold ring design for male under 10,000",9650 +ring design,3,men ring design,17,blue stone ring design for female,5350 +ring design,4,female ring design,15,silver ring design for girl,350 +ring design,5,ring designs,15,umbrella ring design,350 +ring design,6,ear ring design,15,girl finger ring design,300 +ring design,7,ring design for female,14,ear ring design for girl,300 +ring design,8,diamond ring,14,boy ring design silver,250 +ring design,9,design diamond ring,14,gold ear ring design for girl,200 +ring design,10,ear ring,13,today gold price,200 +ring design,11,design engagement ring,13,ring ceremony,200 +ring design,12,silver ring,13,simple gold ring design for female,200 +ring design,13,engagement ring,13,big gold ring design for female,190 +ring design,14,silver ring design,13,toe ring design,160 +ring design,15,stone ring design,13,simple ring design for female,150 +ring design,16,gold price,13,simple ring design for girl,140 +ring design,17,new ring design,12,gold rate today,130 +ring design,18,design of ring,12,small ear ring design,130 +ring design,19,ring design for men,11,queen ring design,110 +ring design,20,female gold ring design,11,latest ring design for girl,110 +ring design,21,ring gold design female,11,boy ring design gold,100 +ring design,22,gold ring design for female,11,chandi ring design for girl,100 +ring design,23,women ring design,11,boys gold ring design,100 +ring design,24,girl ring design,10,finger ring design for girl,100 +, +women ring finger,0,ring finger for women,100,gold ring for women,110 +women ring finger,1,wedding ring finger women,33,gold finger ring for women,40 +women ring finger,2,wedding ring,30,, +women ring finger,3,wedding ring finger,30,, +women ring finger,4,engagement ring finger for women,22,, +women ring finger,5,wedding ring for women,20,, +women ring finger,6,wedding ring finger for women,18,, +women ring finger,7,gold ring for women,18,, +women ring finger,8,gold finger ring for women,16,, +women ring finger,9,ring finger for men,10,, +, +cartier ring,0,love ring,100,cartier 750 ring 52833a real or fake,18050 +cartier ring,1,love ring cartier,97,gucci ghost ring,9100 +cartier ring,2,cartier love,95,cartier 750 ring 52833a leve,6900 +cartier ring,3,cartier gold ring,31,dhgate cartier ring,1450 +cartier ring,4,gold ring,30,chaumet,500 +cartier ring,5,cartier bracelet,28,cartier clash ring,400 +cartier ring,6,diamond ring,26,clash de cartier ring,400 +cartier ring,7,cartier diamond ring,24,cartier ring dupe amazon,350 +cartier ring,8,rings,21,cartier dupe ring,350 +cartier ring,9,cartier rings,21,dhgate,300 +cartier ring,10,tiffany,20,panthère de cartier ring,300 +cartier ring,11,tiffany ring,19,fendi ring,250 +cartier ring,12,wedding ring,18,dior ring,250 +cartier ring,13,engagement ring,18,dior,250 +cartier ring,14,cartier engagement ring,18,cartier love ring dupe,200 +cartier ring,15,cartier wedding ring,17,rings for women,200 +cartier ring,16,cartier ring price,14,chanel bracelet,190 +cartier ring,17,trinity cartier ring,14,cartier 750 ring 52833a,170 +cartier ring,18,trinity ring,14,carters,150 +cartier ring,19,cartier trinity,13,cartier friendship ring,150 +cartier ring,20,trinity,13,christ,140 +cartier ring,21,cartier band,12,cartier live ring,130 +cartier ring,22,cartier love bracelet,12,cartier 750 ring,120 +cartier ring,23,gucci,11,louis vuitton,110 +cartier ring,24,gucci ring,11,gucci rings,110 +, +cartier,0,bracelet cartier,100,adam and cartier love island,5700 +cartier,1,love cartier,72,cartier surjan,5550 +cartier,2,cartier ring,71,adam and cartier,4850 +cartier,3,cartier watch,60,cartier love island,2700 +cartier,4,jacques cartier,42,gabbie cartier,1700 +cartier,5,bracelet love cartier,36,lilou cartier,1400 +cartier,6,santos cartier,27,pasha de cartier parfum,800 +cartier,7,cartier tank,25,cartier skeleton watch,300 +cartier,8,cartier glasses,23,กํา ไล cartier,200 +cartier,9,cartier watches,20,cartier santos skeleton,170 +cartier,10,cartier ring love,19,anel cartier,170 +cartier,11,tiffany,17,cartier frames men,160 +cartier,12,rolex,15,cartier bilezik,140 +cartier,13,cartier panthere,14,bratara cartier,130 +cartier,14,cartier bague,13,cartier crash,130 +cartier,15,cartier necklace,12,van cleef & arpels,130 +cartier,16,gucci,11,cartier bileklik altın,120 +cartier,17,cartier rings,11,cartier brille,120 +cartier,18,cartier pasha,11,pizzeria jacques cartier,110 +cartier,19,cartier perfume,11,bague clou cartier,110 +cartier,20,montre cartier,10,pulseira cartier,110 +cartier,21,parfum cartier,10,bague cartier femme,100 +cartier,22,louis vuitton,10,cartier trinity armband,100 +cartier,23,cartier sunglasses,9,cartier ohrringe,90 +cartier,24,chanel,9,cartier glasses men,90 +, +macys,0,macys near me,100,macys cyber monday 2019,15650 +macys,1,macys sale,66,macys parade 2020,11400 +macys,2,macys hours,59,macys closing stores 2020,4500 +macys,3,macys furniture,59,macys fireworks 2020,2950 +macys,4,nordstrom,53,macys parade 2019,1000 +macys,5,kohls,46,is macys open,250 +macys,6,macys shoes,46,is macys closing,250 +macys,7,jcpenney,45,macys stock,150 +macys,8,macys dresses,42,macys stock price,150 +macys,9,macys coupon,42,coach outlet,130 +macys,10,macys store,41,macys thanksgiving parade time,120 +macys,11,insite macys,41,macys closing,110 +macys,12,macys mens,40,macys outdoor furniture,110 +macys,13,macys card,38,macys open,100 +macys,14,target,37,bath and body works,100 +macys,15,dillards,34,dillards near me,90 +macys,16,macys credit,33,macys palm desert,90 +macys,17,macys login,32,macys radley sectional,90 +macys,18,macys credit card,27,macys pr,80 +macys,19,macys home,26,macys el centro,80 +macys,20,macys men,24,gap factory,80 +macys,21,macys parade,24,macys guam,70 +macys,22,macys boots,21,macys backstage,70 +macys,23,macys online,20,macys bedding sale,70 +macys,24,macys stock,19,macys returns,70 +, +moonstone ring,0,moonstone engagement ring,100,rainbow moonstone engagement ring,300 +moonstone ring,1,moonstone rings,77,vintage moonstone engagement ring,300 +moonstone ring,2,gold moonstone ring,49,moonstone promise ring,250 +moonstone ring,3,moon ring,46,moon magic,140 +moonstone ring,4,silver moonstone ring,42,moon ring,110 +moonstone ring,5,moonstone diamond ring,35,moonstone wedding ring,100 +moonstone ring,6,moonstone wedding ring,31,mens moonstone ring,80 +moonstone ring,7,moonstone meaning,30,moon stone,80 +moonstone ring,8,rainbow moonstone ring,27,raw moonstone ring,70 +moonstone ring,9,etsy moonstone ring,25,moonstone ring uk,60 +moonstone ring,10,etsy,24,rose gold moonstone ring,60 +moonstone ring,11,moonstone ring meaning,24,promise rings,60 +moonstone ring,12,moonstone ring uk,17,silver moonstone ring,40 +moonstone ring,13,rose gold moonstone ring,17,moonstone rings,40 +moonstone ring,14,moon stone,16,, +moonstone ring,15,raw moonstone ring,13,, +moonstone ring,16,vintage moonstone ring,13,, +moonstone ring,17,mens moonstone ring,12,, +moonstone ring,18,alexandrite,12,, +moonstone ring,19,moonstone and diamond ring,11,, +moonstone ring,20,june birthstone,11,, +moonstone ring,21,opal rings,10,, +moonstone ring,22,moonstone engagement ring meaning,7,, +moonstone ring,23,moon magic,6,, +moonstone ring,24,vintage moonstone engagement ring,5,, +, +gold price today,0,gold price in today,100,gold price in pakistan 2020 today,26000 +gold price today,1,gold rate,31,gold price today siliguri,200 +gold price today,2,gold rate today,30,gold price today in siliguri,200 +gold price today,3,today gold price india,20,gold price today kota,180 +gold price today,4,gold price in india today,17,gold price today jammu,120 +gold price today,5,gold price in india,16,yes bank share price,110 +gold price today,6,today price of gold,15,sbi share price,100 +gold price today,7,price of gold,15,gold price today varanasi,100 +gold price today,8,today silver price,14,24ct gold price today,90 +gold price today,9,silver price,14,yes bank share,90 +gold price today,10,gold price today delhi,12,gold price today in moradabad,90 +gold price today,11,today gold price kolkata,9,gold price today jaipur,90 +gold price today,12,today gold price in delhi,9,gold price today patna,90 +gold price today,13,gold price in delhi,9,gold price today in kota,90 +gold price today,14,today gold price hyderabad,8,gold price today amritsar,80 +gold price today,15,today gold price 22k,7,gold price today kanpur,70 +gold price today,16,today 22 carat gold price,7,gold price today in jammu,70 +gold price today,17,22k gold price today,7,ril share price,70 +gold price today,18,gold price today mumbai,6,gold price today 22k kolkata,70 +gold price today,19,gold price mumbai,6,gold price today in meerut,60 +gold price today,20,gold price today pakistan,6,gold price today bbsr,60 +gold price today,21,today gold price in kolkata,6,24 carat gold price in ahmedabad today,60 +gold price today,22,gold price today in hyderabad,6,gold price today jodhpur,60 +gold price today,23,today gold price in hyderabad,6,gold price today in up,60 +gold price today,24,gold price today ahmedabad,5,gold price today gwalior,50 +, +gold rate today,0,today gold price,100,gold rate in pakistan today 2020,22350 +gold rate today,1,gold price,98,gold rate in kerala today 1gm,2900 +gold rate today,2,gold rate today chennai,77,today gold rate in ap,150 +gold rate today,3,today gold rate hyderabad,70,today gold rate mysore,100 +gold rate today,4,gold rate today india,60,today gold rate in karnataka,90 +gold rate today,5,gold rate today in chennai,57,today gold and silver rate in hyderabad,80 +gold rate today,6,gold rate in chennai,54,gold rate today in latur,80 +gold rate today,7,today bangalore gold rate,52,gold rate today bhopal,80 +gold rate today,8,gold silver rate today,48,gold rate today in karnataka,80 +gold rate today,9,silver rate today,48,svbc gold rate today,70 +gold rate today,10,gold rate in india today,48,today gold rate in kakinada,60 +gold rate today,11,gold rate in india,47,gold rate in mysore today,60 +gold rate today,12,gold rate today in india,46,gold rate today kakinada,60 +gold rate today,13,gold rate in hyderabad today,45,today gold rate hyderabad,60 +gold rate today,14,gold rate mumbai today,44,gold silver rate today,50 +gold rate today,15,gold rate in hyderabad,43,today gold rate in up,50 +gold rate today,16,today gold rate delhi,42,gold rate today vizag,50 +gold rate today,17,gold rate delhi today,42,today gold rate visakhapatnam,50 +gold rate today,18,22 carat gold rate,35,gold rate today rajahmundry,50 +gold rate today,19,today gold rate 22 carat,35,silver rate today,50 +gold rate today,20,today gold rate in delhi,33,today gold rate vijayawada,40 +gold rate today,21,gold rate today in bangalore,33,gold rate today nashik,40 +gold rate today,22,gold rate today kerala,32,gold rate today lucknow,40 +gold rate today,23,gold rate in bangalore,32,, +gold rate today,24,gold rate in delhi,32,, +, +how to measure ring size,0,how to measure your ring size,100,how to measure my ring size at home,69350 +how to measure ring size,1,how to measure for ring size,88,how to figure out ring size,25850 +how to measure ring size,2,how to measure ring size at home,75,how to measure a ring size at home,500 +how to measure ring size,3,cm to mm,68,how to measure ring size with string,450 +how to measure ring size,4,rings,63,ring size guide,250 +how to measure ring size,5,how to measure ring size in cm,57,how to measure bra size,200 +how to measure ring size,6,how to measure ring size in mm,56,etsy,200 +how to measure ring size,7,how to measure ring size uk,52,how do you measure ring size,200 +how to measure ring size,8,how to measure ring finger size,50,pandora,180 +how to measure ring size,9,ring size chart,45,how to measure ring size women,170 +how to measure ring size,10,inches to mm,44,how to measure your ring size at home,160 +how to measure ring size,11,how to measure ring size in inches,39,how to measure ring size uk,150 +how to measure ring size,12,ring size in inches,37,how to measure ring size at home,140 +how to measure ring size,13,how to measure ring size men,36,how to measure my ring size,140 +how to measure ring size,14,how to measure ring size women,34,how to measure ring size for men,120 +how to measure ring size,15,how to measure my ring size,33,mejuri,120 +how to measure ring size,16,how to measure finger size for ring,23,how to measure finger size for ring,110 +how to measure ring size,17,how to measure mens ring size,23,cm to mm,110 +how to measure ring size,18,how to measure ring size with tape measure,22,how to measure ring size in cm,100 +how to measure ring size,19,ring sizes,21,how to measure ring size in mm,100 +how to measure ring size,20,how to measure ring size us,19,how to measure ring size in inches,100 +how to measure ring size,21,how to measure your ring size at home,15,how to measure ring size us,100 +how to measure ring size,22,how to measure ring size for men,14,ring size in inches,90 +how to measure ring size,23,how to measure a ring size at home,14,kay jewelers,90 +how to measure ring size,24,pandora,14,how to measure your finger for a ring,80 +, +cartier love ring,0,cartier love bracelet,100,dhgate,23150 +cartier love ring,1,cartier bracelet,91,cartier love ring dupe,350 +cartier love ring,2,love ring cartier gold,71,tiffany and co,160 +cartier love ring,3,cartier love ring diamond,43,cartier love ring silver,50 +cartier love ring,4,cartier rings,32,love ring cartier gold,40 +cartier love ring,5,tiffany ring,31,cartier love ring diamond,40 +cartier love ring,6,cartier love ring price,30,, +cartier love ring,7,tiffany,28,, +cartier love ring,8,fake cartier ring,23,, +cartier love ring,9,cartier love ring dupe,23,, +cartier love ring,10,the love ring cartier,20,, +cartier love ring,11,fake cartier love ring,20,, +cartier love ring,12,cartier white gold love ring,17,, +cartier love ring,13,cartier love ring men,16,, +cartier love ring,14,cartier love necklace,14,, +cartier love ring,15,cartier love ring silver,14,, +cartier love ring,16,louis vuitton,13,, +cartier love ring,17,cartier love ring rose gold,13,, +cartier love ring,18,mens cartier love ring,13,, +cartier love ring,19,tiffany and co,11,, +cartier love ring,20,hermes bracelet,6,, +cartier love ring,21,cartier love bangle,6,, +cartier love ring,22,cartier love ring with diamonds,6,, +cartier love ring,23,cartier trinity ring,5,, +cartier love ring,24,pandora,4,, +, +kay jewelers,0,kay jewelers rings,100,kay jewelers center of me,12900 +kay jewelers,1,rings,99,kays fine jewelry,3250 +kay jewelers,2,kay jewelry,97,kay jewelers hanover pa,2750 +kay jewelers,3,jewelry,96,kay jewelers black friday 2020,1650 +kay jewelers,4,jewelers near me,90,does kay jewelers sell fake diamonds,1600 +kay jewelers,5,kay jewelers near me,88,kayleigh mcenany,1550 +kay jewelers,6,zales jewelers,62,kay jewelers cyber monday 2019,1000 +kay jewelers,7,zales,59,kay jewelers sioux falls,750 +kay jewelers,8,kay jewelers necklace,58,kay jewelers black friday sale,450 +kay jewelers,9,necklace,57,kay jewelers grove city,400 +kay jewelers,10,kay jewelers card,56,kay jewelers virginia beach,350 +kay jewelers,11,kay jewelers credit,55,kay jewelers anklet,350 +kay jewelers,12,kay jewelers credit card,50,kay jewelers salem oregon,250 +kay jewelers,13,pandora,36,kay jewelers mother rings,200 +kay jewelers,14,engagement rings,35,kay jewelers tracking,190 +kay jewelers,15,kay jewelers engagement rings,34,kay pee jewelers,180 +kay jewelers,16,jared jewelers,34,engagement rings for women,170 +kay jewelers,17,jared,33,is kay jewelers good,150 +kay jewelers,18,kay jewelers earrings,31,kay jewelers promo code,140 +kay jewelers,19,kay jewelers outlet,30,kay jewelers memphis,140 +kay jewelers,20,kay outlet,30,zales outlet,130 +kay jewelers,21,kays jewelers,28,kay jewelers rochester mn,120 +kay jewelers,22,kays,27,kay jewelers discount code,120 +kay jewelers,23,kay jewelers sale,26,jewelry stores near me,120 +kay jewelers,24,jewelry stores,24,zales near me,120 +, +swarovski,0,pandora,100,swarovski nl pure,8300 +swarovski,1,swarovski crystal,74,swarovski christmas ornament 2020,5150 +swarovski,2,crystal,74,swarovski 2019 ornament,1700 +swarovski,3,bracelet swarovski,61,swarovski optik dg,1400 +swarovski,4,swarovski necklace,60,swarovski schwanger,750 +swarovski,5,bracelet,60,harga kalung swarovski,650 +swarovski,6,earrings,53,anelli swarovski 2021,650 +swarovski,7,swarovski crystals,52,victoria swarovski schwanger,550 +swarovski,8,swarovski earrings,51,swarovski adalah,250 +swarovski,9,ring swarovski,50,bratara swarovski,250 +swarovski,10,swarovski sale,48,swarovski tennis bracelet,110 +swarovski,11,swarovski victoria,43,swarovski tennis necklace,100 +swarovski,12,outlet swarovski,42,swarovski ireland,90 +swarovski,13,swarovski uk,42,swarovski near me,90 +swarovski,14,jewelry,27,collana swarovski uomo,90 +swarovski,15,swarovski jewelry,27,swarovski store near me,90 +swarovski,16,swarovski canada,26,colar swarovski,90 +swarovski,17,swarovski rings,25,swarovski black friday,80 +swarovski,18,swarovski watch,24,swarovski discount code,80 +swarovski,19,swarovski online,24,idee cadeau femme,80 +swarovski,20,swarovski kette,23,swarovski voucher code,70 +swarovski,21,swarovski jewellery,20,swar,70 +swarovski,22,jewellery,20,swarovski crystals for nails,70 +swarovski,23,tiffany,19,สร้อย swarovski,70 +swarovski,24,swarovski armband,18,swarovski egypt,70 +, +louis vuitton,0,louis vuitton bag,100,louis vuitton california dream,3150 +louis vuitton,1,gucci,89,louis vuitton iphone 11 case,2950 +louis vuitton,2,louis vuitton bags,38,louis vuitton face mask,2900 +louis vuitton,3,lv,36,masque louis vuitton,2250 +louis vuitton,4,chanel,29,louis vuitton filter,1450 +louis vuitton,5,louis vuitton purse,27,multi pochette louis vuitton,1300 +louis vuitton,6,louis vuitton shoes,27,louis vuitton maske,1250 +louis vuitton,7,louis vuitton wallet,27,louis vuitton mask,950 +louis vuitton,8,dior,27,louis partridge,900 +louis vuitton,9,louis vuitton pochette,26,louis vuitton bts,600 +louis vuitton,10,louis vuitton sac,22,on the go louis vuitton,600 +louis vuitton,11,louis vuitton belt,19,louis vuitton ombre nomade,500 +louis vuitton,12,prada,18,louis vuitton league of legends,450 +louis vuitton,13,louis vuitton sale,15,louis vuitton airpod case,400 +louis vuitton,14,neverfull louis vuitton,15,christian dior,250 +louis vuitton,15,louis vuitton backpack,14,bonnet louis vuitton,200 +louis vuitton,16,supreme,13,louis vuitton bucket hat,200 +louis vuitton,17,louis vuitton supreme,13,louis vuitton stencil,200 +louis vuitton,18,louis vuitton tasche,13,louis vuitton pochette accessoires,200 +louis vuitton,19,louis vuitton sneakers,13,louis vuitton felicie pochette,170 +louis vuitton,20,hermes,13,louis vuitton beanie,170 +louis vuitton,21,burberry,12,dior,160 +louis vuitton,22,balenciaga,12,coach outlet,120 +louis vuitton,23,fendi,12,louis vuitton lock necklace,120 +louis vuitton,24,louis vuitton bracelet,11,louis tomlinson,110 +, +the bling ring,0,the real bling ring,100,best movies on netflix,36900 +the bling ring,1,bling ring movie,48,what happened to the bling ring,29050 +the bling ring,2,the bling ring movie,46,the bling ring real story,400 +the bling ring,3,the bling ring netflix,43,the bling ring netflix,400 +the bling ring,4,emma watson,42,the bling ring rotten tomatoes,300 +the bling ring,5,the bling ring emma watson,35,the bling ring real people,250 +the bling ring,6,the bling ring cast,31,the bling ring imdb,250 +the bling ring,7,the bling ring story,29,the bling ring review,150 +the bling ring,8,alexis neiers,24,the real bling ring,130 +the bling ring,9,the bling ring real people,22,the bling ring real life,130 +the bling ring,10,the bling ring film,19,the bling ring cast,70 +the bling ring,11,the bling ring real story,13,, +the bling ring,12,the bling ring streaming,13,, +the bling ring,13,the bling ring real life,12,, +the bling ring,14,the bling ring trailer,11,, +the bling ring,15,the bling ring true story,11,, +the bling ring,16,the bling ring review,6,, +the bling ring,17,best movies on netflix,5,, +the bling ring,18,the bling ring full movie,5,, +the bling ring,19,the bling ring rotten tomatoes,5,, +the bling ring,20,the bling ring imdb,4,, +the bling ring,21,the bling ring 2011,4,, +the bling ring,22,sofia coppola,4,, +the bling ring,23,what happened to the bling ring,4,, +the bling ring,24,is the bling ring a true story,3,, +, diff --git a/004-EmailNotify/README.md b/004-EmailNotify/README.md index f620178..01021c5 100644 --- a/004-EmailNotify/README.md +++ b/004-EmailNotify/README.md @@ -3,6 +3,6 @@ 本小项目主要体现邮箱的作用,如果利用微信绑定收信邮箱,便能实时知道自己想要知道的信息。 注意:发送邮箱的登录密码一般是在设置smtp中生成的授权码。 - 在服务器上启动脚本,与连接无关: -```shell +``` nohup python main.py ``` diff --git a/005-PaidSource/.gitignore b/005-PaidSource/.gitignore new file mode 100644 index 0000000..b6e4761 --- /dev/null +++ b/005-PaidSource/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/005-PaidSource/005-PaidSource.iml b/005-PaidSource/005-PaidSource.iml new file mode 100644 index 0000000..ad3c0a3 --- /dev/null +++ b/005-PaidSource/005-PaidSource.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/005-PaidSource/README.md b/005-PaidSource/README.md new file mode 100644 index 0000000..b671552 --- /dev/null +++ b/005-PaidSource/README.md @@ -0,0 +1,47 @@ +# 这些脚本你肯定会有用到的 + +### 操作已打开的chrome浏览器 + +场景:某些情况我们获取怎么都获取不到cookie,但我们可以使用先在浏览器上登录,然后进行自动化操作。 + +操作指南: + +```shell +需要以该方式启动的浏览器: +win: chrome.exe --remote-debugging-port=9222 +mac:/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --remote-debugging-port=9222& +``` + +实现脚本:[chrome.py](./chrome.py) + +### excel表的常规操作 + +场景:word文档生活使用就不用多说了,学会定能给生活带来很大的便利。 + +操作指南:使用 pandas 开源库实现。 + +实现脚本:1 考勤统计实现 [kaoqin.py](./kaoqin.py) 。2从excel表取数据翻译后重新写入[gtransfer.py](./gtransfer.py) + +### 用ffmpeg批量修改视频的md5值 + +场景:短视频搬运专用。 + +操作指南:需要安装ffmpeg环境。 + +实现脚本:[ff_video.py](./ff_video.py) + +### 文件相关操作:json读写、文件子目录文件获取、html转word等 + +场景:文件的一些操作。 + +操作指南:略。 + +实现脚本:[file_util.py](./file_util.py) + +### 其他站点爬虫与解析 + +场景:注意学会BeautifulSoup解析,取属性值等。 + +操作指南:略。 + +实现脚本:[other_site.py](./other_site.py) \ No newline at end of file diff --git a/005-PaidSource/__init__.py b/005-PaidSource/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/005-PaidSource/chrome.py b/005-PaidSource/chrome.py new file mode 100644 index 0000000..1fe3d62 --- /dev/null +++ b/005-PaidSource/chrome.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 用已经打开的chrome浏览器进行自动化操作。 +在某些应用场景我们获取怎么都获取不到cookie,但我们可以使用先在浏览器上登录,然后进行自动化操作。 +这里实现book118.com网站自动化操作。 +@Date :2022年1月14日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" + +import asyncio +import random +import time + +import aiohttp +import requests +from bs4 import BeautifulSoup +from pyppeteer import launcher + +import file_util as futls +from v2ray_pool import Net + +loop = asyncio.get_event_loop() + + +async def get_cookie(page): + """ + 获取cookie + :param:page page对象 + :return:cookies 处理后的cookie + """ + cookie_list = await page.cookies() + cookies = "" + for cookie in cookie_list: + coo = "{}={};".format(cookie.get("name"), cookie.get("value")) + cookies += coo + return cookies + + +async def main(): + async with aiohttp.ClientSession() as session: + try: + async with session.get("http://localhost:9222/json/version") as response: + chrome = await response.json() + browser = await launcher.connect( + defaultViewport=None, + loop=loop, + browserWSEndpoint=chrome['webSocketDebuggerUrl'] + ) + except aiohttp.ClientConnectorError: + print("start chrome --headless --remote-debugging-port=9222 --disable-gpu") + return + # pages = await browser.pages() + page = await browser.newPage() # "通过 Browser 对象创建页面 Page 对象" + await page.goto('https://max.book118.com/user_center_v1/doc/Doclist/trash.html') + table = await page.waitForSelector('#table') + print(table) + content = await page.content() # 获取页面内容 + futls.write(content, 'test/src.html') + agent = await browser.userAgent() + cookies = await get_cookie(page) + print('agent:[%s]' % agent) + print('cookies:[%s]' % cookies) + results = Book118.parse_page(content) + print(results) + print('需要操作的个数:[%d]' % len(results)) + headers = { + 'Cookie': cookies, + 'User-Agent': agent + } + for result in results: + aids = result.get('aids') + title = result.get('title') + Book118.recycling(headers, aids, title) + time.sleep(random.randint(2, 5)) + Book118.recycling_name(headers, aids) + time.sleep(random.randint(2, 5)) + + +class Book118(Net): + '''https://max.book118.com/user_center_v1/doc/index/index.html#trash''' + + @staticmethod + def recycling(headers, aids, title): + data = { + 'aids': aids, + 'is_optimization': 0, + 'title': title, + 'keywords': '', + 'typeid': 481, + 'dirid': 0, + 'is_original': 0, + 'needmoney': random.randint(3, 35), + 'summary': '' + } + url = 'https://max.book118.com/user_center_v1/doc/Api/updateDocument/docListType/recycling' + r = requests.post(url=url, data=data, headers=headers, allow_redirects=False, verify=False, timeout=15, + stream=True) + if r.status_code == 200: + print('修改[%s]成功' % title) + else: + print(r) + raise Exception('[%s]修改失败!' % title) + + @staticmethod + def recycling_name(headers, aids): + data = { + 'aids': aids, + 'reason': '文件名已修复', + 'status': 1 + } + url = 'https://max.book118.com/user_center_v1/doc/Api/recoverDocument/docListType/recycling' + r = requests.post(url=url, data=data, headers=headers, allow_redirects=False, verify=False, + timeout=15, stream=True) + if r.status_code == 200: + print('提交[%s]成功' % aids) + else: + print(r) + raise Exception('[%s]操作失败!' % aids) + + @staticmethod + def load_page(url): + r = requests.get(url=url, allow_redirects=False, verify=False, + timeout=15, stream=True) + r.encoding = r.apparent_encoding + print('url[%s], code[%d]' % (url, r.status_code)) + if r.status_code == 200: + return r.text + return None + + @staticmethod + def parse_page(content): + soup = BeautifulSoup(content, 'html.parser') + tbody = soup.find('tbody') + results = [] + for tr in tbody.find_all('tr'): + if '文档名不规范' in tr.find('td', class_='col-delete-reason').text: + title: str = tr.get_attribute_list('data-title')[0] + if title.endswith('..docx'): + title = title.replace('..docx', '') + aids = tr.get_attribute_list('data-aid')[0] + results.append({'aids': aids, 'title': title}) + return results + + +if __name__ == "__main__": + ''' + 注意:需要以该方式启动的浏览器: + win: chrome.exe --remote-debugging-port=9222 + mac:/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --remote-debugging-port=9222& + ''' + loop.run_until_complete(main()) diff --git a/005-PaidSource/ff_video.py b/005-PaidSource/ff_video.py new file mode 100644 index 0000000..9178b07 --- /dev/null +++ b/005-PaidSource/ff_video.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 使用ffmpeg去掉最后一帧,改变md5。短视频搬运专用 +@Date :2022年02月17日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import os + + +def cute_video(folder): + files = next(os.walk(folder))[2] # 获取文件 + for file in files: + file_path = os.path.join(folder, file) + shotname, extension = os.path.splitext(file) + if len(shotname) == 0 or len(extension) == 0: + continue + out_file = os.path.join(folder, 'out-{}{}'.format(shotname, extension)) + # 获取时间。输入自己系统安装的ffmpeg,注意斜杠 + time = os.popen( + r"/usr/local/ffmpeg/bin/ffmpeg -i {} 2>&1 | grep 'Duration' | cut -d ' ' -f 4 | sed s/,//".format( + file_path)).read().replace('\n', '').replace(' ', '') + if '.' in time: + match_time = time.split('.')[0] + else: + match_time = time + print(match_time) + ts = match_time.split(':') + sec = int(ts[0]) * 60 * 60 + int(ts[1]) * 60 + int(ts[2]) + # 从0分0秒100毫秒开始截切(目的就是去头去尾) + os.popen(r"/usr/local/ffmpeg/bin/ffmpeg -ss 0:00.100 -i {} -t {} -c:v copy -c:a copy {}".format(file_path, sec, + out_file)) + + +# 主模块执行 +if __name__ == "__main__": + # path = os.path.dirname('/Users/Qincji/Downloads/ffmpeg/') + path = os.path.dirname('需要处理的目录') # 目录下的所有视频 + cute_video(path) diff --git a/005-PaidSource/file_util.py b/005-PaidSource/file_util.py new file mode 100644 index 0000000..961aa0a --- /dev/null +++ b/005-PaidSource/file_util.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 文件相关处理 +@Date :2022年01月22日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" + +import datetime +import json +import os +import re +import shutil + +import cairosvg +import pandas as pd +import pypandoc # 要安装pandoc +from docx import Document + + +def file_name(file_dir): + results = [] + for root, dirs, files in os.walk(file_dir): + # print(root) # 当前目录路径 + # print(dirs) # 当前路径下所有子目录 + # print(files) # 当前路径下所有非目录子文件 + results += files + return results + + +def deal_one_page(): + fs = file_name('100条') + for f in fs: + try: + print('正在检测【%s】' % f) + shotname, extension = os.path.splitext('%s' % f) + print('正在检测【%s】' % shotname) + if '1篇' in shotname: + new_name = re.sub(r'1篇', '', f) + document = Document(r"html/%s" % f) + paragraphs = document.paragraphs + p = paragraphs[0] + p._element.getparent().remove(p._element) + document.save(r"html/%s" % new_name) + os.remove('html/%s' % f) + except Exception as e: + print(e) + + +def copy_doc(): + fs = file_name('all') + i = 1 + k = 1 + temp_dir = '01' + os.makedirs('100条/%s' % temp_dir) + for f in fs: + try: + # print('正在检测【%s】' % f) + shotname, extension = os.path.splitext('%s' % f) + shutil.copyfile(r'all/%s' % f, r'100条/%s/%s' % (temp_dir, f)) + if i % 100 == 0: + temp_dir = '0%d' % k if k < 10 else '%d' % k + k += 1 + os.makedirs('100条/%s' % temp_dir) + i += 1 + except Exception as e: + print(e) + + +'''########文件处理相关#########''' + + +def html_cover_doc(in_path, out_path): + '''将html转化成功doc''' + path, file_name = os.path.split(out_path) + if path and not os.path.exists(path): + os.makedirs(path) + pypandoc.convert_file(in_path, 'docx', outputfile=out_path) + + +def svg_cover_jpg(src, dst): + '''' + drawing = svg2rlg("drawing.svg") + renderPDF.drawToFile(drawing, "drawing.pdf") + renderPM.drawToFile(drawing, "fdrawing.png", fmt="PNG") + renderPM.drawToFile(drawing, "drawing.jpg", fmt="JPG") + ''' + path, file_name = os.path.split(dst) + if path and not os.path.exists(path): + os.makedirs(path) + # drawing = svg2rlg(src) + # renderPM.drawToFile(drawing, dst, fmt="JPG") + cairosvg.svg2png(url=src, write_to=dst) + + +def html_cover_excel(content, out_path): + '''将html转化成excel''' + path, file_name = os.path.split(out_path) + if path and not os.path.exists(path): + os.makedirs(path) + tables = pd.read_html(content, encoding='utf-8') + writer = pd.ExcelWriter(out_path) + for i in range(len(tables)): + tables[i].to_excel(writer, sheet_name='表%d' % (i + 1)) # startrow + writer.save() # 写入硬盘 + + +def write_to_html(content, file_path): + '''将内容写入本地,自动加上head等信息''' + page = ''' + + + + +
''' + page += content + page += '''

AltStyle によって変換されたページ (->オリジナル) /

+ ''' + write(page, file_path) + + +def write_json(content, file_path): + '''写入json''' + path, file_name = os.path.split(file_path) + if path and not os.path.exists(path): + os.makedirs(path) + with open(file_path, 'w') as f: + json.dump(content, f, ensure_ascii=False) + f.close() + + +def read_json(file_path): + '''读取json''' + with open(file_path, 'r') as f: + js_get = json.load(f) + f.close() + return js_get + + +def write(content, file_path): + '''写入txt文本内容''' + path, file_name = os.path.split(file_path) + if path and not os.path.exists(path): + os.makedirs(path) + with open(file_path, 'w') as f: + f.write(content) + f.close() + + +def read(file_path) -> str: + '''读取txt文本内容''' + content = None + try: + with open(file_path, 'r') as f: + content = f.read() + f.close() + except Exception as e: + print(e) + return content + + +def get_next_folder(dst, day_diff, folder, max_size): + '''遍历目录文件,直到文件夹不存在或者数目达到最大(max_size)时,返回路径''' + while True: + day_time = (datetime.date.today() + datetime.timedelta(days=day_diff)).strftime('%Y-%m-%d') # 下一天的目录继续遍历 + folder_path = os.path.join(dst, day_time, folder) + if os.path.exists(folder_path): # 已存在目录 + size = len(next(os.walk(folder_path))[2]) + if size>= max_size: # 该下一个目录了 + day_diff += 1 + continue + else: + os.makedirs(folder_path) + return day_diff, folder_path + + +if __name__ == '__main__': + pass diff --git a/005-PaidSource/gsearch.py b/005-PaidSource/gsearch.py new file mode 100644 index 0000000..daf2102 --- /dev/null +++ b/005-PaidSource/gsearch.py @@ -0,0 +1,216 @@ +import re +import time +import os +from urllib.parse import quote_plus + +import chardet +import requests_html +import pypandoc # 要安装pandoc + +from v2ray_pool import Net +from bs4 import BeautifulSoup +import googlesearch as ggs +import os +import random +import sys +import time +import ssl + +BLACK_DOMAIN = ['www.google.gf', 'www.google.io', 'www.google.com.lc'] +DOMAIN = 'www.google.com' + + +class GSearch(Net): + def search_page(self, url, pause=3): + """ + Google search + :param query: Keyword + :param language: Language + :return: result + """ + time.sleep(random.randint(1, pause)) + try: + r = self.request_en(url) + print('resp code=%d' % r.status_code) + if r.status_code == 200: + charset = chardet.detect(r.content) + content = r.content.decode(charset['encoding']) + return content + elif r.status_code == 301 or r.status_code == 302 or r.status_code == 303: + location = r.headers['Location'] + time.sleep(random.randint(1, pause)) + return self.search_page(location) + # elif r.status_code == 429 or r.status_code == 443: + # time.sleep(3) + # return search_page(url) + return None + except Exception as e: + print(e) + return None + + def parse_html(self, html): + soup = BeautifulSoup(html, 'html.parser') # 声明BeautifulSoup对象 + # find = soup.find('p') # 使用find方法查到第一个p标签 + # print('----->>>>%s' % str(find.text)) + p_s = soup.find_all('p') + results = [] + for p in p_s: + if p.find('img'): # 不要带有图片的标签 + continue + if p.find('a'): # 不要带有链接的标签 + continue + content = str(p) + if "文章来源" in content: + print('过滤[文章来源]>>>>>%s' % content) + continue + if "来源:" in content: + print('过滤[来源:]>>>>>%s' % content) + continue + if len(p.text.replace('\n', '').strip()) < 1: # 过滤空内容 + # print('过滤[空字符]>>>>>!') + continue + results.append(content) + results.append('


') # 隔一下 + return results + # return re.findall(r'()', html, re.DOTALL) + + def get_html(self, url): + session = requests_html.HTMLSession() + html = session.get(url) + html.encoding = html.apparent_encoding + return html.text + + def conver_to_doc(self, in_name, out_name): + try: + pypandoc.convert_file('%s.html' % in_name, 'docx', outputfile="doc/%s.docx" % out_name) + os.remove('%s.html' % in_name) + except Exception as e: + print(e) + + def download_and_merge_page(self, urls, name): + try: + page = [''' + + + + + '''] + k = 0 + size = random.randint(3, 5) # 每次合并成功5篇即可 + for i in range(len(urls)): + if k>= size: + break + try: + temp = self.parse_html(self.get_html(urls[i])) + except Exception as e: + print(e) + continue + if len(temp) < 3: # 篇幅太短 + continue + page.append( + '

第%d篇:

' % ( + k + 1)) # 加入标题 + page += temp + page.append('\n') + k += 1 + page.append('') + with open("%s.html" % name, mode="w") as f: # 写入文件 + for p in page: + f.write(p) + return k + except Exception as e: + print(e) + return 0 + + def get_full_urls(self, html): + a_s = re.findall(r'', html, re.DOTALL) + results = [] + for a in a_s: + try: + # print(a) + # url = re.findall(r'/url\?q=(.*?\.html)', a, re.DOTALL)[0] + url: str = re.findall(r'(http[s]{0,1}://.*?\.html)', a, re.DOTALL)[0] + # title = re.findall(r'(.*?)', a, re.DOTALL)[0] #会有问题 + # print('{"url":"%s","title":"%s"}' % (url, title)) + if 'google.com' in url: + continue + if url in results: + continue + # 过来同一个网站的 + domain = re.findall('http[s]{0,1}://(.*?)/', url, re.DOTALL)[0] + # 含有--的 + if '-' in domain: + continue + # www.sz.gov.cn,'.'超过4个时绝对不行的,像:bbs.jrj.ex3.http.80.ipv6.luzhai.gov.cn + if domain.count('.')> 4: + continue + for u in results: + if domain in u: + continue + results.append(url) + except Exception as e: + # print(e) + pass + return results + + def get_full_titles(self, html): + results = [] + soup = BeautifulSoup(html, "html.parser") + results = [] + for a in soup.find_all(name='a'): + + try: + h3 = a.find(name='h3') + if h3 and h3.has_attr('div'): + div = h3.find(name='div') + results.append(div.getText()) + else: + div = a.find(name='span') + results.append(div.getText()) + + except Exception as e: + print(e) + return results + + def format_common_url(self, search, domain='www.google.com', start=0): + url = 'https://{domain}/search?q={search}&start={start}' + url = url.format(domain=domain, search=quote_plus(search), start=start) + return url + + def format_full_url(self, domain, as_q='', as_epq='', as_oq='', as_eq='', as_nlo='', as_nhi='', lr='', cr='', + as_qdr='', + as_sitesearch='', + as_filetype='', tbs='', start=0, num=10): + """ + https://www.google.com/advanced_search + https://www.google.com/search?as_q=%E8%A1%A3%E6%9C%8D+%E8%A3%A4%E5%AD%90+%E6%9C%8D%E8%A3%85+%E9%A5%B0%E5%93%81+%E7%8F%A0%E5%AE%9D+%E9%93%B6%E9%A5%B0&as_epq=%E5%AE%98%E7%BD%91&as_oq=%E6%9C%8D%E8%A3%85+or+%E9%85%8D%E9%A5%B0&as_eq=%E9%9E%8B%E5%AD%90&as_nlo=20&as_nhi=1000&lr=lang_zh-CN&cr=countryCN&as_qdr=m&as_sitesearch=.com&as_occt=body&safe=active&as_filetype=&tbs= + allintext: 衣服 裤子 服装 饰品 珠宝 银饰 服装 OR or OR 配饰 "官网" -鞋子 site:.com 20..1000 + :param domain: 域名:google.com + :param as_q: 输入重要字词: 砀山鸭梨 + :param as_epq: 用引号将需要完全匹配的字词引起: "鸭梨" + :param as_oq: 在所需字词之间添加 OR: 批发 OR 特价 + :param as_eq: 在不需要的字词前添加一个减号: -山大、-"刺梨" + :param as_nlo: 起点,在数字之间加上两个句号并添加度量单位:0..35 斤、300..500 元、2010..2011 年 + :param as_nhi: 终点,在数字之间加上两个句号并添加度量单位:0..35 斤、300..500 元、2010..2011 年 + :param lr: 查找使用您所选语言的网页。 + :param cr: 查找在特定地区发布的网页。 + :param as_qdr: 查找在指定时间内更新的网页。 + :param as_sitesearch: 搜索某个网站(例如 wikipedia.org ),或将搜索结果限制为特定的域名类型(例如 .edu、.org 或 .gov) + :param as_filetype: 查找采用您指定格式的网页。如:filetype:pdf + :param tbs: 查找可自己随意使用的网页。 + :param start: 第几页,如 90:表示从第9页开始,每一页10条 + :param num: 每一页的条数 + :return: + """ + url = 'https://{domain}/search?as_q={as_q}&as_epq={as_epq}&as_oq={as_oq}&as_eq={as_eq}&as_nlo={as_nlo}&as_nhi={as_nhi}&lr={lr}&cr={cr}&as_qdr={as_qdr}&as_sitesearch={as_sitesearch}&as_occt=body&safe=active&as_filetype={as_filetype}&tbs={tbs}&start={start}&num={num}' + url = url.format(domain=domain, as_q=quote_plus(as_q), as_epq=quote_plus(as_epq), as_oq=quote_plus(as_oq), + as_eq=quote_plus(as_eq), as_nlo=as_nlo, as_nhi=as_nhi, lr=lr, cr=cr, as_qdr=as_qdr, + as_sitesearch=as_sitesearch, start=start, num=num, tbs=tbs, as_filetype=as_filetype) + return url + + +if __name__ == '__main__': + url = 'http://www.sz.gov.cn/cn/zjsz/nj/content/post_1356218.html' + domain: str = re.findall('http[s]{0,1}://(.*?)/', url, re.DOTALL)[0] + print(domain.count('.')) + print(domain) diff --git a/005-PaidSource/gtransfer.py b/005-PaidSource/gtransfer.py new file mode 100644 index 0000000..95434f3 --- /dev/null +++ b/005-PaidSource/gtransfer.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 实现从excel文件获取关键词进行翻译后写入新文件 +@Date :2021年10月22日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" + +import json +import os +import os.path +import random +import time + +import chardet +import pandas as pd + +import file_util as futls +import v2ray_util as utils +from v2ray_pool import Net + +BLACK_DOMAIN = ['www.google.gf', 'www.google.io', 'www.google.com.lc'] +DOMAIN = 'www.google.com' + + +class GTransfer(Net): + def search_page(self, url, pause=3): + """ + Google search + :param query: Keyword + :param language: Language + :return: result + """ + time.sleep(random.randint(1, pause)) + try: + r = self.request_en(url) + print('resp code=%d' % r.status_code) + if r.status_code == 200: + charset = chardet.detect(r.content) + content = r.content.decode(charset['encoding']) + return content + elif r.status_code == 301 or r.status_code == 302 or r.status_code == 303: + location = r.headers['Location'] + time.sleep(random.randint(1, pause)) + return self.search_page(location) + return None + except Exception as e: + print(e) + return None + + def transfer(self, content): + # url = 'http://translate.google.com/translate_a/single?client=gtx&dt=t&dj=1&ie=UTF-8&sl=auto&tl=zh-CN&q=' + content + url = 'http://translate.google.cn/translate_a/single?client=gtx&dt=t&dj=1&ie=UTF-8&sl=en&tl=zh-CN&q=' + content + try: + cache = futls.read_json('data/cache.json') + for c in cache: + if content in c: + print('已存在,跳过:{}'.format(content)) + return c.get(content) + except Exception as e: + pass + try: + result = self.search_page(url) + trans = json.loads(result)['sentences'][0]['trans'] + # 解析获取翻译后的数据 + # print(result) + print(trans) + self.local_cache.append({content: trans}) + futls.write_json(self.local_cache, 'data/cache.json') + # 写入数据吗?下次直接缓存取 + except Exception as e: + print(e) + utils.restart_v2ray() + return self.transfer(content) + return trans + + def init_param(self, file_name): + utils.restart_v2ray() + self.local_cache = [] + # 第一次加载本地的(已翻译的就不再翻译了) + try: + cache = futls.read_json('data/cache.json') + for c in cache: + self.local_cache.append(c) + except Exception as e: + pass + csv_file = os.path.join('data', file_name) + csv_out = os.path.join('data', 'out_' + file_name) + df = pd.read_excel(csv_file, sheet_name='CompetitorWords') + # 代表取出第一行至最后一行,代表取出第四列至最后一列。 + datas = df.values + size = len(df) + print('总共有{}行数据'.format(size)) + titles, titles_zh, keys1, keys2, keys3, pros = [], [], [], [], [], [] + for col in range(0, size): + t = datas[col][0] + titles.append(t) + keys1.append(datas[col][1]) + keys2.append(datas[col][2]) + keys3.append(datas[col][3]) + pros.append(datas[col][4]) + titles_zh.append(self.transfer(t)) + print('总共{},现在到{}'.format(size, col + 1)) + df_write = pd.DataFrame( + {'标题': titles, '中文标题': titles_zh, '关键词1': keys1, '关键词2': keys2, '关键词3': keys3, '橱窗产品': pros}) + df_write.to_excel(csv_out, index=False) + utils.kill_all_v2ray() + + +# http://translate.google.com/translate_a/single?client=gtx&dt=t&dj=1&ie=UTF-8&sl=auto&tl=zh-CN&q=what + + +if __name__ == '__main__': + g = GTransfer() + g.init_param('xxx.xls') + # utils.search_node() diff --git a/005-PaidSource/kaoqin.py b/005-PaidSource/kaoqin.py new file mode 100644 index 0000000..3870292 --- /dev/null +++ b/005-PaidSource/kaoqin.py @@ -0,0 +1,57 @@ +""" +@Description: excel表的常规操作,这里实现统计考勤 +@Date :2022年02月21日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" + +import pandas as pd +import calendar +from pandas._libs.tslibs.timestamps import Timestamp + + +def get_days(year, month): # 获取输出日期的列明 + dates = calendar.monthrange(year, month) + week = dates[0] # 1号那天是星期几 + days = dates[1] # 总共的天数 + print(dates) + index_time = [] + for day in range(1, days): + index_time.append('{}-{}-{} 星期{}'.format(year, month, day, (week + day) % 7)) + print(index_time) + return index_time + + +def parse_excel(csv_file, out_file, names, dates): + df = pd.read_excel(csv_file, sheet_name='Sheet') # 从文件和表格名称读取 + datas = df.values + size = len(df) + print('总共有{}行数据'.format(size)) + results = {} + for name in names: # 我是根据名字统计 + results.update({name: ['' for x in range(len(dates))]}) # 默认生成每个日期的空格 + for col in range(0, size): + s_name = datas[col][2] # 打印一下就知道去的那是哪里列的值了 + t_time: Timestamp = datas[col][6] # 我这里是时间戳,用type(datas[col][6])打印类型可知 + if s_name not in names: + continue + # 获取这天是哪一天的,name_datas是哪个人对应的列表数据 + d, h, m, name_datas = t_time.day, t_time.hour, t_time.minute, results.get(s_name) + # 早上 9:00前打卡,下午18:00后打卡,取一天最早和最晚的一次即可,门禁可能有很多数据 + tt = '2022-1-{} {}:{}'.format(d, h, m) + old = name_datas[d - 1] # 下标 + if len(old) < 5: # 空的 + name_datas[d - 1] = '{} 早 {};'.format(tt, '' if h < 9 else '异常') # 上班打卡 + else: + # 去除第一个: + first = old.split(';')[0] + last = '{} 晚 {}'.format(tt, '' if h>= 18 else '异常') + name_datas[d - 1] = '{};{}'.format(first, last) + print(results) + df_write = pd.DataFrame(results, index=dates) + df_write.to_excel(out_file, index=True) # 写入输出表格数据 + + +if __name__ == '__main__': + names = ['x1', 'x2', 'x3', 'x4', 'x5'] # 要统计那些人 + parse_excel('data/一月考勤.xls', 'data/out_kaoqin.xls', names, get_days(2022, 1)) diff --git a/005-PaidSource/keywords.py b/005-PaidSource/keywords.py new file mode 100644 index 0000000..98d9071 --- /dev/null +++ b/005-PaidSource/keywords.py @@ -0,0 +1,92 @@ +import json +import random +import re +import time +from urllib.parse import quote_plus + +from v2ray_pool import Net + +import chardet +import requests +import urllib3 +from bs4 import BeautifulSoup +from my_fake_useragent import UserAgent + + +class Keywords(Net): + '''url:https://www.5118.com/ciku/index#129''' + + def get_keys_by_net(self) -> []: + try: + r = self.request(r'https://www.5118.com/ciku/index#129') + if r.status_code != 200: + return None + r.encoding = r.apparent_encoding + # 法律
+ soup = BeautifulSoup(r.text, "html.parser") + results = [] + for a in soup.find_all(name='a'): + results += re.findall(r'(.*?) []: + with open('test/key_tag.json', 'r') as f: + js_get = json.load(f) + f.close() + return js_get + + def get_titles_by_local(self) -> []: + with open('test/key_title.json', 'r') as f: + js_get = json.load(f) + f.close() + return js_get + + def get_titles_by_net(self, key): + '''通过网盘搜索检查出 + https://www.alipanso.com/search.html?page=1&keyword=%E7%90%86%E8%B4%A2&search_folder_or_file=2&is_search_folder_content=1&is_search_path_title=1&category=doc&file_extension=doc&search_model=1 + ''' + results = [] + try: + time.sleep(random.randint(1, 4)) + r = self.request_en( + r'https://www.alipanso.com/search.html?page=1&keyword=%s&search_folder_or_file=2&is_search_folder_content=1&is_search_path_title=1&category=doc&file_extension=doc&search_model=1' % key) + if r.status_code != 200: + print(r.status_code) + return None + r.encoding = r.apparent_encoding + soup = BeautifulSoup(r.text, "html.parser") + for a in soup.find_all(name='a'): + ts = re.findall(r'(.*?).doc', str(a.get_text()).replace('\n', ''), re.DOTALL) + for t in ts: + if '公众号' in t or '【' in t or '[' in t or ',' in t or ',' in t or ')' in t or ')' in t or t in results: + continue + if len(t) < 4: + continue + results.append(t) + return results + except Exception as e: + print(e) + return None + + +def test(): + js = ['a', 'b', 'c'] + with open('test/key_tag.json', 'w') as f: + json.dump(js, f) + f.close() + with open('test/key_tag.json', 'r') as f: + js_get = json.load(f) + f.close() + print(js_get) + + +if __name__ == "__main__": + # test() + keys = Keywords().get_keys_by_net() + print(keys) + with open('test/key_tag.json', 'w') as f: + json.dump(keys, f, ensure_ascii=False) + f.close() diff --git a/005-PaidSource/main.py b/005-PaidSource/main.py new file mode 100644 index 0000000..9ffe9f6 --- /dev/null +++ b/005-PaidSource/main.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 关键词获取 +@Date :2021/09/22 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +# from amazon import run_api +import json +import re + +import yagooglesearch + +import v2ray_util as utils +from gsearch import GSearch +from keywords import Keywords + + +def start_task(): + kd = Keywords() + keywords = kd.get_titles_by_local() + name = 'temp' + gs = GSearch() + i = 0 + is_need_start = True + while i < len(keywords): + if is_need_start: + utils.restart_v2ray() + gs.update_agent() + is_need_start = True + key = keywords[i] + query = 'site:gov.cn filetype:html "%s"' % key + client = yagooglesearch.SearchClient( + query, + tbs="li:1", + max_search_result_urls_to_return=100, + http_429_cool_off_time_in_minutes=49, + http_429_cool_off_factor=1.5, + proxy="socks5h://127.0.0.1:1080", + verbosity=5, + ) + client.assign_random_user_agent() + try: + page_urls = client.search() + except Exception: + continue + new_urls = [] + for u1 in page_urls: + domain: str = re.findall('http[s]{0,1}://(.*?)/', u1, re.DOTALL)[0] + # 含有--的 + if '-' in domain: + continue + # www.sz.gov.cn,'.'超过4个时绝对不行的,像:bbs.jrj.ex3.http.80.ipv6.luzhai.gov.cn + if domain.count('.')> 4: + continue + for u2 in new_urls: + if domain in u2: + continue + new_urls.append(u2) + print('过滤器链接数:%d, 过滤后链接数:%d' % (len(page_urls), len(new_urls))) + page_size = len(new_urls) + if page_size == 0: + print('[%s]获取文章链接失败!' % key) + continue + if not gs.download_and_merge_page(page_urls, name): # 合并文章 + print('下载或者合并失败') + continue + doc_name = '%d篇%s' % (page_size, key) + gs.conver_to_doc(name, doc_name) + is_need_start = False + i += 1 + + utils.kill_all_v2ray() + + +def start_proxy_task(): + kd = Keywords() + keywords: [] = kd.get_titles_by_local() + name = 'temp' + gs = GSearch() + key_s = [] + key_s.pop() + + +def start_task2(): + kd = Keywords() + keywords: [] = kd.get_titles_by_local() + name = 'temp' + gs = GSearch() + i = 0 + is_need_start = True + key_size = len(keywords) + key = keywords.pop() + while i < key_size: + if is_need_start: + utils.restart_v2ray() + gs.update_agent() + else: + key = keywords.pop() + is_need_start = True + # key_url = gs.format_full_url(domain='google.com', as_sitesearch='.gov.cn', as_filetype='html', as_epq=key, + # lr='lang_zh-CN', cr='countryCN') + # key_url = gs.format_full_url(domain='search.iwiki.uk', as_sitesearch='gov.cn', as_filetype='html', as_epq=key, + # lr='lang_zh-CN', cr='countryCN') + # key_url = gs.format_common_url('site:gov.cn filetype:html %s' % key, domain='search.iwiki.uk') + key_url = gs.format_common_url('site:gov.cn intitle:%s' % key, domain='www.google.com') + print(key_url) + content = gs.search_page(key_url) + if content is None: + print('[%s]搜索失败,进行重试!' % key) + continue + with open('test/test_search.html', 'w') as f: + f.write(content) + f.close() + page_urls = gs.get_full_urls(content) # 获取文章的url + page_size = len(page_urls) + if page_size == 0: + print('[%s]没有内容,下一个...' % key) + else: + size = gs.download_and_merge_page(page_urls, name) + if size == 0: # 合并文章 + print('下载或者合并失败,跳过!') + else: + doc_name = '%d篇%s' % (size, key) + gs.conver_to_doc(name, doc_name) + print('生成[%s]文章成功!!!' % doc_name) + is_need_start = False + i += 1 + # 重新覆盖本地关键词 + with open('test/key_title.json', 'w') as f: + json.dump(keywords, f, ensure_ascii=False) + f.close() + + utils.kill_all_v2ray() + + +def test_titles(): + kd = Keywords() + keywords = kd.get_titles_by_local() + print('总共需要加载%d个关键词' % len(keywords)) + + +def test_task(): + kd = Keywords() + keywords = kd.get_keys_by_local() + print('总共需要加载%d个关键词' % len(keywords)) + # keywords = ['股市基金'] + i = 0 + search_keys = [] + utils.restart_v2ray() # 第一次用固定agent + is_need_start = False + while i < len(keywords): + if is_need_start: + utils.restart_v2ray() + # kd.update_agent() + is_need_start = True + key = keywords[i] + print('开始搜索:%s' % key) + titles = kd.get_titles_by_net(key) + if titles is None: + print('[%s]获取关键词标题失败!' % key) + continue + print(titles) + for t in titles: + if t not in search_keys: + search_keys.append(t) + # 每次都要更新一次 + with open('test/key_title.json', 'w') as f: + json.dump(search_keys, f, ensure_ascii=False) + f.close() + is_need_start = False + i += 1 + + utils.kill_all_v2ray() + + +def test_get_title(): + with open('search_page.html', 'r') as f: + page = f.read() + f.close() + gs = GSearch() + titles = gs.get_full_titles(page) # 获取文章的标题 + print(titles) + + +if __name__ == "__main__": + # utils.restart_v2ray() + utils.search_node() + # utils.kill_all_v2ray() diff --git a/005-PaidSource/other_site.py b/005-PaidSource/other_site.py new file mode 100644 index 0000000..09c3cbd --- /dev/null +++ b/005-PaidSource/other_site.py @@ -0,0 +1,574 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 获取其他站点信息爬虫 +@Date :2022/1/14 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import os +import random +import re +import time + +import requests +from bs4 import BeautifulSoup + +import file_util as futls +import v2ray_util as utils +from v2ray_pool import Net + + +class Cncic(Net): + '''中华全国商业中心:https://www.cncic.org/''' + + def start_task(self): + utils.restart_v2ray() + cncic = Cncic() + keys = [{'cat': 92, 'name': '专题分析报告'}, {'cat': 95, 'name': '政策法规'}, {'cat': 8, 'name': '月度分析'}, + {'cat': 10, 'name': '黄金周分析'}, {'cat': 16, 'name': '零售百强'}, {'cat': 94, 'name': '市场观察'}, ] + success_datas = [] + for key in keys: + cat = key.get('cat') + name = key.get('name') + datas = cncic.load_list(cat) + while len(datas) == 0: + utils.restart_v2ray() + datas = cncic.load_list(cat) + success_datas.append({'name': name, 'data': datas}) + futls.write_json(success_datas, 'data/cncic/keys.json') # 每次保存到本地 + + success_datas = futls.read_json('data/cncic/keys.json') + key_size = len(success_datas) + is_need_start = False + key = None + for i in range(key_size): + if is_need_start: + utils.restart_v2ray() + else: + key = success_datas.pop() + if key is None: + key = success_datas.pop() + is_need_start = True + folder = key.get('name') + datas = key.get('data') + for data in datas: + try: + load_page = cncic.load_page(data.get('url')) + except Exception as e: + print(e) + continue + title, content = cncic.parse_page(load_page) + html_path = 'data/html/cncic/%s/%s.html' % (folder, title) + doc_path = 'data/doc/cncic/%s/%s.docx' % (folder, title) + futls.write_to_html(content, html_path) + try: + futls.html_cover_doc(html_path, doc_path) + except Exception as e: + print(e) + futls.write_json(success_datas, 'data/cncic/keys.json') # 更新本地数据库 + is_need_start = False + i += 1 + utils.kill_all_v2ray() + + def load_list(self, cat, paged=1) -> []: + results = [] + while True: + url = 'https://www.cncic.org/?cat=%d&paged=%d' % (cat, paged) + try: + page = self.load_page(url) + results += self.parse_list(page) + paged += 1 + time.sleep(random.randint(3, 6)) + except Exception as e: + print(e) + break + return results + + def load_page(self, url): + '''加载页面,如:https://www.cncic.org/?p=3823''' + r = self.request_zh(url) + r.encoding = r.apparent_encoding + print('Cncic[%s] code[%d]' % (url, r.status_code)) + if r.status_code == 200: + return r.text + elif r.status_code == 301 or r.status_code == 302 or r.status_code == 303: + location = r.headers['Location'] + time.sleep(1) + return self.load_page(location) + return None + + def parse_page(self, page): + '''解析页面,返回标题和文章页面内容,如果生成文章则还需要组装''' + soup = BeautifulSoup(page, 'html.parser') + article = soup.find('article') + header = article.find('header') + title = header.text.replace('\n', '').replace(' ', '') + content = article.find('div', class_='single-content') + result = str(header) + result += str(content) + return title, result + + def parse_list(self, page): + '''解析列表(如:https://www.cncic.org/?cat=92)页面,返回标题、连接、日期''' + soup = BeautifulSoup(page, 'html.parser') + main = soup.find('main') + articles = main.find_all('article') + results = [] + for article in articles: + header = article.find('header') + url = header.find('a').get_attribute_list('href')[0] + title = header.text.replace('\n', '').replace(' ', '') + date = article.find('span', class_='date').text + results.append({'url': url, 'title': title, 'date': date}) + return results + + +class Ceicdata(Net): + '''https://www.ceicdata.com/''' + + def start_task_1(self): + cd = Ceicdata() + utils.restart_v2ray() + success_datas = futls.read_json('data/keys/ceicdata.json') + key_size = len(success_datas) + print(key_size) + is_need_start = False + key = None + for i in range(key_size): + if is_need_start: + utils.restart_v2ray() + cd.update_agent() + else: + key = success_datas.pop() + is_need_start = True + if key is None: + key = success_datas.pop() + title = key.get('title') + url = key.get('url') + try: + page = cd.load_page(url) + except Exception as e: + page = None + print(e) + if page is None: + continue + html_path = 'data/html/ceicdata/%s.html' % title + content = cd.parse_page_1(page) + futls.write_to_html(content, html_path) + futls.html_cover_excel(html_path, 'data/doc/ceicdata/%s.xlsx' % title) + futls.write_json(success_datas, 'data/keys/ceicdata.json') # 更新本地数据库 + is_need_start = False + i += 1 + utils.kill_all_v2ray() + + @staticmethod + def start_task2(): + utils.restart_v2ray() + cd = Ceicdata() + keys_path = 'data/keys/ceicdata.json' + keys = futls.read_json(keys_path) + if not keys: + url = 'https://www.ceicdata.com/zh-hans/country/china' + page = cd.load_page(url) + if page is None: + raise Exception('获取页面失败') + keys = cd.parse_main_2(page) + if len(keys) == 0: + raise Exception('获取链接失败') + futls.write_json(keys, keys_path) + key_size = len(keys) + print('下载数量[%d]' % key_size) + is_need_start = False + key = None + for i in range(key_size): + if is_need_start: + utils.restart_v2ray() + cd.update_agent() + else: + key = keys.pop() + is_need_start = True + if key is None: + key = keys.pop() + url = key.get('url') + try: + page = cd.load_page(url) + except Exception as e: + page = None + print(e) + if page is None: + continue + try: + title, content = cd.parse_page_2(page) + except Exception as e: + print(e) + continue + html_path = 'data/html/ceicdata2/%s.html' % title + futls.write_to_html(content, html_path) + futls.html_cover_doc(html_path, 'data/doc/ceicdata/%s.docx' % title) + futls.write_json(keys, keys_path) # 更新本地数据库 + is_need_start = False + i += 1 + utils.kill_all_v2ray() + + def load_page(self, url): + '''加载页面,如:https://www.cncic.org/?p=3823''' + r = self.request_en(url) + # r = self.request(url) + r.encoding = r.apparent_encoding + print('Cncic[%s] code[%d]' % (url, r.status_code)) + if r.status_code == 200: + return r.text + elif r.status_code == 301 or r.status_code == 302 or r.status_code == 303: + location = r.headers['Location'] + time.sleep(1) + return self.load_page(location) + return None + + def parse_main_1(self, page): + '''解析页面,返回标题和文章页面内容,如果生成文章则还需要组装''' + soup = BeautifulSoup(page, 'html.parser') + main = soup.find('main') + lists = main.find('div', class_='indicators-lists') + results = [] + for a in lists.find_all('a'): + # https://www.ceicdata.com/zh-hans/indicator/nominal-gdp + title = a.text.replace(' ', '') + url = 'https://www.ceicdata.com' + a.get_attribute_list('href')[0].replace(' ', '') + results.append({'title': title, 'url': url}) + return results + + def parse_main_2(self, page): + '''解析页面,返回标题和文章页面内容,如果生成文章则还需要组装''' + soup = BeautifulSoup(page, 'html.parser') + main = soup.find('main') + results = [] + for tbody in main.find_all('tbody'): + for a in tbody.find_all('a'): + # https://www.ceicdata.com/zh-hans/indicator/nominal-gdp + title = a.text.replace(' ', '') + url = 'https://www.ceicdata.com' + a.get_attribute_list('href')[0].replace(' ', '') + results.append({'title': title, 'url': url}) + return results + + def parse_page_1(self, page): + '''解析页面,返回标题和文章页面内容,如果生成文章则还需要组装''' + soup = BeautifulSoup(page, 'html.parser') + main = soup.find('main') + clearfix = main.find('div', class_='clearfix') + h1 = clearfix.find('h1') + h2 = clearfix.find('h2') + tables = clearfix.find_all('table') + content = str(h1) + str(tables[0]) + str(h2) + str(tables[1]) + return content + + def parse_page_2(self, page): + '''解析页面,返回标题和文章页面内容,如果生成文章则还需要组装''' + soup = BeautifulSoup(page, 'html.parser') + main = soup.find('main') + left = main.find('div', id='left-col-7') + title = left.find('span', class_='c-purple').text.replace(' ', '').replace('\n', '') + left.find('div', id='breadcrumb').decompose() # 移除节点 + for ele in left.find_all('div', class_='hide'): + ele.decompose() # 移除节点 + for ele in left.find_all('div', class_='div-chart-btns'): + ele.decompose() # 移除节点 + for ele in left.find_all('div', class_='table-buy'): + ele.decompose() # 移除节点 + for ele in left.find_all('div', class_='div-bgr-2'): + if '查看价格选项' in str(ele): + ele.decompose() # 移除节点 + for ele in left.find_all('h4'): + if '购买' in str(ele): + ele.decompose() # 移除节点 + for ele in left.find_all('button'): + if '加载更多' in str(ele): + ele.decompose() # 移除节点 + for ele in left.find_all('div', class_='div-bgr-1'): + if '详细了解我们' in str(ele): + ele.decompose() # 移除节点 + i = 1 + for img in left.find_all('img'): + src = str(img.get('src')) + path = '/Users/Qincji/Desktop/develop/py/project/PythonIsTools/005-PaidSource/data/img/%d.svg' % i + dst = '/Users/Qincji/Desktop/develop/py/project/PythonIsTools/005-PaidSource/data/img/%d.png' % i + if 'www.ceicdata.com' in src: + print('下载图片url[%s]' % src) + r = self.request_zh(src) + # r = self.request(src) + if r.status_code == 200: + with open(path, 'wb') as f: + f.write(r.content) + futls.svg_cover_jpg(path, dst) # 将svg转换成jpg + img['src'] = dst + i += 1 + else: + raise Exception('下载图片失败!') + rs = re.sub(r'href=".*?"', '', str(left)) # 移除href + return title, rs + + +class Cnnic(Net): + '''http://www.cnnic.net.cn/hlwfzyj/hlwxzbg/ , 注意:因为文件过大,使用别人代理下载,固定代理''' + + @staticmethod + def start_task(): + cnnic = Cnnic() + keys_path = 'data/keys/cnnic.json' + all_keys = futls.read_json(keys_path) + if not all_keys: + all_keys = [] + for i in range(7): + if i == 0: + url = 'http://www.cnnic.net.cn/hlwfzyj/hlwxzbg/index.htm' + else: + url = 'http://www.cnnic.net.cn/hlwfzyj/hlwxzbg/index_%d.htm' % i + page = cnnic.load_page(url) + if page: + futls.write(page, 'test/src.html') + all_keys += cnnic.parse_page(page) + futls.write_json(all_keys, keys_path) + size = len(all_keys) + print('将要下载数量[%d]' % size) + for i in range(size): + key = all_keys.pop() + name = key.get('title') + url = key.get('url') + path = 'data/doc/cnnic/%s.pdf' % name + cnnic.download(url, path) + futls.write_json(all_keys, keys_path) + print('已下载[%d] | 还剩[%d]' % (i + 1, size - i - 1)) + + def load_page(self, url): + time.sleep(3) + # r = self.request_en(url) + # r = self.request(url) + proxies = {'http': 'http://11.0.222.4:80', 'https': 'http://11.0.222.4:80'} + r = requests.get(url=url, headers=self._headers, allow_redirects=False, verify=False, + proxies=proxies, timeout=15) + r.encoding = r.apparent_encoding + print('Cnnic[%s] code[%d]' % (url, r.status_code)) + if r.status_code == 200: + return r.text + elif r.status_code == 301 or r.status_code == 302 or r.status_code == 303: + location = r.headers['Location'] + time.sleep(1) + return self.load_page(location) + return None + + def parse_page(self, page): + '''解析页面,返回标题和文章页面内容,如果生成文章则还需要组装''' + soup = BeautifulSoup(page, 'html.parser') + content = soup.find('div', class_='content') + results = [] + for li in content.find_all('li'): + a = li.find('a') + date = li.find('div', class_='date').text[0:4] # 只要年份 + title = a.text.replace('\n', '').replace(' ', '') + # http://www.cnnic.net.cn/hlwfzyj/hlwxzbg/hlwtjbg/202109/P020210915523670981527.pdf + # ./hlwtjbg/202109/P020210915523670981527.pdf + url = 'http://www.cnnic.net.cn/hlwfzyj/hlwxzbg/' + str(a.get('href')).replace('./', '') + if not '年' in title: + title = '%s年发布%s' % (date, title) + results.append({'title': title, 'url': url}) + return results + + def download(self, url, path): + if os.path.exists(path): + os.remove(path) + proxies = {'http': 'http://11.0.222.4:80', 'https': 'http://11.0.222.4:80'} + r = requests.get(url=url, headers=self._headers, allow_redirects=False, verify=False, + proxies=proxies, timeout=15, stream=True) + i = 0 + print('name[%s]|code[%d]' % (path, r.status_code)) + with open(path, "wb") as pdf: + for chunk in r.iter_content(chunk_size=1024): + if chunk: + i += 1 + if i % 30 == 0: + print('.', end='') + pdf.write(chunk) + pdf.close() + time.sleep(random.randint(3, 12)) + + +class Othersite(Net): + def __init__(self): + super(Othersite, self).__init__() + self.dir = 'Othersite' + + @staticmethod + def start_task(): + reqs = [{'title': 'xxx', 'url': 'https://www.xxx.com/wedding/engagement-rings.html'}] + # utils.restart_v2ray() + sl = Othersite() + datas = futls.read_json(os.path.join('Othersite', 'page_urls.json')) + if datas is None: + datas = [] + for req in reqs: + title = req.get('title') + url = req.get('url') + has_write = False + for data in datas: + if title in data.get('title'): + has_write = True + break + if has_write: # 已经请求过了 + print('页面连接 [%s]已存在,跳过!' % title) + continue + start = 1 + page_urs = [] + while True: + if start != 1: + temp = url + '?p=' + str(start) + else: + temp = url + try: + page = sl.load_page(temp) + except Exception as e: + print(e) + print('--------000') + utils.restart_v2ray() + continue + futls.write(page, 'test/src.html') + has_next, results = sl.parse_list(page) + print('has_next: {} | {}'.format(has_next, results)) + page_urs += results + if not has_next: + break + start += 1 + # page = futls.read('test/src.html') + # print(sl.parse_details(page)) + datas.append({'title': title, 'urls': page_urs}) + futls.write_json(datas, os.path.join('Othersite', 'page_urls.json')) + all_results = [] # 总数据表 + size = len(datas) + alls_local = futls.read_json(os.path.join('Othersite', 'all.json')) + for i in range(size): + data = datas.pop() + title = data.get('title') + page_urs = data.get('urls') + has_write_all = False + # for local in alls_local: + # if title in local.get('title'): + # has_write_all = True + # break + # if has_write_all: + # print('[%s]已下载,跳过!' % title) + # continue + sl.dir = os.path.join('Othersite', title) + url_size = len(page_urs) + print('下载数量[%d]' % url_size) + is_need_start = False + url = None + results = [] + for i in range(url_size): + if is_need_start: + utils.restart_v2ray() + sl.update_agent() + else: + url = page_urs.pop() + is_need_start = True + if url is None: + url = page_urs.pop() + # 本地是否也已经存在 + sku1 = url[url.rfind('-') + 1:].replace('.html', '').upper() + if os.path.exists(os.path.join(sl.dir, sku1)): + is_need_start = False + print('ksu [%s]已存在,跳过!' % sku1) + continue + try: + page = sl.load_page(url) + sku = sl.parse_details(page) + results.append(sku) + is_need_start = False + except Exception as e: + print(e) + print('--------333') + all_results.append({'title': title, 'skus': results}) + futls.write_json(all_results, os.path.join('Othersite', 'all.json')) + futls.write_json(datas, os.path.join('Othersite', 'page_urls.json')) + utils.kill_all_v2ray() + + def load_page(self, url): + '''加载页面,如:https://www.cncic.org/?p=3823''' + time.sleep(random.randint(3, 8)) + r = self.request_en(url) + # r = self.request(url) + r.encoding = r.apparent_encoding + print('Othersite code[%d] |url [%s] ' % (r.status_code, url)) + if r.status_code == 200: + return r.text + elif r.status_code == 301 or r.status_code == 302 or r.status_code == 303: + location = r.headers['Location'] + time.sleep(1) + return self.load_page(location) + return None + + def parse_home(self, page): + soup = BeautifulSoup(page, 'html.parser') + nav = soup.find('nav') + results = [] + for a in nav.find_all('a'): + results.append({'title': a.text, 'url': str(a.get('href'))}) + return results + + def parse_list(self, page): + soup = BeautifulSoup(page, 'html.parser') + ol = soup.find('ol') + # 判断是否还有下一页 + next = False + pages = soup.find('div', class_='pages') + if pages: + pages_n = pages.find('li', class_='pages-item-next') + if pages_n: + next = True + results = [] + for a in ol.find_all('a'): + results.append(str(a.get('href'))) + return next, results + + def parse_details(self, page): + soup = BeautifulSoup(page, 'html.parser') + # content = soup.find('div', class_='content') + # main = content.find('main') + right = soup.find('div', class_='product-info-main') + title = right.find('h1').text.replace('Othersite ', '') + sku = right.find('div', class_='value').text + try: + price = right.find('span', id='price-saved').find('span').text + except Exception: + print('没有折扣,继续找..') + price = right.find('span', class_='special-price').find('span', class_='price').text + # 下载图片 + layout = soup.find('amp-layout') + carousel = layout.find('amp-carousel') + imgs = [] + i = 0 + content = '{}\n{}'.format(sku, title) + for img in carousel.find_all('amp-img'): + src = str(img.get('src')) + imgs.append(src) + path = self.download_img(src, sku, i) + content = content + '\n' + path + i += 1 + futls.write(content, os.path.join(self.dir, sku, '{}.txt'.format(sku))) + return {'sku': sku, 'title': title, 'price': price, 'imgs': imgs} + + def download_img(self, src, sku, i): + path = os.path.join(self.dir, sku, '{}-{}.jpg'.format(sku, i)) + pre_path, file_name = os.path.split(path) + if pre_path and not os.path.exists(pre_path): + os.makedirs(pre_path) + time.sleep(random.randint(1, 2)) + r = self.request_en(src) + if r.status_code == 200: + with open(path, 'wb') as f: + f.write(r.content) + else: + raise Exception('下载图片失败!') + return path + + +if __name__ == "__main__": + pass diff --git a/005-PaidSource/v2ray_pool/__init__.py b/005-PaidSource/v2ray_pool/__init__.py new file mode 100644 index 0000000..7a4ce9d --- /dev/null +++ b/005-PaidSource/v2ray_pool/__init__.py @@ -0,0 +1,13 @@ +# 运行时路径。并非__init__.py的路径 +import os +import sys + +BASE_DIR = "../002-V2rayPool" +if os.path.exists(BASE_DIR): + sys.path.append(BASE_DIR) + +from core import utils +from core.conf import Config +from core.client import Creator +from db.db_main import DBManage +from base.net_proxy import Net \ No newline at end of file diff --git a/005-PaidSource/v2ray_pool/_db-checked.txt b/005-PaidSource/v2ray_pool/_db-checked.txt new file mode 100644 index 0000000..7753456 --- /dev/null +++ b/005-PaidSource/v2ray_pool/_db-checked.txt @@ -0,0 +1,4 @@ +ss://YWVzLTI1Ni1nY206cEtFVzhKUEJ5VFZUTHRN@134.195.196.68:443#github.com/freefq%20-%20%E5%8C%97%E7%BE%8E%E5%9C%B0%E5%8C%BA%20%202,134.195.196.68,美国 +ss://YWVzLTI1Ni1nY206ZTRGQ1dyZ3BramkzUVk@134.195.196.81:9101#github.com/freefq%20-%20%E5%8C%97%E7%BE%8E%E5%9C%B0%E5%8C%BA%20%206,134.195.196.81,美国 +ss://YWVzLTI1Ni1nY206WEtGS2wyclVMaklwNzQ@134.195.196.187:8009#github.com/freefq%20-%20%E5%8C%97%E7%BE%8E%E5%9C%B0%E5%8C%BA%20%201,134.195.196.187,美国 +vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIiA6Iue/u+WimeWFmmZhbnFpYW5nZGFuZy5jb20iLCIiIDogIkBTU1JTVUItVjE1LeS7mOi0ueaOqOiNkDpzdW8ueXQvc3Nyc3ViIiwNCiAgImFkZCI6ICI0Mi4xOTMuNDguNjQiLA0KICAicG9ydCI6ICI1MDAwMiIsDQogICJpZCI6ICI0MTgwNDhhZi1hMjkzLTRiOTktOWIwYy05OGNhMzU4MGRkMjQiLA0KICAiYWlkIjogIjY0IiwNCiAgInNjeSI6ICJhdXRvIiwNCiAgIm5ldCI6ICJ0Y3AiLA0KICAidHlwZSI6ICJub25lIiwNCiAgImhvc3QiOiAiNDIuMTkzLjQ4LjY0IiwNCiAgInBhdGgiOiAiIiwNCiAgInRscyI6ICIiLA0KICAic25pIjogIiINCn0=,42.193.48.64,中国 上海 上海市 电信 \ No newline at end of file diff --git a/005-PaidSource/v2ray_pool/_db-uncheck.txt b/005-PaidSource/v2ray_pool/_db-uncheck.txt new file mode 100644 index 0000000..e69de29 diff --git a/005-PaidSource/v2ray_util.py b/005-PaidSource/v2ray_util.py new file mode 100644 index 0000000..80b2a92 --- /dev/null +++ b/005-PaidSource/v2ray_util.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 管理v2ray_pool的工具 +@Date :2022年1月14日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" + +import time + +from v2ray_pool import utils, Config, DBManage + + +def search_node(): + # 如果有系统全局代理,可不需要开启v2ray_core代理,GoogleTrend(proxies=False) + utils.kill_all_v2ray() + Config.set_v2ray_core_path('/Users/Qincji/Desktop/develop/soft/intalled/v2ray-macos-64') # v2ray内核存放路径 + Config.set_v2ray_node_path( + '/Users/Qincji/Desktop/develop/py/project/PythonIsTools/005-PaidSource/v2ray_pool') # 保存获取到节点的路径 + proxy_url = 'ss://YWVzLTI1Ni1nY206WTZSOXBBdHZ4eHptR0M@134.195.196.3:3306#github.com/freefq%20-%20%E5%8C%97%E7%BE%8E%E5%9C%B0%E5%8C%BA%20%201' + dbm = DBManage() + dbm.init() # 必须初始化 + if dbm.check_url_single(proxy_url): + urls = dbm.load_urls_by_net(proxy_url=proxy_url) + dbm.check_and_save(urls, append=False) + # dbm.load_urls_and_save_auto() + # urls = dbm.load_unchecked_urls_by_local() + # dbm.check_and_save(urls, append=False) + utils.kill_all_v2ray() + + +def restart_v2ray(isSysOn=False): + utils.kill_all_v2ray() + Config.set_v2ray_core_path('/Users/Qincji/Desktop/develop/soft/intalled/v2ray-macos-64') # v2ray内核存放路径 + Config.set_v2ray_node_path( + '/Users/Qincji/Desktop/develop/py/project/PythonIsTools/005-PaidSource/v2ray_pool') # 保存获取到节点的路径 + dbm = DBManage() + dbm.init() # 必须初始化 + while 1: + if dbm.start_random_v2ray_by_local(isSysOn=isSysOn): + break + else: + print("启动失败,进行重试!") + time.sleep(1) + + +def kill_all_v2ray(): + utils.kill_all_v2ray() diff --git a/006-TikTok/.gitignore b/006-TikTok/.gitignore new file mode 100644 index 0000000..b6e4761 --- /dev/null +++ b/006-TikTok/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/006-TikTok/006-TikTok.iml b/006-TikTok/006-TikTok.iml new file mode 100644 index 0000000..f4e6189 --- /dev/null +++ b/006-TikTok/006-TikTok.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/006-TikTok/README.md b/006-TikTok/README.md new file mode 100644 index 0000000..6f1950b --- /dev/null +++ b/006-TikTok/README.md @@ -0,0 +1,31 @@ +# App自动化 + +## 效果 + +抖音和tiktok能自动刷App评论。 + +## 实现 +1. 使用android手机,能与电脑正常使用adb连接 +2. 使用[uiautomator2](https://github.com/openatx/uiautomator2) 开源库。 +3. 借助:weditor 来获取元素(xpath等)。 + + +## seo +``` +1、重量副词词汇 +https://gist.github.com/mkulakowski2/4289437 + +2、查看网站排名 +https://www.sdwebseo.com/googlerankcheck/ + +3、看看google浏览器中我的排名,如: +https://www.google.com/search?q=giant stuffed animal caterpillar&num=10&gl=US +https://www.google.com/search?q=4 foot stuffed panda bear&num=100&gl=US + +4、获取外链 +帮记者一个小忙,获取优质的外链:http://app.helpareporter.com/Pitches +https://www.seoactionblog.com/haro-link-building-strategy/ + +``` + +> 其他:注意需要把电脑要把代理关掉 \ No newline at end of file diff --git a/006-TikTok/__init__.py b/006-TikTok/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/006-TikTok/dy_review.py b/006-TikTok/dy_review.py new file mode 100644 index 0000000..c1af3c2 --- /dev/null +++ b/006-TikTok/dy_review.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 抖音app刷评论 +@Date :2021年12月22日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import random +import time + +import uiautomator2 as u2 + +d = u2.connect() +d.implicitly_wait(80) + + +def review_douyin(): # 评论 + d.press("home") + d.app_start('com.ss.android.ugc.aweme', stop=True) + time.sleep(1) + d(resourceId="com.ss.android.ugc.aweme:id/foj").click() # 点击搜索 + time.sleep(1) + d(resourceId="com.ss.android.ugc.aweme:id/et_search_kw").click() # 点击输入框,预防键盘弹不起来 + keys = ['元宵节创意视频'] # , '情人节', '搞笑视频', '真人动漫特效' + comments = ['[比心]', '[强壮]', '[击掌]', '[给力]', '[爱心]', '[派对]', '[不看]', '[炸弹]', '[憨笑]', '[悠闲]', '[嘿哈]', '[西瓜]', '[咖啡]', + '[太阳]', '[月亮]', '[发]', '[红包]', '[拳头]', '[勾引]', '[胜利]', '[抱拳]', '[左边]', '[送心]', '[来看我]', '[来看我]', + '[来看我]', '[灵机一动]', '[耶]', '[色]', '[震惊]', '[小鼓掌]', '[发呆]', '[偷笑]', '[石化]', '[思考]', '[笑哭]', '[奸笑]', + '[坏笑]', '[得意]', '[钱]', '[亲亲]', '[愉快]', '[玫瑰]', '[赞]', '[鼓掌]', '[感谢]', '[666]', '[胡瓜]', '[啤酒]', '[飞吻]', + '[紫薇别走]', '[听歌]', '[绝望的凝视]', '[不失礼貌的微笑]', '[吐舌]', '[呆无辜]', '[看]', '[熊吉]', '[黑脸]', '[吃瓜群众]', '[呲牙]', + '[绿帽子]', '[摸头]', '[皱眉]', '[OK]', '[碰拳]', '[强壮]', '[比心]', '[吐彩虹]', '[奋斗]', '[敲打]', '[惊喜]', '[如花]', '[强]', + '[做鬼脸]', '[尬笑]', '[红脸]', '牛啊', '牛啊牛', 'nb', '666', '赞一个', '赞', '棒', '学到了', '1', '已阅', '板凳', + '插一楼:变戏法的亮手帕', '插一楼:狗吃豆腐脑', '插一楼:癞蛤蟆打伞', '插一楼:离了水晶宫的龙', '插一楼:盲人聊天', '插一楼:五百钱分两下', '插一楼:盲公戴眼镜', + '插一楼:王八倒立', '插一楼:癞蛤蟆背小手', '插一楼:韩湘子吹笛', '插一楼:剥了皮的蛤蟆', '插一楼:马蜂蜇秃子', '插一楼:冷水烫鸡', '插一楼:老孔雀开屏', '插一楼:大姑娘养的', + '插一楼:三角坟地', '插一楼:蜡人玩火', '插一楼:发了霉的葡萄', '插一楼:厥着看天', '插一楼:种地不出苗', '插一楼:老虎头上的苍蝇', '插一楼:雷婆找龙王谈心', + '插一楼:菩萨的胸怀', '插一楼:牛屎虫搬家', '插一楼:变戏法的拿块布', '插一楼:老虎上吊', '插一楼:王八', '插一楼:老虎吃田螺', '插一楼:大肚子踩钢丝', '插一楼:耗子腰粗', + '插一楼:乌龟的', '插一楼:神仙放屁', '插一楼:麻油煎豆腐', '插一楼:汽车坏了方向盘', '插一楼:病床上摘牡丹', '插一楼:芝麻地里撒黄豆', '插一楼:打开棺材喊捉贼', + '插一楼:卤煮寒鸭子', '插一楼:鲤鱼找鲤鱼,鲫鱼找鲫鱼', '插一楼:癞蛤蟆插羽毛', '插一楼:烂伞遮日', '插一楼:老虎头上的苍蝇', '插一楼:三角坟地', '插一楼:卖布兼卖盐', + '插一楼:耗子腰粗', '插一楼:老孔雀开屏', '插一楼:筐中捉鳖', '插一楼:拐子进医院', '插一楼:茅房里打灯笼', '插一楼:癞蛤蟆背小手', '插一楼:肉骨头吹喇叭', + '插一楼:老鼠进棺材', '插一楼:种地不出苗', '插一楼:病床上摘牡丹', '插一楼:裤裆里摸黄泥巴', '插一楼:狗拿耗子', '插一楼:铁匠铺的料', '插一楼:高梁撒在粟地里', + '插一楼:茅厕里题诗', '插一楼:痰盂里放屁', '插一楼:老母猪打喷嚏', '插一楼:厕所里点灯', '插一楼:棺材铺的买卖', '插一楼:老公鸡着火', '插一楼:乌龟翻筋斗', + '插一楼:被窝里的跳蚤', '插一楼:赶着牛车拉大粪', '插一楼:老太婆上鸡窝', '插一楼:狗背上贴膏药', '插一楼:狗咬瓦片', '插一楼:哪吒下凡', '插一楼:二十一天不出鸡', + '插一楼:鞭炮两头点', '插一楼:抱黄连敲门', '插一楼:猫儿踏破油瓶盖', '插一楼:和尚念经', '插一楼:裁缝不带尺', '插一楼:上山钓鱼', '插一楼:狗长犄角', + '插一楼:带着存折进棺材', '插一楼:豁子拜师', '插一楼:宁来看棋', '插一楼:盲公戴眼镜', '插一楼:南来的燕,北来的风', '插一楼:杯水车薪', '插一楼:玉皇大帝放屁', + '插一楼:给刺儿头理发', '插一楼:九月的甘蔗', '插一楼:两只公牛打架', '插一楼:百川归海', '插一楼:挨打的乌龟', '插一楼:和尚挖墙洞', '插一楼:八月十五蒸年糕', + '插一楼:毒蛇钻进竹筒里', '插一楼:苍蝇叮菩萨', '插一楼:白布进染缸', '插一楼:粪堆上开花', '插一楼:癞蛤蟆上蒸笼', + '插楼:沙漠里钓鱼', '插楼:青㭎树雕菩萨', '插楼:看鸭勿上棚', '插楼:下大雨前刮大风', '插楼:在看羊的狗', '插楼:耍大刀里唱小生', '插楼:罗锅上山', '插楼:大车不拉', + '插楼:瞎子白瞪眼', '插楼:铁拐的葫芦', '插楼:苣荬菜炖鲇鱼', '插楼:旅馆里的蚊子', '插楼:石刻底下的冰瘤子', '插楼:吃稀饭摆脑壳', '插楼:叫化子背不起', '插楼:火车拉大粪', + '插楼:寿星玩琵琶', '插楼:六月的腊肉', '插楼:夜叉骂街', '插楼:孩儿的脊梁', '插楼:长了个钱串子脑袋', '插楼:现场看乒乓球比赛', '插楼:寡妇梦丈夫', '插楼:马背上放屁', + '插楼:落雨出太阳', '插楼:猴子捡生姜', '插楼:啄木鸟屙薄屎', '插楼:鸡毛扔火里', '插楼:油火腿子被蛇咬', '插楼:属秦椒的', '插楼:千亩地里一棵草', '插楼:药铺倒了', + '插楼:黄连水做饭', '插楼:卸架的黄烟叶儿', '插楼:螺蛳壳里赛跑', '插楼:躲了和尚躲不了庙', '插楼:驴槽子里面伸出一颗头来', '插楼:老妈妈吃火锅', '插楼:阎王的脸', + '插楼:吃粮勿管事', '插楼:脚跟拴石头', '插楼:麻秸秆儿打狼', '插楼:阎王7粑子', '插楼:画上的美女', '插楼:团鱼下滚汤', '插楼:孔夫子的脸', '插楼:曹操贪慕小乔', + '插楼:蒙住眼睛走路', '插楼:炒菜不放盐', '插楼:三月里的桃花', '插楼:老鼠吃面饽', '插楼:粥锅里煮铁球', '插楼:戴起眼镜喝滚茶', '插楼:吃香油唱曲子', '插楼:过冬的咸菜缸', + '插楼:三个小鬼没抓住', '插楼:对着坛子放屁', '插楼:赤骨肋受棒', '插楼:百灵鸟唱歌', '插楼:雨过天晴放干雷', '插楼:拄着拐棍上炭窑', '插楼:搁着料吃草', '插楼:王八碰桥桩', + '插楼:水上油', '插楼:偷鸡不得摸了一只鸭子', '插楼:黄瓜熬白瓜', '插楼:海瑞的棺材', '插楼:蛤蟆翻田坎', '插楼:乌龟进砂锅', '插楼:夜壶出烟', '插楼:李逵骂宋江', + '插楼:小孩买个花棒槌', '插楼:漏网之虾', '插楼:一口吹灭火焰山', '插楼:冷水调浆'] + for key in keys: + # 不能搜索search + d(resourceId="com.ss.android.ugc.aweme:id/et_search_kw").clear_text() # 清除历史 + d(resourceId="com.ss.android.ugc.aweme:id/et_search_kw").set_text(key) # 输入 + time.sleep(1) + d(text='搜索', className='android.widget.TextView').click() # 点击搜索 + # d(resourceId=" com.ss.android.ugc.aweme:id/d0t").click() # 点击搜索 + time.sleep(1) + d(text='视频', className='android.widget.Button').click() # 点击视频,然后点击第一条 + d.xpath( + '//*[@resource-id="com.ss.android.ugc.aweme:id/gw1"]/android.widget.FrameLayout[1]').click() + stop, index = random.randint(15, 30), 0 + while index < stop: # 随机刷几十条 + print('总共有:{}条 | 现在到:{}条'.format(stop, index)) + try: + time.sleep(random.randint(3, 12)) # 随机停顿1~5秒 + d(resourceId="com.ss.android.ugc.aweme:id/b2b").click() # 点击评论按钮 + except Exception as e: + print(e) + try: + time.sleep(random.randint(1, 2)) # 随机停顿1~2秒 + d(resourceId="com.ss.android.ugc.aweme:id/b1y").click() # 点击弹出键盘 + except Exception as e: + print(e) + time.sleep(random.randint(1, 2)) # 随机停顿1~2秒 + try: + d(resourceId="com.ss.android.ugc.aweme:id/b1y").set_text( + comments[random.randint(0, len(comments) - 1)]) # 输入 + except Exception as e: + print(e) + try: + time.sleep(random.randint(1, 2)) # 随机停顿1~2秒 + d(resourceId="com.ss.android.ugc.aweme:id/b1r").click() # 发送 + except Exception as e: + print(e) + try: + time.sleep(random.randint(1, 2)) # 随机停顿1~2秒 + d(resourceId="com.ss.android.ugc.aweme:id/back_btn").click() # 关闭 + except Exception as e: + print(e) + try: + time.sleep(random.randint(5, 15)) # 随机停顿1~5秒 + d.swipe_ext("up") # 上划,下一个视频 + except Exception as e: + print(e) + index += 1 + d(resourceId="com.ss.android.ugc.aweme:id/back_btn").click() # 返回搜索 + + +''' +教程: +1. 使用 weditor 来查看元素 +2. 当找不到resourceId时,先用d(, className='android.widget.EditText').info找个resourceId,再使用。因为输入框内容会变化,所有不能直接用。 +''' + +if __name__ == "__main__": + review_douyin() diff --git a/006-TikTok/file_util.py b/006-TikTok/file_util.py new file mode 100644 index 0000000..961aa0a --- /dev/null +++ b/006-TikTok/file_util.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 文件相关处理 +@Date :2022/01/22 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" + +import datetime +import json +import os +import re +import shutil + +import cairosvg +import pandas as pd +import pypandoc # 要安装pandoc +from docx import Document + + +def file_name(file_dir): + results = [] + for root, dirs, files in os.walk(file_dir): + # print(root) # 当前目录路径 + # print(dirs) # 当前路径下所有子目录 + # print(files) # 当前路径下所有非目录子文件 + results += files + return results + + +def deal_one_page(): + fs = file_name('100条') + for f in fs: + try: + print('正在检测【%s】' % f) + shotname, extension = os.path.splitext('%s' % f) + print('正在检测【%s】' % shotname) + if '1篇' in shotname: + new_name = re.sub(r'1篇', '', f) + document = Document(r"html/%s" % f) + paragraphs = document.paragraphs + p = paragraphs[0] + p._element.getparent().remove(p._element) + document.save(r"html/%s" % new_name) + os.remove('html/%s' % f) + except Exception as e: + print(e) + + +def copy_doc(): + fs = file_name('all') + i = 1 + k = 1 + temp_dir = '01' + os.makedirs('100条/%s' % temp_dir) + for f in fs: + try: + # print('正在检测【%s】' % f) + shotname, extension = os.path.splitext('%s' % f) + shutil.copyfile(r'all/%s' % f, r'100条/%s/%s' % (temp_dir, f)) + if i % 100 == 0: + temp_dir = '0%d' % k if k < 10 else '%d' % k + k += 1 + os.makedirs('100条/%s' % temp_dir) + i += 1 + except Exception as e: + print(e) + + +'''########文件处理相关#########''' + + +def html_cover_doc(in_path, out_path): + '''将html转化成功doc''' + path, file_name = os.path.split(out_path) + if path and not os.path.exists(path): + os.makedirs(path) + pypandoc.convert_file(in_path, 'docx', outputfile=out_path) + + +def svg_cover_jpg(src, dst): + '''' + drawing = svg2rlg("drawing.svg") + renderPDF.drawToFile(drawing, "drawing.pdf") + renderPM.drawToFile(drawing, "fdrawing.png", fmt="PNG") + renderPM.drawToFile(drawing, "drawing.jpg", fmt="JPG") + ''' + path, file_name = os.path.split(dst) + if path and not os.path.exists(path): + os.makedirs(path) + # drawing = svg2rlg(src) + # renderPM.drawToFile(drawing, dst, fmt="JPG") + cairosvg.svg2png(url=src, write_to=dst) + + +def html_cover_excel(content, out_path): + '''将html转化成excel''' + path, file_name = os.path.split(out_path) + if path and not os.path.exists(path): + os.makedirs(path) + tables = pd.read_html(content, encoding='utf-8') + writer = pd.ExcelWriter(out_path) + for i in range(len(tables)): + tables[i].to_excel(writer, sheet_name='表%d' % (i + 1)) # startrow + writer.save() # 写入硬盘 + + +def write_to_html(content, file_path): + '''将内容写入本地,自动加上head等信息''' + page = ''' + + + + + ''' + page += content + page += ''' + ''' + write(page, file_path) + + +def write_json(content, file_path): + '''写入json''' + path, file_name = os.path.split(file_path) + if path and not os.path.exists(path): + os.makedirs(path) + with open(file_path, 'w') as f: + json.dump(content, f, ensure_ascii=False) + f.close() + + +def read_json(file_path): + '''读取json''' + with open(file_path, 'r') as f: + js_get = json.load(f) + f.close() + return js_get + + +def write(content, file_path): + '''写入txt文本内容''' + path, file_name = os.path.split(file_path) + if path and not os.path.exists(path): + os.makedirs(path) + with open(file_path, 'w') as f: + f.write(content) + f.close() + + +def read(file_path) -> str: + '''读取txt文本内容''' + content = None + try: + with open(file_path, 'r') as f: + content = f.read() + f.close() + except Exception as e: + print(e) + return content + + +def get_next_folder(dst, day_diff, folder, max_size): + '''遍历目录文件,直到文件夹不存在或者数目达到最大(max_size)时,返回路径''' + while True: + day_time = (datetime.date.today() + datetime.timedelta(days=day_diff)).strftime('%Y-%m-%d') # 下一天的目录继续遍历 + folder_path = os.path.join(dst, day_time, folder) + if os.path.exists(folder_path): # 已存在目录 + size = len(next(os.walk(folder_path))[2]) + if size>= max_size: # 该下一个目录了 + day_diff += 1 + continue + else: + os.makedirs(folder_path) + return day_diff, folder_path + + +if __name__ == '__main__': + pass diff --git a/006-TikTok/google_transfer_by_excel.py b/006-TikTok/google_transfer_by_excel.py new file mode 100644 index 0000000..6df21eb --- /dev/null +++ b/006-TikTok/google_transfer_by_excel.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 实现从excel文件获取关键词进行翻译后写入新文件 +@Date :2021年10月22日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" + +import json +import os +import os.path +import random +import time + +import chardet +import pandas as pd +import requests + +import file_util as futls +from my_fake_useragent import UserAgent + + +class GTransfer(object): + def __init__(self, file): + self.file = file + self._ua = UserAgent() + self._agent = self._ua.random() # 随机生成的agent + self.USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0" + self._headers = {"user-agent": self.USER_AGENT, 'Connection': 'close'} + + def request(self, url, allow_redirects=False, verify=False, proxies=None, timeout=8): + """最终的请求实现""" + requests.packages.urllib3.disable_warnings() + if proxies: + return requests.get(url=url, headers=self._headers, allow_redirects=allow_redirects, verify=verify, + proxies=proxies, timeout=timeout) + else: + return requests.get(url=url, headers=self._headers, allow_redirects=allow_redirects, verify=verify, + timeout=timeout) + + def search_page(self, url, pause=3): + """ + Google search + :param query: Keyword + :param language: Language + :return: result + """ + time.sleep(random.randint(1, pause)) + try: + r = self.request(url) + print('resp code=%d' % r.status_code) + if r.status_code == 200: + charset = chardet.detect(r.content) + content = r.content.decode(charset['encoding']) + return content + elif r.status_code == 301 or r.status_code == 302 or r.status_code == 303: + location = r.headers['Location'] + time.sleep(random.randint(1, pause)) + return self.search_page(location) + return None + except Exception as e: + print(e) + return None + + def transfer(self, content): + # url = 'http://translate.google.com/translate_a/single?client=gtx&dt=t&dj=1&ie=UTF-8&sl=auto&tl=zh-CN&q=' + content + url = 'http://translate.google.cn/translate_a/single?client=gtx&dt=t&dj=1&ie=UTF-8&sl=zh-CN&tl=en&q=' + content + try: + cache = futls.read_json('lib/cache.json') + for c in cache: + if content in c: + print('已存在,跳过:{}'.format(content)) + return c.get(content) + except Exception as e: + pass + try: + result = self.search_page(url) + trans = json.loads(result)['sentences'][0]['trans'] + # 解析获取翻译后的数据 + # print(result) + print(trans) + self.local_cache.append({content: trans}) + futls.write_json(self.local_cache, self.file) + # 写入数据吗?下次直接缓存取 + except Exception as e: + print(e) + return self.transfer(content) + return trans + + def transfer_list(self, lists): + self.local_cache = [] + for c in lists: + self.transfer(c) + + +# http://translate.google.com/translate_a/single?client=gtx&dt=t&dj=1&ie=UTF-8&sl=auto&tl=zh-CN&q=what + + +if __name__ == '__main__': + files = next(os.walk('xxx/folder'))[2] + results = [] + for f in files: + name, ext = os.path.splitext(f) + if len(name)> 0 and len(ext): + results.append(name) + g = GTransfer('xxx/en.json') + g.transfer_list(results) diff --git a/006-TikTok/img_2_webp.py b/006-TikTok/img_2_webp.py new file mode 100644 index 0000000..f2453c8 --- /dev/null +++ b/006-TikTok/img_2_webp.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: WordPress 站点 将图片转 webp 格式,实现内容重组 +@Date :2022年10月12日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" + + +import os + + +def get_contain_pics(folder, content): + files = next(os.walk(folder))[2] + results = [] + for f in files: + name, ext = os.path.splitext(f) + if len(name) < 1 or len(ext) < 1 or '.json' == str(ext) or '.txt' == str(ext): # 非视频文件 + continue + if content in f: + results.append(f) + return results + + +def convert_pic_to_webp(folder, sku, pic_files, t, key1, key2, key3, youtube_num): + dst_folder = os.path.join('data', sku) + desc_links = [] + desc_links.append( + '\n'.format( + youtube_num)) + desc_links.append( + '\n') + if not os.path.exists(dst_folder): + os.makedirs(dst_folder) + # TODO - 上传日期需要注意 月份 + full_url = '{} \n' + tail = ''' +

key1 Design Philosophy

+1st: Ideal Size +key1 for girls are great companions, are filled with pp cotton. key2 are many sizes to choose from.This soft friend is irresistible! Anirollz got their name from being key3 shaped like rolls, and they are pure squishy fun. + +2nd: Fine Workmanship +The soft fleece fabric, exquisite embroidery, special color, and full filling of various parts. Therefore, the bear stuffed animal funny plush toy look more lovely and texture. So silky smooth plush toys, you'll never want to let go! + +3rd: Wide Range Of Uses +key2 can meet different needs. For example, put on the bed, car, sofa for holding, lying down, leaning. The cuddle key1 is soft and comfortable. Therefore, you can use this key1 as your sleeping partner! + +4th: Perfect Gift +Our key1 will be loved by kids and adults like. For instance, It's a great gift for family or friends at Christmas, Thanksgiving, birthday, graduation or Valentine's day. + +5th: Develop Empathy And Social Skills +With a cuddling, comforting plush toy companion. It also helps to develop empathy and social skills, and not only stimulates children's imagination and interest in the animal world. And by taking care of their key2 for kids, they build bridges with their friends in the future.in addition, plush toys are the perfect companions for kids three and older and add a unique decorative touch to any nursery, bedroom, or playroom! + +

key2 Buyer notice

+1st: About hair removal +Did you notice hair loss after receiving the goods? Don't worry, Our giant stuffed animal toys don't shed hair themselves.Because it is a new product, it will have a little floating hair and filling fiber, after you get it, tap it a few times, shake it a few times. it will remove the floating hair. + +2nd: About size +All products are measured in kind, and the key2 itself is soft and will have a certain error (about 3 cm). + +3rd: About packaging +We exhaust part of the air for packaging, because key1 itself is relatively large. Only after taking the time to transport can the product be better protected, so when you receive the product, it is slightly off +the description. Small or a little wrinkled, please don't be surprised. because this is a normal phenomenon, shake a few times after opening, under the sun expose for half an hour and recover immediately! + +

Our guarantee

+

Insurance

+Each order includes real-time tracking details and insurance. +

Order And Safe

+We reps ready to help and answer any questions, you have within a 24 hour time frame, 7 days a week. We use state-of-the-art SSL Secure encryption to keep your personal and financial information 100% protected. + +

Other aspects

+we’re constantly reinventing and reimagining our key3. If you have feedback you would like to share with us, please contact us! We genuinely listen to your suggestions when reviewing our existing toys as well as new ones in development. And are you looking for wholesale plush doll for your school or church carnival? We have the perfect mix of darling stuffed toys for carnival prizes all sold at wholesale prices! We have everything from small stuffed animals such as key1 & dogs, fish, turtles, apes, owls, sharks, & bugs to big stuffed doll such as our large stuffed toy frog and our lovable, key3 and puppy dog! + +[画像:large stuffed animals for adults and kids] + ''' + tail = tail.replace('key1', key1).replace('key2', key2).replace('key3', key3) + for i in range(0, len(pic_files)): + f = pic_files[i] + src = os.path.join(folder, f) + # dst_name = '{}-m-{}{}'.format(sku, i + 1, ext) + dst_name = '{}-{}-{}.webp'.format(sku, t, i + 1) # 更新至webp + dst = os.path.join(dst_folder, dst_name) + from PIL import Image + im = Image.open(src) # 读入文件 + print(im.size) + width = 500 + if im.size[0]> width: + h = int(width * im.size[1] / im.size[0]) + print(h) + im.thumbnail((width, h), Image.ANTIALIAS) # 重新设置图片大小 + im.save(dst) # 保存 + print(dst) + if i < 1: + alt_key = key3 + elif i < 3: + alt_key = key2 + else: + alt_key = key1 + if t == 'd': + size = Image.open(dst).size + desc_links.append(full_url.format(dst_name, alt_key, size[0], size[1])) + if t == 'd': + with open(os.path.join('data/desc-html', '{}.txt'.format(sku)), mode='w', encoding='utf-8') as f: + for link in desc_links: + f.write(link) + f.write('\n') + f.write(tail) + f.close() + + +def deal_sku(folder, sku, key1, key2, key3, youtube_num): + folder = os.path.join(folder, sku) + main_pics = sorted(get_contain_pics(folder, '主图')) + desc_pics = sorted(get_contain_pics(folder, '详情')) + convert_pic_to_webp(folder, sku, main_pics, 'm', key1, key2, key3, youtube_num) + convert_pic_to_webp(folder, sku, desc_pics, 'd', key1, key2, key3, youtube_num) + + +def deal_sku_list(): + folder = 'xxx/SKU/' + deal_sku(folder, 'FYTA032DX', 'elephant plush toy', 'stuffed animal elephant', 'elephant stuffed toy', '4kB0ZJBmMkU') # 奶瓶大象 + + +def convert_2_webp(src, dst): + from PIL import Image + im = Image.open(src) # 读入文件 + print(im.size) + # width = 500 + # if im.size[0]> width: + # h = int(width * im.size[1] / im.size[0]) + # print(h) + # im.thumbnail((width, h), Image.ANTIALIAS) # 重新设置图片大小 + im.save(dst) # 保存 + + +def convert_folder(src_folder, dst_folder): + files = next(os.walk(src_folder))[2] + for f in files: + if str(f).startswith('.'): + continue + name, ext = os.path.splitext(f) + convert_2_webp(os.path.join(src_folder, f), os.path.join(dst_folder, '{}.webp'.format(name))) + + +''' +操作说明: +一:使用Fatkun工具下载图片后进行刷选 +1、主图主要4张 +2、详情图看情况删除一些,最好保留产品尺寸相关说明的图,正常不会超过10张 +二:使用本脚本进行进图压缩以及格式转换(压缩了10倍左右) +1.本脚本将图片宽带缩小至500,高度等比例缩小 +2.本脚本将jpg、png格式转换成webp格式 +三:新建产品编辑 +1.本脚本生成描述图片连接,直接拷贝过去 +2.将变量尺寸属性值数字后面加上cm +3.两种产品,定价范围 +A. 长条(如:FYTA004BR,长条兔子),属性名称:Long +70cm 18~21 +90cm 35~40 +110cm 60~70 +130cm 90~100 +150cm 120~130 +B. 圆形(如:FYTA001SP,奶瓶猪),属性名称:Height +35cm 30~35 +45cm 55~59 +55cm 80~88 +65cm 120~130 +75cm 160~180 +''' +if __name__ == '__main__': + # deal_sku_list() + convert_folder('xxx/png', 'xxx/webp') + +#Best Stuffed Pig Toys With A Hat \ No newline at end of file diff --git a/006-TikTok/main.py b/006-TikTok/main.py new file mode 100644 index 0000000..23a5da0 --- /dev/null +++ b/006-TikTok/main.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: tiktok 相关开源库 +@Date :2021年12月22日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +# import TikTokApi +from TikTokAPI import TikTokAPI + +# https://github.com/avilash/TikTokAPI-Python + +if __name__ == "__main__": + pass diff --git a/006-TikTok/post_autotk.py b/006-TikTok/post_autotk.py new file mode 100644 index 0000000..9dbfec1 --- /dev/null +++ b/006-TikTok/post_autotk.py @@ -0,0 +1,263 @@ +""" +@Description: 生成autotk.app(https://github.com/xhunmon/autotk)中post发送内容解析, +@Date :2022年3月20日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import datetime +import os +import random +import re +import time + +import file_util as futls + + +def get_real_files(folder): + ''' + 获取真实文件,去掉(.xxx)等文隐藏文件 + :param files: + :return: + ''' + files = next(os.walk(folder))[2] + results = [] + for f in files: + name, ext = os.path.splitext(f) + if len(name) < 1 or len(ext) < 1 or '.json' == str(ext) or '.txt' == str(ext): # 非视频文件 + print('非视频文件:{}'.format(f)) + continue + results.append(f) + return results + + +def format_num(num): + if num < 10: + return '00{}'.format(num) + elif num < 100: + return '0{}'.format(num) + else: + return '{}'.format(num) + + +def split_en_title(spl, content): + if spl in content: + temps = content.split(spl) + coverDesc, title = "", "" + for temp in temps: + if len(temp) < 35: + coverDesc = temp + else: + title = temp + # print('------->过滤《{}》-> {} | {}'.format(spl, coverDesc, title)) + return coverDesc, title + return None, None + + +def chose_cover_title(transfer: str): # 截取合适的标题和封面内容 + # 大于5个字符才开始截取 + trans = re.findall(r'(.{5,}?\?|.{5,}?\.|.{5,}?#|.{5,}?!|.{5,}?$)', transfer) + title, cover = '', '' + for tran in trans: + tran = tran.replace('#', '').strip() + t_size = len(tran) + if len(title) < t_size: # title取最长的 + title = tran + # cover 优先取10~35个字符的,然后取靠拢的值 + if len(cover) == 0: + cover = tran + elif 35> t_size> 10: + cover = tran + elif 10> t_size> len(cover): + cover = tran + elif 35 < t_size < len(cover): + cover = tran + return cover, title + + +def match_info(pre, files, transfers): + file_name, cover, title, transfer = None, None, None, None + for v in files: # 文件名称 + if str(v).startswith(pre): + file_name = str(v) + break + if not file_name: + raise RuntimeError('没有找到匹配的文件:{}'.format(pre)) + for k in transfers: # 翻译后当前数据 + if str(k).startswith(pre): + transfer = str(k) + break + if not transfer: + raise RuntimeError('没有找到匹配的文件:{}'.format(pre)) + # 处理标题 + transfer = transfer[3:].replace('-', '').strip() + cover, title = chose_cover_title(transfer) + if not cover or not title: + raise RuntimeError('获取数据异常:{} | {} | {}'.format(file_name, cover, title)) + return file_name, cover, title + + +def format_time_by_str(date): # str转换为时间戳 + time_array = time.strptime(date, "%Y-%m-%d %H:%M:%S") + return int(time.mktime(time_array)) + + +def format_time_by_stamp(stamp): # 时间戳转换为时间 + return str(datetime.datetime.fromtimestamp(stamp)) + + +def random_tag(tags): + num = random.randint(1, 2) + if len(tags) < num: + num = len(tags) + results = [] + rsp = '' + while True: + if len(results) == num: + break + tag = random.choice(tags) + if tag in results: + continue + results.append(tag) + rsp += '#{} '.format(tag) + return rsp + + +def write_title_one(file_name, folder): + datas = get_real_files(folder) + with open(file_name, 'a', encoding='utf-8') as txt_f: + for f in datas: + name, ext = os.path.splitext(f) + txt_f.write('{}\n'.format(name)) + # txt_f.write('AAAAAAAA\n') + + +def write_title(): + file_name = 'lib/title.txt' + write_title_one(file_name, 'xxx/bzhsrc/') + + +def read_title_one(file_name): + datas = [] + with open(file_name, 'r', encoding='utf-8') as txt_f: + while True: + txt = txt_f.readline().replace('\n', '') + if not txt: + break + if len(txt)> 5: + datas.append(txt) + return datas + + +def del_content(content, dels): + for d in dels: + content = content.replace(d, '') + return content + + +def is_number(s): + try: + int(s) + return True + except ValueError: + pass + return False + + +def rename_files(src_folder): # 重新生成序列 + dels = ['#热࿆门', '#࿆热࿆门', '#热门', '"', '"', + '-', '-', '[', ']', '《', '》', '/', ':', + ' '] + files = get_real_files(src_folder) + index = 1 + for f in files: + src = os.path.join(src_folder, f) + new_name: str = del_content(f, dels) + num = new_name[0:3] + if is_number(num): # 如果有序号 + new_name = '{}-{}'.format(num, new_name[3:]) # 有序号时使用 + else: + new_name = '{}-{}'.format(format_num(index), new_name) + dst = os.path.join(src_folder, new_name) + print(new_name) + os.rename(src, dst) + index += 1 + print('总共有[{}]个!'.format(len(files))) + + +def start_post(userId, src_folder, date_start, title_tags, transfers, musics, dst='post.txt', index=1, space_start=30, + space_end=40, + num_start=3, num_end=6, half_time=7): + ''' + 开始生成post.txt文件 + :param userId: 用户id + :param src_folder: 视频目录 + :param date_start: 第一个启动的时间,格式如:'2022-3-23 19:05:00' + :param title_tags: + :param transfers: + :param musics: + :param index: 起点位置 + :param space_start: 下一个随机间隔开始发送的视频时间(分钟) + :param space_end: 下一个随机间隔结束发送的视频时间(分钟) + :param num_start: 每天随机发几个开始区间 + :param num_end: 每天随机发几个结束区间 + :param half_time(小时): 保证一半在12点,一半在6点后 + :return: + ''' + posts = [] + stamp = format_time_by_str(date_start) + cover_tags = ["Vector", "Glitch", "Tint", "News", "Retro", "Skew", "Pill", "Pop"] # "Standard" --》有问题 + files = get_real_files(src_folder) + import copy + temp_files: list = copy.deepcopy(files) + size = len(files) + while index <= size: + stamp_1 = stamp + num = random.randint(num_start, num_end) + half_num = int(num / 2) + for j in range(0, num): + if index> size: + print('已经处理完了:{}'.format(index)) + break + stamp_1 += random.randint(1, 30) * 60 + if j == half_num: # 在一半的时候加上秒 + stamp_1 += half_time * 60 * 60 + date = format_time_by_stamp(stamp_1) + sound = random.choice(musics) + # pre = format_num(index) + pre = temp_files.pop()[0:3] + file, coverDesc, title1 = match_info(pre, files, transfers) + # TODO- #foryoutoy 玩具需要 + title = '{} {} #foryoutoy'.format(title1, random_tag(title_tags)) + coverTag = random.choice(cover_tags) + coverPic = random.randint(1, 3) + post = {"date": date, "sound": sound, "title": title, "file": file, "userId": userId, + "coverDesc": coverDesc, "coverTag": coverTag, "coverPic": coverPic} + print(post) + posts.append(post) + stamp_1 += random.randint(space_start, space_end) * 60 + index += 1 + # print('----' * 5) + stamp += 24 * 60 * 60 # 下一天 + futls.write_json(posts, dst) + + +def all_step(): + # 1. 翻译文件,整理成txt + # 2. 标题标签tag,整理成json + # 3. 过滤无效信息,重命名文件 + # 4. post + # 美国比英国晚5个小时 + transfers = futls.read_2_list('lib/furrytoy/fanyi_en.txt') # 英国 19~21,最多8个视频 + title_tags = futls.read_json('lib/furrytoy/tags.json') + musics = futls.read_json('lib/musics.json') + dst = 'lib/furrytoy/post.json' + start_post('kif_nee2022', 'xxx/fyt_toy/', '2022-5-8 10:00:00', transfers=transfers, + title_tags=title_tags, musics=musics, dst=dst, space_start=70, + space_end=83, + num_start=2, num_end=2, half_time=8) # 每天1~2个,12小时*60 + + +if __name__ == '__main__': + # write_title() + # all_step() + rename_files('xxx/beizhihui') # 去掉冗余信息 diff --git a/006-TikTok/tikstar.py b/006-TikTok/tikstar.py new file mode 100644 index 0000000..168c12d --- /dev/null +++ b/006-TikTok/tikstar.py @@ -0,0 +1,29 @@ +""" +@Description: 解析www.tikstar.com网站相关内容,获取tags +@Date :2021年12月22日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +from bs4 import BeautifulSoup +import file_util as futil + + +def parse_tags(page): + '''解析页面,返回标题和文章页面内容,如果生成文章则还需要组装''' + soup = BeautifulSoup(page, 'html.parser') + trs = soup.find('tbody').find_all('tr') + result = [] + for tr in trs: + tds = tr.find_all('td') + tag_name = tds[0].find('h3').text.replace('\n', '').replace(' ', '') + video_num = tds[1].text.replace('\n', '').replace(' ', '') + views = tds[2].text.replace('\n', '').replace(' ', '') + result.append('标签:{} 视频数:{} 观看数:{}'.format(tag_name, video_num, views)) + return result + + +if __name__ == '__main__': + html = futil.read('tags.html') + result = parse_tags(html) + print(result) + futil.write_json(result, 'shoes.json') diff --git a/006-TikTok/tt_review.py b/006-TikTok/tt_review.py new file mode 100644 index 0000000..a785331 --- /dev/null +++ b/006-TikTok/tt_review.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: tiktok app(版本:22.8.2)刷评论脚本 +@Date :2021年12月22日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import random +import time +from datetime import datetime + +import file_util as futil + +import uiautomator2 as u2 + +''' +https://github.com/openatx/uiautomator2 +运行pip3 install -U uiautomator2 安装uiautomator2 +运行python3 -m uiautomator2 init安装包含httprpc服务的apk到安卓手机 +uiautomator2操作:https://python.iitter.com/other/35522.html +借助:weditor 来获取元素,双击找控件id +(注意电脑要把代理关掉) +''' + +d = u2.connect() +print(d.info) +d.implicitly_wait(20) + +comments = ['good job', 'good', 'look me', 'crazy', 'emm...', 'I wish you a happy new year', + 'May you be happy, lucky and happy.', 'I wish you a happy new year and good luck!', + 'to put people and life above everything else', 'heroes in harm’s way', 'spirited', ' behind wave', + 'mythical creatures', 'dagongren, which refers to people who work for others', 'involution', + 'Versailles literature', 'Look at my', 'Too good', 'To learn', 'learned', 'Thank you', 'I got it.', + '666', 'nice', 'Well done', 'Look at my', 'Wonderful', 'Mine is not bad either.', 'Kudos', 'like u', + 'lean it', 'well...', '😊', 'My god!', 'Me too', 'I see', 'Come on', 'See you', 'Allow me', 'Have fun', + 'I\'m home', 'Bless you!', 'Follow me', 'Good luck!', 'Bottoms up!', 'Guess what?', 'Keep it up!', + 'Time is up', 'I like it!', 'That\'s neat', 'Let\'s face it.', 'Let\'s get started', 'Is that so', + 'That\'s something', 'Do you really mean it', 'Mind you', 'I am behind you', 'That depends', + 'What\'s up today?', 'Cut it out', 'What did you say', 'Knock it off', '[angel]', '[astonish]', + '[awkward]', '[blink]', '[complacent]', '[cool]', '[cool][cute]', '[cool][cool]', '[cool][cool][cool]', + '[cry]', '[cute]', '[cute][cute]', '[cute][cute][cute]', '[disdain]', '[drool]', + '[embarrassed]', '[evil]', '[excited]', '[facewithrollingeyes]', '[flushed]', '[funnyface]', '[greedy]', + '[happy]', '[hehe]', '[joyful]', '[laugh]', '[laughwithtears]', '[loveface]', '[lovely]', '[nap]', + '[pride]', '[proud]', '[rage]', '[scream]', '[shock]', '[shout]', '[slap]', '[smile]', '[smileface]', + '[speechless]', '[stun]', '[sulk]', '[surprised]', '[tears]', '[thinking]', '[weep]', '[wicked]', + '[wow]', '[wronged]', '[yummy]', + ' You kick ass.', ' You did a great job.', " You're a really strong person.", ' You read a lot.', + ' That was impressive.', ' Your work on that project was incredible.', ' Keep up the good work!', + " We're so proud of you.", ' How smart of you!', ' You have a real talent.', + ' Well, a good teacher makes good student.', ' I would like to compliment you on your diligence.', + " We're proud of you.", ' He has a long head.', ' You look great today.', " You're beautiful/gorgeous.", + ' You look so healthy.', ' You look like a million dollars.', ' You have a good taste.', + ' I am impressed.', ' You inspire me.', ' You are an amazing friend.', 'You are such a good friend.', + ' You have a good sense of humor.', " You're really talented.", " You're so smart.", + " You've got a great personality.", ' You are just perfect!', ' You are one of a kind.', + ' You make me want to be a better person.', 'brb', 'g2g', 'AMA', 'dbd', 'this look great', + 'we’re so proud of you.', 'nice place.', 'nice going! ', 'emm...amazing!', 'ohh...unbelievable!', + 'yeh,impressive.', 'terrific..', 'fantastic!', 'fabulous.', 'attractive..', 'hei...splendid.', + 'ooh, remarkable', 'gorgeous', 'h.., glamorous', 'marvelous.', 'brilliant..', 'well...glorious', + 'outstanding...', 'stunning!', 'appealing.', 'yeh,impressive[cool]', 'terrific[angel]', 'fantastic[cool]', + 'fabulous[angel]', 'attractive[cool]', 'splendid[angel]', 'remarkable[cool]', 'gorgeous[angel]', + 'glamorous[angel]', 'marvelous[cool]', 'brilliant[angel]', 'glorious[cool]', 'outstanding[angel]', + 'stunning[cool]', 'appealing[angel]', 'Would you like me?[angel]', 'Do you like crafts?[angel]', + 'I have a new creative work, welcome![thinking]'] + +print('总共有{}条随机评论!'.format(len(comments))) + + +def start_vpn(): # 启动代理app + d.press("home") + # d.app_stop('com.v2ray.ang') + d.app_start('com.v2ray.ang') + if 'Not' in d(resourceId="com.v2ray.ang:id/tv_test_state").get_text(): + print_t('正在启动v2ray...') + d(resourceId='com.v2ray.ang:id/fab').click() + if 'Connected' in d(resourceId="com.v2ray.ang:id/tv_test_state").get_text(): + print_t('启动v2ray完成,正在测试速度...') + d(resourceId='com.v2ray.ang:id/layout_test').click() + while 'Testing' in d(resourceId="com.v2ray.ang:id/tv_test_state").get_text(): + time.sleep(1) + print_t(d(resourceId="com.v2ray.ang:id/tv_test_state").get_text()) + + +def review_forYou(): + # d.press("home") + # d.app_start('com.zhiliaoapp.musically', stop=True) + time.sleep(1) + stop, index = random.randint(40, 70), 0 + while index < stop: # 随机刷几十条 + cur_comment = comments[random.randint(0, len(comments) - 1)] + print_t('foryou-总共有:{}条 | 现在到:{}条\t评论:{}'.format(stop, index, cur_comment)) + comment_foryou(cur_comment) + try: + time.sleep(random.randint(7, 35)) # 随机停顿1~5秒 + d.swipe_ext("up") # 上划,下一个视频 + except Exception as e: + print_t(e) + index += 1 + + +def review_tiktok(): # 评论 + keys = ['handmadecraft'] # '#homedecor #flowershower #handmadecraft' + d.press("home") + d.app_start('com.zhiliaoapp.musically', stop=False) + time.sleep(1) + try: + d(, className='android.widget.TextView').click() # 点击发现 + time.sleep(1) + d(resourceId="com.zhiliaoapp.musically:id/fbt").click() # 点击搜索 + except Exception as e: + print_t(e) + for key in keys: + # 不能搜索search + try: + d(resourceId="com.zhiliaoapp.musically:id/b15").clear_text() # 清除历史 + d(resourceId="com.zhiliaoapp.musically:id/b15").set_text(key) # 输入 + d(resourceId="com.zhiliaoapp.musically:id/fap").click() # 点击搜索 + d(resourceId="android:id/text1", ).click() # 点击视频,然后点击第一条 + d.xpath( + '//*[@resource-id="com.zhiliaoapp.musically:id/cfh"]/android.widget.LinearLayout[1]/android.widget.FrameLayout[1]').click() + except Exception as e: + print_t(e) + stop, index = random.randint(35, 60), 0 + while index < stop: # 随机刷几十条 + cur_comment = comments[random.randint(0, len(comments) - 1)] + print_t('{}-总共有:{}条 | 现在到:{}条\t评论:{}'.format(key, stop, index, cur_comment)) + comment(cur_comment) + try: + time.sleep(random.randint(5, 20)) # 随机停顿1~5秒 + d.swipe_ext("up") # 上划,下一个视频 + except Exception as e: + print_t(e) + index += 1 + time.sleep(random.randint(int(0.5 * 60), 5 * 60)) + d(resourceId="com.zhiliaoapp.musically:id/t4").click() # 返回搜索 + + +def comment(content): + try: + time.sleep(random.randint(3, 10)) # 随机停顿1~5秒 + d(resourceId="com.zhiliaoapp.musically:id/acm").click() # 点击评论按钮 + except Exception as e: + print_t(e) + try: + time.sleep(random.randint(1, 3)) # 随机停顿1~2秒 + e_ele = d(, className='android.widget.EditText') + e_ele.click() # 点击弹出键盘 + time.sleep(random.randint(1, 3)) # 随机停顿1~2秒 + e_ele.clear_text() + e_ele.set_text(content) # 输入 + except Exception as e: + print_t(e) + try: + time.sleep(random.randint(1, 3)) # 随机停顿1~2秒 + d(resourceId="com.zhiliaoapp.musically:id/ad6").click() # 发送 + except Exception as e: + print_t(e) + # 关闭系统键盘 + try: + time.sleep(random.randint(1, 3)) # 随机停顿1~2秒 + d(resourceId="com.zhiliaoapp.musically:id/t4").click() # 关闭评论 + except Exception as e: + print_t(e) + + +def comment_foryou(content): + try: + time.sleep(random.randint(3, 10)) # 随机停顿1~5秒 + d(resourceId="com.zhiliaoapp.musically:id/acm").click() # 点击评论按钮 + except Exception as e: + print_t(e) + try: + time.sleep(random.randint(1, 3)) # 随机停顿1~2秒 + e_ele = d(, className='android.widget.EditText') + e_ele.click() # 点击弹出键盘 + time.sleep(random.randint(1, 3)) # 随机停顿1~2秒 + e_ele.clear_text() + e_ele.set_text(content) # 输入 + except Exception as e: + print_t(e) + try: + time.sleep(random.randint(1, 3)) # 随机停顿1~2秒 + d(resourceId="com.zhiliaoapp.musically:id/ad6").click() # 发送 + except Exception as e: + print_t(e) + # 关闭系统键盘 + try: + d.press("back") # 返回1,关闭系统键盘 + except Exception as e: + print_t(e) + + +def print_t(content): + dt = datetime.now() + print(dt.strftime('%H:%M:%S') + '\t' + str(content)) + + +if __name__ == "__main__": + # start_vpn() + # review_tiktok() + # review_forYou() + # comment('Look at my') + d(resourceId="com.zhiliaoapp.musically:id/afa").click() diff --git a/006-TikTok/v2ray_pool/__init__.py b/006-TikTok/v2ray_pool/__init__.py new file mode 100644 index 0000000..d0efa5d --- /dev/null +++ b/006-TikTok/v2ray_pool/__init__.py @@ -0,0 +1,11 @@ +# 运行时路径。并非__init__.py的路径 +import os +import sys + +BASE_DIR = "../002-V2rayPool" +if os.path.exists(BASE_DIR): + sys.path.append(BASE_DIR) + +from core import utils +from core.conf import Config +from db.db_main import DBManage \ No newline at end of file diff --git a/007-CutVideoAudio/.gitignore b/007-CutVideoAudio/.gitignore new file mode 100644 index 0000000..63e603a --- /dev/null +++ b/007-CutVideoAudio/.gitignore @@ -0,0 +1,128 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/007-CutVideoAudio/007-CutVideoAudio.iml b/007-CutVideoAudio/007-CutVideoAudio.iml new file mode 100644 index 0000000..ad3c0a3 --- /dev/null +++ b/007-CutVideoAudio/007-CutVideoAudio.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/007-CutVideoAudio/README.md b/007-CutVideoAudio/README.md new file mode 100644 index 0000000..96a9728 --- /dev/null +++ b/007-CutVideoAudio/README.md @@ -0,0 +1,43 @@ +# FFmpeg批量剪切音视频 + +[V2版本脚本](ff_util_v2.py)更加完善,但是未接入GUI + +本项目主要通过ffmpeg工具进行批量视频剪辑,随机剪辑,从而躲过自媒体平台的检查,从而达到一份视频多个账号运营。 + +使用前提:**必须要安装ffmpeg程序**,安装过程请自行百度。 + + +下载地址: + +MacOS:[QincjiCut1.0.0-mac](https://github.com/xhunmon/PythonIsTools/releases/download/1.0.4/QincjiCut1.0.0.app.zip) 下载后解压后使用 + +Window:QincjiCut1.0.0-win (未打包) + +效果如图: + +![剪辑器截图](./doc/example.png) + +#主要知识点 + +## python GUI(界面) + +本文使用tkinter GUI(界面)框架进行界面显示:[./ui.py](ui.py) ,[学习参考](https://www.cnblogs.com/shwee/p/9427975.html) 。 + +## [pyinstaller](https://pyinstaller.readthedocs.io/en/stable/) 打包 + +使用pyinstaller把python程序打包成window和mac可执行文件,主要命令如下: +```shell +#1 :生成xxx.spec文件;(去掉命令窗口-w) +pyinstaller -F -i res/logo.ico main.py -w +#2:修改xxx.spec,参考main.spec +#3:再次进行打包,参考installer-mac.sh +pyinstaller -F -i res/logo.ico main.spec -w +``` +打包脚本与配置已放在 `doc` 目录下,需要拷贝出根目录进行打包。 + +注意: +pyinstaller打包工具的版本与python版本、python所需第三方库以及操作系统会存在各种问题,所以需要看日志查找问题。例如:打包后运用,发现导入pyppeteer报错,通过降低版本后能正常使用:pip install pyppeteer==0.2.2 + +## 项目 +本项目跟Downloader下载器基本相同,而ffmpeg命令则可以通过 [](https://qincji.gitee.io/2021/01/18/ffmpeg/18_command/) + diff --git a/007-CutVideoAudio/__init__.py b/007-CutVideoAudio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/007-CutVideoAudio/config.ini b/007-CutVideoAudio/config.ini new file mode 100644 index 0000000..dc21c88 --- /dev/null +++ b/007-CutVideoAudio/config.ini @@ -0,0 +1,10 @@ +# 常用配置模块 +[common] +#软件使用截止日期 +expired_time=2025年12月15日 23:59:59 + +#app的版本名称 +version_name=1.0.0 + +#app的版本号 +version_code=1000 \ No newline at end of file diff --git a/007-CutVideoAudio/doc/example.png b/007-CutVideoAudio/doc/example.png new file mode 100644 index 0000000..dbd1710 Binary files /dev/null and b/007-CutVideoAudio/doc/example.png differ diff --git a/007-CutVideoAudio/doc/mac-sh/main.spec b/007-CutVideoAudio/doc/mac-sh/main.spec new file mode 100644 index 0000000..0c0766d --- /dev/null +++ b/007-CutVideoAudio/doc/mac-sh/main.spec @@ -0,0 +1,44 @@ +# -*- mode: python ; coding: utf-8 -*- + + +block_cipher = None + + +a = Analysis(['main.py','type_enum.py','ui.py','utils.py','editors.py','ff_util.py','ff_cut.py'], + pathex=['.'], + binaries=[], + datas=[('res/logo.ico', 'images'),('config.ini', '.')], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) + +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='main', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=False, + disable_windowed_traceback=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None , icon='res/logo.ico') +app = BUNDLE(exe, + name='QincjiCut.app', + icon='res/logo.ico', + bundle_identifier=None) \ No newline at end of file diff --git a/007-CutVideoAudio/doc/mac-sh/pyinstaller.sh b/007-CutVideoAudio/doc/mac-sh/pyinstaller.sh new file mode 100644 index 0000000..762b370 --- /dev/null +++ b/007-CutVideoAudio/doc/mac-sh/pyinstaller.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +pyinstaller -F -i res/logo.ico main.spec main.py -w \ +-p type_enum.py \ +-p ui.py \ +-p utils.py \ +-p ff_util.py \ +-p editors.py \ +-p ff_cut.py \ No newline at end of file diff --git a/007-CutVideoAudio/editors.py b/007-CutVideoAudio/editors.py new file mode 100644 index 0000000..d2f057b --- /dev/null +++ b/007-CutVideoAudio/editors.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description:editing.py 所有视频类的基类,负责与UI界面的绑定 +@Date :2022年03月14日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import time +from threading import Lock + +import requests +from my_fake_useragent import UserAgent + +from type_enum import PrintType +from utils import Config + +ua = UserAgent(family='chrome') + + +class Editors(object): + func_ui_print = None + __mutex_total = Lock() + __mutex_success = Lock() + __mutex_failed = Lock() + __mutex_bmg = Lock() + __count_total = 0 + __count_success = 0 + __count_failed = 0 + __count_bmg = 0 + __beijing_time = 0 # 在线北京时间 + + def __init__(self): + self._headers = {'user-agent': ua.random()} + self.get_beijing_time() + + @staticmethod + def print_hint(): + """显示初始提示信息""" + Editors.print_ui( + """ +程序原理: +\t1、随机裁剪掉视频头尾; +\t2、随机改变视频帧率和比特率; +\t3、随机更换视频bgm; +\t4、保证每次输出不重复。 +使用说明: +\t1、必须安装FFmpeg; +\t2、最短bgm长度大于最大视频长度。 + """ + ) + + def start(self, ffmpeg, video, music, dst): + """业务逻辑由子类实现""" + pass + + @staticmethod + def print_ui(txt): + """在界面显示内容""" + Editors.print_all_ui(txt=txt) # 打印日志 + + @staticmethod + def print_all_ui(txt, print_type: PrintType = PrintType.log): + """通知ui中func_ui_print更新内容""" + if Editors.func_ui_print is not None: + Editors.func_ui_print(txt=txt, print_type=print_type) + + @staticmethod + def get_beijing_time(): + """静态方法:获取在线的北京时间""" + if Editors.__beijing_time> 0: + return Editors.__beijing_time + try: + response = requests.get(url='http://www.beijing-time.org/t/time.asp', headers={'user-agent': ua.random()}) + result = response.text + data = result.split("\r\n") + year = data[1][len("nyear") + 1: len(data[1]) - 1] + month = data[2][len("nmonth") + 1: len(data[2]) - 1] + day = data[3][len("nday") + 1: len(data[3]) - 1] + # wday = data[4][len("nwday")+1 : len(data[4])-1] + hrs = data[5][len("nhrs") + 1: len(data[5]) - 1] + minute = data[6][len("nmin") + 1: len(data[6]) - 1] + sec = data[7][len("nsec") + 1: len(data[7]) - 1] + + beijinTimeStr = "%s/%s/%s %s:%s:%s" % (year, month, day, hrs, minute, sec) + beijinTime = time.strptime(beijinTimeStr, "%Y/%m/%d %X") + Editors.__beijing_time = int(time.mktime(beijinTime)) + except: + pass + return Editors.__beijing_time + + @staticmethod + def is_expired(): + """静态方法:判断是否已过期""" + if Editors.__beijing_time == 0: # 还没获取到时间 + return True + expired_time_str = time.strptime(Config.instance().get_expired_time(), "%Y/%m/%d %X") + expired_time_int = int(time.mktime(expired_time_str)) + return Editors.__beijing_time> expired_time_int + + @staticmethod + def add_total_count(count=1): + """静态方法:添加总下载任务数""" + Editors.__mutex_total.acquire() + Editors.__count_total += count + Editors.__mutex_total.release() + Editors.print_all_ui(txt="视频总数:%d" % Editors.__count_total, print_type=PrintType.total) + + @staticmethod + def add_bgm_count(count=1): + """静态方法:添加总下载任务数""" + Editors.__mutex_bmg.acquire() + Editors.__count_bmg += count + Editors.__mutex_bmg.release() + Editors.print_all_ui(txt="bmg数量:%d" % Editors.__count_bmg, print_type=PrintType.bgm) + + @staticmethod + def get_total_count(): + """静态方法:获取总下载任务数""" + return Editors.__count_total + + @staticmethod + def get_bgm_count(): + """静态方法:获取正在下载任务数""" + return Editors.__count_bmg + + @staticmethod + def add_success_count(): + """静态方法:添加下载成功任务数""" + Editors.__mutex_success.acquire() + Editors.__count_success += 1 + Editors.__mutex_success.release() + # 成功一条,减正在下载的一条 + Editors.print_all_ui(txt="已完成:%d" % Editors.__count_success, print_type=PrintType.success) + + @staticmethod + def get_success_count(): + """静态方法:获取下载成功任务数""" + return Editors.__count_success + + @staticmethod + def add_failed_count(): + """静态方法:添加下载失败任务数""" + Editors.__mutex_failed.acquire() + Editors.__count_failed += 1 + Editors.__mutex_failed.release() + # 失败一条,减正在下载的一条 + Editors.print_all_ui(txt="已失败:%d" % Editors.__count_failed, print_type=PrintType.failed) + + @staticmethod + def get_failed_count(): + """静态方法:获取下载失败任务数""" + return Editors.__count_failed diff --git a/007-CutVideoAudio/ff_cut.py b/007-CutVideoAudio/ff_cut.py new file mode 100644 index 0000000..3bac413 --- /dev/null +++ b/007-CutVideoAudio/ff_cut.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description:editing.py 所有视频类的基类,负责与UI界面的绑定 +@Date :2022年03月14日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import os.path +import random +import shutil +import time + +from editors import Editors +from ff_util import * +from utils import * + + +class ListCut(Editors): + # 初始化 + def __init__(self): + super().__init__() + self.headers = self._headers + # 抓获所有视频 + self.end = False + + def start(self, ffmpeg, video, music, dst): + Editors.print_ui("开始准备处理本地数据") + if not os.path.exists(ffmpeg) or not os.path.exists(video) or not os.path.exists(music) or not os.path.exists( + dst): + Editors.print_ui("文件或者目录不存在!") + return + dst_temp = os.path.join(dst, 'temp') + if not os.path.exists(dst_temp): + os.makedirs(dst_temp) + fps, bit = random.randint(25, 30), random.randint(1200, 1800) + start_v, end_v = random.randint(100, 500), random.randint(100, 500) + v_files, a_files = get_real_files(video), get_real_files(music) + index, size_v, size_a, start_time = 1, len(v_files), len(a_files), time.time() + Editors.add_total_count(size_v) + Editors.add_bgm_count(size_a) + Editors.print_ui("改系列帧率:{} | 比特率:{} | 截切开头:{} | 截切结尾:{}".format(fps, bit, start_v, end_v)) + time.sleep(3) + for i in range(0, size_v): + a_file = random.choice(a_files) + v_file = random.choice(v_files) + v_files.remove(v_file) + Editors.print_ui("{} 与 {} 开始合并!".format(v_file, a_file)) + try: + name_v, ext_v = os.path.splitext(v_file) + name_a, ext_a = os.path.splitext(a_file) + src_v = os.path.join(video, v_file) + dst_file = os.path.join(dst, '{}-{}{}'.format(format_num(index), name_v, ext_v)) + dur_src_v = get_duration(ffmpeg, src_v) # 毫秒 + duration = dur_src_v - start_v - end_v + src_a = os.path.join(music, a_file) + dur_src_a = get_duration(ffmpeg, src_a) + random_a = (dur_src_a - duration) if dur_src_a> duration else 100 + start_a = random.randint(0, random_a) # 随机取音频 + temp_file_v = os.path.join(dst_temp, 'temp{}'.format(ext_v)) + temp_file_a = os.path.join(dst_temp, 'temp{}'.format(ext_a)) + cut_video(ffmpeg, src_v, start_v, duration, fps, bit, temp_file_v) + cut_audio(ffmpeg, src_a, start_a, duration, temp_file_a) + muxer_va(ffmpeg, temp_file_v, temp_file_a, dst_file) + index += 1 + Editors.add_success_count() + except Exception as e: + Editors.print_ui(str(e)) + Editors.add_failed_count() + time.sleep(1) + use_time = time.time() - start_time + Editors.print_ui('已运行{}分{}秒...'.format(int(use_time / 60), int(use_time % 60))) + time.sleep(2) + shutil.rmtree(dst_temp) diff --git a/007-CutVideoAudio/ff_util.py b/007-CutVideoAudio/ff_util.py new file mode 100644 index 0000000..c8a682d --- /dev/null +++ b/007-CutVideoAudio/ff_util.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description:ff_util.py ffmpeg截切命令工具 +@Date :2022年03月14日 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import os + + +def get_duration(ff_file, src): + ''' + 执行命令获取输出这样的:Duration: 00:00:31.63, start: 0.000000, bitrate: 1376 kb/s + :param ff_file: ffmpeg程序路径 + :param src: 音视频文件 + :return: 返回毫秒 + ''' + info = os.popen(r'{} -i "{}" 2>&1 | grep "Duration"'.format(ff_file, src)).read() + dur = info.split(',')[0].replace(' ', '').split(':') + h, m, ss = int(dur[1]) * 60 * 60 * 1000, int(dur[2]) * 60 * 1000, dur[3] + if '.' in ss: + s1 = ss.split('.') + s = int(s1[0]) * 1000 + int(s1[1]) * 10 + else: + s = int(ss) * 1000 + return h + m + s + + +def format_h_m_s(t): + if t < 10: + return '0{}'.format(t) + else: + return '{}'.format(t) + + +def format_ms(t): + if t < 10: + return '00{}'.format(t) + elif t < 100: + return '0{}'.format(t) + else: + return '{}'.format(t) + + +def format_duration_by_ms(ms): + ''' + 通过毫秒转化成 'xx:xx:xx.xxx' 格式 + :param ms: 毫秒 + :return: + ''' + h = format_h_m_s(int(ms / 1000 / 60 / 60)) + m = format_h_m_s(int(ms / 1000 / 60 % 60)) + s = format_h_m_s(int(ms / 1000 % 60)) + m_s = format_ms(int(ms % 1000)) + return '{}:{}:{}.{}'.format(h, m, s, m_s) + + +def cut_audio(ff_file, src, start, dur, dst): + ''' + 裁剪一段音频进行输出 + :param ff_file: ffmpeg程序路径 + :param src: 要裁剪的文件路径,可以是视频文件 + :param start: 开始裁剪点,单位毫秒开始 + :param dur: 裁剪时长,单位秒 + :param dst: 输出路径,包括后缀名 + :return: + ''' + if os.path.exists(dst): + os.remove(dst) + os.system( + r'{} -i "{}" -vn -acodec copy -ss {} -t {} "{}"'.format(ff_file, src, format_duration_by_ms(start), dur, dst)) + + +def cut_video(ff_file, src, start, dur, fps, bit, dst): + ''' + 裁剪一段视频进行输出, -ss xx:xx:xx.xxx + :param ff_file: ffmpeg程序路径 + :param src: 要裁剪的文件路径,可以是视频文件 + :param start: 开始裁剪点,单位毫秒开始 + :param dur: 裁剪时长,单位秒 + :param fps: 帧率,通常是25~30 + :param bit: 比特率,通常是1600~2000即可 + :param dst: 输出路径,包括后缀名 + :return: + ''' + if os.path.exists(dst): + os.remove(dst) + os.system( + r'{} -i "{}" -ss {} -t {} -r {} -b:v {}K -an "{}"'.format(ff_file, src, format_duration_by_ms(start), dur, fps, + bit, dst)) + + +def muxer_va(ff_file, src_v, src_a, dst): + if os.path.exists(dst): + os.remove(dst) + os.system(r'{} -i "{}" -i "{}" -c:v copy -c:a aac -strict experimental "{}"'.format(ff_file, src_v, src_a, dst)) diff --git a/007-CutVideoAudio/ff_util_v2.py b/007-CutVideoAudio/ff_util_v2.py new file mode 100644 index 0000000..9e9f69b --- /dev/null +++ b/007-CutVideoAudio/ff_util_v2.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description:ff_util.py ffmpeg截切命令工具 +@Date :2022/03/14 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import os +import re +import time +from datetime import datetime, timedelta + + +def get_va_infos(ff_file, src): + """ + 获取视频的基本信息 + @param ff_file: ffmpeg路径 + @param src: 视频路径 + @return: 结果:{'duration': '00:11:26.91', 'bitrate': '507', 'v_codec': 'h264', 'v_size': '1280x720', 'v_bitrate': '373', 'v_fps': '25', 'a_codec': 'aac', 'a_bitrate': '128'} + """ + cmd = r'{} -i "{}" -hide_banner 2>&1'.format(ff_file, src) + output = os.popen(cmd).read() + lines = output.splitlines() + result = {} + for line in lines: + if line.strip().startswith('Duration:'): + result['duration'] = line.split(',')[0].split(': ')[-1] + result['bitrate'] = line.split(',')[-1].strip().split(': ')[-1].split(' ')[0] + elif line.strip().startswith('Stream #0'): + line = re.sub(r'\[.*?\]', '', re.sub(r'\(.*?\)', '', line)) + if 'Video' in line: + result['v_codec'] = line.split(',')[0].split(': ')[-1].strip() + result['v_size'] = line.split(',')[2].strip().split(' ')[0].strip() + result['v_bitrate'] = line.split(',')[3].strip().split(' ')[0].strip() + result['v_fps'] = line.split(',')[4].strip().split(' ')[0].strip() + elif 'Audio' in line: + result['a_codec'] = line.split(',')[0].split(': ')[-1].strip() + result['a_bitrate'] = line.split(',')[4].strip().split(' ')[0].strip() + print(result) + return result + + +def get_duration(ff_file, src): + ''' + 执行命令获取输出这样的:Duration: 00:00:31.63, start: 0.000000, bitrate: 1376 kb/s + :param ff_file: ffmpeg程序路径 + :param src: 音视频文件 + :return: 返回毫秒 + ''' + cmd = r'{} -i "{}" 2>&1 | grep "Duration"'.format(ff_file, src) + info = os.popen(cmd).read() + dur = info.split(',')[0].replace(' ', '').split(':') + h, m, ss = int(dur[1]) * 60 * 60 * 1000, int(dur[2]) * 60 * 1000, dur[3] + if '.' in ss: + s1 = ss.split('.') + s = int(s1[0]) * 1000 + int(s1[1]) * 10 + else: + s = int(ss) * 1000 + return h + m + s + + +def format_h_m_s(t): + return f'0{t}' if t < 10 else f'{t}' + + +def format_ms(t): + if t < 10: + return f'00{t}' + return f'0{t}' if t < 100 else f'{t % 1000}' + + +def format_to_time(ms): + ''' + 毫秒 --> 'xx:xx:xx.xxx' + :param ms: 毫秒 + :return: 'xx:xx:xx.xxx' + ''' + # t = timedelta(milliseconds=ms) + # return str(t) + h = format_h_m_s(int(ms / 1000 / 60 / 60)) + m = format_h_m_s(int(ms / 1000 / 60 % 60)) + s = format_h_m_s(int(ms / 1000 % 60)) + m_s = format_ms(int(ms % 1000)) + return '{}:{}:{}.{}'.format(h, m, s, m_s) + + +def format_to_ms(duration: str): + """ + 'xx:xx:xx.xxx' --> 毫秒 + @param duration: 时间长度'xx:xx:xx.xxx' + @return: 毫秒 + """ + hms = duration.split(':') + s_str = hms[2] + ms_str = '0' + if '.' in s_str: + s_ms = s_str.split('.') + s_str = s_ms[0] + ms_str = s_ms[1] + h = int(hms[0]) * 1000 * 60 * 60 + m = int(hms[1]) * 1000 * 60 + s = int(s_str) * 1000 + ms = int(ms_str) + return h + m + s + ms + + +def srt_to_ass(ff_file, src, dst): + os.system(f'{ff_file} -i {src} {dst}') + + +def cut_with_subtitle(ff_file, src, dst, srt, width, height, margin_v, font_size=50, dur_full: str = None, + start='00:00:00.000', tail='00:00:00.000', fps=None, + v_bit=None, a_bit=None): + """ + 添加硬字幕:ffmpeg -i "../output/733316.mp4" -ss "00:02:00" -t 10 -r 23 -b:v 400K -c:v libx264 -b:a 38K -c:a aac -vf "subtitles=../output/input.ass:force_style='PlayResX=1280,PlayResY=720,MarginV=80,Fontsize=50'" ../output/ass.mp4 + """ + t_start = time.time() + dur_ms = format_to_ms(dur_full) - format_to_ms(tail) - format_to_ms(start) + dur = format_to_time(dur_ms) + + filename, ext = os.path.splitext(srt) + ass = f'{filename}.ass' + if os.path.exists(ass): + os.remove(ass) + ass_cmd = '{} -i "{}" "{}"'.format(ff_file, srt, ass) + print(ass_cmd) + os.system(ass_cmd) + # ffmpeg -i "input.mp4" -ss "00:02:10.000" -t 12397 -r 15 -b:v 500K -c:v libx264 -c:a aac + cmd = '{} -i "{}"'.format(ff_file, src) + cmd = '{} -ss "{}" -t {}'.format(cmd, start, int(format_to_ms(dur) / 1000)) + if fps: # 添加裁剪的fps + cmd = '{} -r {}'.format(cmd, fps) + # -c copy 不经过解码,会出现黑屏,因为有可能是P帧和B帧 + if v_bit: # 添加视频bitrate,并且指定用libx264进行编码(ffmpeg必须安装) + cmd = '{} -b:v {}K -c:v libx264'.format(cmd, v_bit) + if a_bit: # 添加音频bitrate,并且指定用aac进行编码 + cmd = '{} -b:a {}K -c:a aac'.format(cmd, a_bit) + # -vf "subtitles=input.ass:force_style='PlayResX=1280,PlayResY=720,MarginV=70,Fontsize=50'" + style = "PlayResX={},PlayResY={},MarginV={},Fontsize={}".format(width, height, margin_v, font_size) + sub_file = "{}".format(ass) + cmd = '''{} -vf "subtitles={}:force_style='{}'"'''.format(cmd, sub_file, style) + # cmd = f'{cmd} -vf "subtitles={ass}:force_style="""PlayResX={width},PlayResY={height},MarginV={margin_v},Fontsize={font_size}""""' + cmd = '{} {}'.format(cmd, dst) + print(cmd) + if os.path.exists(dst): + os.remove(dst) + os.system(cmd) + os.remove(ass) + print('一共花了 {} 秒 进行裁剪并添加字幕 {}'.format(int(time.time() - t_start), src)) + + +def cut_va_full(ff_file, src, dst, dur: str = None, start='00:00:00.000', fps=None, v_bit=None, a_bit=None, + copy_a=False): + """ + ffmpeg -i "input.mp4" -ss "00:02:10.000" -t 12397 -r 15 -b:v 500K -c:v libx264 -c:a aac "凡人修仙传1重制版-国创-高清独家在线观看-bilibili-哔哩哔哩.mp4" + 其他所有的视频裁剪命令都需要通过这个实现 + @param ff_file: ffmpeg路径 + @param src: 输入路径 + @param dst: 输出路径 + @param dur: 裁剪长度,格式为'00:00:00.000' + @param start: 裁剪的起点,如果dur=None,表示需要对时间进行裁剪,只是转换格式罢了 + @param fps: 帧率,通常视频15~18帧即可,动漫一般24帧 + @param v_bit: 单独控制视频的比特率 + @param a_bit: 单独控制音频的比特率 + @param copy_a: 直接复制音频通道数据,当 a_bit=None方有效 + """ + cmd = '{} -i "{}"'.format(ff_file, src) # 输入文件 + if dur is not None: # 添加裁剪时间 + cmd = '{} -ss "{}" -t {}'.format(cmd, start, int(format_to_ms(dur) / 1000)) + if fps: # 添加裁剪的fps + cmd = '{} -r {}'.format(cmd, fps) + # -c copy 不经过解码,会出现黑屏,因为有可能是P帧和B帧 + if v_bit: # 添加视频bitrate,并且指定用libx264进行编码(ffmpeg必须安装) + cmd = '{} -b:v {}K -c:v libx264'.format(cmd, v_bit) + if a_bit: # 添加音频bitrate,并且指定用aac进行编码 + cmd = '{} -b:a {}K -c:a aac'.format(cmd, a_bit) + elif copy_a: # 是否完全复制音频 + cmd = '{} -c:a copy'.format(cmd) + cmd = '{} "{}"'.format(cmd, dst) # 添加输出 + if os.path.exists(dst): + os.remove(dst) + t_start = time.time() + os.system(cmd) + print('一共花了 {} 秒 进行裁剪 {}'.format(int(time.time() - t_start), src)) + + +def cut_va_tail(ff_file, src, dst, dur_full: str = None, start='00:00:00.000', tail='00:00:00.000', fps=None, + v_bit=None, a_bit=None, copy_a=False): + """ + 裁剪头尾 + """ + dur_ms = format_to_ms(dur_full) - format_to_ms(tail) - format_to_ms(start) + dur = format_to_time(dur_ms) + cut_va_full(ff_file, src, dst, dur, start, fps, v_bit, a_bit, copy_a) + + +def cut_audio(ff_file, src, start, dur, dst): + ''' + 裁剪一段音频进行输出 + :param ff_file: ffmpeg程序路径 + :param src: 要裁剪的文件路径,可以是视频文件 + :param start: 开始裁剪点,单位毫秒开始 + :param dur: 裁剪时长,单位秒 + :param dst: 输出路径,包括后缀名 + :return: + ''' + if os.path.exists(dst): + os.remove(dst) + os.system( + r'{} -i "{}" -vn -acodec copy -ss {} -t {} "{}"'.format(ff_file, src, format_to_time(start), dur, dst)) + + +def cut_video(ff_file, src, start, dur, fps, bit, dst): + ''' + 裁剪一段视频进行输出, -ss xx:xx:xx.xxx + :param ff_file: ffmpeg程序路径 + :param src: 要裁剪的文件路径,可以是视频文件 + :param start: 开始裁剪点,单位毫秒开始 + :param dur: 裁剪时长,单位秒 + :param fps: 帧率,通常是25~30 + :param bit: 比特率,通常是1600~2000即可 + :param dst: 输出路径,包括后缀名 + :return: + ''' + if os.path.exists(dst): + os.remove(dst) + os.system( + r'{} -i "{}" -ss {} -t {} -r {} -b:v {}K -an "{}"'.format(ff_file, src, format_to_time(start), dur, fps, + bit, dst)) + + +def cut_va_dur(ff_file, src, dst, start=0, dur=0, fps=None, bit=None): + """ + 根据头尾裁剪视频 + :param ff_file: ffmpeg工具 + :param src: 输入资源 + :param dst: 输出文件 + :param start: 起点 + :param end: 终点 + :param fps: 帧率 + :param bit: 比特率 + :return: + """ + length = get_duration(ff_file, src) + if start + dur> length: + print('裁剪比视频长') + return + if os.path.exists(dst): + os.remove(dst) + + cmd = r'{} -i "{}"'.format(ff_file, src) + cmd = cmd + ' -ss {}'.format(format_to_time(start)) + cmd = cmd + ' -t {}'.format(format_to_time(dur)) + if fps: + cmd = cmd + ' -r {}'.format(fps) + if bit: + cmd = cmd + ' -b:v {}K'.format(bit) + cmd = cmd + ' -c:v libx264 "{}"'.format(dst) # 使用x264解码后重新封装 + # cmd = cmd+' -c copy {}'.format(dst) #不经过解码,会出现黑屏 + os.system(cmd) + + +def cut_va_start_end(ff_file, src, dst, start='00:00:00', end='00:00:00', dur='00:00:00', fps=None, bit=None): + dur = format_to_ms(dur) - format_to_ms(end) + cmd = f'{ff_file} -i "{src}" -ss {start} -t {dur} -c copy "{dst}"' + if os.path.exists(dst): + os.remove(dst) + os.system(cmd) + + +def cut_va_end(ff_file, src, dst, start=0, end=0, fps=None, bit=None): + """ + 根据头尾裁剪视频 + :param ff_file: ffmpeg工具 + :param src: 输入资源 + :param dst: 输出文件 + :param start: 起点 + :param end: 终点 + :param fps: 帧率 + :param bit: 比特率 + :return: + """ + length = get_duration(ff_file, src) + dur = length - (start + end) + if dur <= 0: + print('裁剪比视频长') + return + cut_va_dur(ff_file, src, dst, start, dur, fps, bit) + + +def muxer_va(ff_file, src_v, src_a, dst): + if os.path.exists(dst): + os.remove(dst) + os.system(r'{} -i "{}" -i "{}" -c:v copy -c:a aac -strict experimental "{}"'.format(ff_file, src_v, src_a, dst)) diff --git a/007-CutVideoAudio/main.py b/007-CutVideoAudio/main.py new file mode 100644 index 0000000..13b0b09 --- /dev/null +++ b/007-CutVideoAudio/main.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: 程序主入口 +@Date :2022/03/14 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import os +import sys + +from ui import Ui + +# 主模块执行 +if __name__ == "__main__": + path = os.path.dirname(os.path.realpath(sys.argv[0])) + # ffmpeg = os.path.dirname('/usr/local/ffmpeg/bin/ffmpeg/') + # video = os.path.dirname('/Users/Qincji/Documents/zmt/handmade/') + # music = os.path.dirname('/Users/Qincji/Documents/zmt/music/') + # dst = os.path.dirname('/Users/Qincji/Downloads/ffmpeg/') + app = Ui() + # app.set_dir(ffmpeg, video, music, dst) + app.set_dir(path, path, path, path) + # to do + app.mainloop() diff --git a/007-CutVideoAudio/res/logo.ico b/007-CutVideoAudio/res/logo.ico new file mode 100644 index 0000000..0eff936 Binary files /dev/null and b/007-CutVideoAudio/res/logo.ico differ diff --git a/007-CutVideoAudio/type_enum.py b/007-CutVideoAudio/type_enum.py new file mode 100644 index 0000000..7e1574c --- /dev/null +++ b/007-CutVideoAudio/type_enum.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +""" +@Description:dy_download.py +@Date :2022/03/14 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" + +from enum import Enum + + +class PrintType(Enum): + log = 1 + total = 2 + bgm = 3 + success = 4 + failed = 5 diff --git a/007-CutVideoAudio/ui.py b/007-CutVideoAudio/ui.py new file mode 100644 index 0000000..3824a7d --- /dev/null +++ b/007-CutVideoAudio/ui.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +""" +@Description: 用于GUI界面显示 +@Date :2021/08/14 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +from tkinter import * +from tkinter.filedialog import (askdirectory, askopenfilename) + +from editors import Editors +from ff_cut import ListCut +from type_enum import PrintType +from utils import * + + +# from PIL import Image, ImageTk + + +class Ui(Frame): + def __init__(self, master=None): + global bg_color + bg_color = '#373434' + Frame.__init__(self, master, bg=bg_color) + self.ui_width = 0 + self.pack(expand=YES, fill=BOTH) + self.window_init() + self.createWidgets() + + def window_init(self): + self.master.title( + '欢迎使用-音视频批量截切工具' + Config.instance().get_version_name() + ' 如有疑问请联系:xhunmon@gmail.com') + self.master.bg = bg_color + width, height = self.master.maxsize() + # self.master.geometry("{}x{}".format(width, height)) + self.master.geometry("%dx%d+%d+%d" % (width / 2, height / 2, width / 4, height / 4)) + self.ui_width = width / 2 + + def createWidgets(self): + # fm1 + self.fm1 = Frame(self, bg=bg_color) + self.fm1.pack(fill='y', pady=10) + # window没有原生PIL 64位支持 + # load = Image.open('res/logo.png') + # load.thumbnail((38, 38), Image.ANTIALIAS) + # initIamge = ImageTk.PhotoImage(load) + # self.panel = Label(self.fm1, image=initIamge, bg=bg_color) + # self.panel.image = initIamge + # self.panel.pack(side=LEFT, fill='y', padx=5) + self.titleLabel = Label(self.fm1, , font=('微软雅黑', 28), fg="white", bg=bg_color) + self.titleLabel.pack(side=LEFT, fill='y') + + # fm2 + self.fm2 = Frame(self, bg=bg_color) + self.fm2.pack(side=TOP, fill="y") + self.fm2_right = Frame(self.fm2, bg=bg_color) + self.fm2_right.pack(side=RIGHT, padx=0, pady=10, expand=YES, fill='y') + self.fm2_left = Frame(self.fm2, bg=bg_color) + self.fm2_left.pack(side=LEFT, padx=15, pady=10, expand=YES, fill='x') + self.fm2_left_ffmpeg = Frame(self.fm2_left, bg=bg_color) + self.fm2_left_souce_video = Frame(self.fm2_left, bg=bg_color) + self.fm2_left_souce_music = Frame(self.fm2_left, bg=bg_color) + self.fm2_left_dst = Frame(self.fm2_left, bg=bg_color) + + self.downloadBtn = Button(self.fm2_right, , fg="#aaaaaa", bg=bg_color, + font=('微软雅黑', 18), command=self.start_download) + self.downloadBtn.pack(side=RIGHT) + + self.ffmFileEntry = Entry(self.fm2_left_ffmpeg, font=('微软雅黑', 14), , fg='#ffffff', bg=bg_color, bd=1) + self.ffmFileEntry.config(insert) + self.ffmFileBtn = Button(self.fm2_left_ffmpeg, , bg=bg_color, fg='#aaaaaa', + font=('微软雅黑', 12), width='12', command=self.save_file) + self.ffmFileBtn.pack(side=LEFT) + self.ffmFileEntry.pack(side=LEFT, fill='y') + self.fm2_left_ffmpeg.pack(side=TOP, fill='x') + + self.videoDirEntry = Entry(self.fm2_left_souce_video, font=('微软雅黑', 14), width='72', fg='#ffffff', bg=bg_color, + bd=1) + self.videoDirEntry.config(insert) + self.videoDirBtn = Button(self.fm2_left_souce_video, , bg=bg_color, fg='#aaaaaa', + font=('微软雅黑', 12), width='12', command=self.save_dir) + self.videoDirBtn.pack(side=LEFT) + self.videoDirEntry.pack(side=LEFT, fill='y') + self.fm2_left_souce_video.pack(side=TOP, fill='x') + + self.musicDirEntry = Entry(self.fm2_left_souce_music, font=('微软雅黑', 14), width='72', fg='#ffffff', bg=bg_color, + bd=1) + self.musicDirEntry.config(insert) + self.musicDirBtn = Button(self.fm2_left_souce_music, , bg=bg_color, fg='#aaaaaa', + font=('微软雅黑', 12), width='12', command=self.save_dir) + self.musicDirBtn.pack(side=LEFT) + self.musicDirEntry.pack(side=LEFT, fill='y') + self.fm2_left_souce_music.pack(side=TOP, fill='x') + + self.dstEntry = Entry(self.fm2_left_dst, font=('微软雅黑', 14), width='72', fg='#ffffff', bg=bg_color, bd=1) + self.dstEntry.config(insert) + self.dstBtn = Button(self.fm2_left_dst, , bg=bg_color, fg='#aaaaaa', + font=('微软雅黑', 12), width='12', command=self.download_url) + self.dstBtn.pack(side=LEFT) + self.dstEntry.pack(side=LEFT, fill='y') + self.fm2_left_dst.pack(side=TOP, pady=10, fill='x') + + # fm3 任务数状态 + self.fm3 = Frame(self, bg=bg_color, height="6)" + self.fm3.pack(side=TOP, fill="x") + self.totalLabel = Label(self.fm3, width="10," , font=('微软雅黑', 12), fg="white", bg=bg_color) + self.totalLabel.pack(side=LEFT, fill='y', padx=20) + self.totalBgmLabel = Label(self.fm3, width="10," , font=('微软雅黑', 12), fg="white", bg=bg_color) + self.totalBgmLabel.pack(side=LEFT, fill='y', padx=20) + self.successLabel = Label(self.fm3, width="10," , font=('微软雅黑', 12), fg="white", bg=bg_color) + self.successLabel.pack(side=LEFT, fill='y', padx=20) + self.failLabel = Label(self.fm3, width="10," , font=('微软雅黑', 12), fg="white", bg=bg_color) + self.failLabel.pack(side=LEFT, fill='y', padx=20) + + # fm4 + self.fm4 = Frame(self, bg=bg_color) + self.fm4.pack(side=TOP, expand=YES, fill="both") + self.logLabel = Label(self.fm4, anchor='w', wraplength=self.ui_width - 40, , font=('微软雅黑', 12), + fg="white", + bg=bg_color) + self.logLabel.pack(side=TOP, fill='both', padx=20) + + # 注册回调 + Editors.func_ui_print = self.func_ui_print + # 判断是否有网络 + if Editors.get_beijing_time() == 0: + self.output("获取数据异常,请检查您的网络!") + else: + Editors.print_hint() + + def save_dir(self): + path = askdirectory() + self.set_dir(path) + + def save_file(self): + path = askopenfilename() + self.set_dir(path) + + def set_dir(self, ffmpeg=None, video=None, music=None, dst=None): + if ffmpeg: + self.ffmFileEntry.delete(0, END) + self.ffmFileEntry.insert(0, ffmpeg) + if video: + self.videoDirEntry.delete(0, END) + self.videoDirEntry.insert(0, video) + if music: + self.musicDirEntry.delete(0, END) + self.musicDirEntry.insert(0, music) + if dst: + self.dstEntry.delete(0, END) + self.dstEntry.insert(0, dst) + + def download_url(self): + ground_truth = '' + self.dstEntry.delete(0, END) + self.dstEntry.insert(0, ground_truth) + + def output(self, txt): + self.logLabel.config( + + def func_ui_print(self, txt, print_type: PrintType = None): + if print_type == PrintType.log: + self.logLabel.config( + elif print_type == PrintType.total: + self.totalLabel.config( + elif print_type == PrintType.bgm: + self.totalBgmLabel.config( + elif print_type == PrintType.success: + self.successLabel.config( + elif print_type == PrintType.failed: + self.failLabel.config( + + def start_download(self): + # 判断是否有网络,去掉 + # if Editors.get_beijing_time() == 0: + # self.output("获取数据异常,请检查您的网络!") + # return + if Editors.is_expired(): + self.output("授权证书已到期,请联系客服!") + return + ffmpeg = self.ffmFileEntry.get() + video = self.videoDirEntry.get() + music = self.musicDirEntry.get() + dst = self.dstEntry.get() + editors: Editors = ListCut() + editor = threading.Thread( args=(ffmpeg, video, music, dst)) + editor.setDaemon(True) # 设置守护进程,避免界面卡死 + editor.start() diff --git a/007-CutVideoAudio/utils.py b/007-CutVideoAudio/utils.py new file mode 100644 index 0000000..36230c7 --- /dev/null +++ b/007-CutVideoAudio/utils.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description:工具类 +@Date :2021/08/16 +@Author :xhunmon +@Mail :xhunmon@gmail.com +""" +import configparser +import os +import re +import threading + + +def get_domain(url: str = None): + """ + 获取链接地址的域名 + :param url: + :return: + """ + # http://youtube.com/watch + return re.match(r"(http://|https://).*?\/", url, re.DOTALL).group(0) + + +def get_real_files(folder): + ''' + 获取真实文件,去掉(.xxx)等文隐藏文件 + :param files: + :return: + ''' + files = next(os.walk(folder))[2] + results = [] + for f in files: + name, ext = os.path.splitext(f) + if len(name)> 0 and len(ext): + results.append(f) + return results + + +def format_num(num): + if num < 10: + return '00{}'.format(num) + elif num < 100: + return '0{}'.format(num) + else: + return '{}'.format(num) + + +class Config(object): + """ + 配置文件的单例类 + """ + _instance_lock = threading.Lock() + + def __init__(self): + parent_dir = os.path.dirname(os.path.abspath(__file__)) + conf_path = os.path.join(parent_dir, 'config.ini') + self.conf = configparser.ConfigParser() + self.conf.read(conf_path, encoding="utf-8") + + @classmethod + def instance(cls, *args, **kwargs): + with Config._instance_lock: + if not hasattr(Config, "_instance"): + Config._instance = Config(*args, **kwargs) + return Config._instance + + def get_expired_time(self): + return self.conf.get("common", "expired_time") + + def get_version_name(self): + return self.conf.get("common", "version_name") + + def get_version_code(self): + return self.conf.get("common", "version_code") diff --git a/008-ChatGPT-UI/.gitignore b/008-ChatGPT-UI/.gitignore new file mode 100644 index 0000000..53eb059 --- /dev/null +++ b/008-ChatGPT-UI/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +.idea/ diff --git a/008-ChatGPT-UI/README.md b/008-ChatGPT-UI/README.md new file mode 100644 index 0000000..0fff352 --- /dev/null +++ b/008-ChatGPT-UI/README.md @@ -0,0 +1,9 @@ + +# GPT-UI 2.0 移动☞: [iMedia](https://github.com/xhunmon/iMedia) + +- UI大优化 +- 支持图片生成 + +演示: + +https://user-images.githubusercontent.com/26396755/234748470-6203534e-9845-4a3b-a512-09c315670175.mp4 \ No newline at end of file diff --git a/008-ChatGPT-UI/config.ini b/008-ChatGPT-UI/config.ini new file mode 100644 index 0000000..dc1f1ba --- /dev/null +++ b/008-ChatGPT-UI/config.ini @@ -0,0 +1,10 @@ +[common] +expired_time=2025/12/15 23:59:59 + +title=GPT-UI + +version_name=v1.0.1--github/xhunmon + +version_code=1010 + +email=xhunmon@126.com \ No newline at end of file diff --git a/008-ChatGPT-UI/config.json b/008-ChatGPT-UI/config.json new file mode 100644 index 0000000..061df14 --- /dev/null +++ b/008-ChatGPT-UI/config.json @@ -0,0 +1,10 @@ +{ + "key": "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", + "api_base": "", + "model": "gpt-3.5-turbo", + "stream": true, + "response": true, + "folder": "", + "repeat": true, + "proxy": "socks5://127.0.0.1:7890" +} \ No newline at end of file diff --git a/008-ChatGPT-UI/doc/config.json b/008-ChatGPT-UI/doc/config.json new file mode 100644 index 0000000..76b70cd --- /dev/null +++ b/008-ChatGPT-UI/doc/config.json @@ -0,0 +1,10 @@ +{ + "api_base": "", + "key": "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", + "model": "gpt-3.5-turbo", + "stream": true, + "response": true, + "folder": "/Users/Qincji/Desktop/develop/py/opengpt/gptcli/doc/", + "repeat": true, + "proxy": "socks5://127.0.0.1:7890" +} \ No newline at end of file diff --git a/008-ChatGPT-UI/doc/pyinstaller.sh b/008-ChatGPT-UI/doc/pyinstaller.sh new file mode 100644 index 0000000..6a2cf98 --- /dev/null +++ b/008-ChatGPT-UI/doc/pyinstaller.sh @@ -0,0 +1,7 @@ +#!/bin/bash + + +pyinstaller --windowed --name GPT-UI --add-data "config.ini:." --icon logo.ico main.py gpt.py utils.py + +#if use --onefile, the build file is small, but star very slow. +#pyinstaller --onefile --windowed --name GPT-UI --add-data "config.ini:." --icon logo.ico main.py gpt.py utils.py diff --git a/008-ChatGPT-UI/gpt.py b/008-ChatGPT-UI/gpt.py new file mode 100755 index 0000000..feb6212 --- /dev/null +++ b/008-ChatGPT-UI/gpt.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description:gpt.py +@Date :2023/03/31 +@Author :xhunmon +@Mail :xhunmon@126.com +""" + +import time +from datetime import datetime + +from utils import * + + +class Gpt(object): + func_ui_print = None + + def __init__(self, config: Config): + self.session = [] + self.api_prompt = [] + self.update_config(config) + self.content = "" + self.is_change = False + self.is_finish = True + gpt_t = threading.Thread( + gpt_t.setDaemon(True) + gpt_t.start() + + def update_config(self, config: Config): + self.cfg = config + self.api_key = self.cfg.api_key + self.api_base = self.cfg.api_base + self.api_model = self.cfg.model + self.api_stream = self.cfg.stream + self.api_response = self.cfg.response + self.proxy = self.cfg.proxy + openai.api_key = self.api_key + if self.api_base: + openai.api_base = self.api_base + openai.proxy = self.proxy + + def start(self): + while True: + if self.is_finish: + while not self.is_change: + time.sleep(0.3) + self.print("\nMY:\n{}".format(self.content)) + self.print("\nGPT:\n") + self.is_change = False + self.is_finish = False + self.handle_input(self.content) + time.sleep(1) + + def print(self, content): + Gpt.func_ui_print(content) + + def query_openai_stream(self, data: dict) -> str: + messages = [] + messages.extend(self.api_prompt) + messages.extend(data) + answer = "" + try: + response = openai.ChatCompletion.create( + model=self.api_model, + messages=messages, + stream=True) + for part in response: + finish_reason = part["choices"][0]["finish_reason"] + if "content" in part["choices"][0]["delta"]: + content = part["choices"][0]["delta"]["content"] + answer += content + self.print(content) + elif finish_reason: + pass + + except KeyboardInterrupt: + self.print("Canceled") + except openai.error.OpenAIError as e: + self.print("OpenAIError:{}".format(e)) + answer = "" + return answer + + def content_change(self, content: str): + if not content: + return + if self.content != content: + self.content = content + self.is_change = True + + def handle_input(self, content: str): + if not content: + return + self.is_finish = False + self.session.append({"role": "user", "content": content}) + if self.api_stream: + answer = self.query_openai_stream(self.session) + else: + answer = self.query_openai(self.session) + if not answer: + self.session.pop() + elif self.api_response: + self.session.append({"role": "assistant", "content": answer}) + if answer: + try: + if self.cfg.folder and not os.path.exists(self.cfg.folder): + os.makedirs(self.cfg.folder) + wfile = os.path.join(self.cfg.folder, "gpt.md" if self.cfg.repeat else "gpt_{}.md".format( + datetime.now().strftime("%Y%m%d%H%M:%S"))) + if self.cfg.repeat: + with open(wfile, mode='a', encoding="utf-8") as f: + f.write("MY:\n{}\n".format(content)) + f.write("\nGPT:\n{}\n\n".format(answer)) + f.close() + else: + with open(wfile, mode='w', encoding="utf-8") as f: + f.write("MY:\n{}\n".format(content)) + f.write("\nGPT:{}".format(answer)) + f.close() + except Exception as e: + self.print("Write error: {} ".format(e)) + self.is_finish = True + + def query_openai(self, data: dict) -> str: + messages = [] + messages.extend(self.api_prompt) + messages.extend(data) + try: + response = openai.ChatCompletion.create( + model=self.api_model, + messages=messages + ) + content = response["choices"][0]["message"]["content"] + self.print(content) + return content + except openai.error.OpenAIError as e: + self.print("OpenAI error: {} ".format(e)) + return "" diff --git a/008-ChatGPT-UI/logo.ico b/008-ChatGPT-UI/logo.ico new file mode 100644 index 0000000..3457f2c Binary files /dev/null and b/008-ChatGPT-UI/logo.ico differ diff --git a/008-ChatGPT-UI/main.py b/008-ChatGPT-UI/main.py new file mode 100644 index 0000000..1f4b9d7 --- /dev/null +++ b/008-ChatGPT-UI/main.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: main +@Date :2023年03月31日 +@Author :xhunmon +@Mail :xhunmon@126.com +""" +import sys +import tkinter as tk +from tkinter.filedialog import * + +from gpt import * +from utils import * + + +class EntryWithPlaceholder(tk.Entry): + def __init__(self, master=None, placeholder='', **kwargs): + super().__init__(master, **kwargs) + self.placeholder = placeholder + self.placeholder_color = 'grey' + self.default_fg_color = self['fg'] + self.bind('', self.on_focus_in) + self.bind('', self.on_focus_out) + self.put_placeholder() + + def put_placeholder(self): + self.insert(0, self.placeholder) + self['fg'] = self.placeholder_color + + def remove_placeholder(self): + cur_value = self.get() + if cur_value == self.placeholder: + self.delete(0, tk.END) + self['fg'] = self.default_fg_color + + def on_focus_in(self, event): + self.remove_placeholder() + + def on_focus_out(self, event): + if not self.get(): + self.put_placeholder() + + +class Application(tk.Frame): + def __init__(self, config: Config, master=None): + super().__init__(master) + self.cfg = config + self.gpt = None + self.repeat = False + self.master = master + self.master.title(ConfigIni.instance().get_title()) + self.pack() + self.create_widgets() + + def create_config(self): + row = tk.Frame(self) + row.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5) + button = tk.Button(row, text="Config", width='7', command=self.click_config) + button.pack(side=tk.LEFT, padx=5, pady=5) + self.configEntry = EntryWithPlaceholder(row, placeholder=self.cfg.config_path, width=45) + self.configEntry.pack(side=tk.LEFT, padx=5, pady=5) + button = tk.Button(row, text="Create", width='7', command=self.click_create) + button.pack(side=tk.LEFT, padx=5, pady=5) + + def create_folder(self): + row = tk.Frame(self) + row.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5) + button = tk.Button(row, text="Folder", width='7', command=self.click_folder) + button.pack(side=tk.LEFT, padx=5, pady=5) + self.folderEntry = EntryWithPlaceholder(row, + placeholder=self.cfg.folder if self.cfg.folder else f'{Config.pre_tips} chat output directory, default current', + width=50) + self.folderEntry.pack(side=tk.LEFT, padx=5, pady=5) + + def create_key(self): + row = tk.Frame(self) + row.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5) + label = tk.Label(row, text=f"Key: ", width='7') + label.pack(side=tk.LEFT) + self.keyEntry = EntryWithPlaceholder(row, + placeholder=self.cfg.api_key if self.cfg.api_key else f'{Config.pre_tips} input key id', + width=50) + self.keyEntry.pack(side=tk.LEFT, padx=5, pady=5) + + def create_model(self): + row = tk.Frame(self) + row.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5) + label = tk.Label(row, text=f"Model: ", width='7') + label.pack(side=tk.LEFT) + self.modelEntry = EntryWithPlaceholder(row, + placeholder=self.cfg.model if self.cfg.model else f'{Config.pre_tips} default gpt-3.5-turbo, or: gpt-4/gpt-4-32k', + width=50) + self.modelEntry.pack(side=tk.LEFT, padx=5, pady=5) + + def create_proxy(self): + row = tk.Frame(self) + row.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5) + label = tk.Label(row, text=f"Proxy: ", width='7') + label.pack(side=tk.LEFT) + self.proxyEntry = EntryWithPlaceholder(row, + placeholder=self.cfg.proxy if self.cfg.proxy else f'{Config.pre_tips} default empty, or http/https/socks4a/socks5', + width=50) + self.proxyEntry.pack(side=tk.LEFT, padx=5, pady=5) + + def create_send(self): + row = tk.Frame(self) + row.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5) + self.sendEntry = EntryWithPlaceholder(row, placeholder=f'{Config.pre_tips} say something, then click send.', + width=55) + self.sendEntry.pack(side=tk.LEFT, padx=5, pady=5) + self.sendEntry.bind("", self.on_return_key) + button = tk.Button(row, text="Send", width='7', command=self.click_send) + button.pack(side=tk.LEFT, padx=5, pady=5) + + def create_widgets(self): + self.create_config() + self.create_folder() + self.create_key() + self.create_model() + self.create_proxy() + self.create_send() + # bottom text + text_frame = tk.Frame(self) + text_frame.pack(side=tk.TOP, fill=tk.BOTH, expand=True, padx=5, pady=5) + self.text = tk.Text(text_frame, wrap=tk.WORD, undo=True, font=("Helvetica", 12)) + self.text.pack(side=tk.LEFT, fill=tk.BOTH, expand=True) + scroll_bar = tk.Scrollbar(text_frame, orient=tk.VERTICAL, command=self.text.yview) + scroll_bar.pack(side=tk.RIGHT, fill=tk.Y) + self.text.config(yscrollcommand=scroll_bar.set) + + # email text + email_button = tk.Button(self, text=ConfigIni.instance().get_email()) + email_button.pack(side=tk.LEFT, padx=5, pady=5) + + # version text + version_button = tk.Button(self, text=ConfigIni.instance().get_version_name()) + version_button.pack(side=tk.RIGHT, padx=5, pady=5) + + # clear text + clear_button = tk.Button(self, text="clear", width=10, command=self.clear) + clear_button.pack(side=tk.RIGHT, padx=5, pady=5) + + # copy text + copy_button = tk.Button(self, text="copy", width=10, command=self.copy) + copy_button.pack(side=tk.RIGHT, padx=5, pady=5) + + Gpt.func_ui_print = self.func_ui_print + + def refresh(self): + # self.set_entry(self.configEntry, self.cfg.default) + self.set_entry(self.folderEntry, self.cfg.folder) + self.set_entry(self.keyEntry, self.cfg.api_key) + self.set_entry(self.modelEntry, self.cfg.model) + self.set_entry(self.proxyEntry, self.cfg.proxy) + + def func_ui_print(self, txt): + self.show_text(txt) + + def click_config(self): + path = askopenfilename() + self.set_entry(self.configEntry, path) + if self.cfg.update(path): + self.refresh() + else: + self.show_text("update fail !") + + def click_create(self): + self.cfg.click_create() + self.show_text("create file :{} ".format(self.cfg.config_path)) + + def click_folder(self): + path = askdirectory() + self.set_entry(self.folderEntry, path) + + def set_entry(self, entry: tk.Entry, content): + entry.delete(0, tk.END) + entry.insert(0, content) + + def on_return_key(self, event): + self.click_send() + + def click_send(self): + # config = self.configEntry.get() + self.cfg.update_by_content(self.keyEntry.get(), self.modelEntry.get(), self.folderEntry.get(), + self.proxyEntry.get()) + content: str = self.sendEntry.get() + # self.show_text("me: {}\n".format(content)) + if not self.gpt: + self.gpt: Gpt = Gpt(self.cfg) + else: + self.gpt.update_config(self.cfg) + self.gpt.content_change(content) + + def show_text(self, content): + self.text.insert(tk.END, "{}".format(content)) + self.text.yview_moveto(1.0) # auto scroll to new + + def clear(self): + self.text.delete("1.0", "end") + + def copy(self): + self.master.clipboard_clear() + self.master.clipboard_append(self.text.get("1.0", tk.END)) + + +if __name__ == "__main__": + root = tk.Tk() + folder = os.path.dirname(os.path.realpath(sys.argv[0])) + app = Application(Config(folder), master=root) + app.mainloop() diff --git a/008-ChatGPT-UI/requirements.txt b/008-ChatGPT-UI/requirements.txt new file mode 100644 index 0000000..91d7d5c --- /dev/null +++ b/008-ChatGPT-UI/requirements.txt @@ -0,0 +1,3 @@ +openai +requests[socks] +tkinter \ No newline at end of file diff --git a/008-ChatGPT-UI/utils.py b/008-ChatGPT-UI/utils.py new file mode 100644 index 0000000..7c93c12 --- /dev/null +++ b/008-ChatGPT-UI/utils.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description: tool +@Date :2023年03月31日 +@Author :xhunmon +@Mail :xhunmon@126.com +""" +import configparser +import json +import os +import platform +import re +import threading + +import openai + + +def get_domain(url: str = None): + # http://youtube.com/watch + return re.match(r"(http://|https://).*?\/", url, re.DOTALL).group(0) + + +class ConfigIni(object): + _instance_lock = threading.Lock() + + def __init__(self): + parent_dir = os.path.dirname(os.path.abspath(__file__)) + conf_path = os.path.join(parent_dir, 'config.ini') + self.conf = configparser.ConfigParser() + self.conf.read(conf_path, encoding="utf-8") + + @classmethod + def instance(cls, *args, **kwargs): + with ConfigIni._instance_lock: + if not hasattr(ConfigIni, "_instance"): + ConfigIni._instance = ConfigIni(*args, **kwargs) + return ConfigIni._instance + + def get_expired_time(self): + return self.conf.get("common", "expired_time") + + def get_version_name(self): + return self.conf.get("common", "version_name") + + def get_version_code(self): + return self.conf.get("common", "version_code") + + def get_title(self): + return self.conf.get("common", "title") + + def get_email(self): + return self.conf.get("common", "email") + + +class Config: + sep = "" + pre_tips = "Tips:" + # baseDir = os.path.dirname(os.path.realpath(sys.argv[0])) + base_dir = '' + md_sep = '\n\n' + '-' * 10 + '\n' + encodings = ["utf8", "gbk"] + + api_key = "" + api_base = "" + model = "" + prompt = [] + stream = True + response = False + proxy = "" + folder = "" + config_path = "" + repeat = True + + def __init__(self, dir: str) -> None: + self.base_dir = dir + if platform.system() == 'Darwin': # MacOS:use pyinstaller pack issue. + if '/Contents/MacOS' in dir: # ./GPT-UI.app/Contents/MacOS/ --> ./ + app_path = dir.rsplit('/Contents/MacOS')[0] + self.base_dir = app_path[:app_path.rindex('/')] + self.config_path = os.path.join(self.base_dir, "config.json") + self.cfg = {} + self.load(self.config_path) + + def load(self, file): + if not os.path.exists(file): + return + with open(file, "r") as f: + self.cfg = json.load(f) + c = self.cfg + self.api_key = c.get("api_key", c.get("key", openai.api_key)) # compatible with history key + self.api_base = c.get("api_base", openai.api_base) + self.model = c.get("model", "gpt-3.5-turbo") + self.stream = c.get("stream", True) + self.response = c.get("response", False) + self.proxy = c.get("proxy", "") + self.folder = c.get("folder", self.base_dir) + self.repeat = c.get("repeat", True) + + def get(self, key, default=None): + return self.cfg.get(key, default) + + def click_create(self): + results = { + "key": "", + "api_base": "", + "model": "gpt-3.5-turbo", + "stream": True, + "response": True, + "folder": "", + "repeat": False, + "proxy": "", + "prompt": [] + } + self.write_json(results, self.config_path) + + def write_json(self, content, file_path): + path, file_name = os.path.split(file_path) + if path and not os.path.exists(path): + os.makedirs(path) + with open(file_path, 'w') as f: + json.dump(content, f, ensure_ascii=False) + f.close() + + def update(self, path: str): + if not path.endswith(".json"): + return False + if path and not os.path.exists(path): + return False + self.load(path) + return True + + def update_by_content(self, key: str = None, model: str = None, folder: str = None, proxy: str = None): + if key and len(key.strip())> 0 and not key.startswith(Config.pre_tips): + self.api_key = key + else: + self.api_key = '' + if model and len(model.strip())> 0 and not model.startswith(Config.pre_tips): + self.model = model + else: + self.model = 'gpt-3.5-turbo' + if folder and len(folder.strip())> 0 and not folder.startswith(Config.pre_tips): + self.folder = folder + else: + self.folder = self.base_dir + if proxy.startswith(Config.pre_tips): + self.proxy = None + else: + self.proxy = proxy if len(proxy.strip())> 0 else None diff --git a/009-Translate/README.md b/009-Translate/README.md new file mode 100644 index 0000000..470fe28 --- /dev/null +++ b/009-Translate/README.md @@ -0,0 +1,24 @@ +# 多平台免费翻译神器 + +## 1. 输入框翻译 + +直接在输入框输入内容,选择目标语言,翻译平台即可。 + + +## 2.文件翻译 + +- 1.支持txt翻译,但是文本内容不宜过大 + +- 2.支持html翻译 + +- 3.定制版srt字幕文件翻译,自定义修改[load_srt.py](load_srt.py) + +- 4.其他文本文件也是可以的,但是肯定有bug + + +## 3.打包应用程序 + +可以自行打包 exe和Mac平台的app。打包脚本参考[doc/pyinstall.sh](doc/pyinstaller.sh),注意window平台的路径反斜杠更改。 + + +> 主要引入:translators \ No newline at end of file diff --git a/009-Translate/asset/ch.ini b/009-Translate/asset/ch.ini new file mode 100644 index 0000000..eae8a9a --- /dev/null +++ b/009-Translate/asset/ch.ini @@ -0,0 +1,121 @@ +[Main] +Title = Super86翻译 +Description = 免费多平台多语言翻译... +EnableProxy = 使用代理 +Run = 翻译 +Clear = 清空 +Copy = 复制 +File = 选择文件 +Settings = 设置 +Exit = 退出 +Version = 版本 +Business = 联系我 +Email = 邮箱: +RedBook = 小红书: + +[Settings] +Title = 设置 +Proxy = 代理 +ProxyEnable = 启用 +ProxyDesc = 支持http/https/socks5 +Theme = 主题 +ThemeDesc = 留空使用默认主题 +FullTranslate = 开启全翻译(未验证) +Advanced = 开启高级功能 +Restart = 立刻重启,使所有修改生效 +Ok = 确认修改 +Cancel = 取消 +Reset = 恢复默认 + +[Loading] +Content = 加载中... +Cancel = 关闭 + +[Language] +LanguageCH = 简体中文 +LanguageCHINESE_CHT = 繁体中文 +LanguageEN = 英文 +LanguageJAPAN = 日文 +LanguageKOREAN = 韩文 +LanguageAR = 阿拉伯文 +LanguageFRENCH = 法文 +LanguageGERMAN = 德文 +LanguageRU = 俄罗斯文 +LanguageES = 西班牙文 +LanguagePT = 葡萄牙文 +LanguageIT = 意大利文 +LanguageAF = 南非荷兰文 +LanguageAZ = 阿塞拜疆文 +LanguageBS = 波斯尼亚文 +LanguageCS = 捷克文 +LanguageCY = 威尔士文 +LanguageDA = 丹麦文 +LanguageDE = 德文 +LanguageET = 爱沙尼亚文 +LanguageFR = 法文 +LanguageGA = 爱尔兰文 +LanguageHR = 克罗地亚文 +LanguageHU = 匈牙利文 +LanguageID = 印尼文 +LanguageIS = 冰岛文 +LanguageKU = 库尔德文 +LanguageLA = 拉丁文 +LanguageLT = 立陶宛文 +LanguageLV = 拉脱维亚文 +LanguageMI = 毛利文 +LanguageMS = 马来文 +LanguageMT = 马耳他文 +LanguageNL = 荷兰文 +LanguageNO = 挪威文 +LanguageOC = 欧西坦文 +LanguagePI = 巴利文 +LanguagePL = 波兰文 +LanguageRO = 罗马尼亚文 +LanguageRS_LATIN = 塞尔维亚文(latin) +LanguageSK = 斯洛伐克文 +LanguageSL = 斯洛文尼亚文 +LanguageSQ = 阿尔巴尼亚文 +LanguageSV = 瑞典文 +LanguageSW = 西瓦希里文 +LanguageTL = 塔加洛文 +LanguageTR = 土耳其文 +LanguageUZ = 乌兹别克文 +LanguageVI = 越南文 +LanguageLATIN = 拉丁文 +LanguageFA = 波斯文 +LanguageUG = 维吾尔文 +LanguageUR = 乌尔都文 +LanguageRS_CYRILLIC = 塞尔维亚文(cyrillic) +LanguageBE = 白俄罗斯文 +LanguageBG = 保加利亚文 +LanguageUK = 乌克兰文 +LanguageMN = 蒙古文 +LanguageABQ = 阿巴扎文 +LanguageADY = 阿迪赫文 +LanguageKBD = 卡巴尔达文 +LanguageAVA = 阿瓦尔文 +LanguageDAR = 达尔瓦文 +LanguageINH = 因古什文 +LanguageCHE = 车臣文 +LanguageLBE = 拉克文 +LanguageLEZ = 莱兹甘文 +LanguageTAB = 塔巴萨兰文 +LanguageCYRILLIC = 西里尔文 +LanguageHI = 印地文 +LanguageMR = 马拉地文 +LanguageNE = 尼泊尔文 +LanguageBH = 比尔哈文 +LanguageMAI = 迈蒂利文 +LanguageANG = 昂加文 +LanguageBHO = 孟加拉文 +LanguageMAH = 摩揭陀文 +LanguageSCK = 那格浦尔文 +LanguageNEW = 尼瓦尔文 +LanguageGOM = 保加利亚文 +LanguageSA = 沙特阿拉伯文 +LanguageBGC = 哈里亚纳文 +LanguageDEVANAGARI = 德瓦那加里文 +LanguageTA = 泰米尔文 +LanguageKN = 卡纳达文 +LanguageTE = 泰卢固文 +LanguageKA = 卡纳达文 \ No newline at end of file diff --git a/009-Translate/asset/config.ini b/009-Translate/asset/config.ini new file mode 100644 index 0000000..c97cf47 --- /dev/null +++ b/009-Translate/asset/config.ini @@ -0,0 +1,7 @@ +[Config] +Loading = R0lGODlhoAAYAKEAALy+vOTm5P7+/gAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQJCQACACwAAAAAoAAYAAAC55SPqcvtD6OctNqLs968+w+G4kiW5omm6sq27gvHMgzU9u3cOpDvdu/jNYI1oM+4Q+pygaazKWQAns/oYkqFMrMBqwKb9SbAVDGCXN2G1WV2esjtup3mA5o+18K5dcNdLxXXJ/Ant7d22Jb4FsiXZ9iIGKk4yXgl+DhYqIm5iOcJeOkICikqaUqJavnVWfnpGso6Clsqe2qbirs61qr66hvLOwtcK3xrnIu8e9ar++sczDwMXSx9bJ2MvWzXrPzsHW1HpIQzNG4eRP6DfsSe5L40Iz9PX29/j5+vv8/f7/8PMKDAgf4KAAAh+QQJCQAHACwAAAAAoAAYAIKsqqzU1tTk4uS8urzc3tzk5uS8vrz+/v4D/ni63P4wykmrvTjrzbv/YCiOZGliQKqurHq+cEwBRG3fOAHIfB/TAkJwSBQGd76kEgSsDZ1QIXJJrVpowoF2y7VNF4aweCwZmw3lszitRkfaYbZafnY0B4G8Pj8Q6hwGBYKDgm4QgYSDhg+IiQWLgI6FZZKPlJKQDY2JmVgEeHt6AENfCpuEmQynipeOqWCVr6axrZy1qHZ+oKEBfUeRmLesb7TEwcauwpPItg1YArsGe301pQery4fF2sfcycy44MPezQx3vHmjv5rbjO3A3+Th8uPu3fbxC567odQC1tgsicuGr1zBeQfrwTO4EKGCc+j8AXzH7l5DhRXzXSS4c1EgPY4HIOqR1stLR1nXKKpSCctiRoYvHcbE+GwAAC03u1QDFCaAtJ4D0vj0+RPlT6JEjQ7tuebN0qJKiyYt83SqsyBR/GD1Y82K168htfoZ++QP2LNfn9nAytZJV7RwebSYyyKu3bt48+rdy7ev378NEgAAIfkECQkABQAsAAAAAKAAGACCVFZUtLK05ObkvL68xMLE/v7+AAAAAAAAA/5Yutz+MMpJq7046827/2AojmRpYkCqrqx6vnBMAcRA1LeN74Ds/zGabYgjDnvApBIkLDqNyKV0amkGrtjswBZdDL+1gSRM3hIk5vQQXf6O1WQ0OM2Gbx3CQUC/3ev3NV0KBAKFhoVnEQOHh4kQi4yIaJGSipQCjg+QkZkOm4ydBVZbpKSAA4IFn42TlKEMhK5jl69etLOyEbGceGF+pX1HDruguLyWuY+3usvKyZrNC6PAwYHD0dfP2ccQxKzM2g3ehrWD2KK+v6YBOKmr5MbF4NwP45Xd57D5C/aYvTbqSp1K1a9cgYLxvuELp48hv33mwuUJaEqHO4gHMSKcJ2BvIb1tHeudG8UO2ECQCkU6jPhRnMaXKzNKTJdFC5dhN3LqZKNzp6KePh8BzclzaFGgR3v+C0ONlDUqUKMu1cG0yE2pWKM2AfPkadavS1qIZQG2rNmzaNOqXcu2rdsGCQAAIfkECQkACgAsAAAAAKAAGACDVFZUpKKk1NbUvLq85OLkxMLErKqs3N7cvL685Obk/v7+AAAAAAAAAAAAAAAAAAAABP5QyUmrvTjrzbv/YCiOZGmeaKqubOuCQCzPtCwZeK7v+ev/KkABURgWicYk4HZoOp/QgwFIrYaEgax2ux0sFYYDQUweE8zkqXXNvgAQgYF8TpcHEN/wuEzmE9RtgWxYdYUDd3lNBIZzToATRAiRkxpDk5YFGpKYmwianJQZoJial50Wb3GMc4hMYwMCsbKxA2kWCAm5urmZGbi7ur0Yv8AJwhfEwMe3xbyazcaoBaqrh3iuB7CzsrVijxLJu8sV4cGV0OMUBejPzekT6+6ocNV212BOsAWy+wLdUhbiFXsnQaCydgMRHhTFzldDCoTqtcL3ahs3AWO+KSjnjKE8j9sJQS7EYFDcuY8Q6clBMIClS3uJxGiz2O1PwIcXSpoTaZLnTpI4b6KcgMWAJEMsJ+rJZpGWI2ZDhYYEGrWCzo5Up+YMqiDV0ZZgWcJk0mRmv301NV6N5hPr1qrquMaFC49rREZJ7y2due2fWrl16RYEPFiwgrUED9tV+fLlWHxlBxgwZMtqkcuYP2HO7Gsz52GeL2sOPdqzNGpIrSXa0ydKE42CYr9IxaV2Fr2KWvvxJrv3DyGSggsfjhsNnz4ZfStvUaM5jRs5AvDYIX259evYs2vfzr279+8iIgAAIfkECQkACgAsAAAAAKAAGACDVFZUrKqszMrMvL683N7c5ObklJaUtLK0xMLE5OLk/v7+AAAAAAAAAAAAAAAAAAAABP5QyUmrvTjrzbv/YCiOZGmeaKqubOuCQSzPtCwBeK7v+ev/qgBhSCwaCYEbYoBYNpnOKABIrYaEhqx2u00kFQCm2DkWD6bWtPqCFbjfcLcBqSyT7wj0eq8OJAxxgQIGXjdiBwGIiokBTnoTZktmGpKVA0wal5ZimZuSlJqhmBmilhZtgnBzXwBOAZewsAdijxIIBbi5uAiZurq8pL65wBgDwru9x8QXxsqnBICpb6t1CLOxsrQWzcLL28cF3hW3zhnk3cno5uDiqNKDdGBir9iXs0u1Cue+4hT7v+n4BQS4rlwxds+iCUDghuFCOfFaMblW794ZC/+GUUJYUB2GjMrIOgoUSZCCH4XSqMlbQhFbIyb5uI38yJGmwQsgw228ibHmBHcpI7qqZ89RT57jfB71iFNpUqT+nAJNpTIMS6IDXub5BnVCzn5enUbtaktsWKSoHAqq6kqSyyf5vu5kunRmU7L6zJZFC+0dRFaHGDFSZHRck8MLm3Q6zPDwYsSOSTFurFgy48RgJUCBXNlkX79V7Ry2c5GP6SpYuKjOEpH0nTH5TsteISTBkdtCXZOOPbu3iRrAadzgQVyH7+PIkytfzry58+fQRUQAACH5BAkJAAwALAAAAACgABgAg1RWVKSipMzOzNze3Ly6vNTW1OTm5MTCxKyqrOTi5Ly+vNza3P7+/gAAAAAAAAAAAAT+kMlJq7046827/2AojmRpnmiqrmzrvhUgz3Q9S0iu77wO/8AT4KA4EI3FoxKAGzif0OgAEaz+eljqZBjoer9fApOBGCTM6LM6rbW6V2VptM0AKAKEvH6fDyjGZWdpg2t0b4clZQKLjI0JdFx8kgR+gE4Jk3pPhgxFCp6gGkSgowcan6WoCqepoRmtpRiKC7S1tAJTFHZ4mXqVTWcEAgUFw8YEaJwKBszNzKYZy87N0BjS0wbVF9fT2hbczt4TCAkCtrYCj7p3vb5/TU4ExPPzyGbK2M+n+dmi/OIUDvzblw8gmQHmFhQYoJAhLkjs2lF6dzAYsWH0kCVYwElgQX/+H6MNFBkSg0dsBmfVWngr15YDvNr9qjhA2DyMAuypqwCOGkiUP7sFDTfU54VZLGkVWPBwHS8FBKBKjTrRkhl59OoJ6jjSZNcLJ4W++mohLNGjCFcyvLVTwi6JVeHVLJa1AIEFZ/CVBEu2glmjXveW7YujnFKGC4u5dBtxquO4NLFepHs372DBfglP+KtvLOaAmlUebgkJJtyZcTBhJMZ0QeXFE3p2DgzUc23aYnGftaCoke+2dRpTfYwaTTu8sCUYWc7coIQkzY2wii49GvXq1q6nREMomdPTFOM82Xhu4z1E6BNl4aELJpj3XcITwrsxQX0nnNLrb2Hnk///AMoplwZe9CGnRn77JYiCDQzWgMMOAegQIQ8RKmjhhRhmqOGGHHbo4YcZRAAAIfkECQkADQAsAAAAAKAAGACDVFZUrKqs1NbUvL685ObkxMbE3N7clJaUtLK0xMLE7O7szMrM5OLk/v7+AAAAAAAABP6wyUmrvTjrzbv/YCiOZGmeaKqubOu+VSDPdD1LQK7vvA7/wFPAQCwaj4YALjFIMJ3NpxQQrP4E2KxWSxkevuBwmKFsAJroZxo9oFrfLIFiTq/PBV3DYcHv+/kHSUtraoUJbnCJJ3J8CY2PCngTAQx7f5cHZDhoCAGdn54BT4gTbExsGqeqA00arKtorrCnqa+2rRdyCQy8vbwFkXmWBQvExsULgWUATwGsz88IaKQSCQTX2NcJrtnZ2xkD3djfGOHiBOQX5uLpFIy9BrzxC8GTepeYgmZP0tDR0xbMKbg2EB23ggUNZrCGcFwqghAVliPQUBuGd/HkEWAATJIESv57iOEDpO8ME2f+WEljQq2BtXPtKrzMNjAmhXXYanKD+bCbzlwKdmns1VHYSD/KBiXol3JlGwsvBypgMNVmKYhTLS7EykArhqgUqTKwKkFgWK8VMG5kkLGovWFHk+5r4uwUNFFNWq6bmpWsS4Jd++4MKxgc4LN+owbuavXdULb0PDYAeekYMbkmBzD1h2AUVMCL/ZoTy1d0WNJje4oVa3ojX6qNFSzISMDARgJuP94TORJzs5Ss8B4KeA21xAuKXadeuFi56deFvx5mfVE2W1/z6umGi0zk5ZKcgA8QxfLza+qGCXc9Tlw9Wqjrxb6vIFA++wlyChjTv1/75EpHFXQgQAG+0YVAJ6F84plM0EDBRCqrSCGLLQ7KAkUUDy4UYRTV2eGhZF4g04d3JC1DiBOFAKTIiiRs4WIWwogh4xclpagGIS2xqGMLQ1xnRG1AFmGijVGskeOOSKJgw5I14NDDkzskKeWUVFZp5ZVYZqnllhlEAAAh+QQJCQAMACwAAAAAoAAYAINUVlSkoqTMzszc3ty8urzU1tTk5uTEwsSsqqzk4uS8vrzc2tz+/v4AAAAAAAAAAAAE/pDJSau9OOvNu/9gKI5kaZ5oqq5s674pIM90PUtIru+8Dv/AE+CgOBCNxaMSgBs4n9DoABGs/npY6mQY6Hq/XwKTgRgkzOdEem3WWt+rsjTqZgAUAYJ+z9cHFGNlZ2ZOg4ZOdXCKE0UKjY8YZQKTlJUJdVx9mgR/gYWbe4WJDI9EkBmmqY4HGquuja2qpxgKBra3tqwXkgu9vr0CUxR3eaB7nU1nBAIFzc4FBISjtbi3urTV1q3Zudvc1xcH3AbgFLy/vgKXw3jGx4BNTgTNzPXQT6Pi397Z5RX6/TQArOaPArWAuxII6FVgQIEFD4NhaueOEzwyhOY9cxbtzLRx/gUnDMQVUsJBgvxQogIZacDCXwOACdtyoJg7ZBiV2StQr+NMCiO1rdw3FCGGoN0ynCTZcmHDhhBdrttCkYACq1ivWvRkRuNGaAkWTDXIsqjKo2XRElVrtAICheigSmRnc9NVnHIGzGO2kcACRBaQkhOYNlzhwIcrLBVq4RzUdD/t1NxztTIfvBmf2fPr0cLipGzPGl47ui1i0uZc9nIYledYO1X7WMbclW+zBQs5R5YguCSD3oRR/0sM1Ijx400rKY9MjDLWPpiVGRO7m9Tx67GuG8+u3XeS7izeEkqDps2wybKzbo1XCJ2vNKMWyf+QJUcAH1TB6PdyUdB4NWKpNBFWZ/MVCMQdjiSo4IL9FfJEgGJRB5iBFLpgw4U14IDFfTpwmEOFIIYo4ogklmjiiShSGAEAIfkECQkADQAsAAAAAKAAGACDVFZUrKqs1NbUvL685ObkxMbE3N7clJaUtLK0xMLE7O7szMrM5OLk/v7+AAAAAAAABP6wyUmrvTjrzbv/YCiOZGmeaKqubOu+aSDPdD1LQK7vvA7/wFPAQCwaj4YALjFIMJ3NpxQQrP4E2KxWSxkevuBwmKFsAJroZxo9oFrfLIFiTq/PBV3DYcHv+/kHSUtraoUJbnCJFWxMbBhyfAmRkwp4EwEMe3+bB2Q4aAgBoaOiAU+IE4wDjhmNrqsJGrCzaLKvrBgDBLu8u7EXcgkMw8TDBZV5mgULy83MC4FlAE8Bq9bWCGioEgm9vb+53rzgF7riBOQW5uLpFd0Ku/C+jwoLxAbD+AvIl3qbnILMPMl2DZs2dfESopNFQJ68ha0aKoSIoZvEi+0orOMFL2MDSP4M8OUjwOCYJQmY9iz7ByjgGSbVCq7KxmRbA4vsNODkSLGcuI4Mz3nkllABg3nAFAgbScxkMpZ+og1KQFAmzTYWLMIzanRoA3Nbj/bMWlSsV60NGXQNmtbo2AkgDZAMaYwfSn/PWEoV2KRao2ummthcx/Xo2XhH3XolrNZwULeKdSJurBTDPntMQ+472SDlH2cr974cULUgglNk0yZmsHgXZbWtjb4+TFL22gxgG5P0CElkSJIEnPZTyXKZaGoyVwU+hLC2btpuG59d7Tz267cULF7nXY/uXH12O+Nd+Yy8aFDJB5iqSbaw9Me6sadC7FY+N7HxFzv5C4WepAIAAnjIjHAoZQLVMwcQIM1ApZCCwFU2/RVFLa28IoUts0ChHxRRMBGHHSCG50Ve5QlQgInnubKfKk7YpMiLH2whYxbJiGHjFy5JYY2OargI448sDEGXEQQg4RIjOhLiI5BMCmHDkzTg0MOUOzRp5ZVYZqnlllx26SWTEQAAIfkECQkADAAsAAAAAKAAGACDVFZUpKKkzM7M3N7cvLq81NbU5ObkxMLErKqs5OLkvL683Nrc/v7+AAAAAAAAAAAABP6QyUmrvTjrzbv/YCiOZGmeaKqubOu+cAfMdG3TEqLvfL/HwCAJcFAcikcjcgnIDZ7QqHSAEFpfvmx1Qgx4v2AwoclADBLnNHqt3l7fKfNU6mYAFAGCfs/XBxRkZmhqhGx1cCZGCoqMGkWMjwcYZgKVlpcJdV19nAR/gU8JnXtQhwyQi4+OqaxGGq2RCq8GtLW0khkKtra4FpQLwMHAAlQUd3mje59OaAQCBQXP0gRpprq7t7PYBr0X19jdFgfb3NrgkwMCwsICmcZ4ycqATk8E0Pf31GfW5OEV37v8URi3TeAEgLwc9ZuUQN2CAgMeRiSmCV48T/PKpLEnDdozav4JFpgieC4DyYDmUJpcuLIgOocRIT5sp+kAsnjLNDbDh4/AAjT8XLYsieFkwlwsiyat8KsAsIjDinGxqIBA1atWMYI644xnNAIhpQ5cKo5sBaO1DEpAm22oSl8NgUF0CpHiu5vJcsoZYO/eM2g+gVpAmFahUKWHvZkdm5jCr3XD3E1FhrWyVmZ8o+H7+FPsBLbl3B5FTPQCaLUMTr+UOHdANM+bLuoN1dXjAnWBPUsg3Jb0W9OLPx8ZTvwV8eMvLymXLOGYHstYZ4eM13nk8eK5rg83rh31FQRswoetiHfU7Cgh1yUYZAqR+w9adAT4MTmMfS8ZBan5uX79gmrvBS4YBBGLFGjggfmFckZnITUIoIAQunDDhDbkwMN88mkR4YYcdujhhyCGKOKIKkQAACH5BAkJAA0ALAAAAACgABgAg1RWVKyqrNTW1Ly+vOTm5MTGxNze3JSWlLSytMTCxOzu7MzKzOTi5P7+/gAAAAAAAAT+sMlJq7046827/2AojmRpnmiqrmzrvnAXzHRt0xKg73y/x8AgKWAoGo9IQyCXGCSaTyd0ChBaX4KsdrulEA/gsFjMWDYAzjRUnR5Ur3CVQEGv2+kCr+Gw6Pv/fQdKTGxrhglvcShtTW0ajZADThhzfQmWmAp5EwEMfICgB2U5aQgBpqinAVCJE4ySjY+ws5MZtJEaAwS7vLsJub29vxdzCQzHyMcFmnqfCwV90NELgmYAUAGS2toIaa0SCcG8wxi64gTkF+bi6RbhCrvwvsDy8uiUCgvHBvvHC8yc9kwDFWjUmVLbtnVr8q2BuXrzbBGAGBHDu3jjgAWD165CuI3+94gpMIbMAAEGBv5tktDJGcFAg85ga6PQm7tzIS2K46ixF88MH+EpYFBRXTwGQ4tSqIQymTKALAVKI1igGqEE3RJKWujm5sSJSBl0pPAQrFKPGJPmNHo06dgJxsy6xUfSpF0Gy1Y2+DLwmV+Y1tJk0zpglZOG64bOBXrU7FsJicOu9To07MieipG+/aePqNO8Xjy9/GtVppOsWhGwonwM7GOHuyxrpncs8+uHksU+OhpWt0h9/OyeBB2Qz9S/fkpfczJY6yqG7jxnnozWbNjXcZNe331y+u3YSYe+Zdp6HwGVzfpOg6YcIWHDiCzoyrxdIli13+8TpU72SSMpAzx9EgUj4ylQwIEIQnMgVHuJ9sdxgF11SiqpRNHQGgA2IeAsU+QSSRSvXTHHHSTqxReECgpQVUxoHKKGf4cpImMJXNSoRTNj5AgGi4a8wmFDMwbZQifBHUGAXUUcGViPIBoCpJBQonDDlDbk4MOVPESp5ZZcdunll2CGKaYKEQAAIfkECQkADAAsAAAAAKAAGACDVFZUpKKkzM7M3N7cvLq81NbU5ObkxMLErKqs5OLkvL683Nrc/v7+AAAAAAAAAAAABP6QyUmrvTjrzbv/YCiOZGmeaKqubOu+cAzMdG3TEqLvfL/HwCAJcFAcikcjcgnIDZ7QqHSAEFpfvmx1Qgx4v2AwoclADBLnNHqt3l7fKfNU6mYAFAGCfs/XBxRkZmxsaml1cBJGCoqMGkWMjwcai5GUChhmApqbmwVUFF19ogR/gU8Jo3tQhwyQlpcZlZCTBrW2tZIZCre3uRi7vLiYAwILxsfGAgl1d3mpe6VOaAQCBQXV1wUEhhbAwb4X3rzgFgfBwrrnBuQV5ufsTsXIxwKfXHjP0IBOTwTW//+2nWElrhetdwe/OVIHb0JBWw0RJJC3wFPFBfWYHXCWL1qZNP7+sInclmABK3cKYzFciFBlSwwoxw0rZrHiAIzLQOHLR2rfx2kArRUTaI/CQ3QwV6Z7eSGmQZcpLWQ6VhNjUTs7CSjQynVrT1NnqGX7J4DAmpNKkzItl7ZpW7ZrJ0ikedOmVY0cR231KGeAv6DWCCxAQ/BtO8NGEU9wCpFl1ApTjdW8lvMex62Y+fAFOXaswMqJ41JgjNSt6MWKJZBeN3OexYw68/LJvDkstqCCCcN9vFtmrCPAg08KTnw4ceAzOSkHbWfjnsx9NpfMN/hqouPIdWE/gmiFxDMLCpW82kxU5r0++4IvOa8k8+7wP2jxETuMfS/pxQ92n8C99fgAsipAxCIEFmhgfmmAd4Z71f0X4IMn3CChDTloEYAWEGao4YYcdujhhyB2GAEAIfkECQkADQAsAAAAAKAAGACDVFZUrKqs1NbUvL685ObkxMbE3N7clJaUtLK0xMLE7O7szMrM5OLk/v7+AAAAAAAABP6wyUmrvTjrzbv/YCiOZGmeaKqubOu+cBzMdG3TEqDvfL/HwCApYCgaj0hDIJcYJJpPJ3QKEFpfgqx2u6UQD+CwWMxYNgDONFSdHlSvcJVAQa/b6QKv4bDo+/99B0pMbGuGCW9xFG1NbRqNkANOGpKRaRhzfQmanAp5EwEMfICkB2U5aQgBqqyrAVCJE4yVko+0jJQEuru6Cbm8u74ZA8DBmAoJDMrLygWeeqMFC9LT1QuCZgBQAZLd3QhpsRIJxb2/xcIY5Aq67ObDBO7uBOkX6+3GF5nLBsr9C89A7SEFqICpbKm8eQPXRFwDYvHw0cslLx8GiLzY1bNADpjGc/67PupTsIBBP38EGDj7JCEUH2oErw06s63NwnAcy03M0DHjTnX4FDB4d7EdA6FE7QUd+rPCnGQol62EFvMPNkIJwCmUxNBNzohChW6sAJEd0qYWMIYdOpZCsnhDkbaVFfIo22MlDaQ02Sxgy4HW+sCUibAJt60DXjlxqNYu2godkcp9ZNQusnNrL8MTapnB3Kf89hoAyLKBy4J+qF2l6UTrVgSwvnKGO1cCxM6ai8JF6pkyXLu9ecYdavczyah6Vfo1PXCwNWmrtTk5vPVVQ47E1z52azSlWN+dt9P1Prz2Q6NnjUNdtneqwGipBcA8QKDwANcKFSNKu1vZd3j9JYOV1hONSDHAI1EwYl6CU0xyAUDTFCDhhNIsdxpq08gX3TYItNJKFA6tYWATCNIyhSIrzHHHiqV9EZhg8kE3ExqHqEHgYijmOAIXPGoBzRhAgjGjIbOY6JCOSK5ABF9IEFCEk0XYV2MUsSVpJQs3ZGlDDj50ycOVYIYp5phklmnmmWRGAAAh+QQJCQAMACwAAAAAoAAYAINUVlSkoqTMzszc3ty8urzU1tTk5uTEwsSsqqzk4uS8vrzc2tz+/v4AAAAAAAAAAAAE/pDJSau9OOvNu/9gKI5kaZ5oqq5s675wTAJ0bd+1hOx87/OyoDAEOCgORuQxyQToBtCodDpADK+tn9Y6KQa+4HCY4GQgBgl0OrFuo7nY+OlMncIZAEWAwO/7+QEKZWdpaFCFiFB3JkcKjY8aRo+SBxqOlJcKlpiQF2cCoKGiCXdef6cEgYOHqH2HiwyTmZoZCga3uLeVtbm5uxi2vbqWwsOeAwILysvKAlUUeXutfao6hQQF2drZBIawwcK/FwfFBuIW4L3nFeTF6xTt4RifzMwCpNB609SCT2nYAgoEHNhNkYV46oi5i1Tu3YR0vhTK85QgmbICAxZgdFbqgLR9/tXMRMG2TVu3NN8aMlyYAWHEliphsrRAD+PFjPdK6duXqp/IfwKDZhNAIMECfBUg4nIoQakxDC6XrpwINSZNZMtsNnvWZacCAl/Dgu25Cg3JkgUIHOUKz+o4twfhspPbdmYFBBVvasTJFo9HnmT9DSAQUFthtSjR0X24WELUp2/txpU8gd6CjFlz5pMmtnNgkVDOBlwQEHFfx40ZPDY3NaFMqpFhU6i51ybHzYBDEhosVCDpokdTUoaHpLjxTcaP10quHBjz4vOQiZqOVIKpsZ6/6mY1bS2s59DliJ+9xhAbNJd1fpy2Pc1lo/XYpB9PP4SWAD82i9n/xScdQ2qwMiGfN/UV+EIRjiSo4IL+AVjIURCWB4uBFJaAw4U36LDFDvj5UOGHIIYo4ogklmgiChEAACH5BAkJAA0ALAAAAACgABgAg1RWVKyqrNTW1Ly+vOTm5MTGxNze3JSWlLSytMTCxOzu7MzKzOTi5P7+/gAAAAAAAAT+sMlJq7046827/2AojmRpnmiqrmzrvnBMBnRt37UE7Hzv87KgMBQwGI/IpCGgSwwSTugzSgUMry2BdsvlUoqHsHg8ZjAbgKc6ulYPrNg4SqCo2+91wddwWPj/gH4HS01tbIcJcChuTm4ajZADTxqSkWqUlo0YdH4JnZ8KehMBDH2BpwdmOmoIAa2vrgFRihOMlZKUBLq7ugm5vLu+GQPAwb/FwhZ0CQzNzs0FoXumBQvV13+DZwBRAZLf3whqtBIJxb2PBAq66+jD6uzGGebt7QTJF+bw+/gUnM4GmgVcIG0Un1OBCqTaxgocOHFOyDUgtq9dvwoUea27SEGfxnv+x3ZtDMmLY4N/AQUSYBBNlARSfaohFEQITTc3D8dZ8AjMZLl4Chi4w0AxaNCh+YAKBTlPaVCTywCuhFbw5cGZ2WpyeyLOoSSIb3Y6ZeBzokgGR8syUyc07TGjQssWbRt3k4IFDAxMTdlymh+ZgGRqW+XEm9cBsp5IzAiXKQZ9QdGilXvWKOXIcNXqkiwZqgJmKgUSdNkA5inANLdF6eoVwSyxbOlSZnuUbLrYkdXSXfk0F1y3F/7lXamXZdXSB1FbW75gsM0nhr3KirhTqGTgjzc3ni2Z7ezGjvMt7R7e3+dn1o2TBvO3/Z9qztM4Ye0wcSILxOB2xiSlkpNH/UF7olYkUsgFhYD/BXdXAQw2yOBoX5SCUAECUKiQVt0gAAssUkjExhSXyCGieXiUuF5ygS0Hn1aGIFKgRCPGuEEXNG4xDRk4hoGhIbfccp+MQLpQRF55HUGAXkgawdAhIBaoWJBQroDDlDfo8MOVPUSp5ZZcdunll2CGiUIEACH5BAkJAAwALAAAAACgABgAg1RWVKSipMzOzNze3Ly6vNTW1OTm5MTCxKyqrOTi5Ly+vNza3P7+/gAAAAAAAAAAAAT+kMlJq7046827/2AojmRpnmiqrmzrvnAsW0Bt37gtIXzv/72ZcOgBHBSHYxKpbAJ2g6h0Sh0giNgVcHudGAPgsFhMeDIQg0R6nVC30+pudl5CV6lyBkARIPj/gH4BCmZoamxRh4p5EkgKjpAaR5CTBxqPlZgKl5mRGZ2VGGgCpKWmCXlfgasEg4WJrH9SjAwKBre4t5YZtrm4uxi9vgbAF8K+xRbHuckTowvQ0dACVhR7fbF/rlBqBAUCBd/hAgRrtAfDupfpxJLszRTo6fATy7+iAwLS0gKo1nzZtBGCEsVbuIPhysVR9s7dvHUPeTX8NNHCM2gFBiwosIBaKoD+AVsNPLPGGzhx4MqlOVfxgrxh9CS8ROYQZk2aFxAk0JcRo0aP1g5gC7iNZLeDPBOmWUDLnjqKETHMZHaTKlSbOfNF6znNnxeQBBSEHStW5Ks0BE6K+6bSa7yWFqbeu4pTKtwKcp9a1LpRY0+gX4eyElvUzgCTCBMmWFCtgtN2dK3ajery7lvKFHTq27cRsARVfsSKBlS4ZOKDBBYsxGt5Ql7Ik7HGrlsZszOtPbn2+ygY0OjSaNWCS6m6cbwkyJNzSq6cF/PmwZ4jXy4dn6nrnvWAHR2o9OKAxWnRGd/BUHE3iYzrEbpqNOGRhqPsW3xePPn7orj8+Demfxj4bLQwIeBibYSH34Et7PHIggw2COAaUxBYXBT2IWhhCDlkiMMO+nFx4YcghijiiCSWGGIEACH5BAkJAA0ALAAAAACgABgAg1RWVKyqrNTW1Ly+vOTm5MTGxNze3JSWlLSytMTCxOzu7MzKzOTi5P7+/gAAAAAAAAT+sMlJq7046827/2AojmRpnmiqrmzrvnAsW0Ft37gtAXzv/72ZcOgJGI7IpNIQ2CUGiWcUKq0CiNiVYMvtdinGg3hMJjOaDQB0LWWvB9es3CRQ2O94uwBsOCz+gIF/B0xObm2ICXEUb09vGo6RA1Aak5JrlZeOkJadlBd1fwmipAp7EwEMfoKsB2c7awgBsrSzAVKLEwMEvL28CZW+vsAZu8K/wccExBjGx8wVdQkM1NXUBaZ8qwsFf93cg4VpUgGT5uYIa7kSCQQKvO/Ixe7wvdAW7fHxy5D19Pzz9NnDEIqaAYPUFmRD1ccbK0CE0ACQku4cOnUWnPV6d69CO2H+HJP5CjlPWUcKH0cCtCDNmgECDAwoPCUh1baH4SSuKWdxUron6xp8fKeAgbxm8BgUPXphqDujK5vWK1r0pK6pUK0qXBDT2rWFNRt+wxnRUIKKPX/CybhRqVGr7IwuXQq3gTOqb5PNzZthqFy+LBVwjUng5UFsNBuEcQio27ey46CUc3TuFpSgft0qqHtXM+enmhnU/ejW7WeYeDcTFPzSKwPEYFThDARZzRO0FhHgYvt0qeh+oIv+7vsX9XCkqQFLfWrcakHChgnM1AbOoeOcZnn2tKwIH6/QUXm7fXoaL1N8UMeHr2DM/HoJLV3LBKu44exutWP1nHQLaMYolE1+AckUjYwmyRScAWiJgH0dSAUGWxUg4YSO0WdTdeCMtUBt5CAgiy207DbHiCLUkceJiS2GUwECFHAAATolgqAbQZFoYwZe5MiFNmX0KIY4Ex3SCBs13mikCUbEpERhhiERo5Az+nfklCjkYCUOOwChpQ9Udunll2CGKeaYX0YAACH5BAkJAAsALAAAAACgABgAg1RWVKSipMzOzLy6vNze3MTCxOTm5KyqrNza3Ly+vOTi5P7+/gAAAAAAAAAAAAAAAAT+cMlJq7046827/2AojmRpnmiqrmzrvnAsq0Bt37g977wMFIkCUBgcGgG9pPJyaDqfT8ovQK1arQPkcqs8EL7g8PcgTQQG6LQaHUhoKcFEfK4Bzu0FjRy/T+j5dBmAeHp3fRheAoqLjApkE1NrkgNtbxMJBpmamXkZmJuanRifoAaiF6Sgpxapm6sVraGIBAIItre2AgSPEgBmk2uVFgWlnHrFpnXIrxTExcyXy8rPs7W4twKOZWfAacKw0oLho+Oo5cPn4NRMCtbXCLq8C5HdbG7o6xjOpdAS+6rT+AUEKC5fhUTvcu3aVs+eJQmxjBUUOJGgvnTNME7456paQninCyH9GpCApMmSJb9lNIiP4kWWFTjKqtiR5kwLB9p9jCelALd6KqPBXOnygkyJL4u2tGhUI8KEPEVyQ3nSZFB/GrEO3Zh1wdFkNpE23fr0XdReI4Heiymkrds/bt96iit3FN22cO/mpVuNkd+QaKdWpXqVi2EYXhSIESOPntqHhyOzgELZybYrmKmslcz5sC85oEOL3ty5tJIcqHGYXs26tevXsGMfjgAAIfkECQkACgAsAAAAAKAAGACDlJaUxMbE3N7c7O7svL681NbU5ObkrKqszMrM5OLk/v7+AAAAAAAAAAAAAAAAAAAABP5QyUmrvTjrzbv/YCiOZGmeaKqubOu+cCyrR23fuD3vvHwIwKBwKDj0jshLYclsNik/gHRKpSaMySyyMOh6v90CVABAmM9oM6BoIbjfcA18TpDT3/Z7PaN35+8YXGYBg4UDYhMHCWVpjQBXFgEGBgOTlQZ7GJKUlpOZF5uXl5+RnZyYGqGmpBWqp6wSXAEJtLW0AYdjjAiEvbxqbBUEk8SWsBPDxcZyyst8zZTHEsnKA9IK1MXWgQMItQK04Ai5iWS/jWdrWBTDlQMJ76h87vCUCdcE9PT4+vb89vvk9Ht3TJatBOAS4EIkQdEudMDWTZhlKYE/gRbfxeOXEZ5Fjv4AP2IMKQ9Dvo4buXlDeHChrkIQ1bWx55Egs3ceo92kFW/bM5w98dEMujOnTwsGw7FUSK6hOYi/ZAqrSHSeUZEZZl0tCYpnR66RvNoD20psSiXdDhoQYGAcQwUOz/0ilC4Yu7E58dX0ylGjx757AfsV/JebVnBsbzWF+5TuGV9SKVD0azOrxb1HL5wcem8k0M5WOYP8XDCtrYQuyz2EWVfiNDcB4MSWEzs2bD98CNjejU/3bd92eAPPLXw22gC9kPMitDiu48cFCEXWQl0GFzDY30aBSRey3ergXTgZz0RXlfNSvodfr+UHSyFr47NVz75+jxz4cdjfz7+///8ABgNYXQQAIfkECQkABQAsAAAAAKAAGACCfH58vL685ObkzM7M1NLU/v7+AAAAAAAAA/5Yutz+MMpJq7046827/2AojmRpnmiqrmzrvnAsw0Bt3/es7xZA/MDgDwAJGI9ICXIZUDKPzmczIjVGn1cmxDfoer8E4iMgKJvL0+L5nB6vzW0H+S2IN+ZvOwO/1i/4bFsEA4M/hIUDYnJ0dRIDjH4Kj3SRBZN5jpCZlJuYD1yDX4RdineaVKdqnKirqp6ufUqpDT6hiF2DpXuMA7J0vaxvwLBnw26/vsLJa8YMXLjQuLp/s4utx6/YscHbxHDLgZ+3tl7TCoBmzabI3MXg6e9l6rvs3vJboqOjYfaN7d//0MTz168SOoEBCdJCFMpLrn7zqNXT5i5hxHO8Bl4scE5QQEQADvfZMsdxQACTXU4aVInS5EqUJ106gZnyJUuZVFjGtJKTJk4HoKLpI8mj6I5nDPcRNcqUBo6nNZpKnUq1qtWrWLNq3cq1q1cKCQAAO2ZvZlpFYkliUkxFdG9ZdlpHWWpMU3d6N0VKTDNnVk01aWxQaXBDSXJ2SDMxK3lHMGxMVHJVY0lUU0xvTGdvemw= +Logo = iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyVpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMTQ4IDc5LjE2NDAzNiwgMjAxOS8wOC8xMy0wMTowNjo1NyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIDIxLjAgKE1hY2ludG9zaCkiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6NDA1M0ExN0REQzE1MTFFRDkzOTVGQzQ4OTU1NERGRTciIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6NDA1M0ExN0VEQzE1MTFFRDkzOTVGQzQ4OTU1NERGRTciPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo0MDUzQTE3QkRDMTUxMUVEOTM5NUZDNDg5NTU0REZFNyIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo0MDUzQTE3Q0RDMTUxMUVEOTM5NUZDNDg5NTU0REZFNyIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PojFojMAACEJSURBVHja7F0HdJzVlb5TJY16771bvVqWbVmy3DDYYBsDS1tIAxI2hZPNLptzdje7cE422SUkJEsWEiAQTGxiwOBeZVmyrWL1avU+6hp1zYxm9t0rychGljQz/z8zIr62jn2k0V/e924vTwAckJ2Hk417uN9677jgzTInu3gnfw8/kVjkpdVqndiPJXCPliMN+1IIBIKeoY7erumRiYquiqZrvXVteYOt8l5DLy7Q9xctbGVSv+SITP+ksL2uIT5bLWytQgRCoYSBChr17D3Y9CChSAgMaNCy/6umlB3DbfK8zvLGL1qv15wZ7R0aMgrAjENlMXvSnw3cEPWcjYt9jEajhVmlGrQazT2EOCQEWiQRg1AigunRyc6uisY/VR7L/91Ac3cPbwBH7kjZH/vQ5p/ZuTtFq6aVoJm9x6lGAVsoBImlFJST0721Zwv/q+xo7huqqRk1ZwA7eLo4pT6z61d+SeFPzypVMHtPBJsMaKnMAgZbenKvvXPiez3VrVUr/Y5opQ94xQTF73z5qc+dAj22s13DRLH23kqbirRz6lDmZOsflB7zmHpa2drf0FmtN8AR21O2bHr+weMSmTRAPaO6t8DmYnbPakAoFFj5JoYflNpaDfRUNhehcasTwOHbkjM2v/DgcWbEO96zis2RmbVk2HrHhuy2tLMebr9RX7BqgL2iA2M3v7DvBLuK4z1DyrwJRbZbmM99ysmZJiauK1YE2NrZ3nHHy0+csrCx8teo1PdWcC1wM7OLfBNDd/U1dJ4Z6x3quSvA6Htlfv/ht1xDfLKZAudmh2k1oJ5Vg5J9qTVffmnY99GEFwqEvL68WjMLSrUKlOyeqvlnoBcXCr9GCGtBIBJKPKMCNzbnVb6vnlEqlwQ4fFvSnti9m37O/C2D74mLigtqb2kDQa4+EOcdBsm+URDpGQz+Tp5gx76PwI/NTNDnREIRbTDODBH20tNqJbjZOkKKXzRkhaVCakA0+Dp60v36x0fmokcGbjDShfQHOH1+nZ+DGV4yB1s3C1srUXtR3fmv+MEyR1urPa9++4aVg02kIUYVcgwCt44BmRmaArFeoeAos1vysxPKKajvbYXLjcVwo72WuMtCLDVcalBUTQsPxWXDrsh0sLGQfeXn11vL4VDxKRiaVIBUpHu4HCXSLHtXmcQSLCRSevZJ5TRtLAuxhC2s8cHGDSYUi6bPvPJBQnd1c91tAKc8teP5+H0Zb86MT+l9A+QYBytbOJiwHbLD1+v0knXyFjhcegaqexrBUmKh9wIhR6Fa+M7Gh2FLSNKyn+1S9MHPz74DgxMjIBGJV3d99mdapYRQVz8mFVIgzNWfSSNrArx9WA55TSVQ1FaFqw1iocjoIGPES17XdujUz957At0pegJmUEnTv/XAu0KRyFXfQMaUahoiPQLhx1ufIXGsK0AuNo6wKTiBxGdNT7PeIhs32a51G+Gh2KwVP4vAeDm4wtXmMhAIBSs+M4I7w1TPgzGZ8N2MRxnI/mDPNjRuSGsLK/Cyd4W0wFjwYyqosvsmfZZvG2MpH9nOzTGsq7Lp8MSAYogADtoQnRWWlfiSvsGMadUME8Vh8I/Zz4CTtZ3eD4eLEcs2B/5b0XUTRIyrdIEYxaO11BJe2PwI+9dqVb/jYecCDX1txM0rcRxy7l4G7pMp99MGvBt527tBsIsvXG0pQ/vH6LpZbCEVM508xHzjHNpefskR+/XNBilnVWS4/CDrSZCxxeWCDsRvYyI+jaSCLqRizxLs4gOuTBroQnE+ESTWl7+2GgKcvODRxJ2ruibaIDsj0hkXK8HYhPkCr+igh6QyS5HQ1s1R5hbqsxW/qY++w13/rfR9YHuHIWMoIZf4OXjQwurCwS46gjvHxc4gEohWBDgzNHnVuhoJLXfc9HcLI/IGMDOSbVwdot0j/KOFbuG+acy0DtbooXun1TOQEZIMEe6BnD8kLswBZqyhpaqTDuIhGYK6Fy3lcPcAnX7PnW0cd1tn8iyMblEzHeIR6Zct9I0PzRAIhWJ9uNeKuQjohvBFqf7REMhE7mq5GHV379igzvfpVvQz7p9dLo5ArpRslXr9VpCBPQ9KNo3W+Bk4dHUZ824QypzsEvQRIah7I9iO9nX04M9YYOJ/PQN5tQCj+GwZ7IIeBpgudKO9ZlmjCW0kisbpoU9RBy9lYyFXTzHjFH1n/ML/c8npiKmljSxE7BTg4a9PYAODBTHMcuab1nkEg1QsWSUHC9hiTcGxyhx4ftPBVf0Ouki1vc3LBjvQfUJ11Mb8XH9maK2WhidHmUQZus06xxAtRvm8HdzZuwWxf90oCoabsqqnEbpG+ihQYqh7hZgy5nUVi8QiD304GLklwNmLd4BRj6GYG2fAiVbx0uiT5jQUgR+TLLujNi/72ZvMPXrn+rFVXRddnfymUsgITlz1s19rKYeRqVFSZQtci2A/mfoAbAtff+v7C4ScfL7+Ghwtu3Drs4bpYYGdcL60VbfdwTaEVCylqBXfhP6sraU1M540Omw+CXxQdAL+eO3TJXUyctD5+uvwi/PvEseLVrGQ+L4V3Tcht/HGqvU6SpIFyYCcK2H3+X7m47AnestXwF0wLPfGZMGPsp4iBtJoDSxk1IIMjSu96pZRHBojSoMviiJ6Lpy/+meTiERwpvYqXG+tgAi3ABKJUrEYBicUxLkYVsQFX63bI5g3mt69/hn9zobAuLuHQJmYfT3nz6CYGrsVW8eo1t+n7oEk33Ur3iveJ5xiAX8uPE4SyQASifVifdLBc0kF3q1BrLPWaHQOfeLnrdjiYJStsL0atG2Vt36Cos9Sj6QGcjoafL/N/Qvj5gbYyvxcf0dP2oD4fZQWBa2VtLEwS7YALopb1LXbItJWfa8dEelwob4A+seHDRLV+gHMOEQ5n+rjm6ZV0zA+M0VcqW/405KDDNVikNFmwcXPYzrZzcaR4tBoBQ+Mj1CGDMXyYqMNwY/xCtUpU4aGFhph59l9jA7wnBWthk4miqI8Q3gFGPO2iukxowftV9rgKB0QaDnjWs3onIRB8K2WFKla8LRz0fk+nvauBkfB9F41fKEaeRPvi1krbybuMGUyfTmgkbuQW1EvCzl+RiEHOWW9AUa9U8MWf4gZLXwRGlZoJJkir8o1yUd1j7D1jA0YvLH1BhhF5sjkGPmcfBGmDOv7WvWquDAnQrcNCxl0SZyg1V3d3aRTcoNTgMkQkEjhNLMY+8aGOF8UtNCPlJwF+Bo0UqAE6mBumS7McLbuKvOl+wyWXgYBjH4h+np/uPaJ4U75HXS0/AL5q6sNU64FLv7oxini5JWopLMOPi49y8m7i+L2Zfy7QbuTiZCOETlzZSYh0TeSk8W4eLMQDhWfJLfCHI0rfQNDKKILWqvAkkk+Hwd3WrvFNMHcwdO1+fBewTHynUUc2B5iLh4ew25n2IPhQz2b9qBBOvNkTR5FcLguozUXUa1i7uW71z6Dc3XXIdozmEqGcN0w+lXX20IhTvSBuTIsOQFYMA/yBfbQ+KCPJ9+ncxEARmw+Kj4F+c2lJJr08XsxuoYLCPN1UPSH/YvRsIX6ZVQrYpHIZH413l/EfGX5aD/p5QV/AVcRDSorw0KT/AB8i5OllqQ3Xz3zNqT4R0N22HoIdfNblqPxJfOby8gAGZoc1esFMTeN6Us3G2eIYlzh4+AB9lbWdC2MSU+plDClnIbBiVFoGuiAhv42GFdOMi5hYAvFJuJmsVHuzfkdULwgt2BqDeOyWEoa5OwDXg5ulPbDXYrZnM6RXrbYnQzgnvnwnlRncFGnIXdGugdBqn8Ulazercj+Nv9SMQDF7TWk67sUvSQxRIKvUSvLYun69Ps/5c0RQZGonp0lHYOgo0pFsbmQGUI9i7tYqGf9s4+9B9UobwyO16vXaGx6knThyeorMKHCTSb+2gHM6xsJ5vWKhOOFw8R4os86eGHzQXCQ6Z+TtrWUwf74rRTU/92Vw8wOGOSkdcasrPe19sAYl04LiIUfbn3CIHAXU4RHAPxk2zPgbe9ukjrmewAvAhcT5i9ueYxza9PX0R1e2voUGWpYQH8PYCMTWsrhroHwvYxHeROjmJR/ccvjbPPIlu10oGFv7Of4GbTe8f8Lbtg9gPUx1tjiSYQSKlaz4biD4k4Kc/OF+9Zt/Eq1CgKJ1SHoAcz5rFKwlVpTA5uFyIL8ajQopzkufzVrI4srQr2YEZIC4e5+Rrkf1ntj7Bg5koInDDgXa0dyx2K8QqgXC+u7ZFIL5hkImY89BzwGa8o666kMt2e0j4Isps6EmT3A1DYitoDt4Wm83wu7B7Ge6mzdNep0UKlnyfDKCk0hHxunBSxJ80Il0MULUgOiyP3ChnYM3tT1tYBYJFyx9+lvFmAMZoS7BUEIE5180rXmCvi86jI09rfRprK1sIGdESnwYGwm2FvZ6Ox+ZYYlwSbmn5+tvQ5Hy89Rp6TEBH622QNMHRSewZyXwyxQx1AvfFx2DoraKkGpUVFELcojGB5Lug9CXH0MW1yRCHZHbwR/Jw+qxMQieGOnP80aYOQkiVAMQS4+nF97aGIUTtXkz8fARyiqFsEkxYMxWZDiv47TTFaUVzD8OPtpeO3iBzA0pTCqXjZvgLVzVSNutk6cSoRzdQVwvCoH5GMDtIk87Vzh/qgtkBWWTLlaPijY1Rd+mPUk/PL8exQWNVadmZmL6Ln0Hlcx4oquRvi0/AJUyxuZDzsL1szfxQ4F7CK4qwHFIWFm7bmNB+HXlz8k39kYKUux+XDrXKBgIXe7kDwgUWmguOwbG4YvKnPhcmMRjCsnwJJZ5XEe4bA/PpsGxxiTkvwjYQtz+c7W59FzfK0BRkAp5cfEJjZ3yRhHWTERif/HEqDR6TGDro8lMAjslaYSJo77iWNCXQKYZZzFODcWTFUwsi1iPVxpLqZImJDneVpiUwGLgQFMFYa6+NNknZj5gWkYY8avE9VX4IOiL/Qa7KKhfHQZdfe1DnWSOHSWOcLudZth57p0ClCYktCqjvUMg4L2Sk7baswC4IXa4CTfKBr/gNNolsrl6uszNvZ1wuGSM1DZU0/9U7YW1pDsF0V5Yz8nDzAX2hySCMUd1V8vEY1d8v6O3nAgbhusD4xeUSfryrVHSy/A8erLMKGcJPcq1S8G9sVuhXAPf7MzH0OYVY1D1LCBj89qEqMBjKk+jOWiq+Ao475xfEalJF07xVwQnGxzMH4nZIQmchIgUarVbMNpyGXjijA65mrtSMEPkWiNA4zgRnuGMnCfoOwLX4RiHTM5WSGpFCrkgorbauHTiouUXULjLD0olpProsEns+B/hhbvAGOPTbhbIPwgk19wFxMXBeM9ikH4rPwi5DWXwMzsXJXHG7mHoLC1ivnN2eDr5M4ByPyb8bwCjKk2nAv93c2PUBmrsciQuVTDk2NwsjqPxP3AxBBxWrCzHxmCzQMdcKWlGKrlTZAWGEOzNowRIDFbgNFifiAqAzztXYz6UgK9NqMGrjSWkjjuGpGTK+ds7UAhzO0RaQTwhbpC+KIqB/omBuFkTS6UtNfScNLMsGQqF/6bAhgHZPs4eNLimDtVdTfBJ2VzIUx8bpnECjYGJTDwttC44QXCzFCyfyQFT3KbblDw5I/XjxK3H4jfDgm+YUaTNCYHGKsgcP6zlYmDCstRx3Av07OXoKi9ilwrLCxY7x8Lu6M2UaXlUoSJj2+mPwRbQpNJlBe2VUBtbxP8z8UuOrJgX9xW8Hf2XIVlroKRyXEQ8nx2BC8AY+TI0cpu2VFD/Ijm1QlnDGF+XnmZZmUp5sOhgc4+sD9uO7OSY1bpx/rA9zMfgxvt8dTq2TTQDnktJTRLC6fvoMWNif+70cjUGAyMD/HeUcELwFh85ufmCR52TkYDVwtwq6rxbtWN6JLkNpbA8apcaBvupoQ8pgozQ1JgR+SGZQG5GyX5RcA6z0A4WZUHp+vymZGmgM8qL0JpZx2FRlE/LxWpq+xupP4oi7UYqsQguouNg1mJ41p5KxyruARlXXXMMgaKd9MM6JBkOJCQbdC18Vp4DUxgoC4vaKuE1qEueOvqx3CttYKOF4j2Cr7NoMtrLgVYy26SuViV3SP9zPLNpbZUJfNnMVO1sKzIwV9UXWbc3AOPJu4C70UGlT6EBtmLTGxn9aQS0FXyBijtqoG63mZIC4ij+i4ssK/rbZ2bXiASr12ATUkWornJcyer8+FE9WUKB6IBdac4RJ2tYX8KGJdh8/XOiI1wf/Rmg6s6ojyDIMLdH/KayihY0qGQQ05jAZR31ZPYruxpoNCnANYywCZKtmI9cwVbwOKOGqihMcGiJQd/LgYZ50HiUNLDpafhBvu9Pcy33RAYY9BzYDRtS2gSzZ08VpFDZ0MNTyngo5KTvDTkGR9grWnaOCTMoKliAM/pRqlOgFixr5ahTvjt5UNQ0BIDDzBuNrRcF5MKT69/gM5YOs785/yWUioXWsXRzeYNsMZEAGsBDOIO6XxHw7XWUqY/ayEzNBX2MSPJQWZj0HOh7n0h4yDFBg6XnIWb/S2kCgRrsaIDH3lCOQ1rlRbENnY3nKzOgQqmO/dGZ8LmkAQyzAyhGO8QygVjUcLp2jxSKXwmHXjxslHcdQ73Uh6Vi8U2VX8PlhTJpDJoZz7z7/OPwCun/0AtKYYSRveeSdsLBxN2gEqt4lXaCflZGBHIRweg7fajbHX2pZFwVENlV4PRwcWYNB6w9c/bnoUtwSnEZVgG9NrF9+H1ix9C62CPwffAct2DiTuoNZav1lODB6EtbUDPHWKB/Jfst06va+B4xMK2agp74uQ3HObp5+gJNhZLH22Dse9LN+fKYrmoN8YNZmdhA48n74b1zKL2tHODbrZphydHKApW2FZF9wx09jZI5+P4COxOrJW3GCz+jQYwiQZmzeI0mwSfSJ2bt5BwEhyW3nQp+qm1pHmwE4pJPArowI07F4NrgDXzAGeEJNG9sGAvPTCeWeZWDOh+8q3RWq/uaabvYfO4vu0u2NrS0NdBLadcdzzwBjC+LHbUKaYm9SpzQZGIp7qkB8VRgzWOy8chKRgswCmsmNJDy5Q3gDUaKorbGp56q0KEprB7BkGSXxTMarQkVbpHe+ncpTp5K7jaOul8buLcuwppM19vqSB9zGVfFG8A08XZzsfJdw6W9hDk4q1fVEospR2OkgD7bnFCXC8Duqi9mvSgG1sYPPEU7ZQL9YWccjCeXo6ZoTtLgDApkeQXCevcg6kCpG98gH0NUV7YWWZPYltXwth980A3tI90czogjVeAyccTzM19RpGLYkxfsrOyhg1MEmB3PZ6NMMjEdvtwD4UZp1Uq8HXwpONcx2bGeQd4MSgbmYTBzdXQ307GkpedK8T56HdgGN4HD5dezVnGZgHwgqjFyA2KMU97Nwa0m0HXw9/fFJQAztaOMDAxAv0TQ9RFj6egjDJw8V5cLA7GqO0srCkCtZzxg5vJXmYDOQ3FlCbFk9pivEP1uqebjRMzKOsppMlVntgoQ1gWpqz+75W/wMX6YoOvh/XJOyLT4N/uex72x26nw7NaBjtgUjnJe2RoSQse66Y5uI5ELKb89CyHQ1yMNmUHQUbu+uP1T+DN3L+CXDFo8DWxUvPxlF3wLzu+Ta0wyEHKNT7jyprOG4a1BzDpA4GI6RkBXGosgP84/X9wsiqfzkQylELdfOHlHd+gGVd+Dl40TEXfUUb4e6Ycg6RPVYnZALxgeGFgoI9Zwu8VfgY//eINOF1z1eAKf3Qt0OD52f0vwBPJD4CDlT11VKz2uvg5/Lwr04O+Dh5zp60JBEYfSs71hAGjA4zc4ccs3uzQNGqA7hyRw3sFn8Erp99m/m0zJwu0NzYD/p3p5+ywNPIrV5o/OTMfD8barH/d9Rx1I86oTSPquW5lMWpFB8Vb2d9HEndRDXF6UDx8XplD9chYK4WuBna/74/bCo7WhjWoudo6wHObDkBaQAwl3Ov7W+aOCbgt+qWmDYf+7IGEbTTkbA5wJZjqMAFUL2sWYFzQQCcftpBzBWhxPqGUPrvSWEYdA9isfao2FyqZy7MzMh22hCbq1QC+mNAnxa9j5TlwpOzMnLHHxC8+C/rUWJi/lblCi+PJpjwpgus0q9jY4genxi32K9FP3hKawDg6nBldV+Bs/TVoH+mCdwo+gUs3C2919BmiC682l0NhWyXpfwyf2lvawUOxG2kmpbWFJZgT9Y8Nc5ofNhrAGBlysLKDRN+IpSNVzHp8LHknRauw2+B6WwU0DbbDG7kfUgjwkYQdEOSqWwgQA/ifM/Fc1FHFRLGK6XxLSA9MoDTd4ji2OYHbxPx5MYf1WkYDGNNvOE7BboXMkr+TJ/wg63HY0BrLxPYVppdboKi9kg7SQKPpvqiNKzaQ44mlWIh+qaGQQpcolkNdA+Dh+O0UQzZXwnMk8KAxLi1p43Ew03vutk6rzp2mBkQzsR0BuQ2lcKzyEnSNyuGTinN0WCV2IeyM2ECRn8WEbs7F+kI4UZ1HCQAkPNdhd9Rms+4AXLBP8Ph4jBOsSR2M+tdGRyceN0N2RArpZxTb2NHXpeiBPxUco9QaFpKn+EfRZwtbq+HT8ovQONBGcWQbqTVsC0+jDkF98tHGJrQTWoa6ON+ExgMYQO8AOqYDv5H+IE2mWWg/qelthJacTuoGxPAk1jMrmXtjKbGEBO8IAh8ny60FwrmZR8vOc869pvGDDSAMSf5429NzczMYt6Jve6WpeN4aF0KURwjsi8smjl8rhDVnb+YdoRw3HzOzjAawAECn83OXI2zCxvYQnBaLhzmiEbU3Jov8WanY8FfCgIgxqrpR8ryZexjKmURarvtibQDMOGx0epKz62HpKZ55hB19CAhXszJqe1qoWgSPxONqQy4d0JiC3185Qp2IfIG7ADDWp/Iek8YeWTzTAENxXLoBXM3/6B0doqZwnKozqZpiCyKEIBdfXiodO4f74A/XPoUaeQOv4KJWRIAV7Iv3UTFztdL97KWa7xrsMAXhKWonmL+N3f6DEyOkSwIcvUmX4yQdLkf+YjPA2brr5Oqhf84zuOQ5igUCgZy5MEaZBYSB/eK2arMBGF2ro2UXoHmwfW5gqbUjc63Ww33rNumdl10cUsXOjvGZCajoaqCO/6qeRhpdiGP9jdKtIRCMiofaezudAzwiNbMa3u+HL3ajs5ZxioItpr3JgG3u74aj5efZoteQFYuclOSzDh5J3A4+BoQw0YfFmDce8oznJ43RSORxqutGo03Kw/nAd8VWJITJQcWweHJ4rNI5yHM7GKGIAf3gkUkFhRGfWn+/0YEdGFfcOqdhZEpB5anJvtGwJzqDsloGuQjzrlr/+BAVxmNiA5MG+D1THHiJjQczE9Mt4p6qlny/xLAfgZGyZMjFGCPGI2H1rZXW2R1hovJc3bUvQ5haLXjYucJDMVshOzyVLYZhr05Jeu2XR+aKhCIwNQkZBw82dxeLe6pbc5VTM91iqcRba4SeXtzRU+ppeCv/KPzT9md5mTy7mG6019Fg8LreJqpWxArMTUGJNM6fq0ExC8cRCAQCMCeS17ZfFI909Q8Mt/XmeUQFPKqeMU6ZChoYLUMd8OtLH8JL2U/xMqS0sb+TUoU3OqppQ1mJLWG9fxwTx5shzJ3b+dGKqQkqIpCYyQHTKJ6nFBMt8prWIrGWPVhH6c3PvOJCHoUZ49UhYT1WDeOq1y58QMfFcsVNg0zPYhkQJibQwMGXjfIIoy79RD9+rPf2YTl5COYCsEgqgd7yxtMTQ6OT9EQt12tOx+3f0iuSiN0RcGMRWpS1fc3w6pm34e+SdtO5f/oSFs3l3CymYaLYpYeEB0reH5VB5zTwmSq82dtmlCNydDH6Gi+XfUBgkxEyOT3tFOjp5BrktXlWpTbqs2AAZHR6AgraqqBrpB/sLW114mac+VjSXg8fFJ6gCbA48R3HL+BQ8Oc3H6QEP5+HUGEDHJ5NaIoS26W5VwyjPYO5xR+ee0Wjnv0yFl3xad5vmDX9HYFQ6GxMLkZC0YZGSn7LDShur4I47wg6PxCnw+G8aTy9c0H8USXk7Cxzc8bpKFdsOMMRCxioWAgefHfTI3RaqDHobO01iisby79dEWCRCCo+z/9P1bSSLOZbAA+19chrThe8mnAg87WZiSkTSBUB6WW0SIvaK6CwrZxOAMUgBFZuzIlYAZ3NgFkYLJ7DWY/oW2O8eGGoGG4ABZMIxqDGvg7aYOZSKSKxlEJ3VfMnDZdKz98C/LZAQFP3Dd/E0G0yJ1tfY0S27hbqQ5EqpvMX1LeAxOmsWK+E5/7NqGeIY5GrMYlxe8OZFpr6OyDRN5LXIwRw7MJvLh+iibFiMzCuMHLF1OvQpdc/PjA5Mj5yy6K+TZ9NTqvy3/7iW0x2K4Qi0xsNaLhg0AABX5gOJ54PJNzNqMGfYRvpG5c/okoJPgj96bevHqVCQKmZcK+FzBLK/5r74mCrvPW29bjzgxMDin7l1EyTb1L4w9pZjXl57qvVQwxkTE029LVR4R6X+hHdobfzj8LlpmKQScyjplois4CGi6W/LDp07ld3BquWNC/7G7tqpNaWg95xIbtnleq1iDFxu3xskGZnRHmGgI2F4V17mMt+88rHZgWuhbUV9FS3vnv+v//y4lLG8V39h56qlkJLO9mIW5jPLq1Ga7LZk4aCPDAxDCUddUwf24Kvo4feM1Jx3vTv8z6Gks4a8wCXvYeUcW43Azfn9SPfVk3NLAnQXQFGUNuL6wuYud3imxC6g7lPUmO7T1z52WikFbVV04xmLMB3ktmt6hQzDD829LXDoeJTcKTkDLW88nWAtE62CbqNVhZw81LpLy+9dvhFZjtpl9kHK5NvYlhq+jfvf8vW3SmO7RTQrkFupqNs1WoC2Nvejab24GAYDztnanBDVwcPs5yhwzLGKKeLpbg49Ay7DXEGtTkEMtAVUs0o+0oOX3qp6vjVD1fB6KsjmYOtbfIT234avCnmH4QikYzdBGDt4XzLUFLPF9RJRVJytfALXS+cf4XzRLATA+uUMWdsFhEqiRiEYhGqziMF759+ebClZ1XN1Do/uXdscEz8w5k/cQ/zfZj5XpYY2sSQ2FqlhZ5l7eLFEAjAHNwHphZBLBXT84x09V9gHPuL+vM3zuqoqvUjj3UBkSEZcY96RQfusXFxiBWIBGKNWgOop00VJFnrRLEHBqZIPGcaTSnGG/saOs825VZ81FnWmKeantHHFjPQTLexEnpE+Me5R/pnuYZ6p1nYyEKtnexcBEIBFl3JwFijzWENCxGGJUNCMTk0NsxsnOaBpu4SeV1bjrymrXBiUGFQ3PX/BRgASTi/uQCfTT0AAAAASUVORK5CYII= +Version = 1.2.0 +Date = 2023年4月23日 +Email = xhunmon@126.com +RedBook = Super86 \ No newline at end of file diff --git a/009-Translate/asset/en.ini b/009-Translate/asset/en.ini new file mode 100644 index 0000000..b8f0e53 --- /dev/null +++ b/009-Translate/asset/en.ini @@ -0,0 +1,122 @@ +[Main] +Title = Super86 Translator +Description = Free multi-platform and multi-language translation software... +EnableProxy = enable proxy +Run = Run +Clear = Clear +Copy = Copy +File = File +Settings = Settings +Exit = Exit +Version = Version +Business = Contact Me +Email = Email: +RedBook = RedBook: + +[Settings] +Title = Settings +Proxy = Proxy +ProxyEnable = Enable +ProxyDesc = support http/https/socks5 +Theme = Theme +ThemeDesc = Leave blank to use global default +FullTranslate = full translation (Not verified) +Advanced = Use Advanced Interface +Restart = Restart app now +Ok = Ok +Cancel = Cancel +Reset = Reset + +[Loading] +Content = Loading... +Cancel = Close + + +[Language] +LanguageCH = Simplified Chinese +LanguageCHINESE_CHT = Traditional Chinese +LanguageEN = English +LanguageJAPAN = Japanese +LanguageKOREAN = Korean +LanguageAR = Arabic +LanguageFRENCH = French +LanguageGERMAN = German +LanguageRU = Russian +LanguageES = Spanish +LanguagePT = Portuguese +LanguageIT = Italian +LanguageAF = Afrikaans +LanguageAZ = Azerbaijani +LanguageBS = Bosnian +LanguageCS = Czech +LanguageCY = Welsh +LanguageDA = Danish +LanguageDE = German +LanguageET = Estonian +LanguageFR = French +LanguageGA = Irish +LanguageHR = Croatian +LanguageHU = Hungarian +LanguageID = Indonesian +LanguageIS = Icelandic +LanguageKU = Kurdish +LanguageLA = Latin +LanguageLT = Lithuanian +LanguageLV = Latvian +LanguageMI = Maori +LanguageMS = Malay +LanguageMT = Maltese +LanguageNL = Dutch +LanguageNO = Norwegian +LanguageOC = Occitan +LanguagePI = Pali +LanguagePL = Polish +LanguageRO = Romanian +LanguageRS_LATIN = Serbian(latin) +LanguageSK = Slovak +LanguageSL = Slovenian +LanguageSQ = Albanian +LanguageSV = Swedish +LanguageSW = Swahili +LanguageTL = Tagalog +LanguageTR = Turkish +LanguageUZ = Uzbek +LanguageVI = Vietnamese +LanguageLATIN = Latin +LanguageFA = Persian +LanguageUR = Urdu +LanguageRS_CYRILLIC = Serbian(cyrillic) +LanguageBE = Belarusian +LanguageBG = Bulgarian +LanguageUK = Ukranian +LanguageMN = Mongolian +LanguageABQ = Abaza +LanguageADY = Adyghe +LanguageKBD = Kabardian +LanguageAVA = Avar +LanguageDAR = Dargwa +LanguageINH = Ingush +LanguageCHE = Chechen +LanguageLBE = Lak +LanguageLEZ = Lezghian +LanguageTAB = Tabassaran +LanguageCYRILLIC = Cyrillic +LanguageHI = Hindi +LanguageMR = Marathi +LanguageNE = Nepali +LanguageBH = Bihari +LanguageMAI = Maithili +LanguageANG = Angika +LanguageBHO = Bhojpuri +LanguageMAH = Magahi +LanguageSCK = Nagpur +LanguageNEW = Newari +LanguageGOM = Goan Konkani +LanguageSA = Saudi Arabia +LanguageBGC = Haryanvi +LanguageDEVANAGARI = Devanagari +LanguageTA = Tamil +LanguageKN = Kannada +LanguageUG = Uyghur +LanguageTE = Telugu +LanguageKA = Kannada \ No newline at end of file diff --git a/009-Translate/asset/language.json b/009-Translate/asset/language.json new file mode 100644 index 0000000..0e85f8d --- /dev/null +++ b/009-Translate/asset/language.json @@ -0,0 +1,1250 @@ +[ + { + "en_name": "Chinese(简体)", + "id_name": "zh-CHS", + "zh_name": "简体", + "google": "zh-CN", + "yandex": "zh", + "bing": "zh-Hans", + "baidu": "zh", + "alibaba": "zh", + "tencent": "zh", + "youdao": "Y", + "sogou": "Y", + "deepl": "zh", + "caiyun": "zh", + "argos": "zh" + }, + { + "en_name": "Chinese(繁体)", + "id_name": "zh-CHT", + "zh_name": "繁体", + "google": "zh-TW", + "yandex": "", + "bing": "zh-Hant", + "baidu": "cht", + "alibaba": "zh-TW", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "cnt", + "caiyun": "", + "argos": "" + }, + { + "en_name": "Chinese(文言文)", + "id_name": "wyw", + "zh_name": "文言文", + "google": "", + "yandex": "", + "bing": "", + "baidu": "Y", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "Chinese(粤语)", + "id_name": "yue", + "zh_name": "粤语", + "google": "", + "yandex": "", + "bing": "Y", + "baidu": "Y", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "Y", + "argos": "" + }, + { + "en_name": "Chinese(内蒙语)", + "id_name": "mn", + "zh_name": "内蒙语", + "google": "", + "yandex": "", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "Y", + "argos": "" + }, + { + "en_name": "Chinese(维吾尔语)", + "id_name": "uy", + "zh_name": "维吾尔语", + "google": "", + "yandex": "", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "Chinese(藏语)", + "id_name": "ti", + "zh_name": "藏语", + "google": "", + "yandex": "", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "Chinese(白苗文)", + "id_name": "mww", + "zh_name": "白苗文", + "google": "", + "yandex": "", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "Chinese(彝语)", + "id_name": "ii", + "zh_name": "彝语", + "google": "", + "yandex": "", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "Y", + "argos": "" + }, + { + "en_name": "english", + "id_name": "en", + "zh_name": "英语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "Y", + "alibaba": "Y", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "Y", + "caiyun": "Y", + "argos": "Y" + }, + { + "en_name": "arabic", + "id_name": "ar", + "zh_name": "阿拉伯语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "ara", + "alibaba": "Y", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "Y" + }, + { + "en_name": "russian", + "id_name": "ru", + "zh_name": "俄语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "Y", + "alibaba": "Y", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "Y", + "caiyun": "Y", + "argos": "Y" + }, + { + "en_name": "french", + "id_name": "fr", + "zh_name": "法语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "fra", + "alibaba": "Y", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "Y", + "caiyun": "Y", + "argos": "Y" + }, + { + "en_name": "german", + "id_name": "de", + "zh_name": "德语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "Y", + "alibaba": "", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "Y" + }, + { + "en_name": "spanish", + "id_name": "es", + "zh_name": "西班牙语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "spa", + "alibaba": "Y", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "Y", + "caiyun": "Y", + "argos": "Y" + }, + { + "en_name": "portuguese", + "id_name": "pt", + "zh_name": "葡萄牙语", + "google": "Y", + "yandex": "Y", + "bing": "pt", + "baidu": "Y", + "alibaba": "Y", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "Y" + }, + { + "en_name": "italian", + "id_name": "it", + "zh_name": "意大利语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "Y", + "alibaba": "Y", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "Y" + }, + { + "en_name": "japanese", + "id_name": "ja", + "zh_name": "日语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "jp", + "alibaba": "", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "Y", + "caiyun": "Y", + "argos": "Y" + }, + { + "en_name": "korean", + "id_name": "ko", + "zh_name": "韩语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "kor", + "alibaba": "", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "Y" + }, + { + "en_name": "greek", + "id_name": "el", + "zh_name": "希腊语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "Y", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "dutch", + "id_name": "nl", + "zh_name": "荷兰语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "Y", + "alibaba": "", + "tencent": "", + "youdao": "Y", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "hindi", + "id_name": "hi", + "zh_name": "北印度语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "Y", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "Y" + }, + { + "en_name": "turkish", + "id_name": "tr", + "zh_name": "土耳其语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "Y", + "tencent": "Y", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "Y" + }, + { + "en_name": "malay", + "id_name": "ms", + "zh_name": "马来西亚语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "Y", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "thai", + "id_name": "th", + "zh_name": "泰国语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "Y", + "alibaba": "Y", + "tencent": "Y", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "vietnamese", + "id_name": "vi", + "zh_name": "越南语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "vie", + "alibaba": "Y", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "Y" + }, + { + "en_name": "indonesian", + "id_name": "id", + "zh_name": "印度尼西亚语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "Y", + "tencent": "Y", + "youdao": "Y", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "Y" + }, + { + "en_name": "hebrew", + "id_name": "he", + "zh_name": "希伯来语", + "google": "iw", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "polish", + "id_name": "pl", + "zh_name": "波兰语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "Y", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "Y" + }, + { + "en_name": "mongolian", + "id_name": "mn", + "zh_name": "蒙古语", + "google": "Y", + "yandex": "Y", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "czech", + "id_name": "cs", + "zh_name": "捷克语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "Y", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "hungarian", + "id_name": "hu", + "zh_name": "匈牙利语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "Y", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "estonian", + "id_name": "et", + "zh_name": "爱沙尼亚语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "est", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "bulgarian", + "id_name": "bg", + "zh_name": "保加利亚语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "bul", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "danish", + "id_name": "da", + "zh_name": "丹麦语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "dan", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "finnish", + "id_name": "fi", + "zh_name": "芬兰语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "fin", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "romanian", + "id_name": "ro", + "zh_name": "罗马尼亚语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "rom", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "swedish", + "id_name": "sv", + "zh_name": "瑞典语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "swe", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "slovenian", + "id_name": "sl", + "zh_name": "斯洛文尼亚语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "slo", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "persian", + "id_name": "fa", + "zh_name": "波斯语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "bosnian", + "id_name": "bs", + "zh_name": "波斯尼亚语", + "google": "Y", + "yandex": "Y", + "bing": "bs-Latn", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "bs-Latn", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "serbian", + "id_name": "sr", + "zh_name": "塞尔维亚语", + "google": "Y", + "yandex": "Y", + "bing": "sr-Latn", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "sr-Latn", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "fijian", + "id_name": "fj", + "zh_name": "斐济语", + "google": "", + "yandex": "", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "filipino", + "id_name": "tl", + "zh_name": "菲律宾语", + "google": "Y", + "yandex": "Y", + "bing": "fil", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "fil", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "haitiancreole", + "id_name": "ht", + "zh_name": "海地克里奥尔语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "catalan", + "id_name": "ca", + "zh_name": "加泰罗尼亚语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "croatian", + "id_name": "hr", + "zh_name": "克罗地亚语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "latvian", + "id_name": "lv", + "zh_name": "拉脱维亚语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "lithuanian", + "id_name": "lt", + "zh_name": "立陶宛语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "urdu", + "id_name": "ur", + "zh_name": "乌尔都语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "ukrainian", + "id_name": "uk", + "zh_name": "乌克兰语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "welsh", + "id_name": "cy", + "zh_name": "威尔士语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "tahiti", + "id_name": "ty", + "zh_name": "塔希提岛语", + "google": "", + "yandex": "", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "tongan", + "id_name": "to", + "zh_name": "汤加语", + "google": "", + "yandex": "", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "swahili", + "id_name": "sw", + "zh_name": "斯瓦希里语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "samoan", + "id_name": "sm", + "zh_name": "萨摩亚语", + "google": "Y", + "yandex": "", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "slovak", + "id_name": "sk", + "zh_name": "斯洛伐克", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "Y", + "caiyun": "", + "argos": "" + }, + { + "en_name": "afrikaans", + "id_name": "af", + "zh_name": "南非荷兰语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "norwegian", + "id_name": "no", + "zh_name": "挪威语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "bengali", + "id_name": "bn", + "zh_name": "孟加拉语", + "google": "Y", + "yandex": "Y", + "bing": "bn-BD", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "malagasy", + "id_name": "mg", + "zh_name": "马达加斯加语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "maltese", + "id_name": "mt", + "zh_name": "马耳他语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "queretarootomi", + "id_name": "otq", + "zh_name": "克雷塔罗托米语", + "google": "", + "yandex": "", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "klingon", + "id_name": "tlh", + "zh_name": "克林贡语", + "google": "", + "yandex": "", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "Y", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "gujarati", + "id_name": "gu", + "zh_name": "古吉拉特语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "tamil", + "id_name": "ta", + "zh_name": "泰米尔语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "telugu", + "id_name": "te", + "zh_name": "泰卢固语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "punjabi", + "id_name": "pa", + "zh_name": "旁遮普语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "amharic", + "id_name": "am", + "zh_name": "阿姆哈拉语", + "google": "Y", + "yandex": "Y", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "azerbaijani", + "id_name": "az", + "zh_name": "阿塞拜疆语", + "google": "Y", + "yandex": "Y", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "bashkir", + "id_name": "ba", + "zh_name": "巴什基尔语", + "google": "", + "yandex": "Y", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "belarusian", + "id_name": "be", + "zh_name": "白俄罗斯语", + "google": "Y", + "yandex": "Y", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "cebuano", + "id_name": "ceb", + "zh_name": "切布亚诺", + "google": "Y", + "yandex": "Y", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "chuvash", + "id_name": "cv", + "zh_name": "楚瓦什语", + "google": "", + "yandex": "Y", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "esperanto", + "id_name": "eo", + "zh_name": "世界语", + "google": "Y", + "yandex": "Y", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "basque", + "id_name": "eu", + "zh_name": "巴斯克语", + "google": "Y", + "yandex": "Y", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "irish", + "id_name": "ga", + "zh_name": "爱尔兰语", + "google": "Y", + "yandex": "Y", + "bing": "Y", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + }, + { + "en_name": "emoji", + "id_name": "emj", + "zh_name": "表情", + "google": "", + "yandex": "Y", + "bing": "", + "baidu": "", + "alibaba": "", + "tencent": "", + "youdao": "", + "sogou": "", + "deepl": "", + "caiyun": "", + "argos": "" + } +] \ No newline at end of file diff --git a/009-Translate/asset/logo.png b/009-Translate/asset/logo.png new file mode 100644 index 0000000..995df14 Binary files /dev/null and b/009-Translate/asset/logo.png differ diff --git a/009-Translate/config.py b/009-Translate/config.py new file mode 100644 index 0000000..d114aa7 --- /dev/null +++ b/009-Translate/config.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +""" +@Author : Xhunmon +@Time : 2023年4月23日 08:51 +@FileName: config.py +@desc: +""" +import configparser +import json +import locale +import os + +from utils import * + +__version__ = '1.0.0' +__email__ = 'xhunmon@126.com' +__wechet__ = 'VABlog' + +language, encoding = locale.getdefaultlocale() + + +def is_zh_language(): + cache = get_cache(Key.LANGUAGE, None) + # if cache == 'zh_CN' or language == 'zh_CN': + # return True + # else: + # return False + return True + + +class IniConfig(object): + def __init__(self): + asset_path = os.path.join(os.path.dirname(__file__), 'asset') + language_path = os.path.join(asset_path, 'language.json') + config_path = os.path.join(asset_path, 'config.ini') + ch_ini = os.path.join(asset_path, 'ch.ini') + en_ini = os.path.join(asset_path, 'en.ini') + ini = ch_ini if is_zh_language() else en_ini + self.language = configparser.ConfigParser() + self.language.read(ini, encoding='utf-8') + self.cfg = configparser.ConfigParser() + self.cfg.read(config_path, encoding='utf-8') + with open(language_path, 'r') as f: + self.trl = json.load(f) + f.close() + + def full(self, tag, name): + return self.language[tag][name] + + def main(self, name): + return self.full('Main', name) + + def settings(self, name): + return self.full('Settings', name) + + def loading(self, name): + return self.full('Loading', name) + + def language(self, name): + return self.full('Language', name) + + def config(self, name): + return self.cfg['Config'][name] + + def translate(self): + return self.trl + + +conf = IniConfig() diff --git a/009-Translate/core.py b/009-Translate/core.py new file mode 100644 index 0000000..8062d9d --- /dev/null +++ b/009-Translate/core.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +""" +@Author : Xhunmon +@Time : 2023年4月22日 22:29 +@FileName: core.py +@desc: +""" +import translators as ts + +from config import * +from ui import LoadingWin + + +class Language(object): + + def __init__(self, id_name, full_name, zh_name=None, google=True, yandex=True, bing=True, baidu=True, alibaba=True, + tencent=True, youdao=True, sogou=True, deepl=True, caiyun=True, argos=True): + self.id_name = id_name + self.zh_name = zh_name if zh_name else full_name + self.full_name = full_name + self.google = google + self.yandex = yandex + self.bing = bing + self.baidu = baidu + self.alibaba = alibaba + self.tencent = tencent + self.youdao = youdao + self.sogou = sogou + self.deepl = deepl + self.caiyun = caiyun + self.argos = argos + + def is_enable(self, translator: str): + return eval('self.' + translator) + + +class Translation(object): + def __init__(self): + self.is_full = get_cache(Key.FULL_TRANSLATE, False) # 显示所有翻译平台和语言,但是未校验 + self.is_zh = is_zh_language() # 是否是中文 + self.select_translator = 'baidu' + self.select_from_lang = '自动' if self.is_zh else 'auto' + self.select_to_lang = '英语' if self.is_zh else 'english' + self.languages = conf.translate() + # self.languages.append(Language('en', 'english', '英语')) + # self.languages.append(Language('zh', 'chinese', '中文')) + # self.languages.append(Language('ar', 'arabic', '阿拉伯语', deepl=False, caiyun=False)) + # self.languages.append(Language('ru', 'russian', '俄语')) + # self.languages.append(Language('fr', 'french', '法语')) + # self.languages.append(Language('de', 'german', '德语', alibaba=False, caiyun=False)) + # self.languages.append(Language('es', 'spanish', '西班牙语')) + # self.languages.append(Language('pt', 'portuguese', '葡萄牙语', caiyun=False)) + # self.languages.append(Language('it', 'italian', '意大利语', caiyun=False)) + # self.languages.append(Language('ja', 'japanese', '日本语', alibaba=False)) + # self.languages.append(Language('ko', 'korean', '朝鲜语', alibaba=False, deepl=False, caiyun=False)) + # self.languages.append( + # Language('el', 'greek', '希腊语', alibaba=False, tencent=False, youdao=False, caiyun=False, argos=False)) + + def set_to_lang(self, lang): + if lang: + self.select_to_lang = lang + + def set_from_lang(self, lang): + if lang: + self.select_from_lang = lang + + def check_select_language(self): + languages = self.get_languages() + has_from = False + has_to = False + for lg in languages: # lg为full_name + if lg == self.select_from_lang: + has_from = True + if lg == self.select_to_lang: + has_to = True + if not has_from: + self.select_from_lang = '自动' if self.is_zh else 'auto' + if not has_to: + self.select_to_lang = '英语' if self.is_zh else 'english' + + def get_translators(self): + # 'google', 'yandex', 'bing', 'baidu', 'alibaba', 'tencent', 'youdao', 'sogou', 'deepl', 'caiyun', 'argos', + # 'apertium', 'cloudYi', 'elia', 'iciba', 'iflytek', 'iflyrec', 'itranslate', 'judic', 'languageWire', + # 'lingvanex', 'niutrans', 'mglip', 'modernMt', 'myMemory', 'papago', 'qqFanyi', 'qqTranSmart', 'reverso', + # 'sysTran', 'tilde', 'translateCom', 'translateMe', 'utibet', 'volcEngine', 'yeekit' + # {"en_name": "Chinese(简体)", "id_name": "zh-CHS", "zh_name": "简体", "google": "zh-CN", "yandex": "zh", "bing": "zh-Hans", "baidu": "zh", "alibaba": "zh", "tencent": "zh", "youdao": "Y", "sogou": "Y", "deepl": "zh", "caiyun": "zh", "argos": "zh"} + tors = list(self.languages[0].keys()) + tors.remove('en_name') + tors.remove('id_name') + tors.remove('zh_name') + return tors + + def get_languages(self): + support = [] + for lg in self.languages: # 取出字典 + if lg[self.select_translator] == '': # 不支持 + continue + if self.is_zh: + support.append(lg['zh_name']) + else: + support.append(lg['en_name']) + return support + + def choose_translator(self, tl): + self.select_translator = tl + + def _search_id_name(self, is_from=True): + key = self.select_from_lang if is_from else self.select_to_lang + for lg in self.languages: # 取出字典 + if self.is_zh: # 查找出 zh_name 对应的 + if lg['zh_name'] == key: + if lg[self.select_translator] == '': + continue + elif lg[self.select_translator] == 'Y': + return lg['id_name'] + else: + return lg[self.select_translator] + else: + if lg['en_name'] == key: + if lg[self.select_translator] == '': + continue + elif lg[self.select_translator] == 'Y': + return lg['id_name'] + else: + return lg[self.select_translator] + return None + + def get_id_name(self, is_from=True): + if is_from: + from_key = self.select_from_lang + if from_key == '自动' or from_key == 'auto': + return 'auto' + search = self._search_id_name(True) + return search if search else 'auto' + else: # to_ 目标 + search = self._search_id_name(False) + return search if search else 'en' # 最后还是没有,默认英语 + + def translate(self, window, content, is_html=False, file_path: str = None): + try: + if file_path and file_path.endswith('.srt'): + from load_srt import Translator + import utils, os + filename, ext = os.path.splitext(file_path) + out_path = f'{filename}_output{ext}' + tl = Translator() + proxy = get_cache(Key.PROXY_INPUT, None) if get_cache(Key.PROXY_ENABLE, False) else None + if proxy: + proxy_real = proxy.replace(' ', '').replace('\n', '') + proxy_spit = proxy_real.split('://') + proxy_user = {proxy_spit[0]: proxy_real} + tl.proxy_user = proxy_user + else: + tl.proxy_user = None + tl.translators = {self.select_translator: 3} + tl.translate_file(file_path, out_path, self.get_id_name(is_from=True), self.get_id_name(is_from=False)) + window['OUT_TEXT'].update(f'输出文件:{out_path}') + else: + # ts.preaccelerate() + # 是否使用了代理 export https_proxy=http://127.0.0.1:7890 http_proxy=http://127.0.0.1:7890 all_proxy=socks5://127.0.0.1:7890 + proxy = get_cache(Key.PROXY_INPUT, None) if get_cache(Key.PROXY_ENABLE, False) else None + proxy_user = None + if proxy: + proxy_real = proxy.replace(' ', '').replace('\n', '') + proxy_spit = proxy_real.split('://') + proxy_user = {proxy_spit[0]: proxy_real} + if is_html: + # result = ts.translate_html(content, translator=self.select_translator, + # from_language=self.get_id_name(is_from=True), + # to_language=self.get_id_name(is_from=False), proxies=proxy_user) + result = ts.translate_html(content, translator=self.select_translator, + from_language=self.get_id_name(is_from=True), + to_language=self.get_id_name(is_from=False), proxies=proxy_user, + if_ignore_empty_query=True, if_show_time_stat=True) + else: + result = ts.translate_text(content, translator=self.select_translator, + from_language=self.get_id_name(is_from=True), + to_language=self.get_id_name(is_from=False), proxies=proxy_user) + if file_path is not None: + import utils, os + filename, ext = os.path.splitext(file_path) + out_path = f'{filename}_output{ext}' + utils.write(result, out_path) + window['OUT_TEXT'].update(f'输出文件:{out_path}') + else: + window['OUT_TEXT'].update(result) + except Exception as e: + result = str(e) + LoadingWin.is_loading = False diff --git a/009-Translate/doc/pyinstaller.sh b/009-Translate/doc/pyinstaller.sh new file mode 100644 index 0000000..6f8a67e --- /dev/null +++ b/009-Translate/doc/pyinstaller.sh @@ -0,0 +1,13 @@ +#!/bin/bash + + +#pyinstaller --windowed --name GPT-UI --add-data "config.ini:." --icon logo.ico main.py gpt.py utils.py + +pyinstaller --windowed --name Super86翻译 --add-data "asset/config.ini:asset" --add-data "asset/ch.ini:asset" --add-data "asset/en.ini:asset" --add-data "asset/language.json:asset" --icon asset/logo.png main.py config.py core.py utils.py ui.py load_srt.py + +#https://blog.csdn.net/COCO56/article/details/117452383 +#if use --onefile, the build file is small, but star very slow. +#pyinstaller --onefile --windowed --name GPT-UI --add-data "config.ini:." --icon logo.ico main.py gpt.py utils.py.py + + +pyinstaller --windowed --name Super86翻译 --add-data "asset/config.ini:asset" --add-data "asset/ch.ini:asset" --add-data "asset/en.ini:asset" --add-data "asset/language.json:asset" --icon asset/logo.png main.py config.py core.py utils.py ui.py load_srt.py -p package --paths /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages \ No newline at end of file diff --git a/009-Translate/load_srt.py b/009-Translate/load_srt.py new file mode 100644 index 0000000..61929d8 --- /dev/null +++ b/009-Translate/load_srt.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +""" +@Author : Xhunmon +@Time : 2023年4月18日 14:47 +@FileName: load_srt.py +@desc: 从本地加载srt字幕文件 +""" +import os +import threading +import time + + +class Tag: + def __init__(self): + self.num = None + self.duration = None + self.msg = '' + + +class Translator(object): + STATUS_SUCCESS = 1 + STATUS_FAIL = -1 + + def __init__(self): + # TODO --> 一定要对应自己的代理,可以为None + self.proxy_user = {"socks5": "socks5://127.0.0.1:7890"} + # 翻译平台,deepl尝试3次,失败后,用google尝试2次,再失败后,用百度尝试3次,当所有尝试都失败了,表示该次翻译失败 + self.translators = {"deepl": 3, "google": 2, "baidu": 3} + # 是否支持翻译 + self.__supports = ['.srt', '.txt'] + # 需翻译的语音,可以为自动:auto + self.__from_language = 'zh' + # 目标语言 + self.__to_language = 'en' + # 一次性翻译的字符串数组长度 + self.__one_length = 1024 + # 某些场景需要休眠时间,再进行尝试 + self.__user_sleep = 2 + # 监听的事件 + self.__listener = None + # 需要被翻译的文件路径 + self.__list = [] + + def __callback(self, status, src, dst, msg=None): + """ + 翻译结果回调 + :param status: 1成功,2失败 + :param src: 返回需要翻译的路径 + :param dst: 返回翻译后的文件路径 + :param msg: 返回提示信息 + """ + if self.__listener: + self.__listener(status, src=src, dst=dst, msg=msg) + + def __support_file(self, src: str): + """ + 判断源文件是否支持被翻译 + :param src: 文件路径 + :return: + """ + for item in self.__supports: + if src.endswith(item): + return True + return False + + def __deal_file(self): + """ + 从队列取出文件,进行相关判断和操作 + :return: (是否成功,源文件路径,目标文件路径,操作信息,临时文件路径) + """ + item = self.__list.pop(0) + src, dst = item[0], item[1] + temp = None + if not os.path.exists(src): + return False, src, dst, "源文件不存在", temp + if not self.__support_file(src): + return False, src, dst, "源文件格式不支持", temp + # try: + # folder = os.path.dirname(src) + # filename, ext = os.path.splitext(src) + # if dst is None: + # dst = os.path.join(folder, '{}_out{}'.format(filename, ext)) + # else: + # folder = os.path.dirname(dst) + # if not os.path.exists(folder): + # os.mkdir(folder) + # temp = os.path.join(folder, '{}_temp{}'.format(filename, ext)) + # except Exception as e: + # return False, src, dst, str(e), temp + if dst is None or dst == '': + folder = os.path.dirname(src) + filename, ext = os.path.splitext(src) + dst = os.path.join(folder, '{}_out{}'.format(filename, ext)) + if os.path.exists(dst): + os.remove(dst) + return True, src, dst, '成功', temp + + def __parse_srt(self, src: str): + """ + 获取srt文件内容,一行一行的 + :param src: 路径 + :return: [Line,Line...] + """ + i = 0 + # 遇到空一行方为一组 + tags = [] + tag = Tag() + with open(src, 'r', encoding="utf-8") as f: + for line in f: + line = line.strip().replace('\n', '') + if line == '': # 结束了 + tags.append(tag) + i = 0 + tag = Tag() + continue + if i == 0: + tag.num = line + elif i == 1: + tag.duration = line + else: + tag.msg += line + i += 1 + return tags + + def __parse_file(self, src: str): + """ + 一行一行获取文件内容 + :param src: 文件路径 + :return: 返回[Line, Line...] + """ + if src.endswith('.srt'): + return self.__parse_srt(src) + return [] + + def __merge_content(self, tags): + """ + 合并需要翻译的内容,为了避免请求其次太多,将整个文件的内容进行分组翻译 + :param tags: + :return: + """ + result = [] + item = '' + for tag in tags: + if item == '': + item = tag.msg + else: + item = item + '\n' + tag.msg + if len(item)> self.__one_length: # 开启下一组 + result.append(item) + item = '' + if item != '': # 最后一组数据 + result.append(item) + return result + + def __request_item(self, content): + """ + 真正的请求网络进行翻译 + :param content: 翻译内容 + :return: 是否成功,翻译后的内容 + """ + for key, value in self.translators.items(): + for i in range(1, value + 1): # 每个失败后尝试的次数 + try: + print('使用 {} 翻译,进行次数{}'.format(key, i)) + import translators as ts + item = ts.translate_text(content, translator=key, from_language=self.__from_language, + to_language=self.__to_language, proxies=self.proxy_user) + return True, item + except Exception as e: + print(e) + time.sleep(self.__user_sleep) + + return False, '翻译失败' + + def __request_list(self, contents): + """ + 拆分列表中的数据 + :param contents: + :return: + """ + results = [] + for content in contents: + success, item = self.__request_item(content) + if not success: + return False, results + for x in item.split('\n'): + results.append(x) + return True, results + + def __merge_file(self, tags, items, dst): + with open(dst, 'w', encoding="utf-8") as f: + for tag in tags: + f.write(f'{tag.num}\n') + f.write(f'{tag.duration}\n') + f.write(f'{items.pop(0)}\n') + f.write('\n') + + def __translate(self): + """ + 子线程不断监听翻译文件,进行翻译 + """ + success, src, dst, msg, temp = self.__deal_file() + if not success: + self.__callback(Translator.STATUS_FAIL, src=src, dst=dst, msg=msg) + return + # 解析得到一行行数据 + tags = self.__parse_file(src) + if len(tags) <= 0: + self.__callback(Translator.STATUS_FAIL, src="src," dst=dst, msg="解析文件内容失败") + return + # 将一行行待翻译的文件进行合并,减少翻译次数 + contents = self.__merge_content(tags) + success, results = self.__request_list(contents) + if not success: + # # 缓存已翻译的数据 + # if len(result)> 0: + # self.__merge_file(tags, result, temp) + self.__callback(Translator.STATUS_FAIL, src=src, dst=dst, msg='翻译失败') + return + self.__merge_file(tags, results, dst) + self.__callback(Translator.STATUS_SUCCESS, src=src, dst=dst, msg="成功") + + def add_callback(self, listener): + """ + 添加监听器, + :param listener: 监听器设计模式如:method_listener(status,**kwargs) + :return: + """ + self.__listener = listener + + def translate_file(self, src, dst=None, from_lang='zh', to_lang='en'): + """ + 对外只需要知道传入的文件即可,其余全部在本翻译器处理,较少参数传递 + @param dst: 必传,需要进行翻译的文件。 + @param src: 如果不传,自动根据src所在的目录生成同后缀名的文件 + @param from_lang: 从什么语言翻译 + @param to_lang: 翻译目标语言 + """ + self.__from_language = from_lang + self.__to_language = to_lang + item = (src, dst) + self.__list.append(item) + # self.__event.set() # 唤醒线程 + self.__translate() diff --git a/009-Translate/main.py b/009-Translate/main.py new file mode 100644 index 0000000..588c1cb --- /dev/null +++ b/009-Translate/main.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +""" +@Author : Xhunmon +@Time : 2023年4月21日 23:03 +@FileName: guiv2.py +@desc: +""" +import PySimpleGUI as sg + +from core import * +from ui import * +from ui import settings_show +from utils import * + + +class MainWin(object): + def __init__(self): + self.tl = Translation() + self.lw = LoadingWin() + + def advanced_ui(self, window): + # 高级才显示的UI + advanced_mode = get_cache(Key.ADVANCED_MODE, False) + window[Key.PROXY_ENABLE].update(visible=advanced_mode) + + def make_window(self): + """ + Creates the main window + :return: The main window object + :rtype: (sg.Window) + """ + sg.theme(get_theme()) + + left_col = sg.Column([ + [sg.Multiline(size=(50, 25), write_only=True, expand_x=True, expand_y=True, key='IN_TEXT', + reroute_stdout=True, + echo_stdout_stderr=True, reroute_cprint=True)], + [sg.B(conf.main(Key.M_RUN)), sg.B(conf.main(Key.M_CLEAR)), sg.B(conf.main(Key.M_COPY)), + sg.Input('', key="_FILEBROWSE_", enable_events=True, visible=False), + sg.FileBrowse(conf.main(Key.M_FILE), file_types=(('ALL files', '.*')), + target='_FILEBROWSE_')], + [sg.CB(conf.main('EnableProxy'), default=get_cache(Key.PROXY_ENABLE, False), font='_ 12', + enable_events=True, k=Key.PROXY_ENABLE)]], element_justification='l', expand_x=True, expand_y=True) + + right_col = [ + [sg.Multiline(size=(70, 25), write_only=True, expand_x=True, expand_y=True, key='OUT_TEXT', + reroute_stdout=True, + echo_stdout_stderr=True, reroute_cprint=True)], + [sg.B(conf.main(Key.M_SETTINGS)), sg.Button(conf.main(Key.M_EXIT))], + [sg.T(conf.main('Version') + conf.config('Version'))] + ] + operation_at_bottom = sg.pin(sg.Column([[sg.T(conf.main('Business'), font='Default 12', pad=(0, 0)), + sg.T(conf.main('Email') + conf.config('Email') + ' ', + font='Default 12', pad=(0, 0)), + sg.T(conf.main('RedBook') + conf.config('RedBook') + ' ', + font='Default 12', + pad=(0, 0))]], + pad=(0, 0), k='-OPTIONS BOTTOM-', expand_x=True, expand_y=False), + expand_x=True, expand_y=False) + self.tl.select_translator = get_cache('PLATFORM_TYPES', 'baidu') + self.tl.set_from_lang(get_cache('INPUT_TYPES')) + self.tl.set_to_lang(get_cache('OUTPUT_TYPES')) + self.tl.check_select_language() + lgs1 = self.tl.get_languages() + lgs1.insert(0, '自动') if is_zh_language() else lgs1.insert(0, 'auto') + choose_type_at_top = sg.pin( + # sg.user_settings_get_entry('INPUT_TYPES', lgs1) + sg.Column([[sg.Combo(values=lgs1, default_value=self.tl.select_from_lang, size=(50, 30), + key='IN_TYPE', enable_events=True, readonly=True), + # sg.Combo(sg.user_settings_get_entry('OUTPUT_TYPES', lgs2), + sg.Combo(values=self.tl.get_languages(), default_value=self.tl.select_to_lang, + size=(50, 30), key='OUT_TYPE', enable_events=True, readonly=True), + # sg.Combo(sg.user_settings_get_entry('PLATFORM_TYPES', tls), + sg.Combo(values=self.tl.get_translators(), default_value=self.tl.select_translator, + size=(20, 30), key='PLATFORM_TYPE', enable_events=True, readonly=True) + ]], pad=(0, 0), k='-FOLDER CHOOSE-')) + + # ----- Full layout ----- + + layout = [ + [sg.Text(conf.main('Description'), font='Any 15', pad=(0, 5))], + [choose_type_at_top], + [sg.Pane( + [sg.Column([[left_col]], element_justification='l', expand_x=True, expand_y=True), + sg.Column(right_col, element_justification='c', expand_x=True, expand_y=True)], orientation='h', + relief=sg.RELIEF_SUNKEN, expand_x=True, expand_y=True, k='-PANE-')], + [operation_at_bottom, sg.Sizegrip()]] + + # --------------------------------- Create Window --------------------------------- + window = sg.Window(conf.main('Title'), layout, finalize=True, resizable=True, use_default_focus=False) + window.set_min_size(window.size) + + window.bind('', 'Exit') + # window.bind("", 'Enter') + window['IN_TEXT'].bind('', ' Return') + self.advanced_ui(window) + + window.bring_to_front() + return window + + def show(self): + """ + The main program that contains the event loop. + It will call the make_window function to create the window. + """ + global python_only + # icon = sg.EMOJI_BASE64_HAPPY_WINK + icon = conf.config('Logo').encode('utf-8') + # icon = os.path.join(os.path.dirname(__file__), 'doc', 'logo.ico') + # sg.user_settings_filename('psgdemos.json') + sg.set_options(icon=icon) + window = self.make_window() + window.force_focus() + counter = 0 + + while True: + event, values = window.read() + # print(event, values) + + counter += 1 + if event in (sg.WINDOW_CLOSED, conf.main(Key.M_EXIT)): + break + elif event == conf.main(Key.M_SETTINGS): + change, restart = settings_show() + if change: # settings 可能更改的内容:主题,代理,高级 + if restart: + window.close() + window = self.make_window() + else: + self.advanced_ui(window) + + elif event == conf.main(Key.M_RUN) or event == 'IN_TEXT Return': # IN_TEXT 的回车键监听, 需要从input获取到数据,然后进行翻译 + sg.threading.Thread(target=self.tl.translate, args=(window, window['IN_TEXT'].get(), False), + daemon=True).start() + loading_show() + # result = ts.translate_text(text, translator='deepl', from_language='zh', to_language='en') + # window['OUT_TEXT'].update(result) + # pw.close() + elif event == conf.main(Key.M_CLEAR): + window['IN_TEXT'].update('') + window['OUT_TEXT'].update('') + elif event == conf.main(Key.M_COPY): + sg.clipboard_set(window['OUT_TEXT'].get()) + elif event == '_FILEBROWSE_': + file_path: str = values['_FILEBROWSE_'] + # 不判断了,能解析就用 + # if file_path.endswith('.txt') or file_path.endswith('.html'): + content = read(file_path) + window['IN_TEXT'].update(file_path) + sg.threading.Thread(target=self.tl.translate, + args=(window, content, file_path.endswith('.html'), file_path), daemon=True).start() + loading_show() + elif event == 'IN_TYPE': # 输入 + type1 = values['IN_TYPE'] + self.tl.select_from_lang = type1 + get_cache('INPUT_TYPES', type1) + elif event == 'OUT_TYPE': # 输出 + type1 = values['OUT_TYPE'] + self.tl.select_to_lang = type1 + save_cache('OUTPUT_TYPES', type1) + elif event == 'PLATFORM_TYPE': # 选择平台后,更新输入输出的可选 + type1 = values['PLATFORM_TYPE'] + self.tl.select_translator = type1 + save_cache('PLATFORM_TYPES', type1) + window.close() + window = self.make_window() + elif event == 'Version': + sg.popup_scrolled(sg.get_versions(), keep_on_top=True, non_blocking=True) + elif event == Key.PROXY_ENABLE: # 高级时在本窗口开启或关闭代理,但是需要在设置中设置代理地址 + proxy_enable = values[Key.PROXY_ENABLE] + save_cache(Key.PROXY_ENABLE, proxy_enable) + window[Key.PROXY_ENABLE].update(value=proxy_enable) + window.close() + + +if __name__ == '__main__': + MainWin().show() diff --git a/009-Translate/tran_test.py b/009-Translate/tran_test.py new file mode 100644 index 0000000..308f7ac --- /dev/null +++ b/009-Translate/tran_test.py @@ -0,0 +1,35 @@ +#!/usr/bin/python3.9 +# -*- coding: utf-8 -*- +""" +@Time : 2023年5月18日 14:38 +@Author : xhunmon +@Email : xhunmon@126.com +@File : tran_test.py +@Desc : +""" +import re + +from load_srt import Translator +import time +from utils import * + + +def t_test(srt_source): + sub_start = '00:02:00' + v_start = '00:00:17' + + +def translate_callback(status, **kwargs): + print(status) + print(kwargs["src"]) + print(kwargs["dst"]) + print(kwargs["msg"]) + + +if __name__ == '__main__': + tl = Translator() + tl.add_callback(translate_callback) + try: + tl.translate_file("output/test.srt", "output/test_out.srt") + except Exception as e: + print(e) diff --git a/009-Translate/ui.py b/009-Translate/ui.py new file mode 100644 index 0000000..8836302 --- /dev/null +++ b/009-Translate/ui.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +""" +@Author : Xhunmon +@Time : 2023年4月22日 22:31 +@FileName: ui.py +@desc: +""" +from config import * + + +def loading_show(): + LoadingWin.is_loading = True + layout = [[sg.Text(conf.loading('Content'), font='ANY 15')], + [sg.Image(data=conf.config('Loading').encode('utf-8'), key='_IMAGE_')], + [sg.Button(conf.loading('Cancel'))] + ] + window = sg.Window('').Layout(layout) + while LoadingWin.is_loading: # Event Loop + event, values = window.Read(timeout=25) + if event in (None, 'Exit', conf.loading('Cancel')): + break + window.Element('_IMAGE_').UpdateAnimation(conf.config('Loading').encode('utf-8'), time_between_frames=50) + window.close() + + +class LoadingWin(object): + """ + loading dialog + """ + is_loading = True + + +def settings_show(): + """ + Show the settings window. + This is where the folder paths and program paths are set. + Returns True if settings were changed + + :return: True if settings were changed + :rtype: (bool) + """ + global_theme = get_theme() + proxy_enable = get_cache(Key.PROXY_ENABLE, False) + translate_enable = get_cache(Key.FULL_TRANSLATE, False) + restart_enable = get_cache(Key.RESTART_WINDOW, True) + + layout = [ + [sg.T(' ', font='_ 16', size=(40, 1))], + [sg.T(conf.settings('Proxy'), font='_ 16')], + [sg.CB(conf.settings('ProxyEnable'), default=proxy_enable, enable_events=True, k=Key.PROXY_ENABLE, + font='_ 12')], + [sg.Column([[sg.Input(size=(38, 1), default_text=get_cache(Key.PROXY_INPUT, ''), key=Key.PROXY_INPUT), + sg.T(conf.settings('ProxyDesc'), font='_ 12')]], + expand_x=True, expand_y=True, k=Key.PROXY_LAYOUT, visible=proxy_enable)], + [sg.T(conf.settings('Theme'), font='_ 16')], + [sg.T(conf.settings('ThemeDesc'), font='_ 11'), sg.T(global_theme, font='_ 13')], + [sg.Combo([''] + sg.theme_list(), get_cache(Key.THEME, ''), readonly=True, k=Key.THEME)], + [sg.CB(conf.settings('Advanced'), default=get_cache(Key.ADVANCED_MODE, True), font='_ 12', + k=Key.ADVANCED_MODE)], + [sg.CB(conf.settings('FullTranslate'), visible=False, default=translate_enable, font='_ 12', + k=Key.ADVANCED_MODE)], + [sg.CB(conf.settings('Restart'), default=restart_enable, enable_events=True, k=Key.RESTART_WINDOW, + font='_ 12')], + [sg.B(conf.settings('Ok'), bind_return_key=True), sg.B(conf.settings('Cancel')), sg.B(conf.settings('Reset'))], + ] + + window = sg.Window(conf.settings('Title'), layout, finalize=True) + settings_changed = False + + while True: + event, values = window.read() + if event in (conf.settings('Cancel'), sg.WIN_CLOSED): + break + if event == conf.settings('Ok'): + save_cache(Key.THEME, values[Key.THEME]) + save_cache(Key.ADVANCED_MODE, values[Key.ADVANCED_MODE]) + save_cache(Key.PROXY_ENABLE, proxy_enable) + save_cache(Key.FULL_TRANSLATE, translate_enable) + save_cache(Key.PROXY_INPUT, window[Key.PROXY_INPUT].get()) + save_cache(Key.RESTART_WINDOW, window[Key.RESTART_WINDOW].get()) + settings_changed = True + break + elif event == conf.settings('Reset'): # 恢复所有默认设置 + save_cache(Key.THEME, '') + save_cache(Key.ADVANCED_MODE, False) + save_cache(Key.PROXY_ENABLE, False) + save_cache(Key.FULL_TRANSLATE, False) + save_cache(Key.RESTART_WINDOW, True) + save_cache(Key.PROXY_INPUT, '') + settings_changed = True + break + elif event == Key.PROXY_ENABLE: + proxy_enable = values[Key.PROXY_ENABLE] + window[Key.PROXY_ENABLE].update(value=proxy_enable) + window[Key.PROXY_LAYOUT].update(visible=proxy_enable) + elif event == Key.FULL_TRANSLATE: + translate_enable = values[Key.FULL_TRANSLATE] + window[Key.FULL_TRANSLATE].update(value=translate_enable) + elif event == Key.RESTART_WINDOW: + restart_enable = values[Key.RESTART_WINDOW] + window[Key.RESTART_WINDOW].update(value=restart_enable) + + window.close() + return settings_changed, restart_enable diff --git a/009-Translate/utils.py b/009-Translate/utils.py new file mode 100644 index 0000000..b95fc61 --- /dev/null +++ b/009-Translate/utils.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +""" +@Author : Xhunmon +@Time : 2023年4月22日 21:54 +@FileName: utils.py +@desc: +""" +import PySimpleGUI as sg + + +def get_cache(key: str, default=None): + """ + 获取本地的数据 + :param key: + :param default: + :return: + """ + cache = sg.user_settings_get_entry(key, default) + if cache is None or cache == '': + return default + return cache + + +def save_cache(key: str, value): + """ + 将数据保存到本地 + :param key: + :param value: + :return: + """ + sg.user_settings_set_entry(key, value) + + +def read(file_path) -> str: + '''读取txt文本内容''' + content = None + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + f.close() + except Exception as e: + print(e) + return content + + +def write(content, file_path): + '''写入txt文本内容''' + try: + with open(file_path, mode='w', encoding='utf-8') as f: + f.write(content) + except Exception as e: + print(e) + + +def get_theme(): + """ + Get the theme to use for the program + Value is in this program's user settings. If none set, then use PySimpleGUI's global default theme + :return: The theme + :rtype: str + """ + theme = get_cache(Key.THEME, '') + if theme == '': + theme = sg.OFFICIAL_PYSIMPLEGUI_THEME # 默认主题 + return theme + + +class Key: + """ + 统一管理本地字符串 + """ + PROXY_ENABLE = 'proxy_enable' # settings + PROXY_LAYOUT = 'proxy_layout' # settings + PROXY_INPUT = 'proxy_input' # settings + THEME = 'theme' # settings + RESTART_WINDOW = 'restart_window' # settings + ADVANCED_MODE = 'advanced_mode' # settings + FULL_TRANSLATE = 'full_translate' # settings + LANGUAGE = 'language' # config + + # main + M_CLEAR = 'Clear' + M_RUN = 'Run' + M_COPY = 'Copy' + M_FILE = 'File' + M_SETTINGS = 'Settings' + M_EXIT = 'Exit' diff --git a/010-YouTubeUpload/README.md b/010-YouTubeUpload/README.md new file mode 100644 index 0000000..b79711c --- /dev/null +++ b/010-YouTubeUpload/README.md @@ -0,0 +1,19 @@ +# YouTube uploader 自动上传 + +因为使用API上传是有限制的,每天只能上传6条。因此得使用浏览器操作 + +## 前提 + +1. Chome浏览器,同时需要配置内核,可参考以下文章 + +> selenium复用已打开浏览器:https://developer.aliyun.com/article/1121356 +> 下载chromedriver:https://chromedriver.storage.googleapis.com/index.html?path=112.0.5615.49/ +> 下载安装:https://blog.51cto.com/u_15295315/3042408 + + +2. 通过配置启动debug模式浏览器,然后登录 + + +## 备注 + +脚本目前还不完善,因此有bug,因比较忙,不保证能更新。 \ No newline at end of file diff --git a/010-YouTubeUpload/main.py b/010-YouTubeUpload/main.py new file mode 100644 index 0000000..fd470bb --- /dev/null +++ b/010-YouTubeUpload/main.py @@ -0,0 +1,45 @@ +#!/usr/bin/python3.9 +# -*- coding: utf-8 -*- +""" +@Time : 2023年5月22日 11:44 +@Author : xhunmon +@Email : xhunmon@126.com +@File : main.py +@Desc : 使用案例 +""" +import sys + +from selenium import webdriver +from selenium.webdriver.chrome.options import Options + +from youtube import * + +if __name__ == '__main__': + meta = { + "title": '标题', + "description": "描述内容", + "tags": ['标签1', '标签2', '标签3'], + "edit": False, + "playlist_title": '播放列表1', + "schedule": "" + } + + executable_path = "chromedriver" + options = Options() + # TODO - 以下配置是为了打开现有的浏览器,公用cookie,安全有效,需要提起配置。 + if sys.platform == 'linux': + print("Current OS is Linux.") + elif sys.platform == 'darwin': + print("Current OS is mac OS.") + executable_path = '/usr/local/bin/chromedriver' + options.debugger_address = 'localhost:9222' + elif sys.platform == 'win32': + print("Current OS is Windows.") + + __g_browser = webdriver.Chrome(executable_path=executable_path, chrome_options=options) + __g_browser.implicitly_wait(10) # 设置查找运算智能等待超时时间 + + yt = YtbUploader() + yt.upload(__g_browser, 'xxx/xxx.mp4', options, None) + +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/010-YouTubeUpload/tst.py b/010-YouTubeUpload/tst.py new file mode 100644 index 0000000..c5bc124 --- /dev/null +++ b/010-YouTubeUpload/tst.py @@ -0,0 +1,9 @@ +#!/usr/bin/python3.9 +# -*- coding: utf-8 -*- +""" +@Time : 2023年5月22日 11:44 +@Author : xhunmon +@Email : xhunmon@126.com +@File : tst.py +@Desc : +""" diff --git a/010-YouTubeUpload/youtube.py b/010-YouTubeUpload/youtube.py new file mode 100644 index 0000000..b439a84 --- /dev/null +++ b/010-YouTubeUpload/youtube.py @@ -0,0 +1,327 @@ +#!/usr/bin/python3.9 +# -*- coding: utf-8 -*- +""" +@Time : 2023年5月9日 09:10 +@Author : xhunmon +@Email : xhunmon@126.com +@File : youtube.py +@Desc : +1. 关闭所有浏览器: +2. 命令格式:浏览器名称 --remote-debugging-port=端口号 +例: +windows:chrome --remote-debugging-port=9222 +mac:/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --remote-debugging-port=9222 +""" + +import logging +import platform +import time +from datetime import datetime +from typing import Optional, Tuple + +from selenium.webdriver import Chrome +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys + +logging.basicConfig() + + +class YtbConst: + """A class for storing constants for YoutubeUploader class""" + YOUTUBE_URL = 'https://www.youtube.com' + YOUTUBE_STUDIO_URL = 'https://studio.youtube.com' + YOUTUBE_UPLOAD_URL = 'https://www.youtube.com/upload' + USER_WAITING_TIME = 1 + VIDEO_TITLE = 'title' + VIDEO_DESCRIPTION = 'description' + VIDEO_EDIT = 'edit' + VIDEO_TAGS = 'tags' + TEXTBOX_ID = 'textbox' + TEXT_INPUT = 'text-input' + RADIO_LABEL = 'radioLabel' + UPLOADING_STATUS_CONTAINER = '//*[@id="dialog"]/div[2]/div/ytcp-video-upload-progress/span' + NOT_MADE_FOR_KIDS_LABEL = 'VIDEO_MADE_FOR_KIDS_NOT_MFK' + + UPLOAD_DIALOG = '//ytcp-uploads-dialog' + ADVANCED_BUTTON_ID = 'toggle-button' + TAGS_CONTAINER_ID = 'tags-container' + + TAGS_INPUT = '/html/body/ytcp-uploads-dialog/tp-yt-paper-dialog/div/ytcp-animatable[1]/ytcp-ve/ytcp-video-metadata-editor/div/ytcp-video-metadata-editor-advanced/div[4]/ytcp-form-input-container/div[1]/div/ytcp-free-text-chip-bar/ytcp-chip-bar/div/input' + NEXT_BUTTON = '//*[@id="next-button"]/div' + PUBLIC_BUTTON = '//*[@id="done-button"]/div' + VIDEO_URL_CONTAINER = "//span[@class='video-url-fadeable style-scope ytcp-video-info']" + VIDEO_URL_ELEMENT = '//*[@id="share-url"]' + HREF = 'href' + ERROR_CONTAINER = '//*[@id="error-message"]' + VIDEO_NOT_FOUND_ERROR = 'Could not find video_id' + DONE_BUTTON = 'done-button' + INPUT_FILE_VIDEO = "//input[@type='file']" + INPUT_FILE_THUMBNAIL = "//input[@id='file-loader']" + + # Playlist + VIDEO_PLAYLIST = 'playlist_title' + PL_DROPDOWN_CLASS = 'ytcp-video-metadata-playlists' + PL_SEARCH_INPUT_ID = 'search-input' + PL_ITEMS_CONTAINER_ID = 'items' + PL_ITEM_CONTAINER = '//span[text()="{}"]' + PL_NEW_BUTTON_CLASS = 'new-playlist-button' + PL_CREATE_PLAYLIST_CONTAINER_XPATH = '//*[@id="text-item-0"]/ytcp-ve' + PL_CREATE_BUTTON_XPATH = '//*[@id="create-button"]/div' + PL_DONE_BUTTON_CLASS = 'done-button' + + # Schedule 发布时间 + VIDEO_SCHEDULE = 'schedule' + SCHEDULE_CONTAINER_ID = 'schedule-radio-button' + SCHEDULE_DATE_ID = 'datepicker-trigger' + SCHEDULE_DATE_TEXTBOX = '/html/body/ytcp-date-picker/tp-yt-paper-dialog/div/form/tp-yt-paper-input/tp-yt-paper-input-container/div[2]/div/iron-input/input' + SCHEDULE_TIME = "/html/body/ytcp-uploads-dialog/tp-yt-paper-dialog/div/ytcp-animatable[1]/ytcp-uploads-review/div[2]/div[1]/ytcp-video-visibility-select/div[3]/ytcp-visibility-scheduler/div[1]/ytcp-datetime-picker/div/div[2]/form/ytcp-form-input-container/div[1]/div/tp-yt-paper-input/tp-yt-paper-input-container/div[2]/div/iron-input/input" + + +class YtbUploader: + """ + YouTube上传 + """ + + def __init__(self) -> None: + """ + + @param video_path: 视频路径 + @param metadata_json: 字典数据 + meta = { + "title": "标题", + "description": "描述", + "tags": ["标签1","标签2"], + "edit": False, + "playlist_title": "分栏标题", + "schedule": "" + } + @param new_window: 在现有的的浏览器中打开新的页面,没做登录。需要配置先启动浏览器。 + @param thumbnail_path: + """ + self.video_path = None + self.thumbnail_path = None + self.metadata_dict = None + self.browser: Chrome = None + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.DEBUG) + + self.is_mac = False + if not any(os_name in platform.platform() for os_name in ["Windows", "Linux"]): + self.is_mac = True + + def __login(self): + self.browser.get(YtbConst.YOUTUBE_URL) + time.sleep(YtbConst.USER_WAITING_TIME) + + def __clear_field(self, field): + field.click() + time.sleep(YtbConst.USER_WAITING_TIME) + if self.is_mac: + field.send_keys(Keys.COMMAND + 'a') + else: + field.send_keys(Keys.CONTROL + 'a') + time.sleep(YtbConst.USER_WAITING_TIME) + field.send_keys(Keys.BACKSPACE) + + def __write_in_field(self, field, string, select_all=False): + if select_all: + self.__clear_field(field) + else: + field.click() + time.sleep(YtbConst.USER_WAITING_TIME) + + field.send_keys(string) + + def __get_video_id(self) -> Optional[str]: + video_id = None + try: + video_url_element = self.browser.find_element(By.XPATH, YtbConst.VIDEO_URL_ELEMENT) + video_id = video_url_element.get_attribute( + YtbConst.HREF).split('/')[-1] + except: + self.logger.warning(YtbConst.VIDEO_NOT_FOUND_ERROR) + pass + return video_id + + def upload(self, browser, path, meta, thumbnail=None): + self.browser = browser + self.video_path = path + self.thumbnail_path = thumbnail + self.metadata_dict = meta + # self.__login() + return self.__upload() + + def exit(self): + self.browser.close() + + def __wait_loading(self): + count = 1 + while count <= 10: + time.sleep(YtbConst.USER_WAITING_TIME) + try: + uploading_status_container = self.browser.find_element(By.XPATH, + YtbConst.UPLOADING_STATUS_CONTAINER) + uploading_progress = uploading_status_container.text + self.logger.debug('Upload video progress: {}'.format(uploading_progress)) + if uploading_progress is None or len(uploading_progress) < 5: + break + except: + self.logger.debug('Upload loading trying {}'.format(count)) + count += 1 + + def __upload(self) -> Tuple[bool, Optional[str]]: + edit_mode = self.metadata_dict[YtbConst.VIDEO_EDIT] + if edit_mode: + self.browser.get(edit_mode) + time.sleep(YtbConst.USER_WAITING_TIME) + else: + # 切换到最新的开的页面,然后在该页面直接加载地址 + self.browser.switch_to.window(self.browser.window_handles[0]) + self.browser.get(YtbConst.YOUTUBE_URL) + time.sleep(YtbConst.USER_WAITING_TIME) + self.browser.get(YtbConst.YOUTUBE_UPLOAD_URL) + time.sleep(YtbConst.USER_WAITING_TIME) + absolute_video_path = self.video_path + self.browser.find_element(By.XPATH, YtbConst.INPUT_FILE_VIDEO).send_keys( + absolute_video_path) + self.logger.debug('Attached video {}'.format(self.video_path)) + + # Find status container + # self.__wait_loading() + time.sleep(YtbConst.USER_WAITING_TIME * 2) + + if self.thumbnail_path is not None: + absolute_thumbnail_path = self.thumbnail_path + self.browser.find_element(By.XPATH, YtbConst.INPUT_FILE_THUMBNAIL).send_keys( + absolute_thumbnail_path) + change_display = "document.getElementById('file-loader').style = 'display: block! important'" + self.browser.execute_script(change_display) + self.logger.debug( + 'Attached thumbnail {}'.format(self.thumbnail_path)) + + textboxs = self.browser.find_elements(By.ID, YtbConst.TEXTBOX_ID) + title_field, description_field = textboxs[0], textboxs[1] + # //*[@id="textbox"] + self.__write_in_field( + title_field, self.metadata_dict[YtbConst.VIDEO_TITLE], select_all=True) + self.logger.debug('The video title was set to \"{}\"'.format( + self.metadata_dict[YtbConst.VIDEO_TITLE])) + + video_description = self.metadata_dict[YtbConst.VIDEO_DESCRIPTION] + video_description = video_description.replace("\n", Keys.ENTER) + if video_description: + self.__write_in_field(description_field, video_description, select_all=True) + self.logger.debug('Description filled.') + + kids_section = self.browser.find_element(By.NAME, YtbConst.NOT_MADE_FOR_KIDS_LABEL) + kids_section.location_once_scrolled_into_view + time.sleep(YtbConst.USER_WAITING_TIME) + + self.browser.find_element(By.ID, YtbConst.RADIO_LABEL).click() + self.logger.debug('Selected \"{}\"'.format(YtbConst.NOT_MADE_FOR_KIDS_LABEL)) + + # Playlist + playlist = self.metadata_dict[YtbConst.VIDEO_PLAYLIST] + if playlist: + self.browser.find_element(By.CLASS_NAME, YtbConst.PL_DROPDOWN_CLASS).click() # 点击播放列表 + time.sleep(YtbConst.USER_WAITING_TIME) + self.logger.debug('Playlist xpath: "{}".'.format(YtbConst.PL_ITEM_CONTAINER.format(playlist))) + try: + playlist_item = self.browser.find_element(By.XPATH, YtbConst.PL_ITEM_CONTAINER.format(playlist)) + except: + playlist_item = None + if playlist_item: + self.logger.debug('Playlist found.') + playlist_item.click() + time.sleep(YtbConst.USER_WAITING_TIME) + else: + self.logger.debug('Playlist not found. Creating') + # self.__clear_field(search_field) + time.sleep(YtbConst.USER_WAITING_TIME) + + new_playlist_button = self.browser.find_element(By.CLASS_NAME, YtbConst.PL_NEW_BUTTON_CLASS) + new_playlist_button.click() + + self.browser.find_element(By.XPATH, YtbConst.PL_CREATE_PLAYLIST_CONTAINER_XPATH).click() + playlist_title_textbox = self.browser.find_element(By.XPATH, + '/html/body/ytcp-playlist-creation-dialog/ytcp-dialog/tp-yt-paper-dialog/div[2]/div/ytcp-playlist-metadata-editor/div/div[1]/ytcp-social-suggestions-textbox/ytcp-form-input-container/div[1]/div[2]/div/ytcp-social-suggestion-input/div') + self.__write_in_field(playlist_title_textbox, playlist) + + # //*[@id="textbox"] + time.sleep(YtbConst.USER_WAITING_TIME) + # //*[@id="create-button"]/div //*[@id="dialog"]/div[3]/div/ytcp-button[1]/div + create_playlist_button = self.browser.find_element(By.XPATH, YtbConst.PL_CREATE_BUTTON_XPATH) + create_playlist_button.click() + time.sleep(YtbConst.USER_WAITING_TIME) + + time.sleep(YtbConst.USER_WAITING_TIME * 2) + done_button = self.browser.find_element(By.CLASS_NAME, YtbConst.PL_DONE_BUTTON_CLASS) + self.browser.execute_script("arguments[0].click();", done_button) # js注入实现点击 + # done_button.click() + + # Advanced options + time.sleep(YtbConst.USER_WAITING_TIME) + self.browser.find_element(By.ID, YtbConst.ADVANCED_BUTTON_ID).click() + self.logger.debug('Clicked MORE OPTIONS') + time.sleep(YtbConst.USER_WAITING_TIME) + + # Tags + tags = self.metadata_dict[YtbConst.VIDEO_TAGS] + if tags: + # tags_container = self.browser.find_element(By.ID, Constant.TAGS_CONTAINER_ID) + self.browser.find_element(By.XPATH, '//*[@id="toggle-button"]/div') # 展开 + tags_field = self.browser.find_element(By.XPATH, YtbConst.TAGS_INPUT) + self.__write_in_field(tags_field, ','.join(tags)) + self.logger.debug('The tags were set to \"{}\"'.format(tags)) + + self.browser.find_element(By.XPATH, YtbConst.NEXT_BUTTON).click() + self.logger.debug('Clicked {} one'.format(YtbConst.NEXT_BUTTON)) + time.sleep(YtbConst.USER_WAITING_TIME) + self.browser.find_element(By.XPATH, YtbConst.NEXT_BUTTON).click() + self.logger.debug('Clicked {} two'.format(YtbConst.NEXT_BUTTON)) + time.sleep(YtbConst.USER_WAITING_TIME) + self.browser.find_element(By.XPATH, YtbConst.NEXT_BUTTON).click() + self.logger.debug('Clicked {} three'.format(YtbConst.NEXT_BUTTON)) + time.sleep(YtbConst.USER_WAITING_TIME) + + schedule = self.metadata_dict[YtbConst.VIDEO_SCHEDULE] + if schedule: # 发布时间,暂不设置 + upload_time_object = datetime.strptime(schedule, "%m/%d/%Y, %H:%M") + self.browser.find_element(By.ID, YtbConst.SCHEDULE_CONTAINER_ID).click() + self.browser.find_element(By.ID, YtbConst.SCHEDULE_DATE_ID).click() + self.browser.find_element(By.XPATH, YtbConst.SCHEDULE_DATE_TEXTBOX).clear() + self.browser.find_element(By.XPATH, YtbConst.SCHEDULE_DATE_TEXTBOX).send_keys( + datetime.strftime(upload_time_object, "%b %e, %Y")) + self.browser.find_element(By.XPATH, YtbConst.SCHEDULE_DATE_TEXTBOX).send_keys(Keys.ENTER) + self.browser.find_element(By.XPATH, YtbConst.SCHEDULE_TIME).click() + self.browser.find_element(By.XPATH, YtbConst.SCHEDULE_TIME).clear() + self.browser.find_element(By.XPATH, YtbConst.SCHEDULE_TIME).send_keys( + datetime.strftime(upload_time_object, "%H:%M")) + self.browser.find_element(By.XPATH, YtbConst.SCHEDULE_TIME).send_keys(Keys.ENTER) + self.logger.debug(f"Scheduled the video for {schedule}") + else: + self.browser.find_element(By.XPATH, YtbConst.PUBLIC_BUTTON).click() + self.logger.debug('Made the video {}'.format(YtbConst.PUBLIC_BUTTON)) + + # Check status container and upload progress + self.__wait_loading() + self.logger.debug('Upload container gone.') + + video_id = self.__get_video_id() + + # done_button = self.browser.find_element(By.ID, Constant.DONE_BUTTON) + # + # # Catch such error as + # # "File is a duplicate of a video you have already uploaded" + # if done_button.get_attribute('aria-disabled') == 'true': + # error_message = self.browser.find_element(By.XPATH, Constant.ERROR_CONTAINER).text + # self.logger.error(error_message) + # return False, None + # + # done_button.click() + self.logger.debug( + "Published the video with video_id = {}".format(video_id)) + time.sleep(YtbConst.USER_WAITING_TIME) + # self.browser.get(Constant.YOUTUBE_URL) + return True, video_id diff --git a/LICENSE b/LICENSE index 261eeb9..d159169 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,339 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/README.md b/README.md index ca4a0be..8abd49d 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,29 @@ 学习python最好的方式就是通过不断的实践! 项目地址:[https://github.com/xhunmon/PythonIsTools](https://github.com/xhunmon/PythonIsTools) -加我[qincji]wx好友(请注明来意)进群一起学习一起进步吧! # 实践项目 -- [001-Downloader:抖音、快手、YouTube音视频下载器](./001-Downloader) ——已完成 +- [001-Downloader:抖音、快手音视频下载器](./001-Downloader) ——已完成 - [002-v2ray代理池:爬取vmess、ss、trojan协议节点,进行校验,自更新](./002-V2rayPool) ——已完成 -- 003-电商关键词listing爬取策略 ——进行中 - +- [003-Keywords:获取相关关键词以及其google趋势](./003-Keywords) ——已完成 - [004-EmailNotify:监听虚拟币变化,使用邮箱通知](./004-EmailNotify) ——已完成 +- [005-PaidSource:这些脚本你肯定会有用到的](./005-PaidSource) ——已完成 + +- [006-TikTok:App自动化](./006-TikTok) ——已完成 + +- [007-CutVideoAudio:自媒体运营之视频剪辑,新增V2版本命令](./007-CutVideoAudio) ——已完成 + +- [008-ChatGPT-UI:About A very useful ChatGPT 3.5/5 Tools with OpenAI API. 一个非常实用的聊天工具](https://github.com/xhunmon/iMedia) ——已完成 + +- [009-多平台,多语言,支持文本、文件、srt字幕文件翻译](./009-Translate) ——已完成 + +- [010-YouTube视频上传脚本](./010-YouTubeUpload) ——已完成 + ---------- ##### 声明:本项目仅用于学习交流,禁止任何商业用途,违者自承担相应法律责任!