+ 微信群 + 公众号 + B站 + 知乎 + CSDN + 头条 + 掘金 +
+ +## Python3 网络爬虫教程 2020 +| 文章 | 公众号 | 代码 | +| :------ | :--------: | :--------: | +| Python3 网络爬虫(一):初识网络爬虫之夜探老王家 | [公众号](https://mp.weixin.qq.com/s/1rcq9RQYuAuHFg1w1j8HXg "Python3 网络爬虫(一)") | no | +| Python3 网络爬虫(二):下载小说的正确姿势 | [公众号](https://mp.weixin.qq.com/s/5e2_r0QXUISVp9GdDsqbzg "Python3 网络爬虫(二)") | [Code](https://github.com/Jack-Cherish/python-spider/tree/master/2020/xbqg "Python3 网络爬虫(二)") | +| Python3 网络爬虫(三):漫画下载,动态加载、反爬虫这都不叫事!| [公众号](https://mp.weixin.qq.com/s/wyS-OP04K3Vs9arSelRlyA "Python3网络爬虫(三)") | [Code](https://github.com/Jack-Cherish/python-spider/tree/master/2020/dmzj "Python3 网络爬虫(三)") | +| Python3 网络爬虫(四):视频下载,那些事儿!| [公众号](https://mp.weixin.qq.com/s/_geNA6Dwo4kx25X7trJzlg "Python3 网络爬虫(四)") | [Code](https://github.com/Jack-Cherish/python-spider/tree/master/2020/zycjw "Python3 网络爬虫(四)") | +| Python3 网络爬虫(五):老板,需要特殊服务吗?| [公众号](https://mp.weixin.qq.com/s/PPTSnIHV71b-wB3oRiYnIA "Python3 网络爬虫(五)") | [Code](https://github.com/Jack-Cherish/python-spider/tree/master/2020/api "Python3 网络爬虫(五)") | +| Python3 网络爬虫(六):618,爱他/她,就清空他/她的购物车!| [公众号](https://mp.weixin.qq.com/s/lXXDfzyLVrf3f-aqJN1C3A "Python3 网络爬虫(六)") | [Code](https://github.com/Jack-Cherish/python-spider/tree/master/2020/taobao "Python3 网络爬虫(六)") | +| 宝藏B站UP主,视频弹幕尽收囊中!| [公众号](https://mp.weixin.qq.com/s/aWratg1j9RBAjIghoY66yQ "宝藏B站UP主,视频弹幕尽收囊中!") | [Code](https://github.com/Jack-Cherish/python-spider/tree/master/2020/bilibili "宝藏B站UP主,视频弹幕尽收囊中!") | + +更多精彩,敬请期待! + + + +[画像:wechat] diff --git a/2020/api/api.py b/2020/api/api.py new file mode 100644 index 00000000..4ed08497 --- /dev/null +++ b/2020/api/api.py @@ -0,0 +1,65 @@ +import requests +import base64 +import json +import cv2 +import numpy as np +import matplotlib.pyplot as plt +%matplotlib inline + + +beautify_url = "https://api-cn.faceplusplus.com/facepp/v2/beautify" +# 你创建的应用的 API Key 和 API Secret(也叫 Secret Key) +AK = '' +SK = '' + +# 可选参数,不填写,默认50 +# 美白程度 0 - 100 +whitening = 80 +# 磨皮程度 0 - 100 +smoothing = 80 +# 瘦脸程度 0 - 100 +thinface = 20 +# 小脸程度 0 - 100 +shrink_face = 50 +# 大眼程度 0 - 100 +enlarge_eye = 50 +# 去眉毛程度 0 - 100 +remove_eyebrow = 50 +# 滤镜名称,不填写,默认无滤镜 +filter_type = '' + +# 二进制方式打开图片 +img_name = 'test_1.png' +f = open(img_name, 'rb') +# 转 base64 +img_base64 = base64.b64encode(f.read()) + +# 使用 whitening、smoothing、thinface 三个可选参数,其他用默认值 +data = { + 'api_key': AK, + 'api_secret': SK, + 'image_base64': img_base64, + 'whitening': whitening, + 'smoothing': smoothing, + 'thinface': thinface, + } + +r = requests.post(url=beautify_url, data=data) +html = json.loads(r.text) + +# 解析base64图片 +base64_data = html['result'] +imgData = base64.b64decode(base64_data) +nparr = np.frombuffer(imgData, np.uint8) +img_res = cv2.imdecode(nparr, cv2.IMREAD_COLOR) +img_res_BGR = cv2.cvtColor(img_res, cv2.COLOR_RGB2BGR) + +# 原始图片 +img = cv2.imread(img_name) +img_BGR = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) + +# 显示图片 +fig, axs = plt.subplots(nrows=1, ncols=2, sharex=False, sharey=False, figsize=(10,10)) +axs[0].imshow(img_BGR) +axs[1].imshow(img_res_BGR) +plt.show() diff --git a/2020/api/test_1.png b/2020/api/test_1.png new file mode 100644 index 00000000..38e8def3 Binary files /dev/null and b/2020/api/test_1.png differ diff --git a/2020/bilibili/download.py b/2020/bilibili/download.py new file mode 100644 index 00000000..b8aff376 --- /dev/null +++ b/2020/bilibili/download.py @@ -0,0 +1,120 @@ +# -*-coding:utf-8 -*- +# Website: https://cuijiahua.com +# Author: Jack Cui +# Date: 2020年07月22日 +import requests +import json +import re +import json +import math +import xml2ass +import time +from contextlib import closing + +from bs4 import BeautifulSoup + +import os +from win32com.client import Dispatch + +def addTasktoXunlei(down_url): + flag = False + o = Dispatch('ThunderAgent.Agent64.1') + try: + o.AddTask(down_url, "", "", "", "", -1, 0, 5) + o.CommitTasks() + flag = True + except Exception: + print(Exception.message) + print(" AddTask is fail!") + return flag + +def get_download_url(arcurl): + # 微信搜索 JackCui-AI 关注公众号,后台回复「B 站」获取视频解析地址 + jiexi_url = 'xxx' + payload = {'url': arcurl} + jiexi_req = requests.get(jiexi_url, params=payload) + jiexi_bf = BeautifulSoup(jiexi_req.text) + jiexi_dn_url = jiexi_bf.iframe.get('src') + dn_req = requests.get(jiexi_dn_url) + dn_bf = BeautifulSoup(dn_req.text) + video_script = dn_bf.find('script',src = None) + DPlayer = str(video_script.string) + download_url = re.findall('\'(http[s]?:(?:[a-zA-Z]|[0-9]|[$-_@.&~+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)\'', DPlayer)[0] + download_url = download_url.replace('\\', '') + return download_url + +space_url = 'https://space.bilibili.com/280793434' +search_url = 'https://api.bilibili.com/x/space/arc/search' +mid = space_url.split('/')[-1] +sess = requests.Session() +search_headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36', + 'Accept-Language': 'zh-CN,zh;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'Accept': 'application/json, text/plain, */*'} + +# 获取视频个数 +ps = 1 +pn = 1 +search_params = {'mid': mid, + 'ps': ps, + 'tid': 0, + 'pn': pn} +req = sess.get(url=search_url, headers=search_headers, params=search_params, verify=False) +info = json.loads(req.text) +video_count = info['data']['page']['count'] + +ps = 10 +page = math.ceil(video_count/ps) +videos_list = [] +for pn in range(1, page+1): + search_params = {'mid': mid, + 'ps': ps, + 'tid': 0, + 'pn': pn} + req = sess.get(url=search_url, headers=search_headers, params=search_params, verify=False) + info = json.loads(req.text) + vlist = info['data']['list']['vlist'] + for video in vlist: + title = video['title'] + bvid = video['bvid'] + vurl = 'https://www.bilibili.com/video/' + bvid + videos_list.append([title, vurl]) +print('共 %d 个视频' % len(videos_list)) +all_video = {} +# 下载前 10 个视频 +for video in videos_list[:10]: + download_url = get_download_url(video[1]) + print(video[0] + ':' + download_url) + # 记录视频名字 + xunlei_video_name = download_url.split('?')[0].split('/')[-1] + filename = video[0] + for c in u' ́☆❤◦\/:*?"| ': + filename = filename.replace(c, '') + save_video_name = filename + '.mp4' + all_video[xunlei_video_name] = save_video_name + + addTasktoXunlei(download_url) + # 弹幕下载 + danmu_name = filename + '.xml' + danmu_ass = filename + '.ass' + oid = download_url.split('/')[6] + danmu_url = 'https://api.bilibili.com/x/v1/dm/list.so?oid={}'.format(oid) + danmu_header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36', + 'Accept': '*/*', + 'Accept-Encoding': 'gzip, deflate, br', + 'Accept-Language': 'zh-CN,zh;q=0.9'} + with closing(sess.get(danmu_url, headers=danmu_header, stream=True, verify=False)) as response: + if response.status_code == 200: + with open(danmu_name, 'wb') as file: + for data in response.iter_content(): + file.write(data) + file.flush() + else: + print('链接异常') + time.sleep(0.5) + xml2ass.Danmaku2ASS(danmu_name, danmu_ass, 1280, 720) +# 视频重命名 +for key, item in all_video.items(): + while key not in os.listdir('./'): + time.sleep(1) + os.rename(key, item) diff --git a/2020/bilibili/xml2ass.py b/2020/bilibili/xml2ass.py new file mode 100644 index 00000000..eac3f861 --- /dev/null +++ b/2020/bilibili/xml2ass.py @@ -0,0 +1,802 @@ +# The original author of this program, Danmaku2ASS, is StarBrilliant. +# This file is released under General Public License version 3. +# You should have received a copy of General Public License text alongside with +# this program. If not, you can obtain it at http://gnu.org/copyleft/gpl.html . +# This program comes with no warranty, the author will not be resopnsible for +# any damage or problems caused by this program. + +import argparse +import calendar +import gettext +import io +import json +import logging +import math +import os +import random +import re +import sys +import time +import xml.dom.minidom + + +if sys.version_info < (3,): + raise RuntimeError('at least Python 3.0 is required') + +gettext.install('danmaku2ass', os.path.join(os.path.dirname(os.path.abspath(os.path.realpath(sys.argv[0] or 'locale'))), 'locale')) + +def SeekZero(function): + def decorated_function(file_): + file_.seek(0) + try: + return function(file_) + finally: + file_.seek(0) + return decorated_function + + +def EOFAsNone(function): + def decorated_function(*args, **kwargs): + try: + return function(*args, **kwargs) + except EOFError: + return None + return decorated_function + + +@SeekZero +@EOFAsNone +def ProbeCommentFormat(f): + tmp = f.read(1) + if tmp == '[': + return 'Acfun' + # It is unwise to wrap a JSON object in an array! + # See this: http://haacked.com/archive/2008/11/20/anatomy-of-a-subtle-json-vulnerability.aspx/ + # Do never follow what Acfun developers did! + elif tmp == '{': + tmp = f.read(14) + if tmp == '"status_code":': + return 'Tudou' + elif tmp == '"root":{"total': + return 'sH5V' + elif tmp == '<': + tmp = f.read(1) + if tmp == '?': + tmp = f.read(38) + if tmp == 'xml version="1.0" encoding="UTF-8"?>+ 微信群 + 公众号 + B站 + 知乎 + CSDN + 头条 + 掘金 +
## 声明 -* 软件均仅用于学习交流,请勿用于任何商业用途! +* 代码、教程**仅限于学习交流,请勿用于任何商业用途!** -### 文章首发声明 +## 目录 -* 文章在自己的个人网站首发,其他平台文章均属转发,如想获得最新更新进展,欢迎关注我的个人网站:http://www.cuijiahua.com/ +* [爬虫小工具](#爬虫小工具) + * [文件下载小助手](https://github.com/Jack-Cherish/python-spider/blob/master/downloader.py "悬停显示") +* [爬虫实战](#爬虫实战) + * [笔趣看小说下载](https://github.com/Jack-Cherish/python-spider/blob/master/biqukan.py "悬停显示") + * [百度文库免费文章下载助手_rev1](https://github.com/Jack-Cherish/python-spider/blob/master/baiduwenku.py "悬停显示") + * [百度文库免费文章下载助手_rev2](https://github.com/Jack-Cherish/python-spider/blob/master/baiduwenku_pro_1.py "悬停显示") + * [《帅啊》网帅哥图片下载](https://github.com/Jack-Cherish/python-spider/blob/master/shuaia.py "悬停显示") + * [构建代理IP池](https://github.com/Jack-Cherish/python-spider/blob/master/daili.py "悬停显示") + * [《火影忍者》漫画下载](https://github.com/Jack-Cherish/python-spider/tree/master/cartoon "悬停显示") + * [财务报表下载小助手](https://github.com/Jack-Cherish/python-spider/blob/master/financical.py "悬停显示") + * [一小时入门网络爬虫](https://github.com/Jack-Cherish/python-spider/tree/master/one_hour_spider "悬停显示") + * [抖音App视频下载](https://github.com/Jack-Cherish/python-spider/tree/master/douyin "悬停显示") + * [GEETEST验证码识别](https://github.com/Jack-Cherish/python-spider/blob/master/geetest.py "悬停显示") + * [12306抢票小助手](https://github.com/Jack-Cherish/python-spider/blob/master/12306.py "悬停显示") + * [百万英雄答题辅助系统](https://github.com/Jack-Cherish/python-spider/tree/master/baiwan "悬停显示") + * [网易云音乐免费音乐批量下载](https://github.com/Jack-Cherish/python-spider/tree/master/Netease "悬停显示") + * [B站免费视频和弹幕批量下载](https://github.com/Jack-Cherish/python-spider/tree/master/bilibili "悬停显示") + * [京东商品晒单图下载](https://github.com/Jack-Cherish/python-spider/tree/master/dingdong "悬停显示") + * [正方教务管理系统个人信息查询](https://github.com/Jack-Cherish/python-spider/tree/master/zhengfang_system_spider "悬停显示") +* [其它](#其它) ## 爬虫小工具 @@ -38,39 +69,11 @@ python biqukan.py - * video_downloader:爱奇艺等主流视频网站的VIP视频破解助手(暂只支持PC和手机在线观看VIP视频!) - - 感谢Python3二维码生成器作者:https://github.com/sylnsfar/qrcode - - 编译好的软件下载连接:https://pan.baidu.com/s/1bqSTNJL 密码:p8bs - - 解压密码:`cuijiahua.com` - - 无需Python3环境,在Windows下,解压即用 - - 源码可查看`video_downloader`,运行源码需要搭建Python3环境,并安装相应第三方依赖库: - - 在`video_downloader`文件夹下,安装第三方依赖库: - - pip3 install -r requirements.txt - - 使用方法: - - python movie_downloader.py - - 运行环境: - - Windows, Python3 - - Linux, Python3 - - Mac, Python3 - * baiduwenku.py: 百度文库word文章爬取 原理说明:http://blog.csdn.net/c406495762/article/details/72331737 - 代码不完善,没有进行打包,不具通用性,纯属娱乐,以后有时间会完善。 + 代码不完善,没有进行打包,不具通用性,纯属娱乐。 * shuaia.py: 爬取《帅啊》网,帅哥图片 @@ -122,23 +125,41 @@ * 网络小说下载(静态网站)-biqukan * 优美壁纸下载(动态网站)-unsplash - * 爱奇艺VIP视频下载 + * 视频下载 * douyin.py:抖音App视频下载 - 抖音App的视频下载,就是普通的App爬取,没有原理说明。 + 抖音App的视频下载,就是普通的App爬取。 + + 原理说明: - * geetest.py:GEETEST验证码破解 + * 个人网站:http://cuijiahua.com/blog/2018/03/spider-5.html + + * douyin_pro:抖音App视频下载(升级版) - 爬虫最大的敌人之一是什么?没错,验证码!Geetest作为提供验证码服务的行家,市场占有率还是蛮高的。遇到Geetest提供的滑动验证码怎么破?授人予鱼不如授人予渔,接下来就为大家呈现本教程的精彩内容。 + 抖音App的视频下载,添加视频解析网站,支持无水印视频下载,使用第三方平台解析。 + + 原理说明: + + * 个人网站:http://cuijiahua.com/blog/2018/03/spider-5.html + + * douyin:抖音App视频下载(升级版2) - 原理说明: + 抖音App的视频下载,添加视频解析网站,支持无水印视频下载,通过url解析,无需第三方平台。 - * 个人网站:http://www.cuijiahua.com/blog/2017/11/spider_2_geetest.html + 原理说明: + + * 个人网站:http://cuijiahua.com/blog/2018/03/spider-5.html 动态示意图: -  +  + + * geetest.py:GEETEST验证码识别 + + 原理说明: + + 无 * 12306.py:用Python抢火车票简单代码 @@ -165,10 +186,77 @@ 亲测答题延时在3s左右。 声明:没做过后端和前端,花了一天时间,现学现卖弄好的,javascript也是现看现用,百度的程序,调试调试而已。可能有很多用法比较low的地方,用法不对,请勿见怪,有大牛感兴趣,可以自行完善。 + + * Netease:根据歌单下载网易云音乐 + + 效果图: + +  + + 原理说明: + + 暂无 + + 功能介绍: + + 根据music_list.txt文件里的歌单的信息下载网易云音乐,将自己喜欢的音乐进行批量下载。 + + * bilibili:B站视频和弹幕批量下载 + + 原理说明: + + 暂无 + + 使用说明: + python bilibili.py -d 猫 -k 猫 -p 10 + + 三个参数: + -d 保存视频的文件夹名 + -k B站搜索的关键字 + -p 下载搜索结果前多少页 + + * jingdong:京东商品晒单图下载 + + 效果图: + +  + + 原理说明: + + 暂无 + 使用说明: + python jd.py -k 芒果 + + 三个参数: + -d 保存图片的路径,默认为fd.py文件所在文件夹 + -k 搜索关键词 + -n 下载商品的晒单图个数,即n个商店的晒单图 + * zhengfang_system_spider:对正方教务管理系统个人课表,个人学生成绩,绩点等简单爬取 + + 效果图: +  + + 原理说明: + + 暂无 + + 使用说明: + + cd zhengfang_system_spider + pip install -r requirements.txt + python spider.py + +## 其它 + + * 欢迎 Pull requests,感谢贡献。 + + 更多精彩,敬请期待! + +[画像:wechat] diff --git a/baiduwenku_pro_1.py b/baiduwenku_pro_1.py new file mode 100644 index 00000000..18a79459 --- /dev/null +++ b/baiduwenku_pro_1.py @@ -0,0 +1,101 @@ +import requests +import re +import json +import os + +session = requests.session() + + +def fetch_url(url): + return session.get(url).content.decode('gbk') + + +def get_doc_id(url): + return re.findall('view/(.*).html', url)[0] + + +def parse_type(content): + return re.findall(r"docType.*?\:.*?\'(.*?)\',円", content)[0] + + +def parse_title(content): + return re.findall(r"title.*?\:.*?\'(.*?)\',円", content)[0] + + +def parse_doc(content): + result = '' + url_list = re.findall('(https.*?0.json.*?)\\\\x22}', content) + url_list = [addr.replace("\\\\\\/", "/") for addr in url_list] + for url in url_list[:-5]: + content = fetch_url(url) + y = 0 + txtlists = re.findall('"c":"(.*?)".*?"y":(.*?),', content) + for item in txtlists: + if not y == item[1]: + y = item[1] + n = '\n' + else: + n = '' + result += n + result += item[0].encode('utf-8').decode('unicode_escape', 'ignore') + return result + + +def parse_txt(doc_id): + content_url = 'https://wenku.baidu.com/api/doc/getdocinfo?callback=cb&doc_id=' + doc_id + content = fetch_url(content_url) + md5 = re.findall('"md5sum":"(.*?)"', content)[0] + pn = re.findall('"totalPageNum":"(.*?)"', content)[0] + rsign = re.findall('"rsign":"(.*?)"', content)[0] + content_url = 'https://wkretype.bdimg.com/retype/text/' + doc_id + '?rn=' + pn + '&type=txt' + md5 + '&rsign=' + rsign + content = json.loads(fetch_url(content_url)) + result = '' + for item in content: + for i in item['parags']: + result += i['c'].replace('\\r', '\r').replace('\\n', '\n') + return result + + +def parse_other(doc_id): + content_url = "https://wenku.baidu.com/browse/getbcsurl?doc_id=" + doc_id + "&pn=1&rn=99999&type=ppt" + content = fetch_url(content_url) + url_list = re.findall('{"zoom":"(.*?)","page"', content) + url_list = [item.replace("\\", '') for item in url_list] + if not os.path.exists(doc_id): + os.mkdir(doc_id) + for index, url in enumerate(url_list): + content = session.get(url).content + path = os.path.join(doc_id, str(index) + '.jpg') + with open(path, 'wb') as f: + f.write(content) + print("图片保存在" + doc_id + "文件夹") + + +def save_file(filename, content): + with open(filename, 'w', encoding='utf8') as f: + f.write(content) + print('已保存为:' + filename) + + +# test_txt_url = 'https://wenku.baidu.com/view/cbb4af8b783e0912a3162a89.html?from=search' +# test_ppt_url = 'https://wenku.baidu.com/view/2b7046e3f78a6529657d5376.html?from=search' +# test_pdf_url = 'https://wenku.baidu.com/view/dd6e15c1227916888586d795.html?from=search' +# test_xls_url = 'https://wenku.baidu.com/view/eb4a5bb7312b3169a551a481.html?from=search' +def main(): + url = input('请输入要下载的文库URL地址') + content = fetch_url(url) + doc_id = get_doc_id(url) + type = parse_type(content) + title = parse_title(content) + if type == 'doc': + result = parse_doc(content) + save_file(title + '.txt', result) + elif type == 'txt': + result = parse_txt(doc_id) + save_file(title + '.txt', result) + else: + parse_other(doc_id) + + +if __name__ == "__main__": + main() diff --git a/bilibili/README.md b/bilibili/README.md new file mode 100644 index 00000000..787470fd --- /dev/null +++ b/bilibili/README.md @@ -0,0 +1,24 @@ +## 功能 + +下载B站视频和弹幕,将xml原生弹幕转换为ass弹幕文件,支持plotplayer等播放器的弹幕播放。 + +## 作者 + +* Website: [http://cuijiahua.com](http://cuijiahua.com "悬停显示") +* Author: Jack Cui +* Date: 2018年6月12日 + +## 更新 + +* 2018年09月12日:添加FFmpeg分段视频合并 + +## 使用说明 + +FFmpeg下载,并配置环境变量。http://ffmpeg.org/ + + python bilibili.py -d 猫 -k 猫 -p 10 + + 三个参数: + -d 保存视频的文件夹名 + -k B站搜索的关键字 + -p 下载搜索结果前多少页 diff --git a/bilibili/bilibili.py b/bilibili/bilibili.py new file mode 100644 index 00000000..469766e5 --- /dev/null +++ b/bilibili/bilibili.py @@ -0,0 +1,211 @@ +# -*-coding:utf-8 -*- +# Website: http://cuijiahua.com +# Author: Jack Cui +# Date: 2018年6月9日 + +import requests, json, re, sys, os, urllib, argparse, time +from urllib.request import urlretrieve +from contextlib import closing +from urllib import parse +import xml2ass + +class BiliBili: + def __init__(self, dirname, keyword): + self.dn_headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36', + 'Accept': '*/*', + 'Accept-Encoding': 'gzip, deflate, br', + 'Accept-Language': 'zh-CN,zh;q=0.9', + 'Referer': 'https://search.bilibili.com/all?keyword=%s' % parse.quote(keyword)} + + self.search_headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36', + 'Accept-Language': 'zh-CN,zh;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'Accept': 'application/json, text/plain, */*'} + + self.video_headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36', + 'Accept-Language': 'zh-CN,zh;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'} + + self.danmu_header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36', + 'Accept': '*/*', + 'Accept-Encoding': 'gzip, deflate, br', + 'Accept-Language': 'zh-CN,zh;q=0.9'} + + self.sess = requests.Session() + + self.dir = dirname + + def video_downloader(self, video_url, video_name): + """ + 视频下载 + Parameters: + video_url: 带水印的视频地址 + video_name: 视频名 + Returns: + 无 + """ + size = 0 + with closing(self.sess.get(video_url, headers=self.dn_headers, stream=True, verify=False)) as response: + chunk_size = 1024 + content_size = int(response.headers['content-length']) + if response.status_code == 200: + sys.stdout.write(' [文件大小]:%0.2f MB\n' % (content_size / chunk_size / 1024)) + video_name = os.path.join(self.dir, video_name) + with open(video_name, 'wb') as file: + for data in response.iter_content(chunk_size = chunk_size): + file.write(data) + size += len(data) + file.flush() + + sys.stdout.write(' [下载进度]:%.2f%%' % float(size / content_size * 100) + '\r') + # sys.stdout.flush() + if size / content_size == 1: + print('\n') + else: + print('链接异常') + + def search_video(self, search_url): + """ + 搜索接口 + Parameters: + search_url: 带水印的视频地址 + Returns: + titles:视频名列表 + arcurls: 视频播放地址列表 + """ + req = self.sess.get(url=search_url, headers=self.search_headers, verify=False) + html = json.loads(req.text) + videos = html["data"]['result'] + titles = [] + arcurls = [] + for video in videos: + titles.append(video['title'].replace('','').replace('','')) + arcurls.append(video['arcurl']) + return titles, arcurls + + def get_download_url(self, arcurl): + """ + 获取视频下载地址 + Parameters: + arcurl: 视频播放地址 + oid:弹幕地址参数 + Returns: + download_url:视频下载地址 + """ + req = self.sess.get(url=arcurl, headers=self.video_headers, verify=False) + pattern = '.__playinfo__=(.*)") + tac = _tac_re.search(share_user.text).group(1) + _dytk_re = re.compile(r"dytk\s*:\s*'(.+)'") + dytk = _dytk_re.search(share_user.text).group(1) + _nickname_re = re.compile(r'(.+?)<\/p>')
+ nickname = _nickname_re.search(share_user.text).group(1)
+ data = {
+ 'tac': tac.split('|')[0],
+ 'user_id': user_id,
+ }
+ req = requests.post(sign_api, data=data)
+ while req.status_code != 200:
+ req = requests.post(sign_api, data=data)
+ sign = req.json().get('signature')
+ user_url_prefix = 'https://www.iesdouyin.com/web/api/v2/aweme/like' if type_flag == 'f' else 'https://www.iesdouyin.com/web/api/v2/aweme/post'
+ print('解析视频链接中')
+ while has_more != 0:
+ user_url = user_url_prefix + '/?user_id=%s&sec_uid=&count=21&max_cursor=%s&aid=1128&_signature=%s&dytk=%s' % (user_id, max_cursor, sign, dytk)
+ req = requests.get(user_url, headers=self.headers)
+ while req.status_code != 200:
+ req = requests.get(user_url, headers=self.headers)
+ html = json.loads(req.text)
+ for each in html['aweme_list']:
+ try:
+ url = 'https://aweme.snssdk.com/aweme/v1/play/?video_id=%s&line=0&ratio=720p&media_type=4&vr_type=0&improve_bitrate=0&is_play_url=1&is_support_h265=0&source=PackSourceEnum_PUBLISH'
+ vid = each['video']['vid']
+ video_url = url % vid
+ except:
+ continue
+ share_desc = each['desc']
+ if os.name == 'nt':
+ for c in r'\/:*?"|':
+ nickname = nickname.replace(c, '').strip().strip('\.')
+ share_desc = share_desc.replace(c, '').strip()
+ share_id = each['aweme_id']
+ if share_desc in ['抖音-原创音乐短视频社区', 'TikTok', '']:
+ video_names.append(share_id + '.mp4')
+ else:
+ video_names.append(share_id + '-' + share_desc + '.mp4')
+ share_url = 'https://www.iesdouyin.com/share/video/%s' % share_id
+ share_urls.append(share_url)
+ video_urls.append(video_url)
+ max_cursor = html['max_cursor']
+ has_more = html['has_more']
+
+ return video_names, video_urls, share_urls, nickname
+
+ def get_download_url(self, video_url, watermark_flag):
+ """
+ 获得带水印的视频播放地址
+ Parameters:
+ video_url:带水印的视频播放地址
+ Returns:
+ download_url: 带水印的视频下载地址
+ """
+ # 带水印视频
+ if watermark_flag == True:
+ download_url = video_url.replace('/play/', '/playwm/')
+ # 无水印视频
+ else:
+ download_url = video_url.replace('/playwm/', '/play/')
+
+ return download_url
+
+ def video_downloader(self, video_url, video_name, watermark_flag=False):
+ """
+ 视频下载
+ Parameters:
+ video_url: 带水印的视频地址
+ video_name: 视频名
+ watermark_flag: 是否下载带水印的视频
+ Returns:
+ 无
+ """
+ size = 0
+ video_url = self.get_download_url(video_url, watermark_flag=watermark_flag)
+ with closing(requests.get(video_url, headers=self.headers1, stream=True)) as response:
+ chunk_size = 1024
+ content_size = int(response.headers['content-length'])
+ if response.status_code == 200:
+ sys.stdout.write(' [文件大小]:%0.2f MB\n' % (content_size / chunk_size / 1024))
+
+ with open(video_name, 'wb') as file:
+ for data in response.iter_content(chunk_size = chunk_size):
+ file.write(data)
+ size += len(data)
+ file.flush()
+
+ sys.stdout.write(' [下载进度]:%.2f%%' % float(size / content_size * 100) + '\r')
+ sys.stdout.flush()
+
+ def run(self):
+ """
+ 运行函数
+ Parameters:
+ None
+ Returns:
+ None
+ """
+ self.hello()
+ print('UID取得方式:\n分享用户页面,用浏览器打开短链接,原始链接中/share/user/后的数字即是UID')
+ user_id = input('请输入UID (例如60388937600):')
+ user_id = user_id if user_id else '60388937600'
+ watermark_flag = input('是否下载带水印的视频 (0-否(默认), 1-是):')
+ watermark_flag = watermark_flag if watermark_flag!='' else '0'
+ watermark_flag = bool(int(watermark_flag))
+ type_flag = input('f-收藏的(默认), p-上传的:')
+ type_flag = type_flag if type_flag!='' else 'f'
+ save_dir = input('保存路径 (例如"E:/Download/", 默认"./Download/"):')
+ save_dir = save_dir if save_dir else "./Download/"
+ video_names, video_urls, share_urls, nickname = self.get_video_urls(user_id, type_flag)
+ nickname_dir = os.path.join(save_dir, nickname)
+ if not os.path.exists(save_dir):
+ os.makedirs(save_dir)
+ if nickname not in os.listdir(save_dir):
+ os.mkdir(nickname_dir)
+ if type_flag == 'f':
+ if 'favorite' not in os.listdir(nickname_dir):
+ os.mkdir(os.path.join(nickname_dir, 'favorite'))
+ print('视频下载中:共有%d个作品!\n' % len(video_urls))
+ for num in range(len(video_urls)):
+ print(' 解析第%d个视频链接 [%s] 中,请稍后!\n' % (num + 1, share_urls[num]))
+ if '\\' in video_names[num]:
+ video_name = video_names[num].replace('\\', '')
+ elif '/' in video_names[num]:
+ video_name = video_names[num].replace('/', '')
+ else:
+ video_name = video_names[num]
+ video_path = os.path.join(nickname_dir, video_name) if type_flag!='f' else os.path.join(nickname_dir, 'favorite', video_name)
+ if os.path.isfile(video_path):
+ print('视频已存在')
+ else:
+ self.video_downloader(video_urls[num], video_path, watermark_flag)
+ print('\n')
+ print('下载完成!')
+
+ def hello(self):
+ """
+ 打印欢迎界面
+ Parameters:
+ None
+ Returns:
+ None
+ """
+ print('*' * 100)
+ print('\t\t\t\t抖音App视频下载小助手')
+ print('\t\t作者:Jack Cui、steven7851')
+ print('*' * 100)
+
+
+if __name__ == '__main__':
+ douyin = DouYin()
+ douyin.run()
diff --git a/douyin/fuck-byted-acrawler.js b/douyin/fuck-byted-acrawler.js
new file mode 100644
index 00000000..66be77c8
--- /dev/null
+++ b/douyin/fuck-byted-acrawler.js
@@ -0,0 +1,194 @@
+// Referer:https://raw.githubusercontent.com/loadchange/amemv-crawler/master/fuck-byted-acrawler.js
+function generateSignature(userId) {
+ this.navigator = {
+ userAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"
+ }
+ var e = {}
+
+ var r = (function () {
+ function e(e, a, r) {
+ return (b[e] || (b[e] = t("x,y", "return x " + e + " y")))(r, a)
+ }
+
+ function a(e, a, r) {
+ return (k[r] || (k[r] = t("x,y", "return new x[y](" + Array(r + 1).join(",x[++y]").substr(1) + ")")))(e, a)
+ }
+
+ function r(e, a, r) {
+ var n, t, s = {}, b = s.d = r ? r.d + 1 : 0;
+ for (s["$" + b] = s, t = 0; t < b; t++) s[n = "$" + t] = r[n]; + for (t = 0, b = s.length = a.length; t < b; t++) s[t] = a[t]; + return c(e, 0, s) + } + + function c(t, b, k) { + function u(e) { + v[x++] = e + } + + function f() { + return g = t.charCodeAt(b++) - 32, t.substring(b, b += g) + } + + function l() { + try { + y = c(t, b, k) + } catch (e) { + h = e, y = l + } + } + + for (var h, y, d, g, v = [], x = 0; ;) switch (g = t.charCodeAt(b++) - 32) { + case 1: + u(!v[--x]); + break; + case 4: + v[x++] = f(); + break; + case 5: + u(function (e) { + var a = 0, r = e.length; + return function () { + var c = a < r; + return c && u(e[a++]), c + } + }(v[--x])); + break; + case 6: + y = v[--x], u(v[--x](y)); + break; + case 8: + if (g = t.charCodeAt(b++) - 32, l(), b += g, g = t.charCodeAt(b++) - 32, y === c) b += g; else if (y !== l) return y; + break; + case 9: + v[x++] = c; + break; + case 10: + u(s(v[--x])); + break; + case 11: + y = v[--x], u(v[--x] + y); + break; + case 12: + for (y = f(), d = [], g = 0; g < y.length; g++) d[g] = y.charCodeAt(g) ^ g + y.length; + u(String.fromCharCode.apply(null, d)); + break; + case 13: + y = v[--x], h = delete v[--x][y]; + break; + case 14: + v[x++] = t.charCodeAt(b++) - 32; + break; + case 59: + u((g = t.charCodeAt(b++) - 32) ? (y = x, v.slice(x -= g, y)) : []); + break; + case 61: + u(v[--x][t.charCodeAt(b++) - 32]); + break; + case 62: + g = v[--x], k[0] = 65599 * k[0] + k[1].charCodeAt(g)>>> 0;
+ break;
+ case 65:
+ h = v[--x], y = v[--x], v[--x][y] = h;
+ break;
+ case 66:
+ u(e(t[b++], v[--x], v[--x]));
+ break;
+ case 67:
+ y = v[--x], d = v[--x], u((g = v[--x]).x === c ? r(g.y, y, k) : g.apply(d, y));
+ break;
+ case 68:
+ u(e((g = t[b++]) < "<" ? (b--, f()) : g + g, v[--x], v[--x])); + break; + case 70: + u(!1); + break; + case 71: + v[x++] = n; + break; + case 72: + v[x++] = +f(); + break; + case 73: + u(parseInt(f(), 36)); + break; + case 75: + if (v[--x]) { + b++; + break + } + case 74: + g = t.charCodeAt(b++) - 32 << 16>> 16, b += g;
+ break;
+ case 76:
+ u(k[t.charCodeAt(b++) - 32]);
+ break;
+ case 77:
+ y = v[--x], u(v[--x][y]);
+ break;
+ case 78:
+ g = t.charCodeAt(b++) - 32, u(a(v, x -= g + 1, g));
+ break;
+ case 79:
+ g = t.charCodeAt(b++) - 32, u(k["$" + g]);
+ break;
+ case 81:
+ h = v[--x], v[--x][f()] = h;
+ break;
+ case 82:
+ u(v[--x][f()]);
+ break;
+ case 83:
+ h = v[--x], k[t.charCodeAt(b++) - 32] = h;
+ break;
+ case 84:
+ v[x++] = !0;
+ break;
+ case 85:
+ v[x++] = void 0;
+ break;
+ case 86:
+ u(v[x - 1]);
+ break;
+ case 88:
+ h = v[--x], y = v[--x], v[x++] = h, v[x++] = y;
+ break;
+ case 89:
+ u(function () {
+ function e() {
+ return r(e.y, arguments, k)
+ }
+
+ return e.y = f(), e.x = c, e
+ }());
+ break;
+ case 90:
+ v[x++] = null;
+ break;
+ case 91:
+ v[x++] = h;
+ break;
+ case 93:
+ h = v[--x];
+ break;
+ case 0:
+ return v[--x];
+ default:
+ u((g << 16>> 16) - 16)
+ }
+ }
+
+ var n = this, t = n.Function, s = Object.keys || function (e) {
+ var a = {}, r = 0;
+ for (var c in e) a[r++] = c;
+ return a.length = r, a
+ }, b = {}, k = {};
+ return r
+ })()
+ ('gr$Daten Иb/s!l y͒yĹg,(lfi~ah`{mv,-n|jqewVxp{rvmmx,&effkx[!cs"l".Pq%widthl"@q&heightl"vr*getContextx$"2d[!cs#l#,*;?|u.|uc{uq$fontl#vr(fillTextx$$龘ฑภ경2<[#c}l#2q*shadowblurl#1q-shadowoffsetxl#$$limeq+shadowcolorl#vr#arcx88802[%c}l#vr&strokex[ c}l"v,)}eOmyoZB]mx[ cs!0s$l$Pb
||
')
+ rmtr = re.compile(' ||
