python/douyin/handle_web_share.py · LM-develop/dockerpython

代码拉取完成,页面将自动刷新

扫描微信二维码支付

取消

支付完成

richgiteeai

Watch

不关注关注所有动态仅关注版本发行动态关注但不提醒动态

1 Star 0 Fork 0

LM-develop/dockerpython

代码 Issues 0 Pull Requests 0 Wiki 统计流水线

服务

加入 Gitee

与超过 1400万开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)

免费加入

已有帐号? 立即登录

文件

master

分支 (1)

管理

master

该仓库未声明开源许可证文件(LICENSE),使用请关注具体项目描述及其代码上游依赖。

项目仓库所选许可证以仓库主分支所使用许可证为准

克隆/下载

HTTPS SSH SVN SVN+SSH 下载ZIP

提示

下载代码请复制以下命令到终端执行

为确保你提交的代码身份被 Gitee 正确识别,请执行以下命令完成配置

git config --global user.name userName 
git config --global user.email userEmail

初次使用 SSH 协议进行代码克隆、推送等操作时,需按下述提示完成 SSH 配置

1 生成 RSA 密钥

2 获取 RSA 公钥内容,并配置到 SSH公钥中

在 Gitee 上使用 SVN,请访问使用指南

使用 HTTPS 协议时,命令行会出现如下账号密码验证步骤。基于安全考虑,Gitee 建议配置并使用私人令牌替代登录密码进行克隆、推送等操作

Username for 'https://gitee.com': userName

Password for 'https://userName@gitee.com': # 私人令牌

分支 1

标签 0

dockerpython

python

douyin

handle_web_share.py

handle_web_share.py 3.97 KB

import re
import requests
import time
from lxml import etree
from handle_mongo import handle_get_task
from handle_mongo import handle_insert_douyin

def handle_decode(input_data,share_web_url,task):
 search_douyin_str = re.compile(r'抖音ID:')
 regex_list = [
 {'name':[' &#xe603; ',' &#xe60d; ',' &#xe616; '],'value':0},
 {'name':[' &#xe602; ',' &#xe60e; ',' &#xe618; '],'value':1},
 {'name':[' &#xe605; ',' &#xe610; ',' &#xe617; '],'value':2},
 {'name':[' &#xe604; ',' &#xe611; ',' &#xe61a; '],'value':3},
 {'name':[' &#xe606; ',' &#xe60c; ',' &#xe619; '],'value':4},
 {'name':[' &#xe607; ',' &#xe60f; ',' &#xe61b; '],'value':5},
 {'name':[' &#xe608; ',' &#xe612; ',' &#xe61f; '],'value':6},
 {'name':[' &#xe60a; ',' &#xe613; ',' &#xe61c; '],'value':7},
 {'name':[' &#xe60b; ',' &#xe614; ',' &#xe61d; '],'value':8},
 {'name':[' &#xe609; ',' &#xe615; ',' &#xe61e; '],'value':9},
 ]

for i1 in regex_list:
 for i2 in i1['name']:
 input_data = re.sub(i2,str(i1['value']),input_data)
 share_web_html = etree.HTML(input_data)
 douyin_info = {}
 douyin_info['nick_name'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info1']//p[@class='nickname']/text()")[0]
 douyin_id = ''.join(share_web_html.xpath("//div[@class='personal-card']/div[@class='info1']/p[@class='shortid']/i/text()"))
 douyin_info['douyin_id'] = re.sub(search_douyin_str,'',share_web_html.xpath("//div[@class='personal-card']/div[@class='info1']//p[@class='nickname']/text()")[0]).strip() + douyin_id

try:
 douyin_info['job'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/div[@class='verify-info']/span[@class='info']/text()")[0].strip()
 except:
 pass
 douyin_info['describe'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='signature']/text()")[0].replace('\n',',')
 douyin_info['location'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='extra-info']/span[1]/text()")[0]
 douyin_info['xingzuo'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='extra-info']/span[2]/text()")[0]
 douyin_info['follow_count'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='follow-info']//span[@class='focus block']//i[@class='icon iconfont follow-num']/text()")[0].strip()
 fans_value = ''.join(share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='follow-info']//span[@class='follower block']//i[@class='icon iconfont follow-num']/text()"))
 unit = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='follow-info']//span[@class='follower block']/span[@class='num']/text()")
 if unit[-1].strip() == 'w':
 douyin_info['fans'] = str((int(fans_value)/10))+'w'
 like = ''.join(share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='follow-info']//span[@class='liked-num block']//i[@class='icon iconfont follow-num']/text()"))
 unit = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='follow-info']//span[@class='liked-num block']/span[@class='num']/text()")
 if unit[-1].strip() == 'w':
 douyin_info['like'] = str(int(like)/10)+'w'
 douyin_info['from_url'] = share_web_url

print(douyin_info)
 handle_insert_douyin(douyin_info)

def handle_douyin_web_share(task):
 share_web_url = 'https://www.douyin.com/share/user/'+task["share_id"]
 print(share_web_url)
 share_web_header = {
 'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'
 }
 share_web_response = requests.get(url=share_web_url,headers=share_web_header)
 handle_decode(share_web_response.text,share_web_url,task["share_id"])

if __name__ == '__main__':
 while True:
 task=handle_get_task()
 handle_douyin_web_share(task)
 time.sleep(2)

一键复制编辑原始数据按行查看历史

liming6 提交于 2019年01月31日 11:14 +08:00 . douyin获取人员分享信息

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70

import re
import requests
import time
from lxml import etree
from handle_mongo import handle_get_task
from handle_mongo import handle_insert_douyin


def handle_decode(input_data,share_web_url,task):
 search_douyin_str = re.compile(r'抖音ID:')
 regex_list = [
 {'name':[' &#xe603; ',' &#xe60d; ',' &#xe616; '],'value':0},
 {'name':[' &#xe602; ',' &#xe60e; ',' &#xe618; '],'value':1},
 {'name':[' &#xe605; ',' &#xe610; ',' &#xe617; '],'value':2},
 {'name':[' &#xe604; ',' &#xe611; ',' &#xe61a; '],'value':3},
 {'name':[' &#xe606; ',' &#xe60c; ',' &#xe619; '],'value':4},
 {'name':[' &#xe607; ',' &#xe60f; ',' &#xe61b; '],'value':5},
 {'name':[' &#xe608; ',' &#xe612; ',' &#xe61f; '],'value':6},
 {'name':[' &#xe60a; ',' &#xe613; ',' &#xe61c; '],'value':7},
 {'name':[' &#xe60b; ',' &#xe614; ',' &#xe61d; '],'value':8},
 {'name':[' &#xe609; ',' &#xe615; ',' &#xe61e; '],'value':9},
 ]

 for i1 in regex_list:
 for i2 in i1['name']:
 input_data = re.sub(i2,str(i1['value']),input_data)
 share_web_html = etree.HTML(input_data)
 douyin_info = {}
 douyin_info['nick_name'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info1']//p[@class='nickname']/text()")[0]
 douyin_id = ''.join(share_web_html.xpath("//div[@class='personal-card']/div[@class='info1']/p[@class='shortid']/i/text()"))
 douyin_info['douyin_id'] = re.sub(search_douyin_str,'',share_web_html.xpath("//div[@class='personal-card']/div[@class='info1']//p[@class='nickname']/text()")[0]).strip() + douyin_id


 try:
 douyin_info['job'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/div[@class='verify-info']/span[@class='info']/text()")[0].strip()
 except:
 pass
 douyin_info['describe'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='signature']/text()")[0].replace('\n',',')
 douyin_info['location'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='extra-info']/span[1]/text()")[0]
 douyin_info['xingzuo'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='extra-info']/span[2]/text()")[0]
 douyin_info['follow_count'] = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='follow-info']//span[@class='focus block']//i[@class='icon iconfont follow-num']/text()")[0].strip()
 fans_value = ''.join(share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='follow-info']//span[@class='follower block']//i[@class='icon iconfont follow-num']/text()"))
 unit = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='follow-info']//span[@class='follower block']/span[@class='num']/text()")
 if unit[-1].strip() == 'w':
 douyin_info['fans'] = str((int(fans_value)/10))+'w'
 like = ''.join(share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='follow-info']//span[@class='liked-num block']//i[@class='icon iconfont follow-num']/text()"))
 unit = share_web_html.xpath("//div[@class='personal-card']/div[@class='info2']/p[@class='follow-info']//span[@class='liked-num block']/span[@class='num']/text()")
 if unit[-1].strip() == 'w':
 douyin_info['like'] = str(int(like)/10)+'w'
 douyin_info['from_url'] = share_web_url


 print(douyin_info)
 handle_insert_douyin(douyin_info)


def handle_douyin_web_share(task):
 share_web_url = 'https://www.douyin.com/share/user/'+task["share_id"]
 print(share_web_url)
 share_web_header = {
 'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'
 }
 share_web_response = requests.get(url=share_web_url,headers=share_web_header)
 handle_decode(share_web_response.text,share_web_url,task["share_id"])

if __name__ == '__main__':
 while True:
 task=handle_get_task()
 handle_douyin_web_share(task)
 time.sleep(2)