From a669d913569427de358623ddf118310b6306408f Mon Sep 17 00:00:00 2001 From: Jason Chen Date: Mon, 7 Aug 2017 22:25:10 +0800 Subject: [PATCH 01/36] music.baidu.com.py: fix album only can get 10 songs issue; adjust filename format --- music.baidu.com.py | 51 ++++++++++++++++------------------------------ 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/music.baidu.com.py b/music.baidu.com.py index 0185b46..c2c9016 100755 --- a/music.baidu.com.py +++ b/music.baidu.com.py @@ -85,13 +85,10 @@ def __init__(self, url): self.download = self.play if args.play else self.download - def get_songidlist(self, id_): - html = self.opener.open(self.template_album % id_).read() + def get_songidlist(self, song_id): + html = self.opener.open(self.template_album % song_id).read() songidlist = re.findall(r'/song/(\d+)', html) - api_json = self.opener.open(self.template_api % ','.join(songidlist)).read() - api_json = json.loads(api_json) - infos = api_json['data']['songList'] - return infos + return songidlist def get_cover(self, url): i = 1 @@ -122,53 +119,40 @@ def url_parser(self): elif '/song/' in self.url: self.song_id = re.search(r'/song/(\d+)', self.url).group(1) #print(s % (2, 92, u'\n -- 正在分析歌曲信息 ...')) - self.get_song_infos() + self.get_song_infos(self.song_id) else: print(s % (2, 91, u' 请正确输入baidu网址.')) + self.download() - def get_song_infos(self): - api_json = self.opener.open(self.template_api % self.song_id).read() + def get_song_infos(self, song_id, track_number=''): + api_json = self.opener.open(self.template_api % song_id).read() j = json.loads(api_json) song_info = {} song_info['song_id'] = unicode(j['data']['songList'][0]['songId']) - song_info['track'] = u'' + song_info['track'] = unicode(track_number) song_info['song_url'] = u'http://music.baidu.com/song/' + song_info['song_id'] song_info['song_name'] = modificate_text(j['data']['songList'][0]['songName']).strip() song_info['album_name'] = modificate_text(j['data']['songList'][0]['albumName']).strip() song_info['artist_name'] = modificate_text(j['data']['songList'][0]['artistName']).strip() song_info['album_pic_url'] = j['data']['songList'][0]['songPicRadio'] + song_info['file_name'] = song_info['artist_name'] + ' - ' + song_info['song_name'] + if song_info['track']: + song_info['file_name'] = song_info['track'].zfill(2) + '.' + song_info['file_name'] if args.flac: - song_info['file_name'] = song_info['song_name'] + ' - ' + song_info['artist_name'] + '.flac' + song_info['file_name'] = song_info['file_name'] + '.flac' else: - song_info['file_name'] = song_info['song_name'] + ' - ' + song_info['artist_name'] + '.mp3' + song_info['file_name'] = song_info['file_name'] + '.mp3' song_info['durl'] = j['data']['songList'][0]['songLink'] self.song_infos.append(song_info) - self.download() def get_album_infos(self): songidlist = self.get_songidlist(self.album_id) - z = z_index(songidlist) - ii = 1 + track_number = 1 for i in songidlist: - song_info = {} - song_info['song_id'] = unicode(i['songId']) - song_info['song_url'] = u'http://music.baidu.com/song/' + song_info['song_id'] - song_info['track'] = unicode(ii) - song_info['song_name'] = modificate_text(i['songName']).strip() - song_info['artist_name'] = modificate_text(i['artistName']).strip() - song_info['album_pic_url'] = i['songPicRadio'] - if args.flac: - song_info['file_name'] = song_info['track'].zfill(z) + '.' + song_info['song_name'] + ' - ' + song_info['artist_name'] + '.flac' - else: - song_info['file_name'] = song_info['track'].zfill(z) + '.' + song_info['song_name'] + ' - ' + song_info['artist_name'] + '.mp3' - song_info['album_name'] = modificate_text(i['albumName']).strip() \ - if i['albumName'] else modificate_text(self.song_infos[0]['album_name']) - song_info['durl'] = i['songLink'] - self.song_infos.append(song_info) - ii += 1 - d = modificate_text(self.song_infos[0]['album_name'] + ' - ' + self.song_infos[0]['artist_name']) + self.get_song_infos(i, track_number) + track_number += 1 + d = modificate_text(self.song_infos[0]['artist_name'] + ' - ' + self.song_infos[0]['album_name']) self.dir_ = os.path.join(os.getcwd().decode('utf8'), d) - self.download() def display_infos(self, i): print '\n ----------------' @@ -203,6 +187,7 @@ def download(self): file_name = os.path.join(dir_, t) if os.path.exists(file_name): ## if file exists, no get_durl ii += 1 + print(u'\n 文件已存在~') continue file_name_for_wget = file_name.replace('`', '\`') if 'zhangmenshiting.baidu.com' in i['durl'] or \ From 795b02bb67db3bb9b7339f658f049d04fb260146 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2017年9月25日 18:50:19 +0800 Subject: [PATCH 02/36] [xiami.py] now, xiami requires cookies --- xiami.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/xiami.py b/xiami.py index 72ae035..759dcb0 100755 --- a/xiami.py +++ b/xiami.py @@ -661,7 +661,6 @@ def download_songs(self, song_ids): songs = self.get_song(self.song_id) self.download(songs) - def download_album(self): songs = self.get_songs(self.album_id) song = songs[0] @@ -946,10 +945,14 @@ def play(self, songs, nn=u'1', n=1): cmd = 'mpv --really-quiet ' \ '--cache 8146 ' \ '--user-agent "%s" ' \ - '--http-header-fields="Referer:http://img.xiami.com' \ - '/static/swf/seiya/1.4/player.swf?v=%s" ' \ + '--http-header-fields "Referer: http://img.xiami.com' \ + '/static/swf/seiya/1.4/player.swf?v=%s",' \ + '"Cookie: member_auth=%s" ' \ '"%s"' \ - % (headers['User-Agent'], int(time.time()*1000), durl) + % (headers['User-Agent'], + int(time.time()*1000), + ss.cookies.get('member_auth'), + durl) os.system(cmd) timeout = 1 ii, _, _ = select.select([sys.stdin], [], [], timeout) @@ -1008,9 +1011,13 @@ def download(self, songs, amount_songs=u'1', n=1): '-U "%s" ' \ '--header "Referer:http://img.xiami.com' \ '/static/swf/seiya/1.4/player.swf?v=%s" ' \ + '--header "Cookie: member_auth=%s" ' \ '-O "%s.tmp" %s' \ - % (quiet, headers['User-Agent'], int(time.time()*1000), - file_name_for_wget, durl) + % (quiet, headers['User-Agent'], + int(time.time()*1000), + ss.cookies.get('member_auth'), + file_name_for_wget, + durl) cmd = cmd.encode('utf8') status = os.system(cmd) if status != 0: # other http-errors, such as 302. From 931514f52bbdd84c4ce8483afd3cf770663db2cc Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2017年9月26日 11:19:19 +0800 Subject: [PATCH 03/36] [xiami.py] add new ruler for 320 kbps --- xiami.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xiami.py b/xiami.py index 759dcb0..f30db9f 100755 --- a/xiami.py +++ b/xiami.py @@ -923,7 +923,10 @@ def display_infos(self, i, nn, n, durl): print '—' * int(os.popen('tput cols').read()) def get_mp3_quality(self, durl): - if 'm3.file.xiami.com' in durl or 'm6.file.xiami.com' in durl or '_h.mp3' in durl: + if 'm3.file.xiami.com' in durl \ + or 'm6.file.xiami.com' in durl \ + or '_h.mp3' in durl \ + or 'm320.xiami.net' in durl: return 'h' else: return 'l' From 48db583b456a8aa3b7208dc397532d4236baaa73 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2017年10月24日 11:37:15 +0800 Subject: [PATCH 04/36] [xiami.py] [Update] get all songs from xiami "collect", #127 --- xiami.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/xiami.py b/xiami.py index f30db9f..df2a4eb 100755 --- a/xiami.py +++ b/xiami.py @@ -18,7 +18,7 @@ url_song = "http://www.xiami.com/song/%s" url_album = "http://www.xiami.com/album/%s" -url_collect = "http://www.xiami.com/collect/%s" +url_collect = "http://www.xiami.com/collect/ajax-get-list" url_artist_albums = "http://www.xiami.com/artist/album/id/%s/page/%s" url_artist_top_song = "http://www.xiami.com/artist/top/id/%s" url_lib_songs = "http://www.xiami.com/space/lib-song/u/%s/page/%s" @@ -676,13 +676,28 @@ def download_album(self): self.download(songs, amount_songs, args.from_) def download_collect(self): - html = ss.get(url_collect % self.collect_id).text + page = 1 + song_ids = [] + while True: + params = { + 'id': self.collect_id, + 'p': page, + 'limit': 50, + } + infos = ss.get(url_collect, params=params).json() + for info in infos['result']['data']: + song_ids.append(str(info['song_id'])) + + if infos['result']['total_page'] == page: + break + page += 1 + + html = ss.get('http://www.xiami.com/collect/%s' % self.collect_id).text html = html.split('
(.+?)<', html).group(1) d = collect_name dir_ = os.path.join(os.getcwdu(), d) self.dir_ = modificate_file_name_for_wget(dir_) - song_ids = re.findall(r'/song/(\w+)"\s+title', html) amount_songs = unicode(len(song_ids)) song_ids = song_ids[args.from_ - 1:] print(s % (2, 97, u'\n>> ' + amount_songs + u' 首歌曲将要下载.')) \ @@ -894,8 +909,8 @@ def hack_luoo(self, url): return None cn = r.content songs_info = re.findall(r'

(.+?)

\s+' - r'

Artist: (.+?)

\s+' - r'

Album: (.+?)

', cn) + r'

(?:Artist:|艺人:)(.+?)

\s+' + r'

(?:Album:|专辑:)(.+?)

', cn) # search song at xiami for info in songs_info: From 22cf8e272107c3e7c5d47721a8b87f4700bda814 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2017年11月11日 11:12:48 +0800 Subject: [PATCH 05/36] [xiami.py] support listening ranking --- xiami.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/xiami.py b/xiami.py index df2a4eb..d5872df 100755 --- a/xiami.py +++ b/xiami.py @@ -457,9 +457,12 @@ def get_artist_id(url): elif '/u/' in url: self.user_id = re.search(r'/u/(\w+)', url).group(1) - code = raw_input('>> m # 该用户歌曲库.\n' \ + code = raw_input( + '>> m # 该用户歌曲库.\n' \ '>> c # 最近在听\n' \ '>> s # 分享的音乐\n' + '>> r # 歌曲试听排行 - 一周\n' + '>> rt # 歌曲试听排行 - 全部 \n' '>> rm # 私人电台:来源于"收藏的歌曲","收藏的专辑",\ "喜欢的艺人","收藏的精选集"\n' '>> rc # 虾米猜:基于试听行为所建立的个性电台\n>> ') @@ -472,6 +475,12 @@ def get_artist_id(url): url_shares = 'http://www.xiami.com' \ '/space/feed/u/%s/type/3/page/%s' % (self.user_id, '%s') self.download_user_shares(url_shares) + elif code == 'r': + url = 'http://www.xiami.com/space/charts/u/%s/c/song/t/week' % self.user_id + self.download_ranking_songs(url, 'week') + elif code == 'rt': + url = 'http://www.xiami.com/space/charts/u/%s/c/song/t/all' % self.user_id + self.download_ranking_songs(url, 'all') elif code == 'rm': #print(s % (2, 92, u'\n -- 正在分析该用户的虾米推荐 ...')) url_rndsongs = url_radio_my @@ -808,6 +817,25 @@ def download_user_shares(self, url_shares): if not shares: break page += 1 + def download_ranking_songs(self, url, tp): + d = modificate_text(u'%s 的试听排行 - %s' % (self.user_id, tp)) + dir_ = os.path.join(os.getcwdu(), d) + self.dir_ = modificate_file_name_for_wget(dir_) + page = 1 + n = 1 + while True: + html = ss.get(url + '/page/' + str(page)).text + song_ids = re.findall(r"play\('(\d+)'", html) + if not song_ids: + break + for song_id in song_ids: + songs = self.get_song(song_id) + self.download(songs, n=n) + self.html = '' + self.disc_description_archives = {} + n += 1 + page += 1 + def download_user_radio(self, url_rndsongs): d = modificate_text(u'%s 的虾米推荐' % self.user_id) dir_ = os.path.join(os.getcwdu(), d) From 69d9116c058fd0bbd66a2450614b9ab50cbf4f61 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Thu, 1 Mar 2018 14:55:27 +0800 Subject: [PATCH 06/36] [README.md] [Update] --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 722852a..bb09903 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,8 @@ pan.baidu.com.py 是一个百度网盘的命令行客户端。 **支持多帐号登录** +**现在只支持[用cookie登录](#cookie_login)** + **支持cookie登录** **支持加密上传**, 需要 shadowsocks @@ -507,6 +509,7 @@ bp login username password # 一直用 bp login 即可 ``` + #### cookie 登录: 1. 打开 chrome 隐身模式窗口 From 82397ffb1bdf15bf89ec2f86183304c41af361f3 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2018年3月24日 11:04:35 +0800 Subject: [PATCH 07/36] [pan.baidu.com.py] support https links for `save` --- pan.baidu.com.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index 0e80037..f38df6c 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -1224,7 +1224,7 @@ def _upload_slice(self, piece=0, slice=DefaultSliceSize): fl = cStringIO.StringIO(__slice_block) files = {'file': ('file', fl, '')} data = MultipartEncoder(files) - theaders = headers + theaders = dict(headers) theaders['Content-Type'] = data.content_type url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file' r = ss.post(url, params=p, data=data, verify=VERIFY, headers=theaders) @@ -2362,7 +2362,7 @@ def _get_torrent_info(self, path): } url = 'http://pan.baidu.com/rest/2.0/services/cloud_dl' - r = ss.post(url, params=p) + r = ss.get(url, params=p) j = r.json() if j.get('error_code'): print s % (1, 91, ' !! Error at _get_torrent_info:'), j['error_msg'] @@ -3313,7 +3313,7 @@ def handle_command(comd, xxx): url = xxx[0] x.save_inbox_share(url, remotepath, infos=infos) else: - url = re.search(r'(http://.+?.baidu.com/.+?)(#|$)', xxx[0]).group(1) + url = re.search(r'(https?://.+?.baidu.com/.+?)(#|$)', xxx[0]).group(1) url = url.replace('wap/link', 'share/link') x._secret_or_not(url) x.save_share(url, remotepath, infos=infos) From b700a40fa50b66855c04755af3654e341d445b2b Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2018年4月19日 17:37:55 +0800 Subject: [PATCH 08/36] [xiami.py] [Update] there is a new xiami ablum page html, #133 --- xiami.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/xiami.py b/xiami.py index d5872df..b3bbd2f 100755 --- a/xiami.py +++ b/xiami.py @@ -458,13 +458,13 @@ def get_artist_id(url): elif '/u/' in url: self.user_id = re.search(r'/u/(\w+)', url).group(1) code = raw_input( - '>> m # 该用户歌曲库.\n' \ - '>> c # 最近在听\n' \ + '>> m # 该用户歌曲库.\n' + '>> c # 最近在听\n' '>> s # 分享的音乐\n' '>> r # 歌曲试听排行 - 一周\n' '>> rt # 歌曲试听排行 - 全部 \n' - '>> rm # 私人电台:来源于"收藏的歌曲","收藏的专辑",\ - "喜欢的艺人","收藏的精选集"\n' + '>> rm # 私人电台:来源于"收藏的歌曲","收藏的专辑",' + ' "喜欢的艺人","收藏的精选集"\n' '>> rc # 虾米猜:基于试听行为所建立的个性电台\n>> ') if code == 'm': #print(s % (2, 92, u'\n -- 正在分析用户歌曲库信息 ...')) @@ -558,11 +558,8 @@ def get_songs(self, album_id, song_id=None): t = re.sub(r'<.+?>(\r\n|)', u'\n', t) album_description = t - #t = re.search(r'href="(.+?)" id="albumCover"', html1).group(1) - t = re.search(r'id="albumCover".+?"(http://.+?)" ', html1).group(1) - #tt = t.rfind('.') - #t = '%s_4%s' % (t[:tt], t[tt:]) - t = t.replace('_2.', '_4.') + t = re.search(r'pic:\'//(pic\.xiami\.net.+?)\'', html).group(1) # issue133 + t = 'http://' + t album_pic_url = t songs = [] From c1e410db6913d3af961dc7bd3d05fc26499e056f Mon Sep 17 00:00:00 2001 From: PeterDing Date: Thu, 3 May 2018 22:46:57 +0800 Subject: [PATCH 09/36] [xiami.py] xiami updates web page --- xiami.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xiami.py b/xiami.py index b3bbd2f..50872ae 100755 --- a/xiami.py +++ b/xiami.py @@ -558,7 +558,7 @@ def get_songs(self, album_id, song_id=None): t = re.sub(r'<.+?>(\r\n|)', u'\n', t) album_description = t - t = re.search(r'pic:\'//(pic\.xiami\.net.+?)\'', html).group(1) # issue133 + t = re.search(r'//(pic\.xiami\.net.+?)"', html).group(1) # issue133 t = 'http://' + t album_pic_url = t From 76b4032193010c74bd727077b40be1db690910f8 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2018年5月10日 16:50:30 +0800 Subject: [PATCH 10/36] [xiami.py] request is handled by _request --- xiami.py | 163 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 88 insertions(+), 75 deletions(-) diff --git a/xiami.py b/xiami.py index 50872ae..6edf3f8 100755 --- a/xiami.py +++ b/xiami.py @@ -116,7 +116,7 @@ def z_index(song_infos): class xiami(object): def __init__(self): self.dir_ = os.getcwdu() - self.template_record = 'http://www.xiami.com/count/playrecord?sid=%s' + self.template_record = 'https://www.xiami.com/count/playrecord?sid={song_id}&ishq=1&t={time}&object_id={song_id}&object_name=default&start_point=120&_xiamitoken={token}' self.collect_id = '' self.album_id = '' @@ -150,7 +150,7 @@ def init(self): def check_login(self): #print s % (1, 97, '\n -- check_login') url = 'http://www.xiami.com/task/signin' - r = ss.get(url) + r = self._request(url) if r.content: #print s % (1, 92, ' -- check_login success\n') # self.save_cookies() @@ -159,12 +159,32 @@ def check_login(self): print s % (1, 91, ' -- login fail, please check email and password\n') return False + def _request(self, url, headers=None, data=None, method='GET', timeout=30, retry=2): + for _ in range(retry): + try: + headers = headers or ss.headers + resp = ss.request(method, url, headers=headers, data=data, timeout=timeout) + except Exception, err: + continue + + if not resp.ok: + raise Exception("response is not ok, status_code = %s" % resp.status_code) + + # save cookies + self.save_cookies() + + return resp + raise err + # manually, add cookies # you must know how to get the cookie - def add_member_auth(self, member_auth): - member_auth = member_auth.rstrip(';') - self.save_cookies(member_auth) - ss.cookies.update({'member_auth': member_auth}) + def add_cookies(self, cookies): + _cookies = {} + for item in cookies.strip('; ').split('; '): + k, v = item.split('=', 1) + _cookies[k] = v + self.save_cookies(_cookies) + ss.cookies.update(_cookies) def login(self, email, password): print s % (1, 97, '\n -- login') @@ -189,18 +209,14 @@ def login(self, email, password): 'Cache-Control': 'max-age=1', 'Referer': 'http://www.xiami.com/web/login', 'Connection': 'keep-alive', - } - - cookies = { '_xiamitoken': hashlib.md5(str(time.time())).hexdigest() } url = 'https://login.xiami.com/web/login' for i in xrange(2): - res = ss.post(url, headers=hds, data=data, cookies=cookies) + res = self._request(url, headers=hds, data=data) if ss.cookies.get('member_auth'): - self.save_cookies() return True else: if 'checkcode' not in res.content: @@ -263,7 +279,7 @@ def login_taobao(self, username, password): if err_msg == u'请输入验证码' or err_msg == u'验证码错误,请重新输入': captcha_url = 'http://pin.aliyun.com/get_img?' \ 'identity=passport.alipay.com&sessionID=%s' % data['cid'] - tr = ss.get(captcha_url, headers=theaders) + tr = self._request(captcha_url, headers=theaders) path = os.path.join(os.path.expanduser('~'), 'vcode.jpg') with open(path, 'w') as g: img = tr.content @@ -280,7 +296,7 @@ def login_taobao(self, username, password): url = 'http://www.xiami.com/accounts/back?st=%s' \ % j['content']['data']['st'] - ss.get(url, headers=theaders) + self._request(url, headers=theaders) self.save_cookies() return @@ -292,17 +308,16 @@ def get_validate(self, cn): url = re.search(r'src="(http.+checkcode.+?)"', cn).group(1) path = os.path.join(os.path.expanduser('~'), 'vcode.png') with open(path, 'w') as g: - data = ss.get(url).content + data = self._request(url).content g.write(data) print " ++ 验证码已经保存至", s % (2, 91, path) validate = raw_input(s % (2, 92, ' 请输入验证码: ')) return validate - def save_cookies(self, member_auth=None): - if not member_auth: - member_auth = ss.cookies.get_dict()['member_auth'] + def save_cookies(self, cookies=None): + if not cookies: + cookies = ss.cookies.get_dict() with open(cookie_file, 'w') as g: - cookies = { 'cookies': { 'member_auth': member_auth } } json.dump(cookies, g) def get_durl(self, id_): @@ -310,11 +325,11 @@ def get_durl(self, id_): try: if not args.low: url = 'http://www.xiami.com/song/gethqsong/sid/%s' - j = ss.get(url % id_).json() + j = self._request(url % id_).json() t = j['location'] else: url = 'http://www.xiami.com/song/playlist/id/%s' - cn = ss.get(url % id_).text + cn = self._request(url % id_).text t = re.search(r'location>(.+?)(http.+?)', xml) if not t: return None lyric_url = t.group(1) - data = ss.get(lyric_url).content.replace('\r\n', '\n') + data = self._request(lyric_url).content.replace('\r\n', '\n') data = lyric_parser(data) if data: return data.decode('utf8', 'ignore') @@ -384,7 +404,7 @@ def lyric_parser(data): def get_disc_description(self, album_url, info): if not self.html: - self.html = ss.get(album_url).text + self.html = self._request(album_url).text t = re.findall(re_disc_description, self.html) t = dict([(a, modificate_text(parser.unescape(b))) \ for a, b in t]) @@ -430,7 +450,7 @@ def url_parser(self, urls): elif '/artist/' in url or 'i.xiami.com' in url: def get_artist_id(url): - html = ss.get(url).text + html = self._request(url).text artist_id = re.search(r'artist_id = \'(\w+)\'', html).group(1) return artist_id @@ -531,7 +551,7 @@ def get_artist_id(url): print(s % (2, 91, u' 请正确输入虾米网址.')) def get_songs(self, album_id, song_id=None): - html = ss.get(url_album % album_id).text + html = self._request(url_album % album_id).text html = html.split('
(.+?)<', html).group(1) d = collect_name @@ -720,7 +740,7 @@ def download_artist_albums(self): ii = 1 album_ids = [] while True: - html = ss.get( + html = self._request( url_artist_albums % (self.artist_id, str(ii))).text t = re.findall(r'/album/(\w+)"', html) if album_ids == t: break @@ -737,7 +757,7 @@ def download_artist_albums(self): ii += 1 def download_artist_top_20_songs(self): - html = ss.get(url_artist_top_song % self.artist_id).text + html = self._request(url_artist_top_song % self.artist_id).text song_ids = re.findall(r'/song/(.+?)" title', html) artist_name = re.search( r'

(.+?)<', html).group(1) @@ -756,7 +776,7 @@ def download_artist_top_20_songs(self): n += 1 def download_artist_radio(self): - html = ss.get(url_artist_top_song % self.artist_id).text + html = self._request(url_artist_top_song % self.artist_id).text artist_name = re.search( r'

(.+?)<', html).group(1) d = modificate_text(artist_name + u' - radio') @@ -767,7 +787,7 @@ def download_artist_radio(self): % self.artist_id n = 1 while True: - xml = ss.get(url_artist_radio).text + xml = self._request(url_artist_radio).text song_ids = re.findall(r'(\d+)', xml) for i in song_ids: songs = self.get_song(i) @@ -783,7 +803,7 @@ def download_user_songs(self, url, desc): ii = 1 n = 1 while True: - html = ss.get(url % (self.user_id, str(ii))).text + html = self._request(url % (self.user_id, str(ii))).text song_ids = re.findall(r'/song/(.+?)"', html) if song_ids: for i in song_ids: @@ -802,7 +822,7 @@ def download_user_shares(self, url_shares): self.dir_ = modificate_file_name_for_wget(dir_) page = 1 while True: - html = ss.get(url_shares % page).text + html = self._request(url_shares % page).text shares = re.findall(r'play.*\(\'\d+\'\)', html) for share in shares: if 'album' in share: @@ -821,7 +841,7 @@ def download_ranking_songs(self, url, tp): page = 1 n = 1 while True: - html = ss.get(url + '/page/' + str(page)).text + html = self._request(url + '/page/' + str(page)).text song_ids = re.findall(r"play\('(\d+)'", html) if not song_ids: break @@ -839,7 +859,7 @@ def download_user_radio(self, url_rndsongs): self.dir_ = modificate_file_name_for_wget(dir_) n = 1 while True: - xml = ss.get(url_rndsongs % self.user_id).text + xml = self._request(url_rndsongs % self.user_id).text song_ids = re.findall(r'(\d+)', xml) for i in song_ids: songs = self.get_song(i) @@ -849,14 +869,14 @@ def download_user_radio(self, url_rndsongs): n += 1 def download_chart(self, type_): - html = ss.get('http://www.xiami.com/chart/index/c/%s' \ + html = self._request('http://www.xiami.com/chart/index/c/%s' \ % self.chart_id).text title = re.search(r'(.+?)', html).group(1) d = modificate_text(title) dir_ = os.path.join(os.getcwdu(), d) self.dir_ = modificate_file_name_for_wget(dir_) - html = ss.get( + html = self._request( 'http://www.xiami.com/chart/data?c=%s&limit=200&type=%s' \ % (self.chart_id, type_)).text song_ids = re.findall(r'/song/(\d+)', html) @@ -869,7 +889,7 @@ def download_chart(self, type_): n += 1 def download_genre(self, url_genre): - html = ss.get(url_genre % (self.genre_id, 1)).text + html = self._request(url_genre % (self.genre_id, 1)).text if '/gid/' in url_genre: t = re.search( r'/genre/detail/gid/%s".+?title="(.+?)"' \ @@ -893,11 +913,11 @@ def download_genre(self, url_genre): self.html = '' self.disc_description_archives = {} n += 1 - html = ss.get(url_genre % (self.chart_id, page)).text + html = self._request(url_genre % (self.chart_id, page)).text page += 1 def download_genre_radio(self, url_genre): - html = ss.get(url_genre % (self.genre_id, 1)).text + html = self._request(url_genre % (self.genre_id, 1)).text if '/gid/' in url_genre: t = re.search( r'/genre/detail/gid/%s".+?title="(.+?)"' \ @@ -916,7 +936,7 @@ def download_genre_radio(self, url_genre): n = 1 while True: - xml = ss.get(url_genre_radio).text + xml = self._request(url_genre_radio).text song_ids = re.findall(r'(\d+)', xml) for i in song_ids: songs = self.get_song(i) @@ -941,7 +961,7 @@ def hack_luoo(self, url): for info in songs_info: url = 'http://www.xiami.com/web/search-songs?key=%s' \ % urllib.quote(' '.join(info)) - r = ss.get(url) + r = self._request(url) j = r.json() if not r.ok or not j: print s % (1, 93, ' !! no find:'), ' - '.join(info) @@ -974,13 +994,15 @@ def get_mp3_quality(self, durl): def play(self, songs, nn=u'1', n=1): if args.play == 2: songs = sorted(songs, key=lambda k: k['song_played'], reverse=True) + for i in songs: - self.record(i['song_id']) + self.record(i['song_id'], i['album_id']) durl = self.get_durl(i['song_id']) if not durl: print s % (2, 91, ' !! Error: can\'t get durl'), i['song_name'] continue + cookies = '; '.join(['%s=%s' % (k, v) for k, v in ss.cookies.items()]) mp3_quality = self.get_mp3_quality(durl) i['durl_is_H'] = mp3_quality self.display_infos(i, nn, n, durl) @@ -990,12 +1012,9 @@ def play(self, songs, nn=u'1', n=1): '--user-agent "%s" ' \ '--http-header-fields "Referer: http://img.xiami.com' \ '/static/swf/seiya/1.4/player.swf?v=%s",' \ - '"Cookie: member_auth=%s" ' \ + '"Cookie: %s" ' \ '"%s"' \ - % (headers['User-Agent'], - int(time.time()*1000), - ss.cookies.get('member_auth'), - durl) + % (headers['User-Agent'], int(time.time()*1000), cookies, durl) os.system(cmd) timeout = 1 ii, _, _ = select.select([sys.stdin], [], [], timeout) @@ -1011,6 +1030,7 @@ def download(self, songs, amount_songs=u'1', n=1): if not os.path.exists(dir_): os.mkdir(dir_) + ii = 1 for i in songs: num = random.randint(0, 100) % 8 @@ -1048,6 +1068,7 @@ def download(self, songs, amount_songs=u'1', n=1): else: print ' |--', s % (1, 97, 'MP3-Quality:'), s % (1, 91, 'Low') + cookies = '; '.join(['%s=%s' % (k, v) for k, v in ss.cookies.items()]) file_name_for_wget = file_name.replace('`', '\`') quiet = ' -q' if args.quiet else ' -nv' cmd = 'wget -c%s ' \ @@ -1056,11 +1077,7 @@ def download(self, songs, amount_songs=u'1', n=1): '/static/swf/seiya/1.4/player.swf?v=%s" ' \ '--header "Cookie: member_auth=%s" ' \ '-O "%s.tmp" %s' \ - % (quiet, headers['User-Agent'], - int(time.time()*1000), - ss.cookies.get('member_auth'), - file_name_for_wget, - durl) + % (quiet, headers['User-Agent'], int(time.time()*1000), cookies, file_name_for_wget, durl) cmd = cmd.encode('utf8') status = os.system(cmd) if status != 0: # other http-errors, such as 302. @@ -1088,7 +1105,7 @@ def _save_do(self, id_, type, tags): "_xiamitoken": ss.cookies['_xiamitoken'], } url = 'http://www.xiami.com/ajax/addtag' - r = ss.post(url, data=data) + r = self._request(url, data=data, method='POST') j = r.json() if j['status'] == 'ok': return 0 @@ -1174,30 +1191,26 @@ def main(argv): email = raw_input(s % (1, 97, ' username: ') \ if comd == 'logintaobao' or comd == 'gt' \ else s % (1, 97, ' email: ')) - password = getpass(s % (1, 97, ' password: ')) + cookies = getpass(s % (1, 97, ' cookies: ')) elif len(xxx) == 1: # for add_member_auth - if '@' not in xxx[0]: - x = xiami() - x.add_member_auth(xxx[0]) - x.check_login() - return - - email = xxx[0] - password = getpass(s % (1, 97, ' password: ')) + if '; ' in xxx[0]: + email = None + cookies = xxx[0] + else: + email = xxx[0] + cookies = getpass(s % (1, 97, ' cookies: ')) elif len(xxx) == 2: email = xxx[0] - password = xxx[1] + cookies = xxx[1] else: - print s % (1, 91, - ' login\n login email\n \ - login email password') + msg = ('login: \n' + 'login cookies') + print s % (1, 91, msg) + return x = xiami() - if comd == 'logintaobao' or comd == 'gt': - x.login_taobao(email, password) - else: - x.login(email, password) + x.add_cookies(cookies) is_signin = x.check_login() if is_signin: print s % (1, 92, ' ++ login succeeds.') From bfa5099403ab86fa8e8a8a00f9145a0cbce02816 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2018年5月10日 16:51:07 +0800 Subject: [PATCH 11/36] [README.md] update xiami.py usage --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bb09903..2d8b0a9 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,9 @@ xm login username xm login username password # 手动添加cookie登录 -1. 用浏览器登录后,按F12,然后访问 http://xiami.com/vip -2. 选择‘网络’或network,找到 xiami.com/vip,在其中找到 Cookie: memthod_auth=value -3. value填入 xm g value,再执行。 +1. 用浏览器登录后,按F12,然后访问 https://www.xiami.com/album/123456 +2. 选择‘网络’或network,找到 123456,在其中找到 Cookie: xxx +3. 然后在终端运行 xm g "xxx" # 退出登录 xm signout From 0337e786ac4876ab2c5491c83cc9c545cfb79916 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2018年6月29日 12:01:22 +0800 Subject: [PATCH 12/36] [xiami.py] [Fix] 1. Correcting song file name, not allowing special assic symbols. 2. Adding `params` to `_request`, thank @Harry1993 #135 --- xiami.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/xiami.py b/xiami.py index 6edf3f8..a0ba8e2 100755 --- a/xiami.py +++ b/xiami.py @@ -130,6 +130,7 @@ def __init__(self): self.disc_description_archives = {} self.download = self.play if args.play else self.download + self._is_play = bool(args.play) def init(self): if os.path.exists(cookie_file): @@ -159,11 +160,11 @@ def check_login(self): print s % (1, 91, ' -- login fail, please check email and password\n') return False - def _request(self, url, headers=None, data=None, method='GET', timeout=30, retry=2): + def _request(self, url, headers=None, params=None, data=None, method='GET', timeout=30, retry=2): for _ in range(retry): try: headers = headers or ss.headers - resp = ss.request(method, url, headers=headers, data=data, timeout=timeout) + resp = ss.request(method, url, headers=headers, params=params, data=data, timeout=timeout) except Exception, err: continue @@ -625,7 +626,7 @@ def get_songs(self, album_id, song_id=None): song_info['cd_serial'] = disc song_info['year'] = year song_info['album_pic_url'] = album_pic_url - song_info['song_name'] = song_names[i] + song_info['song_name'] = modificate_text(song_names[i]) song_info['album_name'] = album_name song_info['artist_name'] = artist_name song_info['z'] = z From 63acc9d096b84242a8c83045763021bfee78bf40 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2018年7月24日 19:35:39 +0800 Subject: [PATCH 13/36] [xiami.py] [Update] * Add xiami H5 api and web api. We use web api, because it supports more song's attributions than H5 api. And web api have not need cookies. Album's description is not supported by both web api and H5 api, So we do not take it at mp3's comment. * Display song's length when playing. --- xiami.py | 467 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 344 insertions(+), 123 deletions(-) diff --git a/xiami.py b/xiami.py index a0ba8e2..af88373 100755 --- a/xiami.py +++ b/xiami.py @@ -5,8 +5,10 @@ import sys from getpass import getpass import os +import copy import random import time +import datetime import json import argparse import requests @@ -113,6 +115,314 @@ def z_index(song_infos): ######################################################## +class Song(object): + + def __init__(self): + self.__sure() + self.track = 0 + self.year = 0 + self.cd_serial = 0 + self.disc_description = '' + + # z = len(str(album_size)) + self.z = 1 + + def __sure(self): + __dict__ = self.__dict__ + if '__keys' not in __dict__: + __dict__['__keys'] = {} + + def __getattr__(self, name): + __dict__ = self.__dict__ + return __dict__['__keys'].get(name) + + def __setattr__(self, name, value): + __dict__ = self.__dict__ + __dict__['__keys'][name] = value + + def __getitem__(self, key): + return getattr(self, key) + + def __setitem__(self, key, value): + return setattr(self, key, value) + + def feed(self, **kwargs): + for name, value in kwargs.items(): + setattr(self, name, value) + + +class XiamiH5API(object): + + URL = 'http://api.xiami.com/web' + PARAMS = { + 'v': '2.0', + 'app_key': '1', + } + + def __init__(self): + self.cookies = { + 'user_from': '2', + 'XMPLAYER_addSongsToggler': '0', + 'XMPLAYER_isOpen': '0', + '_xiamitoken': hashlib.md5(str(time.time())).hexdigest() + } + self.sess = requests.session() + self.sess.cookies.update(self.cookies) + + def _request(self, url, method='GET', **kwargs): + try: + resp = self.sess.request(method, url, **kwargs) + except Exception, err: + print 'Error:', err + sys.exit() + + return resp + + def _make_params(self, **kwargs): + params = copy.deepcopy(self.PARAMS) + params.update(kwargs) + return params + + def song(self, song_id): + params = self._make_params(id=song_id, r='song/detail') + url = self.URL + resp = self._request(url, params=params, headers=headers) + + info = resp.json()['data']['song'] + pic_url = re.sub('_\d+\.', '.', info['logo']) + song = Song() + song.feed( + song_id=info['song_id'], + song_name=info['song_name'], + album_id=info['album_id'], + album_name=info['album_name'], + artist_id=info['artist_id'], + artist_name=info['artist_name'], + singers=info['singers'], + album_pic_url=pic_url, + comment='http://www.xiami.com/song/' + str(info['song_id']) + ) + return song + + def album(self, album_id): + url = self.URL + params = self._make_params(id=album_id, r='album/detail') + resp = self._request(url, params=params, headers=headers) + + info = resp.json()['data'] + songs = [] + album_id=info['album_id'], + album_name=info['album_name'], + artist_id = info['artist_id'] + artist_name = info['artist_name'] + pic_url = re.sub('_\d+\.', '.', info['album_logo']) + for track, info_n in enumerate(info['songs'], 1): + song = Song() + song.feed( + song_id=info_n['song_id'], + song_name=info_n['song_name'], + album_id=album_id, + album_name=album_name, + artist_id=artist_id, + artist_name=artist_name, + singers=info_n['singers'], + album_pic_url=pic_url, + track=track, + comment='http://www.xiami.com/song/' + str(info_n['song_id']) + ) + songs.append(song) + return songs + + def collect(self, collect_id): + url = self.URL + params = self._make_params(id=collect_id, r='collect/detail') + resp = self._request(url, params=params, headers=headers) + + info = resp.json()['data'] + collect_name = info['collect_name'] + collect_id = info['list_id'] + songs = [] + for info_n in info['songs']: + pic_url = re.sub('_\d+\.', '.', info['album_logo']) + song = Song() + song.feed( + song_id=info_n['song_id'], + song_name=info_n['song_name'], + album_id=info_n['album_id'], + album_name=info_n['album_name'], + artist_id=info_n['artist_id'], + artist_name=info_n['artist_name'], + singers=info_n['singers'], + album_pic_url=pic_url, + comment='http://www.xiami.com/song/' + str(info_n['song_id']) + ) + songs.append(song) + return collect_id, collect_name, songs + + def artist_top_songs(self, artist_id, page=1, limit=20): + url = self.URL + params = self._make_params(id=artist_id, page=page, limit=limit, r='artist/hot-songs') + resp = self._request(url, params=params, headers=headers) + + info = resp.json()['data'] + for info_n in info['songs']: + song_id = info_n['song_id'] + yield self.song(song_id) + + def search_songs(self, keywords, page=1, limit=20): + url = self.URL + params = self._make_params(key=keywords, page=page, limit=limit, r='search/songs') + resp = self._request(url, params=params, headers=headers) + + info = resp.json()['data'] + for info_n in info['songs']: + pic_url = re.sub('_\d+\.', '.', info['album_logo']) + song = Song() + song.feed( + song_id=info_n['song_id'], + song_name=info_n['song_name'], + album_id=info_n['album_id'], + album_name=info_n['album_name'], + artist_id=info_n['artist_id'], + artist_name=info_n['artist_name'], + singers=info_n['singer'], + album_pic_url=pic_url, + comment='http://www.xiami.com/song/' + str(info_n['song_id']) + ) + yield song + + def get_song_id(self, *song_sids): + song_ids = [] + for song_sid in song_sids: + if isinstance(song_sid, int) or song_sid.isdigit(): + song_ids.append(int(song_sid)) + + url = 'https://www.xiami.com/song/playlist/id/{}/cat/json'.format(song_sid) + resp = self._request(url, headers=headers) + info = resp.json() + song_id = int(str(info['data']['trackList'][0]['song_id'])) + song_ids.append(song_id) + return song_ids + + +class XiamiWebAPI(object): + + URL = 'https://www.xiami.com/song/playlist/' + + def __init__(self): + self.sess = requests.session() + + def _request(self, url, method='GET', **kwargs): + try: + resp = self.sess.request(method, url, **kwargs) + except Exception, err: + print 'Error:', err + sys.exit() + + return resp + + def _make_song(self, info): + song = Song() + song.feed( + song_id=info['song_id'], + song_sub_title=info['song_sub_title'], + songwriters=info['songwriters'], + singers=info['singers'], + song_name=info['name'], + + album_id=info['album_id'], + album_name=info['album_name'], + + artist_id=info['artist_id'], + artist_name=info['artist_name'], + + composer=info['composer'], + lyric_url='http:' + info['lyric_url'], + + track=info['track'], + cd_serial=info['cd_serial'], + album_pic_url='http:' + info['album_pic'], + comment='http://www.xiami.com/song/' + str(info['song_id']), + + length=info['length'], + play_count=info['playCount'], + + location=info['location'] + ) + return song + + def _find_z(self, album): + zs = [] + for i, song in enumerate(album[:-1]): + next_song = album[i+1] + + cd_serial = song.cd_serial + next_cd_serial = next_song.cd_serial + + if cd_serial != next_cd_serial: + z = len(str(song.track)) + zs.append(z) + + z = len(str(song.track)) + zs.append(z) + + for song in album: + song.z = zs[song.cd_serial - 1] + + def song(self, song_id): + url = self.URL + 'id/%s/cat/json' % song_id + resp = self._request(url, headers=headers) + + info = resp.json()['data']['trackList'][0] + song = self._make_song(info) + return song + + def songs(self, *song_ids): + url = self.URL + 'id/%s/cat/json' % '%2C'.join(song_ids) + resp = self._request(url, headers=headers) + + info = resp.json()['data'] + songs = [] + for info_n in info['trackList']: + song = self._make_song(info_n) + songs.append(song) + return songs + + def album(self, album_id): + url = self.URL + 'id/%s/type/1/cat/json' % album_id + resp = self._request(url, headers=headers) + + info = resp.json()['data'] + songs = [] + for info_n in info['trackList']: + song = self._make_song(info_n) + songs.append(song) + + self._find_z(songs) + return songs + + def collect(self, collect_id): + url = self.URL + 'id/%s/type/3/cat/json' % collect_id + resp = self._request(url, headers=headers) + + info = resp.json()['data'] + songs = [] + for info_n in info['trackList']: + song = self._make_song(info_n) + songs.append(song) + return songs + + def search_songs(self, keywords): + url = 'https://www.xiami.com/ajax/search-index?key=%s&_=%s' % ( + urllib.quote(keywords), int(time.time() * 1000)) + resp = self._request(url, headers=headers) + + html = resp.content + song_ids = re.findall(r'song/(\d+)', html) + songs = self.songs(*song_ids) + return songs + + class xiami(object): def __init__(self): self.dir_ = os.getcwdu() @@ -132,6 +442,8 @@ def __init__(self): self.download = self.play if args.play else self.download self._is_play = bool(args.play) + self._api = XiamiWebAPI() + def init(self): if os.path.exists(cookie_file): try: @@ -337,7 +649,7 @@ def get_durl(self, id_): encryed_url = t[1:] durl = decry(row, encryed_url) return durl - except Exception as e: + except Exception, e: print s % (1, 91, ' |-- Error, get_durl --'), e time.sleep(5) @@ -360,7 +672,7 @@ def get_cover(self, info): self.cover_data = self._request(url).content if self.cover_data[:5] != '(.+?)<', html1).group(1) - album_name = modificate_text(t) - - t = re.search(r'"/artist/\w+.+?>(.+?)<', html1).group(1) - artist_name = modificate_text(t) - - t = re.findall(u'(\d+)年(\d+)月(\d+)', html1) - year = '-'.join(t[0]) if t else '' - - album_description = '' - t = re.search(u'专辑介绍:(.+?)

', - html2, re.DOTALL) - if t: - t = t.group(1) - t = re.sub(r'<.+?>', '', t) - t = parser.unescape(t) - t = parser.unescape(t) - t = re.sub(r'\s\s+', u'\n', t).strip() - t = re.sub(r'<.+?(http://.+?)".+?>', r'1円', t) - t = re.sub(r'<.+?>([^\n])', r'1円', t) - t = re.sub(r'<.+?>(\r\n|)', u'\n', t) - album_description = t - - t = re.search(r'//(pic\.xiami\.net.+?)"', html).group(1) # issue133 - t = 'http://' + t - album_pic_url = t - - songs = [] - for c in html2.split('class="trackname"')[1:]: - disc = re.search(r'>disc (\d+)', c).group(1) - - t = re.search(r'>disc .+?\[(.+?)\]', c) - disc_description = modificate_text(t.group(1)) if t else '' - - # find track - t = re.findall(r'"trackid">(\d+)', c) - tracks = [i.lstrip('0') for i in t] - z = len(str(len(tracks))) - - # find all song_ids and song_names - t = re.findall(r'(.+?) i: - song_info['song_played'] = song_played[i] - else: - song_info['song_played'] = 0 - - song_info['album_id'] = album_id - song_info['song_url'] = u'http://www.xiami.com/song/' \ - + song_ids[i] - song_info['track'] = tracks[i] - song_info['cd_serial'] = disc - song_info['year'] = year - song_info['album_pic_url'] = album_pic_url - song_info['song_name'] = modificate_text(song_names[i]) - song_info['album_name'] = album_name - song_info['artist_name'] = artist_name - song_info['z'] = z - song_info['disc_description'] = disc_description - t = '%s\n\n%s%s' % (song_info['song_url'], - disc_description + u'\n\n' \ - if disc_description else '', - album_description) - song_info['comment'] = t - - songs.append(song_info) + songs = self._api.album(album_id) cd_serial_auth = int(songs[-1]['cd_serial'])> 1 - for i in xrange(len(songs)): - z = songs[i]['z'] - file_name = songs[i]['track'].zfill(z) + '.' \ - + songs[i]['song_name'] \ - + ' - ' + songs[i]['artist_name'] + '.mp3' - if cd_serial_auth: - songs[i]['file_name'] = ''.join([ - '[Disc-', - songs[i]['cd_serial'], - ' # ' + songs[i]['disc_description'] \ - if songs[i]['disc_description'] else '', '] ', - file_name]) - else: - songs[i]['file_name'] = file_name - - t = [i for i in songs if i['song_id'] == song_id] \ - if song_id else songs - songs = t + for song in songs: + self.make_file_name(song, cd_serial_auth=cd_serial_auth) + songs = [i for i in songs if i['song_id'] == song_id] \ + if song_id else songs return songs def get_song(self, song_id): - html = self._request(url_song % song_id).text - html = html.split('
(?:Album:|专辑:)(.+?)

', cn) # search song at xiami - for info in songs_info: - url = 'http://www.xiami.com/web/search-songs?key=%s' \ - % urllib.quote(' '.join(info)) - r = self._request(url) - j = r.json() - if not r.ok or not j: - print s % (1, 93, ' !! no find:'), ' - '.join(info) + for name, artist, album in songs_info: + name = name.strip() + artist = artist.strip() + album = album.strip() + + songs = self._api.search_songs(name + ' ' + artist) + if not songs: + print s % (1, 93, ' !! no find:'), ' - '.join([name, artist, album]) continue - self.song_id = j[0]['id'] - self.download_song() + + self.make_file_name(songs[0]) + self.download(songs[:1], n=1) def display_infos(self, i, nn, n, durl): length = datetime.datetime.fromtimestamp(i['length']).strftime('%M:%S') From 1cb0272e3dd2fb08ef446159a2c9eb4d309198c3 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2018年7月25日 13:37:34 +0800 Subject: [PATCH 15/36] [xiami.py] [Fix] * fix xiami `save` --- xiami.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/xiami.py b/xiami.py index 3b07566..50945d2 100755 --- a/xiami.py +++ b/xiami.py @@ -334,6 +334,12 @@ def _request(self, url, method='GET', **kwargs): def _make_song(self, info): song = Song() + + location=info['location'] + row = location[0] + encryed_url = location[1:] + durl = decry(row, encryed_url) + song.feed( song_id=info['song_id'], song_sub_title=info['song_sub_title'], @@ -358,7 +364,8 @@ def _make_song(self, info): length=info['length'], play_count=info['playCount'], - location=info['location'] + location=info['location'], + location_url=durl ) return song @@ -1339,7 +1346,7 @@ def _save_do(self, id_, type, tags): "shareTo": "all", "_xiamitoken": ss.cookies['_xiamitoken'], } - url = 'http://www.xiami.com/ajax/addtag' + url = 'https://www.xiami.com/ajax/addtag' r = self._request(url, data=data, method='POST') j = r.json() if j['status'] == 'ok': @@ -1351,27 +1358,31 @@ def save(self, urls): tags = args.tags for url in urls: if '/collect/' in url: - collect_id = re.search(r'/collect/(\d+)', url).group(1) + collect_id = re.search(r'/collect/(\w+)', url).group(1) print s % (1, 97, u'\n ++ save collect:'), \ 'http://www.xiami.com/song/collect/' + collect_id result = self._save_do(collect_id, 4, tags) elif '/album/' in url: - album_id = re.search(r'/album/(\d+)', url).group(1) + album_id = re.search(r'/album/(\w+)', url).group(1) + album = self._api.album(album_id) + album_id = album[0].album_id print s % (1, 97, u'\n ++ save album:'), \ - 'http://www.xiami.com/album/' + album_id + 'http://www.xiami.com/album/' + str(album_id) result = self._save_do(album_id, 5, tags) elif '/artist/' in url: - artist_id = re.search(r'/artist/(\d+)', url).group(1) + artist_id = re.search(r'/artist/(\w+)', url).group(1) print s % (1, 97, u'\n ++ save artist:'), \ 'http://www.xiami.com/artist/' + artist_id result = self._save_do(artist_id, 6, tags) elif '/song/' in url: - song_id = re.search(r'/song/(\d+)', url).group(1) + song_id = re.search(r'/song/(\w+)', url).group(1) + song = self._api.song(song_id) + song_id = song.song_id print s % (1, 97, u'\n ++ save song:'), \ - 'http://www.xiami.com/song/' + song_id + 'http://www.xiami.com/song/' + str(song_id) result = self._save_do(song_id, 3, tags) elif '/u/' in url: From 17f674c17c5d47c40c4370233d204e47fe280352 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Wed, 1 Aug 2018 14:45:51 +0800 Subject: [PATCH 16/36] [xiami.py] [Fix] * song_name need to unescape html * Get correct song_ids of artist top list --- xiami.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xiami.py b/xiami.py index 50945d2..ee7915e 100755 --- a/xiami.py +++ b/xiami.py @@ -22,7 +22,7 @@ url_album = "http://www.xiami.com/album/%s" url_collect = "http://www.xiami.com/collect/ajax-get-list" url_artist_albums = "http://www.xiami.com/artist/album/id/%s/page/%s" -url_artist_top_song = "http://www.xiami.com/artist/top/id/%s" +url_artist_top_song = "http://www.xiami.com/artist/top-%s" url_lib_songs = "http://www.xiami.com/space/lib-song/u/%s/page/%s" url_recent = "http://www.xiami.com/space/charts-recent/u/%s/page/%s" @@ -345,7 +345,7 @@ def _make_song(self, info): song_sub_title=info['song_sub_title'], songwriters=info['songwriters'], singers=info['singers'], - song_name=info['name'], + song_name=parser.unescape(info['name']), album_id=info['album_id'], album_name=info['album_name'], @@ -371,6 +371,8 @@ def _make_song(self, info): def _find_z(self, album): zs = [] + song = album[0] + for i, song in enumerate(album[:-1]): next_song = album[i+1] @@ -996,7 +998,7 @@ def download_artist_albums(self): def download_artist_top_20_songs(self): html = self._request(url_artist_top_song % self.artist_id).text - song_ids = re.findall(r'/song/(.+?)" title', html) + song_ids = re.findall(r'/music/send/id/(\d+)', html) artist_name = re.search( r'

(.+?)<', html).group(1) d = modificate_text(artist_name + u' - top 20') From 6fdc7eff9072dc59810c115d7b89b69e1bcf15e2 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2018年8月11日 21:59:57 +0800 Subject: [PATCH 17/36] [xiami.py] [Update] handling missing songs --- xiami.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/xiami.py b/xiami.py index ee7915e..7709f82 100755 --- a/xiami.py +++ b/xiami.py @@ -393,6 +393,10 @@ def song(self, song_id): url = self.URL + 'id/%s/cat/json' % song_id resp = self._request(url, headers=HEADERS2) + # there is no song + if not resp.json().get('data'): + return None + info = resp.json()['data']['trackList'][0] song = self._make_song(info) return song @@ -401,6 +405,10 @@ def songs(self, *song_ids): url = self.URL + 'id/%s/cat/json' % '%2C'.join(song_ids) resp = self._request(url, headers=HEADERS2) + # there is no song + if not resp.json().get('data'): + return None + info = resp.json()['data'] songs = [] for info_n in info['trackList']: @@ -412,6 +420,10 @@ def album(self, album_id): url = self.URL + 'id/%s/type/1/cat/json' % album_id resp = self._request(url, headers=HEADERS2) + # there is no album + if not resp.json().get('data'): + return None + info = resp.json()['data'] songs = [] for info_n in info['trackList']: @@ -901,6 +913,9 @@ def make_file_name(self, song, cd_serial_auth=False): def get_songs(self, album_id, song_id=None): songs = self._api.album(album_id) + if not songs: + return [] + cd_serial_auth = int(songs[-1]['cd_serial'])> 1 for song in songs: self.make_file_name(song, cd_serial_auth=cd_serial_auth) @@ -911,6 +926,10 @@ def get_songs(self, album_id, song_id=None): def get_song(self, song_id): song = self._api.song(song_id) + + if not song: + return [] + self.make_file_name(song) return [song] @@ -929,6 +948,9 @@ def download_songs(self, song_ids): def download_album(self): songs = self.get_songs(self.album_id) + if not songs: + return + song = songs[0] d = song['album_name'] + ' - ' + song['artist_name'] From 3c4dfad9ed6ff1c265cead44766e8c7095f389c2 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Fri, 7 Sep 2018 10:06:16 +0800 Subject: [PATCH 18/36] [tumblr.py] support proxy, #125 * using proxy: -x 'protocol://address:port' protocol can be http, https, socks5 --- tumblr.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/tumblr.py b/tumblr.py index 8effbb2..4329c63 100755 --- a/tumblr.py +++ b/tumblr.py @@ -62,6 +62,8 @@ ss = requests.session() ss.headers.update(headers) +PROXY = None + class Error(Exception): def __init__(self, msg): self.msg = msg @@ -119,12 +121,19 @@ def download_run(item): # num = random.randint(0, 7) % 8 # col = s % (1, num + 90, filepath) # print ' ++ download: %s' % col - cmd = ' '.join([ - 'wget', '-c', '-q', '-T', '10', - '-O', '"%s.tmp"' % filepath, - '--user-agent', '"%s"' % headers['User-Agent'], - '"%s"' % item['durl'].replace('http:', 'https:') - ]) + + if PROXY: + cmd = ' '.join([ + 'curl', '-s', '-x', '"%s"' % PROXY, '-o', '"%s.tmp"' % filepath, + '-H', '"User-Agent: %s"' % headers['User-Agent'], + '"%s"' % item['durl'] + ]) + else: + cmd = ' '.join([ + 'curl', '-s', '-o', '"%s.tmp"' % filepath, + '-H', '"User-Agent: %s"' % headers['User-Agent'], + '"%s"' % item['durl'] + ]) status = os.system(cmd) return status, filepath @@ -165,16 +174,20 @@ def _request(self, base_hostname, target, type, params): api_url = '/'.join(['https://api.tumblr.com/v2/blog', base_hostname, target, type]) params['api_key'] = API_KEY + if PROXY: + proxies = {'http': PROXY, 'https': PROXY} + else: + proxies = None while True: try: - res = ss.get(api_url, params=params, timeout=10) + res = ss.get(api_url, params=params, proxies=proxies, timeout=10) json_data = res.json() break except KeyboardInterrupt: sys.exit() except Exception as e: NET_ERRORS.value += 1 # count errors - # print s % (1, 93, '[Error at requests]:'), e + print s % (1, 93, '[Error at requests]:'), e, '\n' time.sleep(5) if json_data['meta']['msg'].lower() != 'ok': raise Error(s % (1, 91, json_data['meta']['msg'])) @@ -506,9 +519,19 @@ def args_handler(argv): help='update new things') p.add_argument('--redownload', action='store_true', help='redownload all things') + p.add_argument('-x', '--proxy', type=str, + help='redownload all things') args = p.parse_args(argv[1:]) xxx = args.xxx + if args.proxy: + if args.proxy[:4] not in ('http', 'sock'): + print s % (1, 91, '[Error]:'), 'proxy must have a protocol:// prefix' + sys.exit(1) + else: + global PROXY + PROXY = args.proxy + if args.redownload: args.update = True return args, xxx From d07ee93f9ba1b9c4e4014a2e9fa2ff8fb314d817 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Fri, 7 Sep 2018 10:12:54 +0800 Subject: [PATCH 19/36] [README.md] [tumblr.py] support proxies --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 2d8b0a9..6c4b9e5 100644 --- a/README.md +++ b/README.md @@ -1301,6 +1301,7 @@ python2-requests (https://github.com/kennethreitz/requests) --update 下载新发布的东西 --redownload 重新遍历所有的东西,如果有漏掉的东西则下载 +--proxy protocol://address:port 设置代理 -f OFFSET, --offset OFFSET 从第offset个开始,只对 -V 有用。 ``` @@ -1314,6 +1315,10 @@ tm是tumblr.py的马甲 (alias tm='python2 /path/to/tumblr.py') tm http://sosuperawesome.tumblr.com tm http://sosuperawesome.tumblr.com -t beautiful +# 下载图片(使用代理) +tm http://sosuperawesome.tumblr.com -x socks5://127.0.0.1:1024 +tm http://sosuperawesome.tumblr.com -t beautiful -x socks5://127.0.0.1:1024 + # 下载单张图片 tm http://sosuperawesome.tumblr.com/post/121467716523/murosvur-on-etsy From ca39081a7265c32c2438b5ec00e33b63ce5ad128 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2019年1月21日 18:06:33 +0800 Subject: [PATCH 20/36] [pan.baidu.com.py] Update params of `transfer` api, #141 --- pan.baidu.com.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index f38df6c..25cbe82 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -1504,8 +1504,8 @@ def _share_transfer(self, surl, info): j = {'errno': 'file has exist'} return j - data = ('filelist=' \ - + urllib.quote_plus('["%s"]' % info['path'].encode('utf8')) \ + data = ('fsidlist=' \ + + urllib.quote_plus('[%s]' % info['fs_id']) \ + '&path=' \ + urllib.quote_plus(info['remotepath'].encode('utf8')) ) @@ -1582,15 +1582,17 @@ def _get_share_infos(self, url, remotepath, infos): j = info['file_list']['list'] isdirs = [x['isdir'] for x in j] paths = [x['path'] for x in j] - z = zip(isdirs, paths) + fs_ids = [x['fs_id'] for x in j] + z = zip(fs_ids, isdirs, paths) if not infos: infos = [ { - 'isdir': x, - 'path': y, + 'fs_id': a, + 'isdir': b, + 'path': c, 'remotepath': remotepath \ if remotepath[-1] != '/' else remotepath[:-1] - } for x, y in z + } for a, b, c in z ] return infos From ddef9c6be7cc1e7e89f3913824d9501700e88143 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2019年1月24日 12:37:40 +0800 Subject: [PATCH 21/36] Remove cookie from headers of `_secret_or_not`, try to fix #142 --- pan.baidu.com.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index 25cbe82..9ca9009 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -1639,8 +1639,8 @@ def save_share(self, url, remotepath, infos=None): def _secret_or_not(url): ss.headers['Referer'] = 'http://pan.baidu.com' r = ss.get(url, headers=headers) + if r.status_code != 200 and r.status_code != 302: - print('cookies', ss.cookies.get_dict()) ss.headers['Cookie'] = ';'.join(['{}={}'.format(k, v) for k, v in ss.cookies.get_dict().items()]) r = ss.get(url, headers=headers, cookies=r.cookies) @@ -1663,8 +1663,8 @@ def _secret_or_not(url): 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Accept': '*/*', 'X-Requested-With': 'XMLHttpRequest', - 'Cookie': 'BAIDUID=0F38C66B2C9AC2FC887BD3FEB059F5AC:FG=1; PANWEB=1', 'Connection': 'keep-alive', + 'Referer': 'http://pan.baidu.com' } r = ss.post(url, data=data, headers=theaders) if r.json()['errno']: From 3942e89273c39827adc1451bebc2832f0bed5fcf Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2019年3月18日 17:32:24 +0800 Subject: [PATCH 22/36] [pan.baidu.com.py] Update `get_vcode` api --- pan.baidu.com.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index 9ca9009..72d35ba 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -2940,20 +2940,40 @@ def get_params(self, path): 'fs_id': j[0]['fs_id'] }) + def get_vcode(self): + url = ( + 'https://pan.baidu.com/api/getvcode' + '?prod=pan' + '&t={}' + '&channel=chunlei' + '&web=1' + '&app_id=250528' + '&bdstoken={}' + ).format(random.random(), self.bdstoken) + + r = ss.get(url) + j = r.json() + return j + def get_infos(self): url = ('https://pan.baidu.com/api/sharedownload?' 'sign={}×tamp={}&bdstoken={}' '&channel=chunlei&clienttype=0&web=1').format( self.sign, self.timestamp, self.bdstoken) - data = ('encrypt=0&product=share' - + '&uk=' + self.uk - + '&primaryid=' + self.shareid - + '&fid_list=' + urllib.quote_plus('["%s"]' % self.infos['fs_id']) - ) + data = { + 'encrypt': '0', + 'product': 'share', + 'uk': self.uk, + 'primaryid': self.shareid, + 'fid_list': urllib.quote_plus('[%s]' % self.infos['fs_id']), + 'path_list': '', + 'vip': '0', + } while True: - r = ss.post(url, data=data) + data_str = '&'.join(['{}={}'.format(k, v) for k, v in data.items()]) + r = ss.post(url, data=data_str) j = r.json() if not j['errno']: dlink = fast_pcs_server(j['list'][0]['dlink'].encode('utf8')) @@ -2964,9 +2984,10 @@ def get_infos(self): panbaiducom_HOME._download_do(self.infos) break else: + j = self.get_vcode() vcode = j['vcode'] input_code = panbaiducom_HOME.save_img(j['img'], 'jpg') - self.params.update({'input': input_code, 'vcode': vcode}) + data.update({'vcode_input': input_code, 'vcode_str': vcode}) def get_infos2(self, path): while True: From 90b786dfe2eab3d7316536ba0357b7c13ac2a3f5 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2019年3月18日 18:24:35 +0800 Subject: [PATCH 23/36] [pan.baidu.com] Fix `verify` error --- pan.baidu.com.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index 72d35ba..81ccf8b 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -1637,6 +1637,8 @@ def save_share(self, url, remotepath, infos=None): @staticmethod def _secret_or_not(url): + surl = url.split('?')[0].split('/1')[1].strip('/') + ss.headers['Referer'] = 'http://pan.baidu.com' r = ss.get(url, headers=headers) @@ -1649,12 +1651,17 @@ def _secret_or_not(url): secret = raw_input(s % (2, 92, " 请输入提取密码: ")) else: secret = args.secret + data = 'pwd=%s&vcode=&vcode_str=' % secret - query = 'bdstoken=null&channel=chunlei&clienttype=0&web=1&app_id=250528' - url = "%s&t=%d&%s" % ( - r.url.replace('init', 'verify'), - int(time.time()*1000), - query + url = ( + 'https://pan.baidu.com/share/verify?' + + 'surl=' + surl + + '&t=' + str(int(time.time()*1000)) + + '&channel=chunlei' + + '&web=1' + + '&app_id=250528' + + '&bdstoken=null' + + '&clienttype=0' ) theaders = { 'Accept-Encoding': 'gzip, deflate', @@ -2975,7 +2982,8 @@ def get_infos(self): data_str = '&'.join(['{}={}'.format(k, v) for k, v in data.items()]) r = ss.post(url, data=data_str) j = r.json() - if not j['errno']: + errno = j['errno'] + if errno == 0: dlink = fast_pcs_server(j['list'][0]['dlink'].encode('utf8')) self.infos['dlink'] = dlink if args.play: @@ -2983,6 +2991,9 @@ def get_infos(self): else: panbaiducom_HOME._download_do(self.infos) break + elif errno == 118: + print s % (1, 91, ' !! 没有下载权限!, 请转存网盘后,从网盘地址下载') + sys.exit(1) else: j = self.get_vcode() vcode = j['vcode'] @@ -3008,7 +3019,7 @@ def get_infos2(self, path): panbaiducom_HOME._download_do(self.infos) break else: - print s % (1, ' !! Error at get_infos2, can\'t get dlink') + print s % (1, 91, ' !! Error at get_infos2, can\'t get dlink') def do(self, paths): for path in paths: From 264cc4de11a0bea351098600f26ce1837fe3fa9d Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2019年9月14日 12:41:13 +0800 Subject: [PATCH 24/36] [pan.baidu.com.py] Update User-Agent --- pan.baidu.com.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index 81ccf8b..581c895 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -114,7 +114,7 @@ "Accept-Language":"en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2", "Referer":"http://pan.baidu.com/disk/home", "X-Requested-With": "XMLHttpRequest", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36", "Connection": "keep-alive", } @@ -758,7 +758,7 @@ def sign2(j, r): self.timestamp = timestamp def _get_dlink(self, path): - dlink = ('http://d.pcs.baidu.com/rest/2.0/pcs/file?method=download' + dlink = ('http://c.pcs.baidu.com/rest/2.0/pcs/file?method=download' '&app_id=250528&path={}&ver=2.0&clienttype=1').format( urllib.quote(path)) @@ -939,25 +939,31 @@ def _download_do(infos): cookie = 'Cookie: ' + '; '.join([ k + '=' + v for k, v in ss.cookies.get_dict().items()]) - user_agent = "netdisk;5.3.1.3;PC;PC-Windows;5.1.2600;WindowsBaiduYunGuanJia" - # user_agent = "netdisk;7.15.1;HUAWEI+G750-T01;android-android;4.2.2" - # user_agent = headers['User-Agent'] + + # Netdisk user agents: + # + # "netdisk;6.7.1.9;PC;PC-Windows;10.0.17763;WindowsBaiduYunGuanJia" + # "netdisk;5.3.1.3;PC;PC-Windows;5.1.2600;WindowsBaiduYunGuanJia" + # "netdisk;7.15.1;HUAWEI+G750-T01;android-android;4.2.2" + # "netdisk;4.4.0.6;PC;PC-Windows;6.2.9200;WindowsBaiduYunGuanJia" + # "netdisk;5.3.1.3;PC;PC-Windows;5.1.2600;WindowsBaiduYunGuanJia" + # + # Recently all downloading requests using above user-agents are limited by baidu + + user_agent = headers['User-Agent'] if args.aget_s: quiet = ' --quiet=true' if args.quiet else '' - #'--user-agent "netdisk;4.4.0.6;PC;PC-Windows;6.2.9200;WindowsBaiduYunGuanJia" ' \ - #'--user-agent "netdisk;5.3.1.3;PC;PC-Windows;5.1.2600;WindowsBaiduYunGuanJia" ' \ - #'--header "Referer:http://pan.baidu.com/disk/home " ' \ - cmd = 'aget -k %s -s %s ' \ + cmd = 'aget ' \ + '"%s" ' \ '-o "%s.tmp" ' \ '-H "User-Agent: %s" ' \ - '-H "Content-Type: application/x-www-form-urlencoded" ' \ + '-H "Referer: http://pan.baidu.com/disk/home" ' \ '-H "Connection: Keep-Alive" ' \ '-H "Accept-Encoding: gzip" ' \ '-H "%s" ' \ - '"%s"' \ - % (args.aget_k, args.aget_s, infos['file'], - user_agent, cookie, infos['dlink']) + '-s %s -k %s' \ + % (infos['dlink'], infos['file'], user_agent, cookie, args.aget_s, args.aget_k) elif args.aria2c: quiet = ' --quiet=true' if args.quiet else '' taria2c = ' -x %s -s %s' % (args.aria2c, args.aria2c) From b9a8edc7539743506e74873e536796fd961838e7 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2019年9月14日 14:05:53 +0800 Subject: [PATCH 25/36] [pan.baidu.com.py] Upload api needs Netdisk user agent --- pan.baidu.com.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index 581c895..83a2f8c 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -118,6 +118,8 @@ "Connection": "keep-alive", } +NETDISK_UA = 'netdisk;8.12.9;;android-android;7.0;JSbridge3.0.0' + ss = requests.session() ss.headers.update(headers) @@ -1143,8 +1145,13 @@ def _rapidupload_file(self, lpath, rpath): "content-crc32" : content_crc32, "ondup" : self.ondup } + + # WARNING: here needs netdist user-agent + theaders = dict(ss.headers) + theaders['User-Agent'] = NETDISK_UA + url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file' - r = ss.post(url, params=p, data=data, verify=VERIFY) + r = ss.post(url, params=p, data=data, verify=VERIFY, headers=theaders) if r.ok: return ENoError else: @@ -1204,8 +1211,13 @@ def _combine_file(self, lpath, rpath): {'block_list': self.upload_datas[lpath]['slice_md5s']} ) } + + # WARNING: here needs netdist user-agent + theaders = dict(ss.headers) + theaders['User-Agent'] = NETDISK_UA + url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file' - r = ss.post(url, params=p, data=data, verify=VERIFY) + r = ss.post(url, params=p, data=data, verify=VERIFY, headers=theaders) if r.ok: return ENoError else: @@ -1232,6 +1244,8 @@ def _upload_slice(self, piece=0, slice=DefaultSliceSize): data = MultipartEncoder(files) theaders = dict(headers) theaders['Content-Type'] = data.content_type + theaders['User-Agent'] = NETDISK_UA + url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file' r = ss.post(url, params=p, data=data, verify=VERIFY, headers=theaders) j = r.json() From c97d7bd521847cc78e53450043ccccfdad475f99 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2019年10月11日 22:01:56 +0800 Subject: [PATCH 26/36] [pan.baidu.com.py] Fix save shared url error --- pan.baidu.com.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index 83a2f8c..6811cbe 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -1659,7 +1659,7 @@ def save_share(self, url, remotepath, infos=None): def _secret_or_not(url): surl = url.split('?')[0].split('/1')[1].strip('/') - ss.headers['Referer'] = 'http://pan.baidu.com' + ss.headers['Referer'] = 'https://pan.baidu.com' r = ss.get(url, headers=headers) if r.status_code != 200 and r.status_code != 302: @@ -1691,12 +1691,14 @@ def _secret_or_not(url): 'Accept': '*/*', 'X-Requested-With': 'XMLHttpRequest', 'Connection': 'keep-alive', - 'Referer': 'http://pan.baidu.com' + 'Sec-Fetch-Mode': 'cors', + 'Referer': 'https://pan.baidu.com/share/init?surl=' + surl } r = ss.post(url, data=data, headers=theaders) if r.json()['errno']: - print s % (2, 91, " !! 提取密码错误\n") + print s % (2, 91, " !! 提取密码错误, %s\n" % r.text) sys.exit(1) + ss.cookies.update(r.cookies.get_dict()) ####################################################################### # for saveing inbox shares @@ -2916,7 +2918,6 @@ def cd_do(path): class panbaiducom(object): @staticmethod def get_web_fileinfo(cm, url): - info = {} if 'shareview' in url: info['uk'] = re.search(r'uk="(\d+)"', cm).group(1) info['shareid'] = re.search(r'shareid="(\d+)"', cm).group(1) From 6594900e2b6318e35b3b70337b1cbb498ecae014 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2020年4月24日 13:50:25 +0800 Subject: [PATCH 27/36] [pan.baidu.com.py] Update mpv arguments --- pan.baidu.com.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index 6811cbe..f855063 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -1033,12 +1033,10 @@ def _play_do(infos): cookie = 'Cookie: ' + '; '.join([ k + '=' + v for k, v in ss.cookies.get_dict().items()]) + user_agent = 'User-Agent: ' + headers['User-Agent'] quiet = ' --really-quiet' if args.quiet else '' - cmd = 'mpv%s --no-ytdl --cache-default 20480 --cache-secs 120 ' \ - '--http-header-fields "%s" ' \ - '--http-header-fields "%s" ' \ - '"%s"' \ - % (quiet, headers['User-Agent'], cookie, infos['dlink']) + cmd = 'mpv%s --no-ytdl --http-header-fields="%s","%s" "%s"' \ + % (quiet, user_agent, cookie, infos['dlink']) os.system(cmd) timeout = 1 From a167b875ee9030e5c1a5e8cfdbdfbf441c02736e Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2020年6月23日 10:50:09 +0800 Subject: [PATCH 28/36] Support to setting `appid` --- pan.baidu.com.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index f855063..a126557 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -760,9 +760,11 @@ def sign2(j, r): self.timestamp = timestamp def _get_dlink(self, path): + # use app_id: 778750 + # reference: [3个方法解决百度网盘限速](https://www.runningcheese.com/baiduyun) dlink = ('http://c.pcs.baidu.com/rest/2.0/pcs/file?method=download' - '&app_id=250528&path={}&ver=2.0&clienttype=1').format( - urllib.quote(path)) + '&app_id={}&path={}&ver=2.0&clienttype=1').format( + args.appid, urllib.quote(path)) dlink = fast_pcs_server(dlink) return dlink @@ -956,7 +958,7 @@ def _download_do(infos): if args.aget_s: quiet = ' --quiet=true' if args.quiet else '' - cmd = 'aget ' \ + cmd = 'ag ' \ '"%s" ' \ '-o "%s.tmp" ' \ '-H "User-Agent: %s" ' \ @@ -3109,6 +3111,8 @@ def handle_args(argv): type=int, help='aget 分段下载数量') p.add_argument('-k', '--aget_k', action='store', default='200K', \ type=str, help='aget 分段大小') + p.add_argument('--appid', action='store', default='250528', type=str, \ + help='设置 app-id. 如果无法下载或下载慢, 尝试设置为 778750') p.add_argument('-p', '--play', action='store_true', help='play with mpv') p.add_argument('-v', '--view', action='count', help='view details') p.add_argument('-V', '--VERIFY', action='store_true', help='verify') From 0b16f45a3d4d4f89cbec2a26ca4f1f6da5fbe2e9 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2020年6月23日 10:50:25 +0800 Subject: [PATCH 29/36] Update --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c4b9e5..1c8cb81 100644 --- a/README.md +++ b/README.md @@ -443,6 +443,7 @@ jca 或 jobclearall # 清除 *全部任务* -k num, --aget_k size aget 分段大小: eg: -k 200K -k 1M -k 2M +--appid num 设置 app-id. 如果无法下载或下载慢, 尝试设置为 778750 -p, --play play with mpv -P password, --passwd password 分享密码,加密密码 -y, --yes yes # 用于 rmre, mvre, cpre, rnre !!慎用 @@ -558,7 +559,8 @@ bp cd ... ``` ## 下载、播放速度慢? -如果wiki中的速度解决方法不管用,可以试试加该参数 -t fs +如果无法下载或下载慢, 尝试设置参数 --appid 778750 +bp d /path/file --appid 778750 # 下载当前工作目录 (递归) bp d . -R @@ -780,6 +782,10 @@ ls、重命名、移动、删除、复制、使用正则表达式进行文件操 > https://github.com/houtianze/bypy + +> 3个方法解决百度网盘限速: https://www.runningcheese.com/baiduyun + + --- @@ -907,6 +913,7 @@ bt c magnet_link -t be64 > http://en.wikipedia.org/wiki/Torrent_file + --- From 341e0fcf68dd9e24ac2b26a0b9a7f1a872e965f2 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2020年10月18日 19:57:50 +0800 Subject: [PATCH 30/36] Add option `outdir` --- pan.baidu.com.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index a126557..69befff 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -489,7 +489,7 @@ def _get_bdstoken(self): html_string = resp.content - mod = re.search(r'"bdstoken":"(.+?)"', html_string) + mod = re.search(r"bdstoken', '(.+?)'", html_string) if mod: self.bdstoken = mod.group(1) return self.bdstoken @@ -873,7 +873,7 @@ def download(self, paths): t = i['path'].encode('utf8') t = t.replace(base_dir, '') t = t[1:] if t[0] == '/' else t - t = os.path.join(os.getcwd(), t) + t = os.path.join(args.outdir, t) i['dlink'] = self._get_dlink(i['path'].encode('utf8')) @@ -896,7 +896,7 @@ def download(self, paths): elif not meta['info'][0]['isdir']: t = os.path.join( - os.getcwd(), meta['info'][0]['server_filename'].encode('utf8') + args.outdir, meta['info'][0]['server_filename'].encode('utf8') ) infos = { 'file': t, @@ -952,17 +952,18 @@ def _download_do(infos): # "netdisk;4.4.0.6;PC;PC-Windows;6.2.9200;WindowsBaiduYunGuanJia" # "netdisk;5.3.1.3;PC;PC-Windows;5.1.2600;WindowsBaiduYunGuanJia" # + # 'LogStatistic' + # Recently all downloading requests using above user-agents are limited by baidu user_agent = headers['User-Agent'] if args.aget_s: quiet = ' --quiet=true' if args.quiet else '' - cmd = 'ag ' \ + cmd = 'aget ' \ '"%s" ' \ '-o "%s.tmp" ' \ '-H "User-Agent: %s" ' \ - '-H "Referer: http://pan.baidu.com/disk/home" ' \ '-H "Connection: Keep-Alive" ' \ '-H "Accept-Encoding: gzip" ' \ '-H "%s" ' \ @@ -978,7 +979,7 @@ def _download_do(infos): '--header "%s" ' \ '"%s"' \ % (quiet, taria2c, tlimit, infos['name'], - infos['dir_'], headers['User-Agent'], + infos['dir_'], user_agent, cookie, infos['dlink']) else: quiet = ' -q' if args.quiet else '' @@ -990,7 +991,7 @@ def _download_do(infos): '--header "%s" ' \ '"%s"' \ % (quiet, tlimit, infos['file'], - headers['User-Agent'], cookie, infos['dlink']) + user_agent, cookie, infos['dlink']) status = os.system(cmd) exit = True @@ -1001,7 +1002,15 @@ def _download_do(infos): pass else: exit = False - if status != 0: # other http-errors, such as 302. + + content_length_matched = False + saved_path = '%s.tmp' % infos['file'] + if os.path.exists(saved_path): + meta = os.stat(saved_path) + if meta.st_size == infos['size']: + content_length_matched = True + + if status != 0 or not content_length_matched: # other http-errors, such as 302. #wget_exit_status_info = wget_es[status] print('\n\n ---### \x1b[1;91mEXIT STATUS\x1b[0m ==> '\ '\x1b[1;91m%d\x1b[0m ###--- \n\n' % status) @@ -2962,9 +2971,9 @@ def get_params(self, path): self.infos.update({ 'name': j[0]['server_filename'].encode('utf8'), 'file': os.path.join( - os.getcwd(), j[0]['server_filename'].encode('utf8') + args.outdir, j[0]['server_filename'].encode('utf8') ), - 'dir_': os.getcwd(), + 'dir_': args.outdir, 'fs_id': j[0]['fs_id'] }) @@ -3030,8 +3039,8 @@ def get_infos2(self, path): if dlink: self.infos = { 'name': name, - 'file': os.path.join(os.getcwd(), name), - 'dir_': os.getcwd(), + 'file': os.path.join(args.outdir, name), + 'dir_': args.outdir, 'dlink': fast_pcs_server(dlink.group(1)) } if args.play: @@ -3061,8 +3070,8 @@ def do4(self, paths): name = urllib.unquote_plus(t) self.infos = { 'name': name, - 'file': os.path.join(os.getcwd(), name), - 'dir_': os.getcwd(), + 'file': os.path.join(args.outdir, name), + 'dir_': args.outdir, 'dlink': fast_pcs_server(path) } @@ -3113,6 +3122,8 @@ def handle_args(argv): type=str, help='aget 分段大小') p.add_argument('--appid', action='store', default='250528', type=str, \ help='设置 app-id. 如果无法下载或下载慢, 尝试设置为 778750') + p.add_argument('-o', '--outdir', action='store', default=os.getcwd(), \ + type=str, help='保存目录') p.add_argument('-p', '--play', action='store_true', help='play with mpv') p.add_argument('-v', '--view', action='count', help='view details') p.add_argument('-V', '--VERIFY', action='store_true', help='verify') From 94ca2179bba84460ca50fbaffccbc2034f854bab Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2020年10月18日 20:01:01 +0800 Subject: [PATCH 31/36] Update --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1c8cb81..5106f7d 100644 --- a/README.md +++ b/README.md @@ -444,6 +444,7 @@ jca 或 jobclearall # 清除 *全部任务* -k 1M -k 2M --appid num 设置 app-id. 如果无法下载或下载慢, 尝试设置为 778750 +-o path, --outdir path 指定下周目录: eg: -o /path/to/directory -p, --play play with mpv -P password, --passwd password 分享密码,加密密码 -y, --yes yes # 用于 rmre, mvre, cpre, rnre !!慎用 From e21f0f12444cfc072001964073af53e4efdf7209 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2020年11月16日 16:21:30 +0800 Subject: [PATCH 32/36] Update `_get_dlink` api using code from BaiduPCS-Go. Using aget-rs --- pan.baidu.com.py | 469 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 464 insertions(+), 5 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index 69befff..86e1124 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -3,6 +3,7 @@ import os import sys +import hashlib import functools import requests requests.packages.urllib3.disable_warnings() # disable urllib3's warnings https://urllib3.readthedocs.org/en/latest/security.html#insecurerequestwarning @@ -102,6 +103,348 @@ ".zoo", ".zpaq", ".zz" ] +PHONE_MODEL_DATABASE = [ + "1501_M02", # 360 F4 + "1503-M02", # 360 N4 + "1505-A01", # 360 N4S + "303SH", # 夏普 Aquos Crystal Xx Mini 303SH + "304SH", # 夏普 Aquos Crystal Xx SoftBank + "305SH", # 夏普 Aquos Crystal Y + "306SH", # 夏普 Aquos Crystal 306SH + "360 Q5 Plus", # 360 Q5 Plus + "360 Q5", # 360 Q5 + "402SH", # 夏普 Aquos Crystal X + "502SH", # 夏普 Aquos Crystal Xx2 + "6607", # OPPO U3 + "A1001", # 一加手机1 + "ASUS_A001", # 华硕 ZenFone 3 Ultra + "ASUS_A001", # 华硕 ZenFone 3 Ultra + "ASUS_Z00ADB", # 华硕 ZenFone 2 + "ASUS_Z00UDB", # 华硕 Zenfone Selfie + "ASUS_Z00XSB", # 华硕 ZenFone Zoom + "ASUS_Z012DE", # 华硕 ZenFone 3 + "ASUS_Z012DE", # 华硕 ZenFone 3 + "ASUS_Z016D", # 华硕 ZenFone 3 尊爵 + "ATH-TL00H", # 华为 荣耀 7i + "Aster T", # Vertu Aster T + "BLN-AL10", # 华为 荣耀 畅玩6X + "BND-AL10", # 荣耀7X + "BTV-W09", # 华为 M3 + "CAM-UL00", # 华为 荣耀 畅玩5A + "Constellation V", # Vertu Constellation V + "D6683", # 索尼 Xperia Z3 Dual TD + "DIG-AL00", # 华为 畅享 6S + "E2312", # 索尼 Xperia M4 Aqua + "E2363 ", # 索尼 Xperia M4 Aqua Dual + "E5363", # 索尼 Xperia C4 + "E5563", # 索尼 Xperia C5 + "E5663", # 索尼 Xperia M5 + "E5823", # 索尼 Xperia Z5 Compact + "E6533", # 索尼 Xperia Z3+ + "E6683", # 索尼 Xperia Z5 + "E6883", # 索尼 Xperia Z5 Premium + "EBEN M2", # 8848 M2 + "EDI-AL10", # 华为 荣耀 Note 8 + "EVA-AL00", # 华为 P9 + "F100A", # 金立 F100 + "F103B", # 金立 F103B + "F3116", # 索尼 Xperia XA + "F3216", # 索尼 Xperia XA Ultra + "F5121 / F5122", # 索尼 Xperia X + "F5321", # 索尼 Xperia X Compact + "F8132", # 索尼 Xperia X Performance + "F8332", # 索尼 Xperia XZ + "FRD-AL00", # 华为 荣耀 8 + "FS8001", # 夏普 C1 + "FS8002", # 夏普 A1 + "G0111", # 格力手机 1 + "G0215", # 格力手机 2 + "G8142", # 索尼Xperia XZ Premium G8142 + "G8342", # 索尼Xperia XZ1 + "GIONEE S9", # 金立 S9 + "GN5001S", # 金立 金钢 + "GN5003", # 金立 大金钢 + "GN8002S", # 金立 M6 Plus + "GN8003", # 金立 M6 + "GN9011", # 金立 S8 + "GN9012", # 金立 S6 Pro + "GRA-A0", # Coolpad Cool Play 6C + "H60-L11", # 华为 荣耀 6 + "HN3-U01", # 华为 荣耀 3 + "HTC D10w", # HTC Desire 10 Pro + "HTC E9pw", # HTC One E9+ + "HTC M10u", # HTC 10 + "HTC M8St", # HTC One M8 + "HTC M9PT", # HTC One M9+ + "HTC M9e", # HTC One M9 + "HTC One A9", # HTC One A9 + "HTC U-1w", # HTC U Ultra + "HTC X9u", # HTC One X9 + "HTC_M10h", # HTC 10 国际版 + "HUAWEI CAZ-AL00", # 华为 Nova + "HUAWEI CRR-UL00", # 华为 Mate S + "HUAWEI GRA-UL10", # 华为 P8 + "HUAWEI MLA-AL10", # 华为 麦芒 5 + "HUAWEI MT7-AL00", # 华为 mate 7 + "HUAWEI MT7-TL00", # 华为 Mate 7 + "HUAWEI NXT-AL10", # 华为 Mate 8 + "HUAWEI P7-L00", # 华为 P7 + "HUAWEI RIO-AL00", # 华为 麦芒 4 + "HUAWEI TAG-AL00", # 华为 畅享 5S + "HUAWEI VNS-AL00", # 华为 G9 + "IUNI N1", # 艾优尼 N1 + "IUNI i1", # 艾优尼 i1 + "KFAPWI", # Amazon Kindle Fire HDX 8.9 + "KFSOWI", # Amazon Kindle Fire HDX 7 + "KFTHWI", # Amazon Kindle Fire HD + "KIW-TL00H", # 华为 荣耀 畅玩5X + "KNT-AL10", # 华为 荣耀 V8 + "L55t", # 索尼 Xperia Z3 + "L55u", # 索尼 Xperia Z3 + "LEX626", # 乐视 乐S3 + "LEX720", # 乐视 乐Pro3 + "LG-D858", # LG G3 + "LG-H818", # LG G4 + "LG-H848", # LG G5 SE + "LG-H868", # LG G5 + "LG-H968", # LG V10 + "LON-AL00", # 华为 Mate 9 Pro + "LON-AL00-PD", # 华为 Mate 9 Porsche Design + "LT18i", # Sony Ericsson Xperia Arc S + "LT22i", # Sony Ericsson Xperia P + "LT26i", # Sony Ericsson Xperia S + "LT26ii", # Sony Ericsson Xperia SL + "LT26w", # Sony Ericsson Xperia Acro S + "Le X520", # 乐视 乐2 + "Le X620", # 乐视 乐2Pro + "Le X820", # 乐视 乐Max2 + "Lenovo A3580", # 联想 黄金斗士 A8 畅玩 + "Lenovo A7600-m", # 联想 黄金斗士 S8 + "Lenovo A938t", # 联想 黄金斗士 Note8 + "Lenovo K10e70", # 联想 乐檬K10 + "Lenovo K30-T", # 联想 乐檬 K3 + "Lenovo K32C36", # 联想 乐檬3 + "Lenovo K50-t3s", # 联想 乐檬 K3 Note + "Lenovo K52-T38", # 联想 乐檬 K5 Note + "Lenovo K52e78", # Lenovo K5 Note + "Lenovo P2c72", # 联想 P2 + "Lenovo X3c50", # 联想 乐檬 X3 + "Lenovo Z90-3", # 联想 VIBE Shot大拍 + "M040", # 魅族 MX 2 + "M1 E", # 魅蓝 E + "M2-801w", # 华为 M2 + "M2017", # 金立 M2017 + "M3", # EBEN M3 + "M355", # 魅族 MX 3 + "MHA-AL00", # 华为 Mate 9 + "MI 4LTE", # 小米手机4 + "MI 4S", # 小米手机4S + "MI 5", # 小米手机5 + "MI 5s Plus", # 小米手机5s Plus + "MI 5s", # 小米手机5s + "MI MAX", # 小米Max + "MI Note Pro", # 小米Note顶配版 + "MI PAD 2", # 小米平板 2 + "MIX", # 小米MIX + "MLA-UL00", # 华为 G9 Plus + "MP1503", # 美图 M6 + "MP1512", # 美图 M6s + "MT27i", # Sony Ericsson Xperia Sola + "MX4 Pro", # 魅族 MX 4 Pro + "MX4", # 魅族 MX 4 + "MX5", # 魅族 MX 5 + "MX6", # 魅族 MX 6 + "Meitu V4s", # 美图 V4s + "Meizu M3 Max", # 魅蓝max + "Meizu U20", # 魅蓝U20 + "Mi 5", + "Mi 6", + "Mi A1", # MI androidone + "Mi Note 2", # 小米Note2 + "MiTV2S-48", # 小米电视2s + "Moto G (4)", # 摩托罗拉 G4 Plus + "N1", # Nokia N1 + "NCE-AL00", # 华为 畅享 6 + "NTS-AL00", # 华为 荣耀 Magic + "NWI-AL10", # nova2s + "NX508J", # 努比亚 Z9 + "NX511J", # 努比亚 小牛4 Z9 Mini + "NX512J", # 努比亚 大牛 Z9 Max + "NX513J", # 努比亚 My 布拉格 + "NX513J", # 努比亚 布拉格S + "NX523J", # 努比亚 Z11 Max + "NX529J", # 努比亚 小牛5 Z11 Mini + "NX531J", # 努比亚 Z11 + "NX549J", # 努比亚 小牛6 Z11 MiniS + "NX563J", # 努比亚Z17 + "Nexus 4", + "Nexus 5X", + "Nexus 6", + "Nexus 6P", + "Nexus 7", + "Nexus 9", + "Nokia_X", # Nokia X + "Nokia_XL_4G", # Nokia XL + "ONE A2001", # 一加手机2 + "ONE E1001", # 一加手机X + "ONEPLUS A5010", # 一加5T + "OPPO A53", # OPPO A53 + "OPPO A59M", # OPPO A59 + "OPPO A59s", # OPPO A59s + "OPPO R11", + "OPPO R7", # OPPO R7 + "OPPO R7Plus", # OPPO R7Plus + "OPPO R7S", # OPPO R7S + "OPPO R7sPlus", # OPPO R7sPlus + "OPPO R9 Plustm A", # OPPO R9Plus + "OPPO R9s Plus", # OPPO R9s Plus + "OPPO R9s", + "OPPO R9s", # OPPO R9s + "OPPO R9tm", # OPPO R9 + "PE-TL10", # 华为 荣耀 6 Plus + "PLK-TL01H", # 华为 荣耀 7 + "Pro 5", # 魅族 Pro 5 + "Pro 6", # 魅族 Pro 6 + "Pro 6s", # 魅族 Pro 6s + "RM-1010", # Nokia Lumia 638 + "RM-1018", # Nokia Lumia 530 + "RM-1087", # Nokia Lumia 930 + "RM-1090", # Nokia Lumia 535 + "RM-867", # Nokia Lumia 920 + "RM-875", # Nokia Lumia 1020 + "RM-887", # Nokia Lumia 720 + "RM-892", # Nokia Lumia 925 + "RM-927", # Nokia Lumia 929 + "RM-937", # Nokia Lumia 1520 + "RM-975", # Nokia Lumia 635 + "RM-977", # Nokia Lumia 630 + "RM-984", # Nokia Lumia 830 + "RM-996", # Nokia Lumia 1320 + "Redmi 3S", # 红米3s + "Redmi 4", # 小米 红米4 + "Redmi 4A", # 小米 红米4A + "Redmi Note 2", # 小米 红米Note2 + "Redmi Note 3", # 小米 红米Note3 + "Redmi Note 4", # 小米 红米Note4 + "Redmi Pro", # 小米 红米Pro + "S3", # 佳域S3 + "SCL-TL00H", # 华为 荣耀 4A + "SD4930UR", # Amazon Fire Phone + "SH-03G", # 夏普 Aquos Zeta SH-03G + "SH-04F", # 夏普 Aquos Zeta SH-04F + "SHV31", # 夏普 Aquos Serie Mini SHV31 + "SM-A5100", # Samsung Galaxy A5 + "SM-A7100", # Samsung Galaxy A7 + "SM-A8000", # Samsung Galaxy A8 + "SM-A9000", # Samsung Galaxy A9 + "SM-A9100", # Samsung Galaxy A9 高配版 + "SM-C5000", # Samsung Galaxy C5 + "SM-C5010", # Samsung Galaxy C5 Pro + "SM-C7000", # Samsung Galaxy C7 + "SM-C7010", # Samsung Galaxy C7 Pro + "SM-C9000", # Samsung Galaxy C9 Pro + "SM-G1600", # Samsung Galaxy Folder + "SM-G5500", # Samsung Galaxy On5 + "SM-G6000", # Samsung Galaxy On7 + "SM-G7100", # Samsung Galaxy On7(2016) + "SM-G7200", # Samsung Galasy Grand Max + "SM-G9198", # Samsung 领世旗舰III + "SM-G9208", # Samsung Galaxy S6 + "SM-G9250", # Samsung Galasy S7 Edge + "SM-G9280", # Samsung Galaxy S6 Edge+ + "SM-G9300", # Samsung Galaxy S7 + "SM-G9350", # Samsung Galaxy S7 Edge + "SM-G9500", # Samsung Galaxy S8 + "SM-G9550", # Samsung Galaxy S8+ + "SM-G9600", # Samsung Galaxy S9 + "SM-G960F", # Galaxy S9 Dual SIM + "SM-G9650", # Samsung Galaxy S9+ + "SM-G965F", # Galaxy S9+ Dual SIM + "SM-J3109", # Samsung Galaxy J3 + "SM-J3110", # Samsung Galaxy J3 Pro + "SM-J327A", # Samsung Galaxy J3 Emerge + "SM-J5008", # Samsung Galaxy J5 + "SM-J7008", # Samsung Galaxy J7 + "SM-N9108V", # Samsung Galasy Note4 + "SM-N9200", # Samsung Galaxy Note5 + "SM-N9300", # Samsung Galaxy Note 7 + "SM-N935S", # Samsung Galaxy Note Fan Edition + "SM-N9500", # Samsung Galasy Note8 + "SM-W2015", # Samsung W2015 + "SM-W2016", # Samsung W2016 + "SM-W2017", # Samsung W2017 + "SM705", # 锤子 T1 + "SM801", # 锤子 T2 + "SM901", # 锤子 M1 + "SM919", # 锤子 M1L + "ST18i", # Sony Ericsson Xperia Ray + "ST25i", # Sony Ericsson Xperia U + "STV100-1", # 黑莓Priv + "Signature Touch", # Vertu Signature Touch + "TA-1000", # Nokia 6 + "TA-1000", # HMD Nokia 6 + "TA-1041", # Nokia 7 + "VERTU Ti", # Vertu Ti + "VIE-AL10", # 华为 P9 Plus + "VIVO X20", + "VIVO X20A", + "W909", # 金立 天鉴 W909 + "X500", # 乐视 乐1S + "X608", # 乐视 乐1 + "X800", # 乐视 乐1Pro + "X900", # 乐视 乐Max + "XT1085", # 摩托罗拉 X + "XT1570", # 摩托罗拉 X Style + "XT1581", # 摩托罗拉 X 极 + "XT1585", # 摩托罗拉 Droid Turbo 2 + "XT1635", # 摩托罗拉 Z Play + "XT1635-02", # 摩托罗拉 Z Play + "XT1650", # 摩托罗拉 Z + "XT1650-05", # 摩托罗拉 Z + "XT1706", # 摩托罗拉 E3 POWER + "YD201", # YotaPhone2 + "YD206", # YotaPhone2 + "YQ60", # 锤子 坚果 + "ZTE A2015", # 中兴 AXON 天机 + "ZTE A2017", # 中兴 AXON 天机 7 + "ZTE B2015", # 中兴 AXON 天机 MINI + "ZTE BV0720", # 中兴 Blade A2 + "ZTE BV0730", # 中兴 Blade A2 Plus + "ZTE C2016", # 中兴 AXON 天机 MAX + "ZTE C2017", # 中兴 AXON 天机 7 MAX + "ZTE G720C", # 中兴 星星2号 + "ZUK Z2121", # ZUK Z2 Pro + "ZUK Z2131", # ZUK Z2 + "ZUK Z2151", # ZUK Edge + "ZUK Z2155", # ZUK Edge L + "m030", # 魅族mx + "m1 metal", # 魅蓝metal + "m1 note", # 魅蓝 Note + "m1", # 魅蓝 + "m2 note", # 魅蓝 Note 2 + "m2", # 魅蓝 2 + "m3 note", # 魅蓝 Note 3 + "m3", # 魅蓝 3 + "m3s", # 魅蓝 3S + "m9", # 魅族m9 + "marlin", # Google Pixel XL + "sailfish", # Google Pixel + "vivo V3Max", # vivo V3Max + "vivo X6D", # vivo X6 + "vivo X6PlusD", # vivo X6Plus + "vivo X6S", # vivo X6S + "vivo X6SPlus", # vivo X6SPlus + "vivo X7", # vivo X7 + "vivo X7Plus", # vivo X7Plus + "vivo X9", # vivo X9 + "vivo X9Plus", # vivo X9Plus + "vivo Xplay5A 金", # vivo Xplay5 + "vivo Xplay6", # vivo Xplay6 + "vivo Y66", # vivo Y66 + "vivo Y67", # vivo Y67 + "z1221", # ZUK Z1 +] + s = '\x1b[%s;%sm%s\x1b[0m' # terminual color template cookie_file = os.path.join(os.path.expanduser('~'), '.bp.cookies') @@ -123,6 +466,39 @@ ss = requests.session() ss.headers.update(headers) +def to_md5(buff): + assert isinstance(buff, (str, unicode)) + if isinstance(buff, unicode): + buff = buff.encode('utf-8') + return hashlib.md5(buff).hexdigest() + + +def to_sha1(buff): + assert isinstance(buff, (str, unicode)) + if isinstance(buff, unicode): + buff = buff.encode('utf-8') + return hashlib.sha1(buff).hexdigest() + +# 根据key计算出imei +def sum_IMEI(key): + hs = 53202347234687234 + for k in key: + hs += (hs << 5) + ord(k) + hs %= int(1e15) + if hs < int(1e14): + hs += int(1e14) + return str(int(hs)) + +# 根据key, 从 PHONE_MODEL_DATABASE 中取出手机型号 +def get_phone_model(key): + if len(PHONE_MODEL_DATABASE) <= 0: + return "S3" + hs = 2134 + for k in key: + hs += (hs << 4) + ord(k) + hs %= len(PHONE_MODEL_DATABASE) + return PHONE_MODEL_DATABASE[hs] + def import_shadowsocks(): try: global encrypt @@ -239,6 +615,7 @@ def __init__(self): self.accounts = self._check_cookie_file() self.dsign = None self.timestamp = None + self.user_id = None self.highlights = [] if any([args.tails, args.heads, args.includes]): @@ -309,6 +686,8 @@ def init(self): user = u[0] self.user = user self.cwd = j[user]['cwd'] if j[user].get('cwd') else '/' + self.user_id = j[user].get('user_id') + self.bduss = j[user]['cookies']['BDUSS'] ss.cookies.update(j[user]['cookies']) else: print s % (1, 91, ' !! no account is online, please login or userchange') @@ -320,6 +699,10 @@ def init(self): with open(cookie_file, 'w') as g: pk.dump(j, g) sys.exit(1) + + if not self.user_id: + info = self._user_info(self.bduss) + self.user_id = info['user']['id'] else: print s % (1, 97, ' no account, please login') sys.exit(1) @@ -465,6 +848,7 @@ def save_cookies(self, username=None, on=0, tocwd=False): accounts[username]['cookies'] = \ accounts[username].get('cookies', ss.cookies.get_dict()) accounts[username]['on'] = on + accounts[username]['user_id'] = self.user_id quota = self._get_quota() capacity = '%s/%s' % (sizeof_fmt(quota['used']), sizeof_fmt(quota['total'])) accounts[username]['capacity'] = capacity @@ -499,6 +883,44 @@ def _get_bdstoken(self): # self.bdstoken = md5.new(str(time.time())).hexdigest() + def _user_info(self, bduss): + timestamp = str(int(time.time())) + model = get_phone_model(bduss) + phoneIMEIStr = sum_IMEI(bduss) + + data = { + 'bdusstoken': bduss + '|null', + 'channel_id': '', + 'channel_uid': '', + 'stErrorNums': '0', + 'subapp_type': 'mini', + 'timestamp': timestamp + '922', + } + data['_client_type'] = '2' + data['_client_version'] = '7.0.0.0' + data['_phone_imei'] = phoneIMEIStr + data['from'] = 'mini_ad_wandoujia' + data['model'] = model + data['cuid'] = to_md5( + bduss + '_' + data['_client_version'] + '_' + data['_phone_imei'] + '_' + data['from'] + ).upper() + '|' + phoneIMEIStr[::-1] + data['sign'] = to_md5( + ''.join([k + '=' + data[k] for k in sorted(data.keys())]) + 'tiebaclient!!!' + ).upper() + + headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Cookie': 'ka=open', + 'net': '1', + 'User-Agent': 'bdtb for Android 6.9.2.1', + 'client_logid': timestamp + '416', + 'Connection': 'Keep-Alive', + } + + resp = requests.post('http://tieba.baidu.com/c/s/login', headers=headers, data=data) + info = resp.json() + return info + #def _sift(self, fileslist, name=None, size=None, time=None, head=None, tail=None, include=None, exclude=None): def _sift(self, fileslist, **arguments): """ @@ -760,6 +1182,35 @@ def sign2(j, r): self.timestamp = timestamp def _get_dlink(self, path): + bduss = self.bduss + uid = self.user_id + + timestamp = str(int(time.time() * 1000)) + devuid = '0|' + to_md5(bduss).upper() + + enc = to_sha1(bduss) + rand = to_sha1( + enc + str(uid) + 'ebrcUYiuxaZv2XGu7KIYKxUrqfnOfpDF' + str(timestamp) + devuid + ) + + url = ( + 'https://pcs.baidu.com/rest/2.0/pcs/file?app_id=' + args.appid \ + + '&method=locatedownload&ver=2' \ + + '&path=' + urllib.quote(path) + '&time=' \ + + timestamp + '&rand=' + rand + '&devuid=' + devuid + ) + + headers = dict(ss.headers) + headers['User-Agent'] = 'netdisk;2.2.51.6;netdisk;10.0.63;PC;android-android' + resp = self._request('GET', url, '_get_dlink', headers=headers) + info = resp.json() + if info.get('urls'): + return info['urls'][0]['url'].encode('utf8') + else: + print s % (1, 91, ' !! Error at _get_dlink, can\'t get dlink') + sys.exit(1) + + def _get_dlink4(self, path): # use app_id: 778750 # reference: [3个方法解决百度网盘限速](https://www.runningcheese.com/baiduyun) dlink = ('http://c.pcs.baidu.com/rest/2.0/pcs/file?method=download' @@ -956,11 +1407,12 @@ def _download_do(infos): # Recently all downloading requests using above user-agents are limited by baidu - user_agent = headers['User-Agent'] + # user_agent = headers['User-Agent'] + user_agent = 'netdisk;2.2.51.6;netdisk;10.0.63;PC;android-android' if args.aget_s: quiet = ' --quiet=true' if args.quiet else '' - cmd = 'aget ' \ + cmd = 'ag ' \ '"%s" ' \ '-o "%s.tmp" ' \ '-H "User-Agent: %s" ' \ @@ -976,18 +1428,25 @@ def _download_do(infos): cmd = 'aria2c -c%s%s%s ' \ '-o "%s.tmp" -d "%s" ' \ '--user-agent "%s" ' \ + '--header "Connection: Keep-Alive" ' \ + '--header "Accept-Encoding: gzip" ' \ '--header "%s" ' \ '"%s"' \ % (quiet, taria2c, tlimit, infos['name'], infos['dir_'], user_agent, cookie, infos['dlink']) else: + if infos['size']>= 100 * OneM: + print '\x1b[1;91mWarning\x1b[0m: '\ + '\x1b[1;91m%s\x1b[0m\n\n' % "File size is large, please use aget or aria2 to download\naget: https://github.com/PeterDing/aget-rs\naria2: https://github.com/aria2/aria2" + quiet = ' -q' if args.quiet else '' tlimit = ' --limit-rate %s' % args.limit if args.limit else '' cmd = 'wget -c%s%s ' \ '-O "%s.tmp" ' \ - '--user-agent "%s" ' \ - '--header "Referer:http://pan.baidu.com/disk/home" ' \ + '--header "User-Agent: %s" ' \ + '--header "Connection: Keep-Alive" ' \ + '--header "Accept-Encoding: gzip" ' \ '--header "%s" ' \ '"%s"' \ % (quiet, tlimit, infos['file'], @@ -3120,7 +3579,7 @@ def handle_args(argv): type=int, help='aget 分段下载数量') p.add_argument('-k', '--aget_k', action='store', default='200K', \ type=str, help='aget 分段大小') - p.add_argument('--appid', action='store', default='250528', type=str, \ + p.add_argument('--appid', action='store', default='778750', type=str, \ help='设置 app-id. 如果无法下载或下载慢, 尝试设置为 778750') p.add_argument('-o', '--outdir', action='store', default=os.getcwd(), \ type=str, help='保存目录') From 25a4c334c691a508c9ab473c9a4d41676a770517 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2020年11月16日 16:22:04 +0800 Subject: [PATCH 33/36] Update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5106f7d..7171cc0 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ wget aria2 (~ 1.18) -aget # 需要 python>= 3.5, 安装 pip3 install aget +aget-rs (https://github.com/PeterDing/aget-rs/releases) pip2 install rsa pyasn1 requests requests-toolbelt From 65131cda47750cda281164cd04de5ed7e1a2785e Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2020年12月10日 18:52:36 +0800 Subject: [PATCH 34/36] Update `_get_bdstoken`; fix the error of mpv playing m3u8 --- pan.baidu.com.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index 86e1124..a13c270 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -873,7 +873,8 @@ def _get_bdstoken(self): html_string = resp.content - mod = re.search(r"bdstoken', '(.+?)'", html_string) + open('/tmp/t/sss', 'w').write(html_string) + mod = re.search(r'bdstoken[\'":\s]+([0-9a-f]{32})', html_string) if mod: self.bdstoken = mod.group(1) return self.bdstoken @@ -1205,7 +1206,8 @@ def _get_dlink(self, path): resp = self._request('GET', url, '_get_dlink', headers=headers) info = resp.json() if info.get('urls'): - return info['urls'][0]['url'].encode('utf8') + dlink = info['urls'][0]['url'].encode('utf8') + return dlink else: print s % (1, 91, ' !! Error at _get_dlink, can\'t get dlink') sys.exit(1) @@ -1417,7 +1419,6 @@ def _download_do(infos): '-o "%s.tmp" ' \ '-H "User-Agent: %s" ' \ '-H "Connection: Keep-Alive" ' \ - '-H "Accept-Encoding: gzip" ' \ '-H "%s" ' \ '-s %s -k %s' \ % (infos['dlink'], infos['file'], user_agent, cookie, args.aget_s, args.aget_k) @@ -1505,8 +1506,14 @@ def _play_do(infos): k + '=' + v for k, v in ss.cookies.get_dict().items()]) user_agent = 'User-Agent: ' + headers['User-Agent'] quiet = ' --really-quiet' if args.quiet else '' - cmd = 'mpv%s --no-ytdl --http-header-fields="%s","%s" "%s"' \ - % (quiet, user_agent, cookie, infos['dlink']) + cmd = 'mpv%s --no-ytdl --http-header-fields="%s","%s" ' \ + % (quiet, user_agent, cookie) + + if infos.get('m3u8'): + # https://github.com/mpv-player/mpv/issues/6928#issuecomment-532198445 + cmd += ' --stream-lavf-o-append="protocol_whitelist=file,http,https,tcp,tls,crypto,hls,applehttp" ' + + cmd += "%s" % infos['dlink'] os.system(cmd) timeout = 1 @@ -3301,7 +3308,7 @@ def _share(self, paths, pwd=None): r = ss.post(url, params=params, data=data) j = r.json() - if j['errno'] != 0: + if not j.get('shorturl'): print s % (1, 91, ' !! Error at _share'), j sys.exit(1) else: From a2af64e8dfcf9906f4c145ce84a1fe2682eab6c1 Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2020年12月14日 14:58:39 +0800 Subject: [PATCH 35/36] Remove unused code --- pan.baidu.com.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pan.baidu.com.py b/pan.baidu.com.py index a13c270..2617123 100755 --- a/pan.baidu.com.py +++ b/pan.baidu.com.py @@ -873,7 +873,6 @@ def _get_bdstoken(self): html_string = resp.content - open('/tmp/t/sss', 'w').write(html_string) mod = re.search(r'bdstoken[\'":\s]+([0-9a-f]{32})', html_string) if mod: self.bdstoken = mod.group(1) From 556ad287dc3b0879a539549fed776f3ef2ef4a1a Mon Sep 17 00:00:00 2001 From: PeterDing Date: 2021年1月21日 16:12:50 +0800 Subject: [PATCH 36/36] Update --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7171cc0..81fa0af 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,8 @@ -## iScript +# iScript + +## pan.baidu.com.py 已经重构,不再维护 + +[**BaiduPCS-Py**](https://github.com/PeterDing/BaiduPCS-Py) 是 pan.baidu.com.py 的重构版,运行在 Python>= 3.6 [![Join the chat at https://gitter.im/PeterDing/iScript](https://badges.gitter.im/PeterDing/iScript.svg)](https://gitter.im/PeterDing/iScript?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -206,6 +210,10 @@ xm s http://www.xiami.com/artist/23460?spm=a1z1s.6928801.1561534521.115.ShW08b ### pan.baidu.com.py - 百度网盘的下载、离线下载、上传、播放、转存、文件操作 +**pan.baidu.com.py 已经重构,不再维护** + +[**BaiduPCS-Py**](https://github.com/PeterDing/BaiduPCS-Py) 是 pan.baidu.com.py 的重构版,运行在 Python>= 3.6 + #### 1. 依赖 ```

AltStyle によって変換されたページ (->オリジナル) /