Hello community, here is the log from the commit of package you-get for openSUSE:Factory checked in at 2018-01-16 09:42:26 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/you-get (Old) and /work/SRC/openSUSE:Factory/.you-get.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "you-get" Tue Jan 16 09:42:26 2018 rev:2 rq:564040 version:0.4.1011 Changes: -------- --- /work/SRC/openSUSE:Factory/you-get/you-get.changes 2018-01-13 21:43:20.458119477 +0100 +++ /work/SRC/openSUSE:Factory/.you-get.new/you-get.changes 2018-01-16 09:42:29.237361299 +0100 @@ -1,0 +2,5 @@ +Sat Dec 23 19:13:51 UTC 2017 - aloi...@gmx.com + +- Update to version 0.4.1011 (no changelog supplied) + +------------------------------------------------------------------- Old: ---- you-get-0.4.995.tar.gz New: ---- you-get-0.4.1011.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ you-get.spec ++++++ --- /var/tmp/diff_new_pack.cgbihF/_old 2018-01-16 09:42:29.937328535 +0100 +++ /var/tmp/diff_new_pack.cgbihF/_new 2018-01-16 09:42:29.941328348 +0100 @@ -17,7 +17,7 @@ Name: you-get -Version: 0.4.995 +Version: 0.4.1011 Release: 0 Summary: Dumb downloader that scrapes the web License: MIT ++++++ you-get-0.4.995.tar.gz -> you-get-0.4.1011.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/README.md new/you-get-0.4.1011/README.md --- old/you-get-0.4.995/README.md 2017-12-04 04:18:45.000000000 +0100 +++ new/you-get-0.4.1011/README.md 2017-12-23 18:27:08.000000000 +0100 @@ -413,6 +413,9 @@ | 火猫TV | <http://www.huomao.com/> |✓| | | | 全民直播 | <http://www.quanmin.tv/> |✓| | | | 阳光宽频网 | <http://www.365yg.com/> |✓| | | +| 西瓜视频 | <https://www.ixigua.com/> |✓| | | +| 快手 | <https://www.kuaishou.com/> |✓|✓| | +| 抖音 | <https://www.douyin.com/> |✓| | | For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/src/you_get/common.py new/you-get-0.4.1011/src/you_get/common.py --- old/you-get-0.4.995/src/you_get/common.py 2017-12-04 04:18:45.000000000 +0100 +++ new/you-get-0.4.1011/src/you_get/common.py 2017-12-23 18:27:08.000000000 +0100 @@ -38,6 +38,7 @@ 'dailymotion' : 'dailymotion', 'dilidili' : 'dilidili', 'douban' : 'douban', + 'douyin' : 'douyin', 'douyu' : 'douyutv', 'ehow' : 'ehow', 'facebook' : 'facebook', @@ -61,11 +62,13 @@ 'interest' : 'interest', 'iqilu' : 'iqilu', 'iqiyi' : 'iqiyi', + 'ixigua' : 'ixigua', 'isuntv' : 'suntv', 'joy' : 'joy', 'kankanews' : 'bilibili', 'khanacademy' : 'khan', 'ku6' : 'ku6', + 'kuaishou' : 'kuaishou', 'kugou' : 'kugou', 'kuwo' : 'kuwo', 'le' : 'le', @@ -1021,8 +1024,11 @@ global output_filename if output_filename: dotPos = output_filename.rfind('.') - title = output_filename[:dotPos] - ext = output_filename[dotPos+1:] + if dotPos > 0: + title = output_filename[:dotPos] + ext = output_filename[dotPos+1:] + else: + title = output_filename title = tr(get_filename(title)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/src/you_get/extractors/__init__.py new/you-get-0.4.1011/src/you_get/extractors/__init__.py --- old/you-get-0.4.995/src/you_get/extractors/__init__.py 2017-12-04 04:18:45.000000000 +0100 +++ new/you-get-0.4.1011/src/you_get/extractors/__init__.py 2017-12-23 18:27:08.000000000 +0100 @@ -15,6 +15,7 @@ from .dailymotion import * from .dilidili import * from .douban import * +from .douyin import * from .douyutv import * from .ehow import * from .facebook import * @@ -85,3 +86,4 @@ from .ted import * from .khan import * from .zhanqi import * +from .kuaishou import * diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/src/you_get/extractors/bilibili.py new/you-get-0.4.1011/src/you_get/extractors/bilibili.py --- old/you-get-0.4.995/src/you_get/extractors/bilibili.py 2017-12-04 04:18:45.000000000 +0100 +++ new/you-get-0.4.1011/src/you_get/extractors/bilibili.py 2017-12-23 18:27:08.000000000 +0100 @@ -32,6 +32,7 @@ SEC2 = '9b288147e5474dd2aa67085f716c560d' stream_types = [ {'id': 'hdflv'}, + {'id': 'flv720'}, {'id': 'flv'}, {'id': 'hdmp4'}, {'id': 'mp4'}, @@ -43,13 +44,15 @@ @staticmethod def bilibili_stream_type(urls): url = urls[0] - if 'hd.flv?' in url or '-112.flv' in url: + if 'hd.flv' in url or '-112.flv' in url: return 'hdflv', 'flv' - if '.flv?' in url: + if '-64.flv' in url: + return 'flv720', 'flv' + if '.flv' in url: return 'flv', 'flv' - if 'hd.mp4?' in url or '-48.mp4' in url: + if 'hd.mp4' in url or '-48.mp4' in url: return 'hdmp4', 'mp4' - if '.mp4?' in url: + if '.mp4' in url: return 'mp4', 'mp4' raise Exception('Unknown stream type') @@ -122,7 +125,7 @@ self.referer = self.url self.page = get_content(self.url) - m = re.search(r'<h1\s*title="([^"]+)"', self.page) + m = re.search(r'<h1.*?>(.*?)</h1>', self.page) if m is not None: self.title = m.group(1) if self.title is None: @@ -162,7 +165,10 @@ qq_download_by_vid(tc_flashvars, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only']) return - cid = re.search(r'cid=(\d+)', self.page).group(1) + try: + cid = re.search(r'cid=(\d+)', self.page).group(1) + except: + cid = re.search(r'"cid":(\d+)', self.page).group(1) if cid is not None: self.download_by_vid(cid, False, **kwargs) else: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/src/you_get/extractors/douyin.py new/you-get-0.4.1011/src/you_get/extractors/douyin.py --- old/you-get-0.4.995/src/you_get/extractors/douyin.py 1970-01-01 01:00:00.000000000 +0100 +++ new/you-get-0.4.1011/src/you_get/extractors/douyin.py 2017-12-23 18:27:08.000000000 +0100 @@ -0,0 +1,38 @@ +# coding=utf-8 + +import re +import json + +from ..common import ( + url_size, + print_info, + get_content, + download_urls, + playlist_not_supported, +) + + +__all__ = ['douyin_download_by_url'] + + +def douyin_download_by_url(url, **kwargs): + page_content = get_content(url) + match_rule = re.compile(r'var data = \[(.*?)\];') + video_info = json.loads(match_rule.findall(page_content)[0]) + video_url = video_info['video']['play_addr']['url_list'][0] + title = video_info['cha_list'][0]['cha_name'] + video_format = 'mp4' + size = url_size(video_url) + print_info( + site_info='douyin.com', title=title, + type=video_format, size=size + ) + if not kwargs['info_only']: + download_urls( + urls=[video_url], title=title, ext=video_format, total_size=size, + **kwargs + ) + + +download = douyin_download_by_url +download_playlist = playlist_not_supported('douyin') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/src/you_get/extractors/ixigua.py new/you-get-0.4.1011/src/you_get/extractors/ixigua.py --- old/you-get-0.4.995/src/you_get/extractors/ixigua.py 1970-01-01 01:00:00.000000000 +0100 +++ new/you-get-0.4.1011/src/you_get/extractors/ixigua.py 2017-12-23 18:27:08.000000000 +0100 @@ -0,0 +1,85 @@ +#!/usr/bin/env python +__all__ = ['ixigua_download', 'ixigua_download_playlist'] +import base64 +import random +import binascii +from ..common import * + +def get_video_id(text): + re_id = r"videoId: '(.*?)'" + return re.findall(re_id, text)[0] + +def get_r(): + return str(random.random())[2:] + +def right_shift(val, n): + return val >> n if val >= 0 else (val + 0x100000000) >> n + +def get_s(text): + """get video info""" + id = get_video_id(text) + p = get_r() + url = 'http://i.snssdk.com/video/urls/v/1/toutiao/mp4/%s' % id + n = parse.urlparse(url).path + '?r=%s' % p + c = binascii.crc32(n.encode('utf-8')) + s = right_shift(c, 0) + title = ''.join(re.findall(r"title: '(.*?)',", text)) + return url + '?r=%s&s=%s' % (p, s), title + +def get_moment(url, user_id, base_url, video_list): + """Recursively obtaining a video list""" + video_list_data = json.loads(get_content(url)) + if not video_list_data['next']['max_behot_time']: + return video_list + [video_list.append(i["display_url"]) for i in video_list_data["data"]] + max_behot_time = video_list_data['next']['max_behot_time'] + _param = { + 'user_id': user_id, + 'base_url': base_url, + 'video_list': video_list, + 'url': base_url.format(user_id=user_id, max_behot_time=max_behot_time), + } + return get_moment(**_param) + +def ixigua_download(url, output_dir='.', info_only=False, **kwargs): + """ Download a single video + Sample URL: https://www.ixigua.com/a6487187567887254029/#mid=59051127876 + """ + try: + video_info_url, title = get_s(get_content(url)) + video_info = json.loads(get_content(video_info_url)) + except Exception: + raise NotImplementedError(url) + try: + video_url = base64.b64decode(video_info["data"]["video_list"]["video_1"]["main_url"]).decode() + except Exception: + raise NotImplementedError(url) + filetype, ext, size = url_info(video_url) + print_info(site_info, title, filetype, size) + if not info_only: + download_urls([video_url], title, ext, size, output_dir=output_dir) + +def ixigua_download_playlist(url, output_dir='.', info_only=False, **kwargs): + """Download all video from the user's video list + Sample URL: https://www.ixigua.com/c/user/71141690831/ + """ + if 'user' not in url: + raise NotImplementedError(url) + user_id = url.split('/')[-2] + max_behot_time = 0 + if not user_id: + raise NotImplementedError(url) + base_url = "https://www.ixigua.com/c/user/article/?user_id={user_id}"; \ + "&max_behot_time={max_behot_time}&max_repin_time=0&count=20&page_type=0" + _param = { + 'user_id': user_id, + 'base_url': base_url, + 'video_list': [], + 'url': base_url.format(user_id=user_id, max_behot_time=max_behot_time), + } + for i in get_moment(**_param): + ixigua_download(i, output_dir, info_only, **kwargs) + +site_info = "ixigua.com" +download = ixigua_download +download_playlist = ixigua_download_playlist \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/src/you_get/extractors/kuaishou.py new/you-get-0.4.1011/src/you_get/extractors/kuaishou.py --- old/you-get-0.4.995/src/you_get/extractors/kuaishou.py 1970-01-01 01:00:00.000000000 +0100 +++ new/you-get-0.4.1011/src/you_get/extractors/kuaishou.py 2017-12-23 18:27:08.000000000 +0100 @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +import urllib.request +import urllib.parse +import json +import re + +from ..util import log +from ..common import get_content, download_urls, print_info, playlist_not_supported, url_size + +__all__ = ['kuaishou_download_by_url'] + + +def kuaishou_download_by_url(url, info_only=False, **kwargs): + page = get_content(url) + # size = video_list[-1]['size'] + # result wrong size + try: + og_video_url = re.search(r"<meta\s+property=\"og:video:url\"\s+content=\"(.+?)\"/>", page).group(1) + video_url = og_video_url + title = url.split('/')[-1] + size = url_size(video_url) + video_format = video_url.split('.')[-1] + print_info(site_info, title, video_format, size) + if not info_only: + download_urls([video_url], title, video_format, size, **kwargs) + except:# extract image + og_image_url = re.search(r"<meta\s+property=\"og:image\"\s+content=\"(.+?)\"/>", page).group(1) + image_url = og_image_url + title = url.split('/')[-1] + size = url_size(image_url) + image_format = image_url.split('.')[-1] + print_info(site_info, title, image_format, size) + if not info_only: + download_urls([image_url], title, image_format, size, **kwargs) + +site_info = "kuaishou.com" +download = kuaishou_download_by_url +download_playlist = playlist_not_supported('kuaishou') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/src/you_get/extractors/le.py new/you-get-0.4.1011/src/you_get/extractors/le.py --- old/you-get-0.4.995/src/you_get/extractors/le.py 2017-12-04 04:18:45.000000000 +0100 +++ new/you-get-0.4.1011/src/you_get/extractors/le.py 2017-12-23 18:27:08.000000000 +0100 @@ -65,9 +65,6 @@ if "stream_id" in kwargs and kwargs["stream_id"].lower() in support_stream_id: stream_id = kwargs["stream_id"] else: - print("Current Video Supports:") - for i in support_stream_id: - print("\t--format",i,"<URL>") if "1080p" in support_stream_id: stream_id = '1080p' elif "720p" in support_stream_id: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/src/you_get/extractors/youku.py new/you-get-0.4.1011/src/you_get/extractors/youku.py --- old/you-get-0.4.995/src/you_get/extractors/youku.py 2017-12-04 04:18:45.000000000 +0100 +++ new/you-get-0.4.1011/src/you_get/extractors/youku.py 2017-12-23 18:27:08.000000000 +0100 @@ -78,7 +78,7 @@ self.api_error_code = None self.api_error_msg = None - self.ccode = '0507' + self.ccode = '0512' self.utid = None def youku_ups(self): @@ -154,7 +154,7 @@ log.wtf('Cannot fetch vid') if kwargs.get('src') and kwargs['src'] == 'tudou': - self.ccode = '0501' + self.ccode = '0512' if kwargs.get('password') and kwargs['password']: self.password_protected = True diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/src/you_get/version.py new/you-get-0.4.1011/src/you_get/version.py --- old/you-get-0.4.995/src/you_get/version.py 2017-12-04 04:18:45.000000000 +0100 +++ new/you-get-0.4.1011/src/you_get/version.py 2017-12-23 18:27:08.000000000 +0100 @@ -1,4 +1,4 @@ #!/usr/bin/env python script_name = 'you-get' -__version__ = '0.4.995' +__version__ = '0.4.1011' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.995/tests/test.py new/you-get-0.4.1011/tests/test.py --- old/you-get-0.4.995/tests/test.py 2017-12-04 04:18:45.000000000 +0100 +++ new/you-get-0.4.1011/tests/test.py 2017-12-23 18:27:08.000000000 +0100 @@ -2,25 +2,51 @@ import unittest -from you_get.extractors import * +from you_get.extractors import ( + imgur, + magisto, + youtube, + yixia, + bilibili, + douyin, +) class YouGetTests(unittest.TestCase): def test_imgur(self): - imgur.download("http://imgur.com/WVLk5nD";, info_only=True) - imgur.download("http://imgur.com/gallery/WVLk5nD";, info_only=True) + imgur.download('http://imgur.com/WVLk5nD', info_only=True) + imgur.download('http://imgur.com/gallery/WVLk5nD', info_only=True) def test_magisto(self): - magisto.download("http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA";, info_only=True) + magisto.download( + 'http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA', + info_only=True + ) def test_youtube(self): - youtube.download("http://www.youtube.com/watch?v=pzKerr0JIPA";, info_only=True) - youtube.download("http://youtu.be/pzKerr0JIPA";, info_only=True) - youtube.download("http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare";, - info_only=True) + youtube.download( + 'http://www.youtube.com/watch?v=pzKerr0JIPA', info_only=True + ) + youtube.download('http://youtu.be/pzKerr0JIPA', info_only=True) + youtube.download( + 'http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare', # noqa + info_only=True + ) def test_yixia(self): - yixia_download("http://m.miaopai.com/show/channel/vlvreCo4OZiNdk5Jn1WvdopmAvdIJwi8";, info_only=True) + yixia.download( + 'http://m.miaopai.com/show/channel/vlvreCo4OZiNdk5Jn1WvdopmAvdIJwi8', # noqa + info_only=True + ) + + def test_bilibili(self): + bilibili.download( + 'https://www.bilibili.com/video/av16907446/', info_only=True + ) + bilibili.download( + 'https://www.bilibili.com/video/av13228063/', info_only=True + ) + if __name__ == '__main__': unittest.main()
