Hello community, here is the log from the commit of package you-get for openSUSE:Factory checked in at 2020-01-27 20:19:29 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/you-get (Old) and /work/SRC/openSUSE:Factory/.you-get.new.26092 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "you-get" Mon Jan 27 20:19:29 2020 rev:28 rq:767830 version:0.4.1403 Changes: -------- --- /work/SRC/openSUSE:Factory/you-get/you-get.changes 2020-01-01 14:59:35.413971593 +0100 +++ /work/SRC/openSUSE:Factory/.you-get.new.26092/you-get.changes 2020-01-27 20:19:36.084601788 +0100 @@ -1,0 +2,5 @@ +Mon Jan 27 16:50:42 UTC 2020 - Luigi Baldoni <[email protected]> + +- Update to version 0.4.1403 (no changelog supplied) + +------------------------------------------------------------------- Old: ---- you-get-0.4.1388.tar.gz New: ---- you-get-0.4.1403.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ you-get.spec ++++++ --- /var/tmp/diff_new_pack.Iwd4Zo/_old 2020-01-27 20:19:36.856602259 +0100 +++ /var/tmp/diff_new_pack.Iwd4Zo/_new 2020-01-27 20:19:36.864602264 +0100 @@ -1,7 +1,7 @@ # # spec file for package you-get # -# Copyright (c) 2019 SUSE LLC +# Copyright (c) 2020 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -17,7 +17,7 @@ Name: you-get -Version: 0.4.1388 +Version: 0.4.1403 Release: 0 Summary: Dumb downloader that scrapes the web License: MIT ++++++ you-get-0.4.1388.tar.gz -> you-get-0.4.1403.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1388/setup.py new/you-get-0.4.1403/setup.py --- old/you-get-0.4.1388/setup.py 2019-12-28 21:35:25.000000000 +0100 +++ new/you-get-0.4.1403/setup.py 2020-01-27 00:27:15.000000000 +0100 @@ -41,5 +41,9 @@ classifiers = proj_info['classifiers'], - entry_points = {'console_scripts': proj_info['console_scripts']} + entry_points = {'console_scripts': proj_info['console_scripts']}, + + extras_require={ + 'socks': ['PySocks'], + } ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1388/src/you_get/extractors/bilibili.py new/you-get-0.4.1403/src/you_get/extractors/bilibili.py --- old/you-get-0.4.1388/src/you_get/extractors/bilibili.py 2019-12-28 21:35:25.000000000 +0100 +++ new/you-get-0.4.1403/src/you_get/extractors/bilibili.py 2020-01-27 00:27:15.000000000 +0100 @@ -542,7 +542,7 @@ self.url = url kwargs['playlist'] = True - html_content = get_content(self.url, headers=self.bilibili_headers()) + html_content = get_content(self.url, headers=self.bilibili_headers(referer=self.url)) # sort it out if re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url): @@ -615,8 +615,12 @@ for choice in node_info['data']['edges']['choices']: search_node_list.append(choice['node_id']) if not choice['cid'] in download_cid_set: - download_cid_set.add(choice['cid'] ) + download_cid_set.add(choice['cid']) self.prepare_by_cid(aid,choice['cid'],initial_state['videoData']['title']+('P{}. {}'.format(len(download_cid_set),choice['option'])),html_content,playinfo,playinfo_,url) + try: + self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams] + except: + self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams] self.extract(**kwargs) self.download(**kwargs) else: @@ -626,8 +630,13 @@ html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16')) playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>') # FIXME playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None - for pi in range(pn): + p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or '1')-1 + for pi in range(p,pn): self.prepare_by_cid(aid,initial_state['videoData']['pages'][pi]['cid'],'%s (P%s. %s)' % (initial_state['videoData']['title'], pi+1, initial_state['videoData']['pages'][pi]['part']),html_content,playinfo,playinfo_,url) + try: + self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams] + except: + self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams] self.extract(**kwargs) self.download(**kwargs) # purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi+1) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1388/src/you_get/extractors/ixigua.py new/you-get-0.4.1403/src/you_get/extractors/ixigua.py --- old/you-get-0.4.1388/src/you_get/extractors/ixigua.py 2019-12-28 21:35:25.000000000 +0100 +++ new/you-get-0.4.1403/src/you_get/extractors/ixigua.py 2020-01-27 00:27:15.000000000 +0100 @@ -5,8 +5,10 @@ from ..common import * import random +import string import ctypes from json import loads +from urllib import request __all__ = ['ixigua_download', 'ixigua_download_playlist_by_url'] @@ -80,7 +82,29 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs): # example url: https://www.ixigua.com/i6631065141750268420/#mid=63024814422 - html = get_html(url, faker=True) + resp = urlopen_with_retry(request.Request(url)) + html = resp.read().decode('utf-8') + + _cookies = [] + for c in resp.getheader('Set-Cookie').split("httponly,"): + _cookies.append(c.strip().split(' ')[0]) + headers['cookie'] = ' '.join(_cookies) + + conf = loads(match1(html, r"window\.config = (.+);")) + if not conf: + log.e("Get window.config from url failed, url: {}".format(url)) + return + verify_url = conf['prefix'] + conf['url'] + '?key=' + conf['key'] + '&psm=' + conf['psm'] \ + + '&_signature=' + ''.join(random.sample(string.ascii_letters + string.digits, 31)) + try: + ok = get_content(verify_url) + except Exception as e: + ok = e.msg + if ok != 'OK': + log.e("Verify failed, verify_url: {}, result: {}".format(verify_url, ok)) + return + html = get_content(url, headers=headers) + video_id = match1(html, r"\"vid\":\"([^\"]+)") title = match1(html, r"\"player__videoTitle\">.*?<h1.*?>(.*)<\/h1><\/div>") if not video_id: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1388/src/you_get/extractors/sohu.py new/you-get-0.4.1403/src/you_get/extractors/sohu.py --- old/you-get-0.4.1388/src/you_get/extractors/sohu.py 2019-12-28 21:35:25.000000000 +0100 +++ new/you-get-0.4.1403/src/you_get/extractors/sohu.py 2020-01-27 00:27:15.000000000 +0100 @@ -26,7 +26,7 @@ vid = r1('id=(\d+)', url) else: html = get_html(url) - vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html) + vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html) or r1(r'bid:\'(\d+)\',', html) or r1(r'bid=(\d+)', html) assert vid if extractor_proxy: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1388/src/you_get/extractors/youtube.py new/you-get-0.4.1403/src/you_get/extractors/youtube.py --- old/you-get-0.4.1388/src/you_get/extractors/youtube.py 2019-12-28 21:35:25.000000000 +0100 +++ new/you-get-0.4.1403/src/you_get/extractors/youtube.py 2020-01-27 00:27:15.000000000 +0100 @@ -94,7 +94,8 @@ f1 = match1(js, r'\.set\(\w+\.sp,encodeURIComponent\(([$\w]+)') or \ match1(js, r'\.set\(\w+\.sp,\(0,window\.encodeURIComponent\)\(([$\w]+)') or \ match1(js, r'\.set\(\w+\.sp,([$\w]+)\(\w+\.s\)\)') or \ - match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)') + match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)') or \ + match1(js, r'=([$\w]+)\(decodeURIComponent\(') f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \ match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1)) f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def) @@ -222,7 +223,7 @@ except: if 'url_encoded_fmt_stream_map' not in video_info: stream_list = json.loads(video_info['player_response'][0])['streamingData']['formats'] - else: + else: stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',') if re.search('([^"]*/base\.js)"', video_page): self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1) @@ -451,6 +452,8 @@ for afmt in video_info['adaptive_fmts'][0].split(',')] else: streams = json.loads(video_info['player_response'][0])['streamingData']['adaptiveFormats'] + # streams without contentLength got broken urls, just remove them (#2767) + streams = [stream for stream in streams if 'contentLength' in stream] for stream in streams: stream['itag'] = str(stream['itag']) if 'qualityLabel' in stream: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1388/src/you_get/version.py new/you-get-0.4.1403/src/you_get/version.py --- old/you-get-0.4.1388/src/you_get/version.py 2019-12-28 21:35:25.000000000 +0100 +++ new/you-get-0.4.1403/src/you_get/version.py 2020-01-27 00:27:15.000000000 +0100 @@ -1,4 +1,4 @@ #!/usr/bin/env python script_name = 'you-get' -__version__ = '0.4.1388' +__version__ = '0.4.1403' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1388/tests/test.py new/you-get-0.4.1403/tests/test.py --- old/you-get-0.4.1388/tests/test.py 2019-12-28 21:35:25.000000000 +0100 +++ new/you-get-0.4.1403/tests/test.py 2020-01-27 00:27:15.000000000 +0100 @@ -14,7 +14,6 @@ class YouGetTests(unittest.TestCase): def test_imgur(self): imgur.download('http://imgur.com/WVLk5nD', info_only=True) - imgur.download('http://imgur.com/gallery/WVLk5nD', info_only=True) def test_magisto(self): magisto.download( @@ -40,7 +39,7 @@ ) def test_acfun(self): - acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True) + acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True) if __name__ == '__main__': unittest.main()
