Hello community, here is the log from the commit of package you-get for openSUSE:Factory checked in at 2020-11-19 16:47:40 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/you-get (Old) and /work/SRC/openSUSE:Factory/.you-get.new.5913 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "you-get" Thu Nov 19 16:47:40 2020 rev:34 rq:849400 version:0.4.1488 Changes: -------- --- /work/SRC/openSUSE:Factory/you-get/you-get.changes 2020-10-28 10:01:13.143263839 +0100 +++ /work/SRC/openSUSE:Factory/.you-get.new.5913/you-get.changes 2020-11-23 10:54:51.874864773 +0100 @@ -1,0 +2,5 @@ +Thu Nov 19 12:08:01 UTC 2020 - [email protected] + +- Update to version 0.4.1488 (no changelog supplied) + +------------------------------------------------------------------- Old: ---- you-get-0.4.1475.tar.gz New: ---- you-get-0.4.1488.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ you-get.spec ++++++ --- /var/tmp/diff_new_pack.F8McTS/_old 2020-11-23 10:54:52.630865529 +0100 +++ /var/tmp/diff_new_pack.F8McTS/_new 2020-11-23 10:54:52.634865534 +0100 @@ -17,7 +17,7 @@ Name: you-get -Version: 0.4.1475 +Version: 0.4.1488 Release: 0 Summary: Dumb downloader that scrapes the web License: MIT ++++++ you-get-0.4.1475.tar.gz -> you-get-0.4.1488.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/.github/workflows/python-package.yml new/you-get-0.4.1488/.github/workflows/python-package.yml --- old/you-get-0.4.1475/.github/workflows/python-package.yml 1970-01-01 01:00:00.000000000 +0100 +++ new/you-get-0.4.1488/.github/workflows/python-package.yml 2020-11-18 16:35:59.000000000 +0100 @@ -0,0 +1,39 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: develop + +on: + push: + branches: [ develop ] + pull_request: + branches: [ develop ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8, pypy3] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with unittest + run: | + make test diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/.travis.yml new/you-get-0.4.1488/.travis.yml --- old/you-get-0.4.1475/.travis.yml 2020-10-27 18:19:28.000000000 +0100 +++ new/you-get-0.4.1488/.travis.yml 1970-01-01 01:00:00.000000000 +0100 @@ -1,22 +0,0 @@ -# https://travis-ci.org/soimort/you-get -language: python -python: - - "3.4" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - #- "nightly" (flake8 not working in python 3.9 yet, module 'ast' has no attribute 'AugLoad') - - "pypy3" -before_install: - - pip install flake8 -before_script: - - flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics -script: make test -notifications: - webhooks: - urls: - - https://webhooks.gitter.im/e/43cd57826e88ed8f2152 - on_success: change # options: [always|never|change] default: always - on_failure: always # options: [always|never|change] default: always - on_start: never # options: [always|never|change] default: always diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/README.md new/you-get-0.4.1488/README.md --- old/you-get-0.4.1475/README.md 2020-10-27 18:19:28.000000000 +0100 +++ new/you-get-0.4.1488/README.md 2020-11-18 16:35:59.000000000 +0100 @@ -1,7 +1,7 @@ # You-Get +[](https://github.com/soimort/you-get/actions) [](https://pypi.python.org/pypi/you-get/) -[](https://travis-ci.org/soimort/you-get) [](https://gitter.im/soimort/you-get?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) **NOTICE: Read [this](https://github.com/soimort/you-get/blob/develop/CONTRIBUTING.md) if you are looking for the conventional "Issues" tab.** diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/src/you_get/common.py new/you-get-0.4.1488/src/you_get/common.py --- old/you-get-0.4.1475/src/you_get/common.py 2020-10-27 18:19:28.000000000 +0100 +++ new/you-get-0.4.1488/src/you_get/common.py 2020-11-18 16:35:59.000000000 +0100 @@ -143,7 +143,7 @@ 'Accept-Charset': 'UTF-8,*;q=0.5', 'Accept-Encoding': 'gzip,deflate,sdch', 'Accept-Language': 'en-US,en;q=0.8', - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0', # noqa + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.74 Safari/537.36 Edg/79.0.309.43', # noqa } if sys.stdout.isatty(): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/src/you_get/extractors/netease.py new/you-get-0.4.1488/src/you_get/extractors/netease.py --- old/you-get-0.4.1475/src/you_get/extractors/netease.py 2020-10-27 18:19:28.000000000 +0100 +++ new/you-get-0.4.1488/src/you_get/extractors/netease.py 2020-11-18 16:35:59.000000000 +0100 @@ -123,10 +123,10 @@ output_dir=output_dir, info_only=info_only) def netease_download_common(title, url_best, output_dir, info_only): - songtype, ext, size = url_info(url_best) + songtype, ext, size = url_info(url_best, faker=True) print_info(site_info, title, songtype, size) if not info_only: - download_urls([url_best], title, ext, size, output_dir) + download_urls([url_best], title, ext, size, output_dir, faker=True) def netease_download(url, output_dir = '.', merge = True, info_only = False, **kwargs): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/src/you_get/extractors/pptv.py new/you-get-0.4.1488/src/you_get/extractors/pptv.py --- old/you-get-0.4.1475/src/you_get/extractors/pptv.py 2020-10-27 18:19:28.000000000 +0100 +++ new/you-get-0.4.1488/src/you_get/extractors/pptv.py 2020-11-18 16:35:59.000000000 +0100 @@ -174,7 +174,7 @@ src = [] for i, seg in enumerate(stream['segs']): url = 'http://{}/{}/{}?key={}&k={}'.format(host, i, rid, key, key_expr) - url += '&fpp.ver=1.3.0.4&type=' + url += '&type=web.fpp' src.append(url) return src @@ -189,17 +189,27 @@ ] def prepare(self, **kwargs): + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/69.0.3497.100 Safari/537.36" + } + self.vid = match1(self.url, r'https?://sports.pptv.com/vod/(\d+)/*') if self.url and not self.vid: if not re.match(r'https?://v.pptv.com/show/(\w+)\.html', self.url): raise('Unknown url pattern') - page_content = get_content(self.url,{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"}) + page_content = get_content(self.url, headers) + self.vid = match1(page_content, r'webcfg\s*=\s*{"id":\s*(\d+)') + if not self.vid: + request = urllib.request.Request(self.url, headers=headers) + response = urllib.request.urlopen(request) + self.vid = match1(response.url, r'https?://sports.pptv.com/vod/(\d+)/*') if not self.vid: raise('Cannot find id') api_url = 'http://web-play.pptv.com/webplay3-0-{}.xml'.format(self.vid) - api_url += '?appplt=flp&appid=pptv.flashplayer.vod&appver=3.4.2.28&type=&version=4' - dom = parseString(get_content(api_url,{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"})) + api_url += '?type=web.fpp¶m=type=web.fpp&version=4' + dom = parseString(get_content(api_url, headers)) self.title, m_items, m_streams, m_segs = parse_pptv_xml(dom) xml_streams = merge_meta(m_items, m_streams, m_segs) for stream_id in xml_streams: @@ -212,146 +222,6 @@ 'src': src } -''' -def constructKey(arg): - - def str2hex(s): - r="" - for i in s[:8]: - t=hex(ord(i))[2:] - if len(t)==1: - t="0"+t - r+=t - for i in range(16): - r+=hex(int(15*random()))[2:] - return r - - #ABANDONED Because SERVER_KEY is static - def getkey(s): - #returns 1896220160 - l2=[i for i in s] - l4=0 - l3=0 - while l4<len(l2): - l5=l2[l4] - l6=ord(l5) - l7=l6<<((l4%4)*8) - l3=l3^l7 - l4+=1 - return l3 - pass - - def rot(k,b): ##>>> in as3 - if k>=0: - return k>>b - elif k<0: - return (2**32+k)>>b - pass - - def lot(k,b): - return (k<<b)%(2**32) - - #WTF? - def encrypt(arg1,arg2): - delta=2654435769 - l3=16; - l4=getkey(arg2) #1896220160 - l8=[i for i in arg1] - l10=l4; - l9=[i for i in arg2] - l5=lot(l10,8)|rot(l10,24)#101056625 - # assert l5==101056625 - l6=lot(l10,16)|rot(l10,16)#100692230 - # assert 100692230==l6 - l7=lot(l10,24)|rot(l10,8) - # assert 7407110==l7 - l11="" - l12=0 - l13=ord(l8[l12])<<0 - l14=ord(l8[l12+1])<<8 - l15=ord(l8[l12+2])<<16 - l16=ord(l8[l12+3])<<24 - l17=ord(l8[l12+4])<<0 - l18=ord(l8[l12+5])<<8 - l19=ord(l8[l12+6])<<16 - l20=ord(l8[l12+7])<<24 - - l21=(((0|l13)|l14)|l15)|l16 - l22=(((0|l17)|l18)|l19)|l20 - - l23=0 - l24=0 - while l24<32: - l23=(l23+delta)%(2**32) - l33=(lot(l22,4)+l4)%(2**32) - l34=(l22+l23)%(2**32) - l35=(rot(l22,5)+l5)%(2**32) - l36=(l33^l34)^l35 - l21=(l21+l36)%(2**32) - l37=(lot(l21,4)+l6)%(2**32) - l38=(l21+l23)%(2**32) - l39=(rot(l21,5))%(2**32) - l40=(l39+l7)%(2**32) - l41=((l37^l38)%(2**32)^l40)%(2**32) - l22=(l22+l41)%(2**32) - - l24+=1 - - l11+=chr(rot(l21,0)&0xff) - l11+=chr(rot(l21,8)&0xff) - l11+=chr(rot(l21,16)&0xff) - l11+=chr(rot(l21,24)&0xff) - l11+=chr(rot(l22,0)&0xff) - l11+=chr(rot(l22,8)&0xff) - l11+=chr(rot(l22,16)&0xff) - l11+=chr(rot(l22,24)&0xff) - - return l11 - - - loc1=hex(int(arg))[2:]+(16-len(hex(int(arg))[2:]))*"\x00" - SERVER_KEY="qqqqqww"+"\x00"*9 - res=encrypt(loc1,SERVER_KEY) - return str2hex(res) - - -def pptv_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False): - xml = get_html('http://web-play.pptv.com/webplay3-0-%s.xml?type=web.fpp' % id) - #vt=3 means vod mode vt=5 means live mode - host = r1(r'<sh>([^<>]+)</sh>', xml) - k = r1(r'<key expire=[^<>]+>([^<>]+)</key>', xml) - rid = r1(r'rid="([^"]+)"', xml) - title = r1(r'nm="([^"]+)"', xml) - - st=r1(r'<st>([^<>]+)</st>',xml)[:-4] - st=time.mktime(time.strptime(st))*1000-60*1000-time.time()*1000 - st+=time.time()*1000 - st=st/1000 - - key=constructKey(st) - - pieces = re.findall('<sgm no="(\d+)"[^<>]+fs="(\d+)"', xml) - numbers, fs = zip(*pieces) - urls=["http://{}/{}/{}?key={}&fpp.ver=1.3.0.4&k={}&type=web.fpp".format(host,i,rid,key,k) for i in range(max(map(int,numbers))+1)] - - total_size = sum(map(int, fs)) - assert rid.endswith('.mp4') - print_info(site_info, title, 'mp4', total_size) - - if not info_only: - try: - download_urls(urls, title, 'mp4', total_size, output_dir = output_dir, merge = merge) - except urllib.error.HTTPError: - #for key expired - pptv_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only) - -def pptv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs): - assert re.match(r'http://v.pptv.com/show/(\w+)\.html', url) - html = get_html(url) - id = r1(r'webcfg\s*=\s*{"id":\s*(\d+)', html) - assert id - pptv_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only) -''' site = PPTV() #site_info = "PPTV.com" #download = pptv_download diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/src/you_get/extractors/xinpianchang.py new/you-get-0.4.1488/src/you_get/extractors/xinpianchang.py --- old/you-get-0.4.1475/src/you_get/extractors/xinpianchang.py 2020-10-27 18:19:28.000000000 +0100 +++ new/you-get-0.4.1488/src/you_get/extractors/xinpianchang.py 2020-11-18 16:35:59.000000000 +0100 @@ -20,7 +20,7 @@ def prepare(self, **kwargs): # find key page_content = get_content(self.url) - match_rule = r"vid: \"(.+?)\"," + match_rule = r"vid = \"(.+?)\";" key = re.findall(match_rule, page_content)[0] # get videos info diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/src/you_get/extractors/youku.py new/you-get-0.4.1488/src/you_get/extractors/youku.py --- old/you-get-0.4.1475/src/you_get/extractors/youku.py 2020-10-27 18:19:28.000000000 +0100 +++ new/you-get-0.4.1488/src/you_get/extractors/youku.py 2020-11-18 16:35:59.000000000 +0100 @@ -41,7 +41,6 @@ mobile_ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36' dispatcher_url = 'vali.cp31.ott.cibntv.net' - # Last updated: 2017-10-13 stream_types = [ {'id': 'hd3', 'container': 'flv', 'video_profile': '1080P'}, {'id': 'hd3v2', 'container': 'flv', 'video_profile': '1080P'}, @@ -78,7 +77,7 @@ self.api_error_code = None self.api_error_msg = None - self.ccode = '0519' + self.ccode = '0590' # Found in http://g.alicdn.com/player/ykplayer/0.5.64/youku-player.min.js # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/src/you_get/extractors/youtube.py new/you-get-0.4.1488/src/you_get/extractors/youtube.py --- old/you-get-0.4.1475/src/you_get/extractors/youtube.py 2020-10-27 18:19:28.000000000 +0100 +++ new/you-get-0.4.1488/src/you_get/extractors/youtube.py 2020-11-18 16:35:59.000000000 +0100 @@ -68,7 +68,7 @@ 'audio_encoding': 'AAC', 'audio_bitrate': '24'}, ] - def decipher(js, s): + def s_to_sig(js, s): # Examples: # - https://www.youtube.com/yts/jsbin/player-da_DK-vflWlK-zq/base.js # - https://www.youtube.com/yts/jsbin/player-vflvABTsY/da_DK/base.js @@ -204,13 +204,22 @@ video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid) try: ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1)) - self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js'] + # Workaround: get_video_info returns bad s. Why? if 'url_encoded_fmt_stream_map' not in ytplayer_config['args']: stream_list = json.loads(ytplayer_config['args']['player_response'])['streamingData']['formats'] else: stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',') #stream_list = ytplayer_config['args']['adaptive_fmts'].split(',') + + if 'assets' in ytplayer_config: + self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js'] + elif re.search('([^"]*/base\.js)"', video_page): + self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1) + self.html5player = self.html5player.replace('\/', '/') # unescape URL + else: + self.html5player = None + except: if 'url_encoded_fmt_stream_map' not in video_info: stream_list = json.loads(video_info['player_response'][0])['streamingData']['formats'] @@ -481,13 +490,13 @@ if stream['type'].startswith('audio/mp4'): dash_mp4_a_url = stream['url'] if 's' in stream: - sig = self.__class__.decipher(self.js, stream['s']) + sig = self.__class__.s_to_sig(self.js, stream['s']) dash_mp4_a_url += '&sig={}'.format(sig) dash_mp4_a_size = stream['clen'] elif stream['type'].startswith('audio/webm'): dash_webm_a_url = stream['url'] if 's' in stream: - sig = self.__class__.decipher(self.js, stream['s']) + sig = self.__class__.s_to_sig(self.js, stream['s']) dash_webm_a_url += '&sig={}'.format(sig) dash_webm_a_size = stream['clen'] for stream in streams: # video @@ -496,7 +505,7 @@ mimeType = 'video/mp4' dash_url = stream['url'] if 's' in stream: - sig = self.__class__.decipher(self.js, stream['s']) + sig = self.__class__.s_to_sig(self.js, stream['s']) dash_url += '&sig={}'.format(sig) dash_size = stream['clen'] itag = stream['itag'] @@ -515,7 +524,7 @@ mimeType = 'video/webm' dash_url = stream['url'] if 's' in stream: - sig = self.__class__.decipher(self.js, stream['s']) + sig = self.__class__.s_to_sig(self.js, stream['s']) dash_url += '&sig={}'.format(sig) dash_size = stream['clen'] itag = stream['itag'] @@ -564,7 +573,7 @@ if not hasattr(self, 'js'): self.js = get_content(self.html5player) s = self.streams[stream_id]['s'] - sig = self.__class__.decipher(self.js, s) + sig = self.__class__.s_to_sig(self.js, s) src += '&sig={}'.format(sig) self.streams[stream_id]['src'] = [src] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/src/you_get/version.py new/you-get-0.4.1488/src/you_get/version.py --- old/you-get-0.4.1475/src/you_get/version.py 2020-10-27 18:19:28.000000000 +0100 +++ new/you-get-0.4.1488/src/you_get/version.py 2020-11-18 16:35:59.000000000 +0100 @@ -1,4 +1,4 @@ #!/usr/bin/env python script_name = 'you-get' -__version__ = '0.4.1475' +__version__ = '0.4.1488' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1475/tests/test.py new/you-get-0.4.1488/tests/test.py --- old/you-get-0.4.1475/tests/test.py 2020-10-27 18:19:28.000000000 +0100 +++ new/you-get-0.4.1488/tests/test.py 2020-11-18 16:35:59.000000000 +0100 @@ -33,14 +33,14 @@ 'http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare', # noqa info_only=True ) - youtube.download( - 'https://www.youtube.com/watch?v=Fpr4fQSh1cc', info_only=True - ) + #youtube.download( + # 'https://www.youtube.com/watch?v=Fpr4fQSh1cc', info_only=True + #) def test_acfun(self): acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True) - def test_bilibil(self): + def test_bilibili(self): bilibili.download( "https://www.bilibili.com/watchlater/#/BV1PE411q7mZ/p6";, info_only=True ) @@ -58,10 +58,10 @@ # 'https://soundcloud.com/anthony-flieger/sets/cytus', info_only=True #) - def tests_tiktok(self): - tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True) - tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True) - tiktok.download('https://vt.tiktok.com/UGJR4R/', info_only=True) + #def tests_tiktok(self): + # tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True) + # tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True) + # tiktok.download('https://vt.tiktok.com/UGJR4R/', info_only=True) if __name__ == '__main__': _______________________________________________ openSUSE Commits mailing list -- [email protected] To unsubscribe, email [email protected] List Netiquette: https://en.opensuse.org/openSUSE:Mailing_list_netiquette List Archives: https://lists.opensuse.org/archives/list/[email protected]
