Hello community, here is the log from the commit of package you-get for openSUSE:Factory checked in at 2018-10-29 14:23:10 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/you-get (Old) and /work/SRC/openSUSE:Factory/.you-get.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "you-get" Mon Oct 29 14:23:10 2018 rev:13 rq:645201 version:0.4.1164 Changes: -------- --- /work/SRC/openSUSE:Factory/you-get/you-get.changes 2018-10-15 09:40:35.751530529 +0200 +++ /work/SRC/openSUSE:Factory/.you-get.new/you-get.changes 2018-10-29 14:32:42.666687069 +0100 @@ -1,0 +2,5 @@ +Mon Oct 29 09:38:50 UTC 2018 - Luigi Baldoni <[email protected]> + +- Update to version 0.4.1164 (no changelog supplied) + +------------------------------------------------------------------- Old: ---- you-get-0.4.1148.tar.gz New: ---- you-get-0.4.1164.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ you-get.spec ++++++ --- /var/tmp/diff_new_pack.3qBeKO/_old 2018-10-29 14:32:43.338670273 +0100 +++ /var/tmp/diff_new_pack.3qBeKO/_new 2018-10-29 14:32:43.338670273 +0100 @@ -17,12 +17,12 @@ Name: you-get -Version: 0.4.1148 +Version: 0.4.1164 Release: 0 Summary: Dumb downloader that scrapes the web License: MIT Group: Productivity/Networking/Web/Utilities -Url: https://you-get.org +URL: https://you-get.org Source0: https://github.com/soimort/you-get/archive/v%{version}.tar.gz#/%{name}-%{version}.tar.gz BuildRequires: bash-completion BuildRequires: fdupes ++++++ you-get-0.4.1148.tar.gz -> you-get-0.4.1164.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/.travis.yml new/you-get-0.4.1164/.travis.yml --- old/you-get-0.4.1148/.travis.yml 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/.travis.yml 2018-10-28 13:33:28.000000000 +0100 @@ -8,9 +8,10 @@ - "3.6" - "nightly" - "pypy3" -before_install: pip install flake8 +before_install: + - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'* ]]; then pip install flake8; fi before_script: - - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* ]]; then flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics; fi + - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'* ]]; then flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics; fi script: make test sudo: false notifications: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/README.md new/you-get-0.4.1164/README.md --- old/you-get-0.4.1148/README.md 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/README.md 2018-10-28 13:33:28.000000000 +0100 @@ -113,6 +113,14 @@ $ brew install you-get ``` +### Option 8: pkg (FreeBSD only) + +You can install `you-get` easily via: + +``` +# pkg install you-get +``` + ### Shell completion Completion definitions for Bash, Fish and Zsh can be found in [`contrib/completion`](https://github.com/soimort/you-get/tree/develop/contrib/completion). Please consult your shell's manual for how to take advantage of them. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/common.py new/you-get-0.4.1164/src/you_get/common.py --- old/you-get-0.4.1148/src/you_get/common.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/common.py 2018-10-28 13:33:28.000000000 +0100 @@ -622,7 +622,7 @@ if not is_part: if bar: bar.done() - print( + log.w( 'Skipping {}: file already exists'.format( tr(os.path.basename(filepath)) ) @@ -648,7 +648,10 @@ print('Changing name to %s' % tr(os.path.basename(filepath)), '...') continue_renameing = True continue - print('Overwriting %s' % tr(os.path.basename(filepath)), '...') + if log.yes_or_no('File with this name already exists. Overwrite?'): + log.w('Overwriting %s ...' % tr(os.path.basename(filepath))) + else: + return elif not os.path.exists(os.path.dirname(filepath)): os.mkdir(os.path.dirname(filepath)) @@ -925,7 +928,7 @@ if total_size: if not force and os.path.exists(output_filepath) and not auto_rename\ and os.path.getsize(output_filepath) >= total_size * 0.9: - print('Skipping %s: file already exists' % output_filepath) + log.w('Skipping %s: file already exists' % output_filepath) print() return bar = SimpleProgressBar(total_size, len(urls)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/extractors/bilibili.py new/you-get-0.4.1164/src/you_get/extractors/bilibili.py --- old/you-get-0.4.1148/src/you_get/extractors/bilibili.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/extractors/bilibili.py 2018-10-28 13:33:28.000000000 +0100 @@ -192,7 +192,12 @@ index_id = int(re.search(r'index_(\d+)', self.url).group(1)) cid = page_list[index_id-1]['cid'] # change cid match rule except: - cid = re.search(r'"cid":(\d+)', self.page).group(1) + page = re.search(r'p=(\d+)', self.url) + if page is None: + p = 1 + else: + p = int(page.group(1)) + cid = re.search(r'"cid":(\d+),"page":%s' % p, self.page).group(1) if cid is not None: self.download_by_vid(cid, re.search('bangumi', self.url) is not None, **kwargs) else: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/extractors/naver.py new/you-get-0.4.1164/src/you_get/extractors/naver.py --- old/you-get-0.4.1148/src/you_get/extractors/naver.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/extractors/naver.py 2018-10-28 13:33:28.000000000 +0100 @@ -7,31 +7,35 @@ from ..util import log from ..common import get_content, download_urls, print_info, playlist_not_supported, url_size +from .universal import * __all__ = ['naver_download_by_url'] -def naver_download_by_url(url, info_only=False, **kwargs): +def naver_download_by_url(url, output_dir='.', merge=True, info_only=False, **kwargs): ep = 'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{}?key={}' page = get_content(url) - og_video_url = re.search(r"<meta\s+property=\"og:video:url\"\s+content='(.+?)'>", page).group(1) - params_dict = urllib.parse.parse_qs(urllib.parse.urlparse(og_video_url).query) - vid = params_dict['vid'][0] - key = params_dict['outKey'][0] - meta_str = get_content(ep.format(vid, key)) - meta_json = json.loads(meta_str) - if 'errorCode' in meta_json: - log.wtf(meta_json['errorCode']) - title = meta_json['meta']['subject'] - videos = meta_json['videos']['list'] - video_list = sorted(videos, key=lambda video: video['encodingOption']['width']) - video_url = video_list[-1]['source'] - # size = video_list[-1]['size'] - # result wrong size - size = url_size(video_url) - print_info(site_info, title, 'mp4', size) - if not info_only: - download_urls([video_url], title, 'mp4', size, **kwargs) + try: + og_video_url = re.search(r"<meta\s+property=\"og:video:url\"\s+content='(.+?)'>", page).group(1) + params_dict = urllib.parse.parse_qs(urllib.parse.urlparse(og_video_url).query) + vid = params_dict['vid'][0] + key = params_dict['outKey'][0] + meta_str = get_content(ep.format(vid, key)) + meta_json = json.loads(meta_str) + if 'errorCode' in meta_json: + log.wtf(meta_json['errorCode']) + title = meta_json['meta']['subject'] + videos = meta_json['videos']['list'] + video_list = sorted(videos, key=lambda video: video['encodingOption']['width']) + video_url = video_list[-1]['source'] + # size = video_list[-1]['size'] + # result wrong size + size = url_size(video_url) + print_info(site_info, title, 'mp4', size) + if not info_only: + download_urls([video_url], title, 'mp4', size, **kwargs) + except: + universal_download(url, output_dir, merge=merge, info_only=info_only, **kwargs) site_info = "naver.com" download = naver_download_by_url diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/extractors/pptv.py new/you-get-0.4.1164/src/you_get/extractors/pptv.py --- old/you-get-0.4.1148/src/you_get/extractors/pptv.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/extractors/pptv.py 2018-10-28 13:33:28.000000000 +0100 @@ -192,14 +192,14 @@ if self.url and not self.vid: if not re.match(r'http://v.pptv.com/show/(\w+)\.html', self.url): raise('Unknown url pattern') - page_content = get_content(self.url) + page_content = get_content(self.url,{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"}) self.vid = match1(page_content, r'webcfg\s*=\s*{"id":\s*(\d+)') if not self.vid: raise('Cannot find id') api_url = 'http://web-play.pptv.com/webplay3-0-{}.xml'.format(self.vid) api_url += '?appplt=flp&appid=pptv.flashplayer.vod&appver=3.4.2.28&type=&version=4' - dom = parseString(get_content(api_url)) + dom = parseString(get_content(api_url,{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"})) self.title, m_items, m_streams, m_segs = parse_pptv_xml(dom) xml_streams = merge_meta(m_items, m_streams, m_segs) for stream_id in xml_streams: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/extractors/sohu.py new/you-get-0.4.1164/src/you_get/extractors/sohu.py --- old/you-get-0.4.1148/src/you_get/extractors/sohu.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/extractors/sohu.py 2018-10-28 13:33:28.000000000 +0100 @@ -15,9 +15,9 @@ new api ''' -def real_url(host,vid,tvid,new,clipURL,ck): - url = 'http://'+host+'/?prot=9&prod=flash&pt=1&file='+clipURL+'&new='+new +'&key='+ ck+'&vid='+str(vid)+'&uid='+str(int(time.time()*1000))+'&t='+str(random())+'&rb=1' - return json.loads(get_html(url))['url'] +def real_url(fileName, key, ch): + url = "https://data.vod.itc.cn/ip?new="; + fileName + "&num=1&key=" + key + "&ch=" + ch + "&pt=1&pg=2&prod=h5n" + return json.loads(get_html(url))['servers'][0]['url'] def sohu_download(url, output_dir = '.', merge = True, info_only = False, extractor_proxy=None, **kwargs): if re.match(r'http://share.vrs.sohu.com', url): @@ -51,9 +51,8 @@ title = data['tvName'] size = sum(data['clipsBytes']) assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su']) - for new,clip,ck, in zip(data['su'], data['clipsURL'], data['ck']): - clipURL = urlparse(clip).path - urls.append(real_url(host,hqvid,tvid,new,clipURL,ck)) + for fileName, key in zip(data['su'], data['ck']): + urls.append(real_url(fileName, key, data['ch'])) # assert data['clipsURL'][0].endswith('.mp4') else: @@ -66,9 +65,8 @@ title = data['tvName'] size = sum(map(int,data['clipsBytes'])) assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su']) - for new,clip,ck, in zip(data['su'], data['clipsURL'], data['ck']): - clipURL = urlparse(clip).path - urls.append(real_url(host,vid,tvid,new,clipURL,ck)) + for fileName, key in zip(data['su'], data['ck']): + urls.append(real_url(fileName, key, data['ch'])) print_info(site_info, title, 'mp4', size) if not info_only: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/extractors/tumblr.py new/you-get-0.4.1164/src/you_get/extractors/tumblr.py --- old/you-get-0.4.1148/src/you_get/extractors/tumblr.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/extractors/tumblr.py 2018-10-28 13:33:28.000000000 +0100 @@ -33,7 +33,7 @@ 'X-Requested-With': 'XMLHttpRequest' }, post_data_raw='{"eu_resident":true,"gdpr_is_acceptable_age":true,"gdpr_consent_core":true,"gdpr_consent_first_party_ads":true,"gdpr_consent_third_party_ads":true,"gdpr_consent_search_history":true,"redirect_to":"%s","gdpr_reconsent":false}' % url) - page = get_html(url) + page = get_html(url, faker=True) html = parse.unquote(page).replace('\/', '/') feed = r1(r'<meta property="og:type" content="tumblr-feed:(\w+)" />', html) @@ -43,9 +43,9 @@ page_title = r1(r'<meta name="description" content="([^"\n]+)', html) or \ r1(r'<meta property="og:description" content="([^"\n]+)', html) or \ r1(r'<title>([^<\n]*)', html) - urls = re.findall(r'(https?://[^;"&]+/tumblr_[^;"]+_\d+\.jpg)', html) +\ - re.findall(r'(https?://[^;"&]+/tumblr_[^;"]+_\d+\.png)', html) +\ - re.findall(r'(https?://[^;"&]+/tumblr_[^";]+_\d+\.gif)', html) + urls = re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.jpg)', html) +\ + re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.png)', html) +\ + re.findall(r'(https?://[^;"&]+/tumblr_[^";&]+_\d+\.gif)', html) tuggles = {} for url in urls: @@ -121,11 +121,15 @@ r1(r'<meta property="og:description" content="([^"]*)" />', html) or r1(r'<title>([^<\n]*)', html) or url.split("/")[4]).replace('\n', '') - type, ext, size = url_info(real_url) + # this is better + vcode = r1(r'tumblr_(\w+)', real_url) + real_url = 'https://vt.media.tumblr.com/tumblr_%s.mp4' % vcode + + type, ext, size = url_info(real_url, faker=True) print_info(site_info, title, type, size) if not info_only: - download_urls([real_url], title, ext, size, output_dir, merge = merge) + download_urls([real_url], title, ext, size, output_dir, merge=merge) site_info = "Tumblr.com" download = tumblr_download diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/extractors/universal.py new/you-get-0.4.1164/src/you_get/extractors/universal.py --- old/you-get-0.4.1148/src/you_get/extractors/universal.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/extractors/universal.py 2018-10-28 13:33:28.000000000 +0100 @@ -67,9 +67,9 @@ urls = [] for i in media_exts: - urls += re.findall(r'(https?://[^ ;"\'\\]+' + i + r'[^ ;"\'\\]*)', page) + urls += re.findall(r'(https?://[^ ;&"\'\\]+' + i + r'[^ ;&"\'\\]*)', page) - p_urls = re.findall(r'(https?%3A%2F%2F[^;&]+' + i + r'[^;&]*)', page) + p_urls = re.findall(r'(https?%3A%2F%2F[^;&"]+' + i + r'[^;&"]*)', page) urls += [parse.unquote(url) for url in p_urls] q_urls = re.findall(r'(https?:\\\\/\\\\/[^ ;"\']+' + i + r'[^ ;"\']*)', page) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/extractors/yixia.py new/you-get-0.4.1164/src/you_get/extractors/yixia.py --- old/you-get-0.4.1148/src/you_get/extractors/yixia.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/extractors/yixia.py 2018-10-28 13:33:28.000000000 +0100 @@ -8,6 +8,24 @@ import re #---------------------------------------------------------------------- +def miaopai_download_by_smid(smid, output_dir = '.', merge = True, info_only = False): + """""" + api_endpoint = 'https://n.miaopai.com/api/aj_media/info.json?smid={smid}'.format(smid = smid) + + html = get_content(api_endpoint) + + api_content = loads(html) + + video_url = api_content['data']['meta_data'][0]['play_urls']['l'] + title = api_content['data']['description'] + + type, ext, size = url_info(video_url) + + print_info(site_info, title, type, size) + if not info_only: + download_urls([video_url], title, ext, size, output_dir, merge=merge) + +#---------------------------------------------------------------------- def yixia_miaopai_download_by_scid(scid, output_dir = '.', merge = True, info_only = False): """""" api_endpoint = 'http://api.miaopai.com/m/v2_channel.json?fillType=259&scid={scid}&vend=miaopai'.format(scid = scid) @@ -47,7 +65,11 @@ def yixia_download(url, output_dir = '.', merge = True, info_only = False, **kwargs): """wrapper""" hostname = urlparse(url).hostname - if 'miaopai.com' in hostname: #Miaopai + if 'n.miaopai.com' == hostname: + smid = match1(url, r'n\.miaopai\.com/media/([^.]+)') + miaopai_download_by_smid(smid, output_dir, merge, info_only) + return + elif 'miaopai.com' in hostname: #Miaopai yixia_download_by_scid = yixia_miaopai_download_by_scid site_info = "Yixia Miaopai" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/util/fs.py new/you-get-0.4.1164/src/you_get/util/fs.py --- old/you-get-0.4.1148/src/you_get/util/fs.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/util/fs.py 2018-10-28 13:33:28.000000000 +0100 @@ -13,6 +13,7 @@ ord('|'): '-', }) + # FIXME: do some filesystem detection if os == 'windows' or os == 'cygwin' or os == 'wsl': # Windows (non-POSIX namespace) text = text.translate({ @@ -28,6 +29,7 @@ ord('>'): '-', ord('['): '(', ord(']'): ')', + ord('\t'): ' ', }) else: # *nix diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/util/log.py new/you-get-0.4.1164/src/you_get/util/log.py --- old/you-get-0.4.1148/src/you_get/util/log.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/util/log.py 2018-10-28 13:33:28.000000000 +0100 @@ -96,3 +96,9 @@ print_log(message, RED, BOLD) if exit_code is not None: sys.exit(exit_code) + +def yes_or_no(message): + ans = str(input('%s (y/N) ' % message)).lower().strip() + if ans == 'y': + return True + return False diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/you-get-0.4.1148/src/you_get/version.py new/you-get-0.4.1164/src/you_get/version.py --- old/you-get-0.4.1148/src/you_get/version.py 2018-09-11 23:51:43.000000000 +0200 +++ new/you-get-0.4.1164/src/you_get/version.py 2018-10-28 13:33:28.000000000 +0100 @@ -1,4 +1,4 @@ #!/usr/bin/env python script_name = 'you-get' -__version__ = '0.4.1148' +__version__ = '0.4.1164'
