Hello community, here is the log from the commit of package youtube-dl for openSUSE:Factory checked in at 2019-09-09 12:35:24 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/youtube-dl (Old) and /work/SRC/openSUSE:Factory/.youtube-dl.new.7948 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "youtube-dl" Mon Sep 9 12:35:24 2019 rev:114 rq:729186 version:2019.09.01 Changes: -------- --- /work/SRC/openSUSE:Factory/youtube-dl/python-youtube-dl.changes 2019-08-14 11:37:54.500691099 +0200 +++ /work/SRC/openSUSE:Factory/.youtube-dl.new.7948/python-youtube-dl.changes 2019-09-09 12:35:43.277469020 +0200 @@ -1,0 +2,7 @@ +Sun Sep 1 06:19:42 UTC 2019 - Jan Engelhardt <[email protected]> + +- Update to release 2019.09.01 + * Add support for oload.vip, lequipe.fr, woof.tube + * Add support for squarespace embeds + +------------------------------------------------------------------- youtube-dl.changes: same change Old: ---- youtube-dl-2019.08.13.tar.gz youtube-dl-2019.08.13.tar.gz.sig New: ---- youtube-dl-2019.09.01.tar.gz youtube-dl-2019.09.01.tar.gz.sig ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-youtube-dl.spec ++++++ --- /var/tmp/diff_new_pack.sGNmBK/_old 2019-09-09 12:35:47.373468349 +0200 +++ /var/tmp/diff_new_pack.sGNmBK/_new 2019-09-09 12:35:47.377468349 +0200 @@ -19,7 +19,7 @@ %define modname youtube-dl %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-youtube-dl -Version: 2019.08.13 +Version: 2019.09.01 Release: 0 Summary: A Python module for downloading from video sites for offline watching License: SUSE-Public-Domain AND CC-BY-SA-3.0 ++++++ youtube-dl.spec ++++++ --- /var/tmp/diff_new_pack.sGNmBK/_old 2019-09-09 12:35:47.393468346 +0200 +++ /var/tmp/diff_new_pack.sGNmBK/_new 2019-09-09 12:35:47.393468346 +0200 @@ -17,7 +17,7 @@ Name: youtube-dl -Version: 2019.08.13 +Version: 2019.09.01 Release: 0 Summary: A tool for downloading from video sites for offline watching License: SUSE-Public-Domain AND CC-BY-SA-3.0 ++++++ youtube-dl-2019.08.13.tar.gz -> youtube-dl-2019.09.01.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/ChangeLog new/youtube-dl/ChangeLog --- old/youtube-dl/ChangeLog 2019-08-13 18:18:35.000000000 +0200 +++ new/youtube-dl/ChangeLog 2019-08-31 22:32:58.000000000 +0200 @@ -1,3 +1,25 @@ +version 2019.09.01 + +Core ++ [extractor/generic] Add support for squarespace embeds (#21294, #21802, + #21859) ++ [downloader/external] Respect mtime option for aria2c (#22242) + +Extractors ++ [xhamster:user] Add support for user pages (#16330, #18454) ++ [xhamster] Add support for more domains ++ [verystream] Add support for woof.tube (#22217) ++ [dailymotion] Add support for lequipe.fr (#21328, #22152) ++ [openload] Add support for oload.vip (#22205) ++ [bbccouk] Extend URL regular expression (#19200) ++ [youtube] Add support for invidious.nixnet.xyz and yt.elukerio.org (#22223) +* [safari] Fix authentication (#22161, #22184) +* [usanetwork] Fix extraction (#22105) ++ [einthusan] Add support for einthusan.ca (#22171) +* [youtube] Improve unavailable message extraction (#22117) ++ [piksel] Extract subtitles (#20506) + + version 2019.08.13 Core diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/docs/supportedsites.md new/youtube-dl/docs/supportedsites.md --- old/youtube-dl/docs/supportedsites.md 2019-08-13 18:18:38.000000000 +0200 +++ new/youtube-dl/docs/supportedsites.md 2019-08-31 22:33:02.000000000 +0200 @@ -1100,6 +1100,7 @@ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo, RapidVideo.TV, FastVideo.me - **XHamster** - **XHamsterEmbed** + - **XHamsterUser** - **xiami:album**: 虾米音乐 - 专辑 - **xiami:artist**: 虾米音乐 - 歌手 - **xiami:collection**: 虾米音乐 - 精选集 Binary files old/youtube-dl/youtube-dl and new/youtube-dl/youtube-dl differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/downloader/external.py new/youtube-dl/youtube_dl/downloader/external.py --- old/youtube-dl/youtube_dl/downloader/external.py 2019-08-13 18:18:19.000000000 +0200 +++ new/youtube-dl/youtube_dl/downloader/external.py 2019-08-31 22:32:53.000000000 +0200 @@ -194,6 +194,7 @@ cmd += self._option('--interface', 'source_address') cmd += self._option('--all-proxy', 'proxy') cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=') + cmd += self._bool_option('--remote-time', 'updatetime', 'true', 'false', '=') cmd += ['--', info_dict['url']] return cmd diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/bbc.py new/youtube-dl/youtube_dl/extractor/bbc.py --- old/youtube-dl/youtube_dl/extractor/bbc.py 2019-08-13 18:18:19.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/bbc.py 2019-08-31 22:32:53.000000000 +0200 @@ -40,6 +40,7 @@ iplayer(?:/[^/]+)?/(?:episode/|playlist/)| music/(?:clips|audiovideo/popular)[/#]| radio/player/| + sounds/play/| events/[^/]+/play/[^/]+/ ) (?P<id>%s)(?!/(?:episodes|broadcasts|clips)) @@ -70,7 +71,7 @@ 'info_dict': { 'id': 'b039d07m', 'ext': 'flv', - 'title': 'Leonard Cohen, Kaleidoscope - BBC Radio 4', + 'title': 'Kaleidoscope, Leonard Cohen', 'description': 'The Canadian poet and songwriter reflects on his musical career.', }, 'params': { @@ -221,6 +222,20 @@ 'skip_download': True, }, }, { + 'url': 'https://www.bbc.co.uk/sounds/play/m0007jzb', + 'note': 'Audio', + 'info_dict': { + 'id': 'm0007jz9', + 'ext': 'mp4', + 'title': 'BBC Proms, 2019, Prom 34: West–Eastern Divan Orchestra', + 'description': "Live BBC Proms. West–Eastern Divan Orchestra with Daniel Barenboim and Martha Argerich.", + 'duration': 9840, + }, + 'params': { + # rtmp download + 'skip_download': True, + } + }, { 'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4', 'only_matching': True, }, { @@ -609,7 +624,7 @@ 'url': 'http://www.bbc.com/news/world-europe-32668511', 'info_dict': { 'id': 'world-europe-32668511', - 'title': 'Russia stages massive WW2 parade despite Western boycott', + 'title': 'Russia stages massive WW2 parade', 'description': 'md5:00ff61976f6081841f759a08bf78cc9c', }, 'playlist_count': 2, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/dailymotion.py new/youtube-dl/youtube_dl/extractor/dailymotion.py --- old/youtube-dl/youtube_dl/extractor/dailymotion.py 2019-08-13 18:18:20.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/dailymotion.py 2019-08-31 22:32:53.000000000 +0200 @@ -48,7 +48,14 @@ class DailymotionIE(DailymotionBaseInfoExtractor): - _VALID_URL = r'(?i)https?://(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:(?:embed|swf|#)/)?video|swf)/(?P<id>[^/?_]+)' + _VALID_URL = r'''(?ix) + https?:// + (?: + (?:(?:www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:(?:embed|swf|\#)/)?video|swf)| + (?:www\.)?lequipe\.fr/video + ) + /(?P<id>[^/?_]+) + ''' IE_NAME = 'dailymotion' _FORMATS = [ @@ -133,6 +140,12 @@ }, { 'url': 'http://www.dailymotion.com/swf/x3ss1m_funny-magic-trick-barry-and-stuart_fun', 'only_matching': True, + }, { + 'url': 'https://www.lequipe.fr/video/x791mem', + 'only_matching': True, + }, { + 'url': 'https://www.lequipe.fr/video/k7MtHciueyTcrFtFKA2', + 'only_matching': True, }] @staticmethod diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/einthusan.py new/youtube-dl/youtube_dl/extractor/einthusan.py --- old/youtube-dl/youtube_dl/extractor/einthusan.py 2019-08-13 18:18:20.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/einthusan.py 2019-08-31 22:32:53.000000000 +0200 @@ -19,7 +19,7 @@ class EinthusanIE(InfoExtractor): - _VALID_URL = r'https?://(?P<host>einthusan\.(?:tv|com))/movie/watch/(?P<id>[^/?#&]+)' + _VALID_URL = r'https?://(?P<host>einthusan\.(?:tv|com|ca))/movie/watch/(?P<id>[^/?#&]+)' _TESTS = [{ 'url': 'https://einthusan.tv/movie/watch/9097/', 'md5': 'ff0f7f2065031b8a2cf13a933731c035', @@ -36,6 +36,9 @@ }, { 'url': 'https://einthusan.com/movie/watch/9097/', 'only_matching': True, + }, { + 'url': 'https://einthusan.ca/movie/watch/4E9n/?lang=hindi', + 'only_matching': True, }] # reversed from jsoncrypto.prototype.decrypt() in einthusan-PGMovieWatcher.js diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/extractors.py new/youtube-dl/youtube_dl/extractor/extractors.py --- old/youtube-dl/youtube_dl/extractor/extractors.py 2019-08-13 18:18:20.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/extractors.py 2019-08-31 22:32:53.000000000 +0200 @@ -1425,6 +1425,7 @@ from .xhamster import ( XHamsterIE, XHamsterEmbedIE, + XHamsterUserIE, ) from .xiami import ( XiamiSongIE, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/generic.py new/youtube-dl/youtube_dl/extractor/generic.py --- old/youtube-dl/youtube_dl/extractor/generic.py 2019-08-13 18:18:20.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/generic.py 2019-08-31 22:32:53.000000000 +0200 @@ -2076,6 +2076,22 @@ 'playlist_count': 6, }, { + # Squarespace video embed, 2019-08-28 + 'url': 'http://ootboxford.com', + 'info_dict': { + 'id': 'Tc7b_JGdZfw', + 'title': 'Out of the Blue, at Childish Things 10', + 'ext': 'mp4', + 'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f', + 'uploader_id': 'helendouglashouse', + 'uploader': 'Helen & Douglas House', + 'upload_date': '20140328', + }, + 'params': { + 'skip_download': True, + }, + }, + { # Zype embed 'url': 'https://www.cookscountry.com/episode/554-smoky-barbecue-favorites', 'info_dict': { @@ -2395,6 +2411,12 @@ # Unescaping the whole page allows to handle those cases in a generic way webpage = compat_urllib_parse_unquote(webpage) + # Unescape squarespace embeds to be detected by generic extractor, + # see https://github.com/ytdl-org/youtube-dl/issues/21294 + webpage = re.sub( + r'<div[^>]+class=[^>]*?\bsqs-video-wrapper\b[^>]*>', + lambda x: unescapeHTML(x.group(0)), webpage) + # it's tempting to parse this further, but you would # have to take into account all the variations like # Video Title - Site Name diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/openload.py new/youtube-dl/youtube_dl/extractor/openload.py --- old/youtube-dl/youtube_dl/extractor/openload.py 2019-08-13 18:18:21.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/openload.py 2019-08-31 22:32:53.000000000 +0200 @@ -243,7 +243,13 @@ class OpenloadIE(InfoExtractor): - _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|best|biz|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|press|pw|life|live|space|services|website)|oladblock\.(?:services|xyz|me)|openloed\.co)' + _DOMAINS = r''' + (?: + openload\.(?:co|io|link|pw)| + oload\.(?:tv|best|biz|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|press|pw|life|live|space|services|website|vip)| + oladblock\.(?:services|xyz|me)|openloed\.co + ) + ''' _VALID_URL = r'''(?x) https?:// (?P<host> @@ -383,12 +389,15 @@ }, { 'url': 'https://openloed.co/f/b8NWEgkqNLI/', 'only_matching': True, + }, { + 'url': 'https://oload.vip/f/kUEfGclsU9o', + 'only_matching': True, }] @classmethod def _extract_urls(cls, webpage): return re.findall( - r'<iframe[^>]+src=["\']((?:https?://)?%s/%s/[a-zA-Z0-9-_]+)' + r'(?x)<iframe[^>]+src=["\']((?:https?://)?%s/%s/[a-zA-Z0-9-_]+)' % (cls._DOMAINS, cls._EMBED_WORD), webpage) def _extract_decrypted_page(self, page_url, webpage, video_id): @@ -454,7 +463,7 @@ class VerystreamIE(OpenloadIE): IE_NAME = 'verystream' - _DOMAINS = r'(?:verystream\.com)' + _DOMAINS = r'(?:verystream\.com|woof\.tube)' _VALID_URL = r'''(?x) https?:// (?P<host> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/piksel.py new/youtube-dl/youtube_dl/extractor/piksel.py --- old/youtube-dl/youtube_dl/extractor/piksel.py 2019-08-13 18:18:21.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/piksel.py 2019-08-31 22:32:53.000000000 +0200 @@ -18,15 +18,14 @@ _VALID_URL = r'https?://player\.piksel\.com/v/(?P<id>[a-z0-9]+)' _TESTS = [ { - 'url': 'http://player.piksel.com/v/nv60p12f', - 'md5': 'd9c17bbe9c3386344f9cfd32fad8d235', + 'url': 'http://player.piksel.com/v/ums2867l', + 'md5': '34e34c8d89dc2559976a6079db531e85', 'info_dict': { - 'id': 'nv60p12f', + 'id': 'ums2867l', 'ext': 'mp4', - 'title': 'فن الحياة - الحلقة 1', - 'description': 'احدث برامج الداعية الاسلامي " مصطفي حسني " فى رمضان 2016علي النهار نور', - 'timestamp': 1465231790, - 'upload_date': '20160606', + 'title': 'GX-005 with Caption', + 'timestamp': 1481335659, + 'upload_date': '20161210' } }, { @@ -39,7 +38,7 @@ 'title': 'WAW- State of Washington vs. Donald J. Trump, et al', 'description': 'State of Washington vs. Donald J. Trump, et al, Case Number 17-CV-00141-JLR, TRO Hearing, Civil Rights Case, 02/3/2017, 1:00 PM (PST), Seattle Federal Courthouse, Seattle, WA, Judge James L. Robart presiding.', 'timestamp': 1486171129, - 'upload_date': '20170204', + 'upload_date': '20170204' } } ] @@ -113,6 +112,13 @@ }) self._sort_formats(formats) + subtitles = {} + for caption in video_data.get('captions', []): + caption_url = caption.get('url') + if caption_url: + subtitles.setdefault(caption.get('locale', 'en'), []).append({ + 'url': caption_url}) + return { 'id': video_id, 'title': title, @@ -120,4 +126,5 @@ 'thumbnail': video_data.get('thumbnailUrl'), 'timestamp': parse_iso8601(video_data.get('dateadd')), 'formats': formats, + 'subtitles': subtitles, } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/safari.py new/youtube-dl/youtube_dl/extractor/safari.py --- old/youtube-dl/youtube_dl/extractor/safari.py 2019-08-13 18:18:21.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/safari.py 2019-08-31 22:32:53.000000000 +0200 @@ -68,9 +68,10 @@ raise ExtractorError( 'Unable to login: %s' % credentials, expected=True) - # oreilly serves two same groot_sessionid cookies in Set-Cookie header - # and expects first one to be actually set - self._apply_first_set_cookie_header(urlh, 'groot_sessionid') + # oreilly serves two same instances of the following cookies + # in Set-Cookie header and expects first one to be actually set + for cookie in ('groot_sessionid', 'orm-jwt', 'orm-rt'): + self._apply_first_set_cookie_header(urlh, cookie) _, urlh = self._download_webpage_handle( auth.get('redirect_uri') or next_uri, None, 'Completing login',) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/usanetwork.py new/youtube-dl/youtube_dl/extractor/usanetwork.py --- old/youtube-dl/youtube_dl/extractor/usanetwork.py 2019-08-13 18:18:22.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/usanetwork.py 2019-08-31 22:32:53.000000000 +0200 @@ -1,11 +1,9 @@ # coding: utf-8 from __future__ import unicode_literals -import re - from .adobepass import AdobePassIE from ..utils import ( - extract_attributes, + NO_DEFAULT, smuggle_url, update_url_query, ) @@ -31,22 +29,22 @@ display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) - player_params = extract_attributes(self._search_regex( - r'(<div[^>]+data-usa-tve-player-container[^>]*>)', webpage, 'player params')) - video_id = player_params['data-mpx-guid'] - title = player_params['data-episode-title'] - - account_pid, path = re.search( - r'data-src="(?:https?)?//player\.theplatform\.com/p/([^/]+)/.*?/(media/guid/\d+/\d+)', - webpage).groups() + def _x(name, default=NO_DEFAULT): + return self._search_regex( + r'data-%s\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1' % name, + webpage, name, default=default, group='value') + + video_id = _x('mpx-guid') + title = _x('episode-title') + mpx_account_id = _x('mpx-account-id', '2304992029') query = { 'mbr': 'true', } - if player_params.get('data-is-full-episode') == '1': + if _x('is-full-episode', None) == '1': query['manifest'] = 'm3u' - if player_params.get('data-entitlement') == 'auth': + if _x('is-entitlement', None) == '1': adobe_pass = {} drupal_settings = self._search_regex( r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);', @@ -57,7 +55,7 @@ adobe_pass = drupal_settings.get('adobePass', {}) resource = self._get_mvpd_resource( adobe_pass.get('adobePassResourceId', 'usa'), - title, video_id, player_params.get('data-episode-rating', 'TV-14')) + title, video_id, _x('episode-rating', 'TV-14')) query['auth'] = self._extract_mvpd_auth( url, video_id, adobe_pass.get('adobePassRequestorId', 'usa'), resource) @@ -65,11 +63,11 @@ info.update({ '_type': 'url_transparent', 'url': smuggle_url(update_url_query( - 'http://link.theplatform.com/s/%s/%s' % (account_pid, path), + 'http://link.theplatform.com/s/HNK2IC/media/guid/%s/%s' % (mpx_account_id, video_id), query), {'force_smil_url': True}), 'id': video_id, 'title': title, - 'series': player_params.get('data-show-title'), + 'series': _x('show-title', None), 'episode': title, 'ie_key': 'ThePlatform', }) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/xhamster.py new/youtube-dl/youtube_dl/extractor/xhamster.py --- old/youtube-dl/youtube_dl/extractor/xhamster.py 2019-08-13 18:18:22.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/xhamster.py 2019-08-31 22:32:53.000000000 +0200 @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import itertools import re from .common import InfoExtractor @@ -8,6 +9,7 @@ clean_html, determine_ext, dict_get, + extract_attributes, ExtractorError, int_or_none, parse_duration, @@ -18,21 +20,21 @@ class XHamsterIE(InfoExtractor): + _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster[27]\.com)' _VALID_URL = r'''(?x) https?:// - (?:.+?\.)?xhamster\.(?:com|one)/ + (?:.+?\.)?%s/ (?: movies/(?P<id>\d+)/(?P<display_id>[^/]*)\.html| videos/(?P<display_id_2>[^/]*)-(?P<id_2>\d+) ) - ''' - + ''' % _DOMAINS _TESTS = [{ - 'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html', - 'md5': '8281348b8d3c53d39fffb377d24eac4e', + 'url': 'https://xhamster.com/videos/femaleagent-shy-beauty-takes-the-bait-1509445', + 'md5': '98b4687efb1ffd331c4197854dc09e8f', 'info_dict': { 'id': '1509445', - 'display_id': 'femaleagent_shy_beauty_takes_the_bait', + 'display_id': 'femaleagent-shy-beauty-takes-the-bait', 'ext': 'mp4', 'title': 'FemaleAgent Shy beauty takes the bait', 'timestamp': 1350194821, @@ -40,13 +42,12 @@ 'uploader': 'Ruseful2011', 'duration': 893, 'age_limit': 18, - 'categories': ['Fake Hub', 'Amateur', 'MILFs', 'POV', 'Beauti', 'Beauties', 'Beautiful', 'Boss', 'Office', 'Oral', 'Reality', 'Sexy', 'Taking'], }, }, { - 'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd', + 'url': 'https://xhamster.com/videos/britney-spears-sexy-booty-2221348?hd=', 'info_dict': { 'id': '2221348', - 'display_id': 'britney_spears_sexy_booty', + 'display_id': 'britney-spears-sexy-booty', 'ext': 'mp4', 'title': 'Britney Spears Sexy Booty', 'timestamp': 1379123460, @@ -54,13 +55,12 @@ 'uploader': 'jojo747400', 'duration': 200, 'age_limit': 18, - 'categories': ['Britney Spears', 'Celebrities', 'HD Videos', 'Sexy', 'Sexy Booty'], }, 'params': { 'skip_download': True, }, }, { - # empty seo + # empty seo, unavailable via new URL schema 'url': 'http://xhamster.com/movies/5667973/.html', 'info_dict': { 'id': '5667973', @@ -71,7 +71,6 @@ 'uploader': 'parejafree', 'duration': 72, 'age_limit': 18, - 'categories': ['Amateur', 'Blowjobs'], }, 'params': { 'skip_download': True, @@ -94,6 +93,18 @@ }, { 'url': 'https://xhamster.one/videos/femaleagent-shy-beauty-takes-the-bait-1509445', 'only_matching': True, + }, { + 'url': 'https://xhamster.desi/videos/femaleagent-shy-beauty-takes-the-bait-1509445', + 'only_matching': True, + }, { + 'url': 'https://xhamster2.com/videos/femaleagent-shy-beauty-takes-the-bait-1509445', + 'only_matching': True, + }, { + 'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html', + 'only_matching': True, + }, { + 'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd', + 'only_matching': True, }] def _real_extract(self, url): @@ -285,7 +296,7 @@ class XHamsterEmbedIE(InfoExtractor): - _VALID_URL = r'https?://(?:.+?\.)?xhamster\.com/xembed\.php\?video=(?P<id>\d+)' + _VALID_URL = r'https?://(?:.+?\.)?%s/xembed\.php\?video=(?P<id>\d+)' % XHamsterIE._DOMAINS _TEST = { 'url': 'http://xhamster.com/xembed.php?video=3328539', 'info_dict': { @@ -322,3 +333,49 @@ video_url = dict_get(vars, ('downloadLink', 'homepageLink', 'commentsLink', 'shareUrl')) return self.url_result(video_url, 'XHamster') + + +class XHamsterUserIE(InfoExtractor): + _VALID_URL = r'https?://(?:.+?\.)?%s/users/(?P<id>[^/?#&]+)' % XHamsterIE._DOMAINS + _TESTS = [{ + # Paginated user profile + 'url': 'https://xhamster.com/users/netvideogirls/videos', + 'info_dict': { + 'id': 'netvideogirls', + }, + 'playlist_mincount': 267, + }, { + # Non-paginated user profile + 'url': 'https://xhamster.com/users/firatkaan/videos', + 'info_dict': { + 'id': 'firatkaan', + }, + 'playlist_mincount': 1, + }] + + def _entries(self, user_id): + next_page_url = 'https://xhamster.com/users/%s/videos/1' % user_id + for pagenum in itertools.count(1): + page = self._download_webpage( + next_page_url, user_id, 'Downloading page %s' % pagenum) + for video_tag in re.findall( + r'(<a[^>]+class=["\'].*?\bvideo-thumb__image-container[^>]+>)', + page): + video = extract_attributes(video_tag) + video_url = url_or_none(video.get('href')) + if not video_url or not XHamsterIE.suitable(video_url): + continue + video_id = XHamsterIE._match_id(video_url) + yield self.url_result( + video_url, ie=XHamsterIE.ie_key(), video_id=video_id) + mobj = re.search(r'<a[^>]+data-page=["\']next[^>]+>', page) + if not mobj: + break + next_page = extract_attributes(mobj.group(0)) + next_page_url = url_or_none(next_page.get('href')) + if not next_page_url: + break + + def _real_extract(self, url): + user_id = self._match_id(url) + return self.playlist_result(self._entries(user_id), user_id) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/youtube.py new/youtube-dl/youtube_dl/extractor/youtube.py --- old/youtube-dl/youtube_dl/extractor/youtube.py 2019-08-13 18:18:22.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/youtube.py 2019-08-31 22:32:53.000000000 +0200 @@ -387,8 +387,10 @@ (?:www\.)?invidious\.enkirton\.net/| (?:www\.)?invidious\.13ad\.de/| (?:www\.)?invidious\.mastodon\.host/| + (?:www\.)?invidious\.nixnet\.xyz/| (?:www\.)?tube\.poal\.co/| (?:www\.)?vid\.wxzm\.sx/| + (?:www\.)?yt\.elukerio\.org/| youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains (?:.*?\#/)? # handle anchor (#/) redirect urls (?: # the various things that can precede the ID: @@ -1809,10 +1811,15 @@ break def extract_unavailable_message(): - return self._html_search_regex( - (r'(?s)<div[^>]+id=["\']unavailable-submessage["\'][^>]+>(.+?)</div', - r'(?s)<h1[^>]+id=["\']unavailable-message["\'][^>]*>(.+?)</h1>'), - video_webpage, 'unavailable message', default=None) + messages = [] + for tag, kind in (('h1', 'message'), ('div', 'submessage')): + msg = self._html_search_regex( + r'(?s)<{tag}[^>]+id=["\']unavailable-{kind}["\'][^>]*>(.+?)</{tag}>'.format(tag=tag, kind=kind), + video_webpage, 'unavailable %s' % kind, default=None) + if msg: + messages.append(msg) + if messages: + return '\n'.join(messages) if not video_info: unavailable_message = extract_unavailable_message() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/version.py new/youtube-dl/youtube_dl/version.py --- old/youtube-dl/youtube_dl/version.py 2019-08-13 18:18:35.000000000 +0200 +++ new/youtube-dl/youtube_dl/version.py 2019-08-31 22:32:58.000000000 +0200 @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2019.08.13' +__version__ = '2019.09.01'
