Hello community,
here is the log from the commit of package youtube-dl for openSUSE:Factory
checked in at 2020-11-03 15:16:35
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/youtube-dl (Old)
and /work/SRC/openSUSE:Factory/.youtube-dl.new.3463 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "youtube-dl"
Tue Nov 3 15:16:35 2020 rev:140 rq:845526 version:2020.11.01.1
Changes:
--------
--- /work/SRC/openSUSE:Factory/youtube-dl/python-youtube-dl.changes
2020-09-21 17:47:26.861140570 +0200
+++ /work/SRC/openSUSE:Factory/.youtube-dl.new.3463/python-youtube-dl.changes
2020-11-03 15:17:00.796065886 +0100
@@ -1,0 +2,9 @@
+Mon Nov 2 12:59:41 UTC 2020 - Jan Engelhardt <jeng...@inai.de>
+
+- Update to release 2020.11.01.1
+ * youtube: Fix JS player URL extraction
+ * ytsearch: Fix extraction
+ * ustream: Add support for video.ibm.com
+ * expressen: Add support for di.se
+
+-------------------------------------------------------------------
youtube-dl.changes: same change
Old:
----
youtube-dl-2020.09.20.tar.gz
youtube-dl-2020.09.20.tar.gz.sig
New:
----
youtube-dl-2020.11.01.1.tar.gz
youtube-dl-2020.11.01.1.tar.gz.sig
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-youtube-dl.spec ++++++
--- /var/tmp/diff_new_pack.oG0WeY/_old 2020-11-03 15:17:05.640070548 +0100
+++ /var/tmp/diff_new_pack.oG0WeY/_new 2020-11-03 15:17:05.644070552 +0100
@@ -19,7 +19,7 @@
%define modname youtube-dl
%{?!python_module:%define python_module() python-%{**} python3-%{**}}
Name: python-youtube-dl
-Version: 2020.09.20
+Version: 2020.11.01.1
Release: 0
Summary: A Python module for downloading from video sites for offline
watching
License: SUSE-Public-Domain AND CC-BY-SA-3.0
++++++ youtube-dl.spec ++++++
--- /var/tmp/diff_new_pack.oG0WeY/_old 2020-11-03 15:17:05.664070572 +0100
+++ /var/tmp/diff_new_pack.oG0WeY/_new 2020-11-03 15:17:05.668070575 +0100
@@ -17,7 +17,7 @@
Name: youtube-dl
-Version: 2020.09.20
+Version: 2020.11.01.1
Release: 0
Summary: A tool for downloading from video sites for offline watching
License: SUSE-Public-Domain AND CC-BY-SA-3.0
++++++ youtube-dl-2020.09.20.tar.gz -> youtube-dl-2020.11.01.1.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/ChangeLog new/youtube-dl/ChangeLog
--- old/youtube-dl/ChangeLog 2020-09-20 07:30:42.000000000 +0200
+++ new/youtube-dl/ChangeLog 2020-11-01 02:58:37.000000000 +0100
@@ -1,3 +1,21 @@
+version 2020.11.01
+
+Core
+* [utils] Don't attempt to coerce JS strings to numbers in js_to_json (#26851)
+* [downloader/http] Properly handle missing message in SSLError (#26646)
+* [downloader/http] Fix access to not yet opened stream in retry
+
+Extractors
+* [youtube] Fix JS player URL extraction
+* [ytsearch] Fix extraction (#26920)
+* [afreecatv] Fix typo (#26970)
+* [23video] Relax URL regular expression (#26870)
++ [ustream] Add support for video.ibm.com (#26894)
+* [iqiyi] Fix typo (#26884)
++ [expressen] Add support for di.se (#26670)
+* [iprima] Improve video id extraction (#26507, #26494)
+
+
version 2020.09.20
Core
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/README.md new/youtube-dl/README.md
--- old/youtube-dl/README.md 2020-09-20 07:30:44.000000000 +0200
+++ new/youtube-dl/README.md 2020-11-01 02:58:39.000000000 +0100
@@ -545,7 +545,7 @@
- `extractor` (string): Name of the extractor
- `extractor_key` (string): Key name of the extractor
- `epoch` (numeric): Unix epoch when creating the file
- - `autonumber` (numeric): Five-digit number that will be increased with each
download, starting at zero
+ - `autonumber` (numeric): Number that will be increased with each download,
starting at `--autonumber-start`
- `playlist` (string): Name or id of the playlist that contains the video
- `playlist_index` (numeric): Index of the video in the playlist padded with
leading zeros according to the total length of the playlist
- `playlist_id` (string): Playlist identifier
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/README.txt new/youtube-dl/README.txt
--- old/youtube-dl/README.txt 2020-09-20 07:31:22.000000000 +0200
+++ new/youtube-dl/README.txt 2020-11-01 02:58:52.000000000 +0100
@@ -634,8 +634,8 @@
- extractor (string): Name of the extractor
- extractor_key (string): Key name of the extractor
- epoch (numeric): Unix epoch when creating the file
-- autonumber (numeric): Five-digit number that will be increased with
- each download, starting at zero
+- autonumber (numeric): Number that will be increased with each
+ download, starting at --autonumber-start
- playlist (string): Name or id of the playlist that contains the
video
- playlist_index (numeric): Index of the video in the playlist padded
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/devscripts/release_gitlab.sh
new/youtube-dl/devscripts/release_gitlab.sh
--- old/youtube-dl/devscripts/release_gitlab.sh 1970-01-01 01:00:00.000000000
+0100
+++ new/youtube-dl/devscripts/release_gitlab.sh 2020-11-01 02:58:20.000000000
+0100
@@ -0,0 +1,141 @@
+#!/bin/bash
+
+# IMPORTANT: the following assumptions are made
+# * the GH repo is on the origin remote
+# * the gh-pages branch is named so locally
+# * the git config user.signingkey is properly set
+
+# You will need
+# pip install coverage nose rsa wheel
+
+# TODO
+# release notes
+# make hash on local files
+
+set -e
+
+skip_tests=true
+gpg_sign_commits=""
+buildserver='localhost:8142'
+
+while true
+do
+case "$1" in
+ --run-tests)
+ skip_tests=false
+ shift
+ ;;
+ --gpg-sign-commits|-S)
+ gpg_sign_commits="-S"
+ shift
+ ;;
+ --buildserver)
+ buildserver="$2"
+ shift 2
+ ;;
+ --*)
+ echo "ERROR: unknown option $1"
+ exit 1
+ ;;
+ *)
+ break
+ ;;
+esac
+done
+
+if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0
1994.09.06"; exit 1; fi
+version="$1"
+major_version=$(echo "$version" | sed -n
's#^\([0-9]*\.[0-9]*\.[0-9]*\).*#\1#p')
+if test "$major_version" '!=' "$(date '+%Y.%m.%d')"; then
+ echo "$version does not start with today's date!"
+ exit 1
+fi
+
+if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already
present'; exit 1; fi
+#if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR:
the working directory is not clean; commit or stash changes'; exit 1; fi
+useless_files=$(find youtube_dl -type f -not -name '*.py')
+if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl:
$useless_files"; exit 1; fi
+if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing';
exit 1; fi
+if ! type pandoc >/dev/null 2>/dev/null; then echo 'ERROR: pandoc is missing';
exit 1; fi
+if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is
missing'; exit 1; fi
+if ! python3 -c 'import wheel' 2>/dev/null; then echo 'ERROR: wheel is
missing'; exit 1; fi
+
+read -p "Is ChangeLog up to date? (y/n) " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
+
+/bin/echo -e "\n### First of all, testing..."
+make clean
+if $skip_tests ; then
+ echo 'SKIPPING TESTS'
+else
+ nosetests --verbose --with-coverage --cover-package=youtube_dl
--cover-html test --stop || exit 1
+fi
+
+/bin/echo -e "\n### Changing version in version.py..."
+sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
+
+/bin/echo -e "\n### Changing version in ChangeLog..."
+sed -i "s/<unreleased>/$version/" ChangeLog
+
+/bin/echo -e "\n### Committing documentation, templates and
youtube_dl/version.py..."
+make README.md CONTRIBUTING.md issuetemplates supportedsites
+git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE/1_broken_site.md
.github/ISSUE_TEMPLATE/2_site_support_request.md
.github/ISSUE_TEMPLATE/3_site_feature_request.md
.github/ISSUE_TEMPLATE/4_bug_report.md
.github/ISSUE_TEMPLATE/5_feature_request.md
.github/ISSUE_TEMPLATE/6_question.md docs/supportedsites.md
youtube_dl/version.py ChangeLog
+git commit $gpg_sign_commits -m "release $version"
+
+/bin/echo -e "\n### Now tagging, signing and pushing..."
+git tag -s -m "Release $version" "$version"
+git show "$version"
+read -p "Is it good, can I push? (y/n) " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
+echo
+MASTER=$(git rev-parse --abbrev-ref HEAD)
+git push dstftw-gitlab $MASTER:master
+git push dstftw-gitlab "$version"
+
+/bin/echo -e "\n### OK, now it is time to build the binaries..."
+REV=$(git rev-parse HEAD)
+make youtube-dl youtube-dl.tar.gz
+read -p "VM running? (y/n) " -n 1
+wget "http://$buildserver/build/ytdl-org/youtube-dl/youtube-dl.exe?rev=$REV";
-O youtube-dl.exe
+mkdir -p "build/$version"
+mv youtube-dl youtube-dl.exe "build/$version"
+mv youtube-dl.tar.gz "build/$version/youtube-dl-$version.tar.gz"
+RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz"
+(cd build/$version/ && md5sum $RELEASE_FILES > MD5SUMS)
+(cd build/$version/ && sha1sum $RELEASE_FILES > SHA1SUMS)
+(cd build/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS)
+(cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS)
+
+/bin/echo -e "\n### Signing and uploading the new binaries to GitHub..."
+for f in $RELEASE_FILES; do gpg --passphrase-repeat 5 --detach-sig
"build/$version/$f"; done
+
+ROOT=$(pwd)
+#python devscripts/create-github-release.py ChangeLog $version
"$ROOT/build/$version"
+
+#ssh y...@yt-dl.org "sh html/update_latest.sh $version"
+
+/bin/echo -e "\n### Now switching to gh-pages..."
+git clone --branch gh-pages --single-branch . build/gh-pages
+(
+ set -e
+ ORIGIN_URL=$(git config --get remote.dstftw-gitlab.url)
+ cd build/gh-pages
+ "$ROOT/devscripts/gh-pages/add-version.py" $version
+ "$ROOT/devscripts/gh-pages/update-feed.py"
+ "$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem"
+ "$ROOT/devscripts/gh-pages/generate-download.py"
+ "$ROOT/devscripts/gh-pages/update-copyright.py"
+ "$ROOT/devscripts/gh-pages/update-sites.py"
+ git add *.html *.html.in update
+ git commit $gpg_sign_commits -m "release $version"
+ git push "$ROOT" gh-pages
+ git push "$ORIGIN_URL" gh-pages
+)
+#rm -rf build
+
+make pypi-files
+echo "Uploading to PyPi ..."
+python setup.py sdist bdist_wheel upload
+#make clean
+
+/bin/echo -e "\n### DONE!"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/test/test_utils.py
new/youtube-dl/test/test_utils.py
--- old/youtube-dl/test/test_utils.py 2020-09-20 07:29:46.000000000 +0200
+++ new/youtube-dl/test/test_utils.py 2020-11-01 02:52:19.000000000 +0100
@@ -994,6 +994,12 @@
on = js_to_json('{42:4.2e1}')
self.assertEqual(json.loads(on), {'42': 42.0})
+ on = js_to_json('{ "0x40": "0x40" }')
+ self.assertEqual(json.loads(on), {'0x40': '0x40'})
+
+ on = js_to_json('{ "040": "040" }')
+ self.assertEqual(json.loads(on), {'040': '040'})
+
def test_js_to_json_malformed(self):
self.assertEqual(js_to_json('42a1'), '42"a1"')
self.assertEqual(js_to_json('42a-1'), '42"a"-1')
Binary files old/youtube-dl/youtube-dl and new/youtube-dl/youtube-dl differ
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube-dl.1 new/youtube-dl/youtube-dl.1
--- old/youtube-dl/youtube-dl.1 2020-09-20 07:31:23.000000000 +0200
+++ new/youtube-dl/youtube-dl.1 2020-11-01 02:58:53.000000000 +0100
@@ -1141,8 +1141,8 @@
.IP \[bu] 2
\f[C]epoch\f[] (numeric): Unix epoch when creating the file
.IP \[bu] 2
-\f[C]autonumber\f[] (numeric): Five\-digit number that will be increased
-with each download, starting at zero
+\f[C]autonumber\f[] (numeric): Number that will be increased with each
+download, starting at \f[C]\-\-autonumber\-start\f[]
.IP \[bu] 2
\f[C]playlist\f[] (string): Name or id of the playlist that contains the
video
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube_dl/downloader/http.py
new/youtube-dl/youtube_dl/downloader/http.py
--- old/youtube-dl/youtube_dl/downloader/http.py 2020-09-20
07:29:51.000000000 +0200
+++ new/youtube-dl/youtube_dl/downloader/http.py 2020-11-01
02:52:19.000000000 +0100
@@ -223,9 +223,10 @@
def retry(e):
to_stdout = ctx.tmpfilename == '-'
- if not to_stdout:
- ctx.stream.close()
- ctx.stream = None
+ if ctx.stream is not None:
+ if not to_stdout:
+ ctx.stream.close()
+ ctx.stream = None
ctx.resume_len = byte_counter if to_stdout else
os.path.getsize(encodeFilename(ctx.tmpfilename))
raise RetryDownload(e)
@@ -240,7 +241,7 @@
except socket.error as e:
# SSLError on python 2 (inherits socket.error) may have
# no errno set but this error message
- if e.errno in (errno.ECONNRESET, errno.ETIMEDOUT) or
getattr(e, 'message') == 'The read operation timed out':
+ if e.errno in (errno.ECONNRESET, errno.ETIMEDOUT) or
getattr(e, 'message', None) == 'The read operation timed out':
retry(e)
raise
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/afreecatv.py
new/youtube-dl/youtube_dl/extractor/afreecatv.py
--- old/youtube-dl/youtube_dl/extractor/afreecatv.py 2020-09-20
07:29:46.000000000 +0200
+++ new/youtube-dl/youtube_dl/extractor/afreecatv.py 2020-11-01
02:52:19.000000000 +0100
@@ -275,7 +275,7 @@
video_element = video_xml.findall(compat_xpath('./track/video'))[-1]
if video_element is None or video_element.text is None:
raise ExtractorError(
- 'Video %s video does not exist' % video_id, expected=True)
+ 'Video %s does not exist' % video_id, expected=True)
video_url = video_element.text.strip()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/expressen.py
new/youtube-dl/youtube_dl/extractor/expressen.py
--- old/youtube-dl/youtube_dl/extractor/expressen.py 2020-09-20
07:29:46.000000000 +0200
+++ new/youtube-dl/youtube_dl/extractor/expressen.py 2020-11-01
02:52:19.000000000 +0100
@@ -15,7 +15,7 @@
class ExpressenIE(InfoExtractor):
_VALID_URL = r'''(?x)
https?://
- (?:www\.)?expressen\.se/
+ (?:www\.)?(?:expressen|di)\.se/
(?:(?:tvspelare/video|videoplayer/embed)/)?
tv/(?:[^/]+/)*
(?P<id>[^/?#&]+)
@@ -42,13 +42,16 @@
}, {
'url':
'https://www.expressen.se/videoplayer/embed/tv/ditv/ekonomistudion/experterna-har-ar-fragorna-som-avgor-valet/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di',
'only_matching': True,
+ }, {
+ 'url':
'https://www.di.se/videoplayer/embed/tv/ditv/borsmorgon/implantica-rusar-70--under-borspremiaren-hor-styrelsemedlemmen/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di',
+ 'only_matching': True,
}]
@staticmethod
def _extract_urls(webpage):
return [
mobj.group('url') for mobj in re.finditer(
-
r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?expressen\.se/(?:tvspelare/video|videoplayer/embed)/tv/.+?)\1',
+
r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?(?:expressen|di)\.se/(?:tvspelare/video|videoplayer/embed)/tv/.+?)\1',
webpage)]
def _real_extract(self, url):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/iprima.py
new/youtube-dl/youtube_dl/extractor/iprima.py
--- old/youtube-dl/youtube_dl/extractor/iprima.py 2020-09-20
07:29:46.000000000 +0200
+++ new/youtube-dl/youtube_dl/extractor/iprima.py 2020-11-01
02:52:19.000000000 +0100
@@ -86,7 +86,8 @@
(r'<iframe[^>]+\bsrc=["\'](?:https?:)?//(?:api\.play-backend\.iprima\.cz/prehravac/embedded|prima\.iprima\.cz/[^/]+/[^/]+)\?.*?\bid=(p\d+)',
r'data-product="([^"]+)">',
r'id=["\']player-(p\d+)"',
- r'playerId\s*:\s*["\']player-(p\d+)'),
+ r'playerId\s*:\s*["\']player-(p\d+)',
+ r'\bvideos\s*=\s*["\'](p\d+)'),
webpage, 'real id')
playerpage = self._download_webpage(
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/iqiyi.py
new/youtube-dl/youtube_dl/extractor/iqiyi.py
--- old/youtube-dl/youtube_dl/extractor/iqiyi.py 2020-09-20
07:29:46.000000000 +0200
+++ new/youtube-dl/youtube_dl/extractor/iqiyi.py 2020-11-01
02:52:19.000000000 +0100
@@ -150,7 +150,7 @@
elif function in other_functions:
other_functions[function]()
else:
- raise ExtractorError('Unknown funcion %s' % function)
+ raise ExtractorError('Unknown function %s' % function)
return sdk.target
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/twentythreevideo.py
new/youtube-dl/youtube_dl/extractor/twentythreevideo.py
--- old/youtube-dl/youtube_dl/extractor/twentythreevideo.py 2020-09-20
07:29:46.000000000 +0200
+++ new/youtube-dl/youtube_dl/extractor/twentythreevideo.py 2020-11-01
02:52:19.000000000 +0100
@@ -8,8 +8,8 @@
class TwentyThreeVideoIE(InfoExtractor):
IE_NAME = '23video'
- _VALID_URL =
r'https?://video\.(?P<domain>twentythree\.net|23video\.com|filmweb\.no)/v\.ihtml/player\.html\?(?P<query>.*?\bphoto(?:_|%5f)id=(?P<id>\d+).*)'
- _TEST = {
+ _VALID_URL =
r'https?://(?P<domain>[^.]+\.(?:twentythree\.net|23video\.com|filmweb\.no))/v\.ihtml/player\.html\?(?P<query>.*?\bphoto(?:_|%5f)id=(?P<id>\d+).*)'
+ _TESTS = [{
'url':
'https://video.twentythree.net/v.ihtml/player.html?showDescriptions=0&source=site&photo%5fid=20448876&autoPlay=1',
'md5': '75fcf216303eb1dae9920d651f85ced4',
'info_dict': {
@@ -21,11 +21,14 @@
'uploader_id': '12258964',
'uploader': 'Rasmus Bysted',
}
- }
+ }, {
+ 'url':
'https://bonnier-publications-danmark.23video.com/v.ihtml/player.html?token=f0dc46476e06e13afd5a1f84a29e31e8&source=embed&photo%5fid=36137620',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
domain, query, photo_id = re.match(self._VALID_URL, url).groups()
- base_url = 'https://video.%s' % domain
+ base_url = 'https://%s' % domain
photo_data = self._download_json(
base_url + '/api/photo/list?' + query, photo_id, query={
'format': 'json',
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/ustream.py
new/youtube-dl/youtube_dl/extractor/ustream.py
--- old/youtube-dl/youtube_dl/extractor/ustream.py 2020-09-20
07:29:46.000000000 +0200
+++ new/youtube-dl/youtube_dl/extractor/ustream.py 2020-11-01
02:52:19.000000000 +0100
@@ -19,7 +19,7 @@
class UstreamIE(InfoExtractor):
- _VALID_URL =
r'https?://(?:www\.)?ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<id>\d+)'
+ _VALID_URL =
r'https?://(?:www\.)?(?:ustream\.tv|video\.ibm\.com)/(?P<type>recorded|embed|embed/recorded)/(?P<id>\d+)'
IE_NAME = 'ustream'
_TESTS = [{
'url': 'http://www.ustream.tv/recorded/20274954',
@@ -67,12 +67,15 @@
'params': {
'skip_download': True, # m3u8 download
},
+ }, {
+ 'url':
'https://video.ibm.com/embed/recorded/128240221?&autoplay=true&controls=true&volume=100',
+ 'only_matching': True,
}]
@staticmethod
def _extract_url(webpage):
mobj = re.search(
-
r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1',
webpage)
+
r'<iframe[^>]+?src=(["\'])(?P<url>http://(?:www\.)?(?:ustream\.tv|video\.ibm\.com)/embed/.+?)\1',
webpage)
if mobj is not None:
return mobj.group('url')
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/youtube.py
new/youtube-dl/youtube_dl/extractor/youtube.py
--- old/youtube-dl/youtube_dl/extractor/youtube.py 2020-09-20
07:29:46.000000000 +0200
+++ new/youtube-dl/youtube_dl/extractor/youtube.py 2020-11-01
02:52:19.000000000 +0100
@@ -2086,7 +2086,10 @@
if cipher:
if 's' in url_data or
self._downloader.params.get('youtube_include_dash_manifest', True):
- ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
+ ASSETS_RE = (
+
r'<script[^>]+\bsrc=("[^"]+")[^>]+\bname=["\']player_ias/base',
+ r'"jsUrl"\s*:\s*("[^"]+")',
+ r'"assets":.+?"js":\s*("[^"]+")')
jsplayer_url_json = self._search_regex(
ASSETS_RE,
embed_webpage if age_gate else video_webpage,
@@ -3181,54 +3184,94 @@
_MAX_RESULTS = float('inf')
IE_NAME = 'youtube:search'
_SEARCH_KEY = 'ytsearch'
- _EXTRA_QUERY_ARGS = {}
+ _SEARCH_PARAMS = None
_TESTS = []
- def _get_n_results(self, query, n):
- """Get a specified number of results for a query"""
-
- videos = []
- limit = n
-
- url_query = {
- 'search_query': query.encode('utf-8'),
+ def _entries(self, query, n):
+ data = {
+ 'context': {
+ 'client': {
+ 'clientName': 'WEB',
+ 'clientVersion': '2.20201021.03.00',
+ }
+ },
+ 'query': query,
}
- url_query.update(self._EXTRA_QUERY_ARGS)
- result_url = 'https://www.youtube.com/results?' +
compat_urllib_parse_urlencode(url_query)
-
- for pagenum in itertools.count(1):
- data = self._download_json(
- result_url, video_id='query "%s"' % query,
- note='Downloading page %s' % pagenum,
- errnote='Unable to download API page',
- query={'spf': 'navigate'})
- html_content = data[1]['body']['content']
-
- if 'class="search-message' in html_content:
- raise ExtractorError(
- '[youtube] No video results', expected=True)
-
- new_videos = list(self._process_page(html_content))
- videos += new_videos
- if not new_videos or len(videos) > limit:
+ if self._SEARCH_PARAMS:
+ data['params'] = self._SEARCH_PARAMS
+ total = 0
+ for page_num in itertools.count(1):
+ search = self._download_json(
+
'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+ video_id='query "%s"' % query,
+ note='Downloading page %s' % page_num,
+ errnote='Unable to download API page', fatal=False,
+ data=json.dumps(data).encode('utf8'),
+ headers={'content-type': 'application/json'})
+ if not search:
break
- next_link = self._html_search_regex(
-
r'href="(/results\?[^"]*\bsp=[^"]+)"[^>]*>\s*<span[^>]+class="[^"]*\byt-uix-button-content\b[^"]*"[^>]*>Next',
- html_content, 'next link', default=None)
- if next_link is None:
+ slr_contents = try_get(
+ search,
+ (lambda x:
x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
+ lambda x:
x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
+ list)
+ if not slr_contents:
break
- result_url = compat_urlparse.urljoin('https://www.youtube.com/',
next_link)
+ isr_contents = try_get(
+ slr_contents,
+ lambda x: x[0]['itemSectionRenderer']['contents'],
+ list)
+ if not isr_contents:
+ break
+ for content in isr_contents:
+ if not isinstance(content, dict):
+ continue
+ video = content.get('videoRenderer')
+ if not isinstance(video, dict):
+ continue
+ video_id = video.get('videoId')
+ if not video_id:
+ continue
+ title = try_get(video, lambda x:
x['title']['runs'][0]['text'], compat_str)
+ description = try_get(video, lambda x:
x['descriptionSnippet']['runs'][0]['text'], compat_str)
+ duration = parse_duration(try_get(video, lambda x:
x['lengthText']['simpleText'], compat_str))
+ view_count_text = try_get(video, lambda x:
x['viewCountText']['simpleText'], compat_str) or ''
+ view_count = int_or_none(self._search_regex(
+ r'^(\d+)', re.sub(r'\s', '', view_count_text),
+ 'view count', default=None))
+ uploader = try_get(video, lambda x:
x['ownerText']['runs'][0]['text'], compat_str)
+ total += 1
+ yield {
+ '_type': 'url_transparent',
+ 'ie_key': YoutubeIE.ie_key(),
+ 'id': video_id,
+ 'url': video_id,
+ 'title': title,
+ 'description': description,
+ 'duration': duration,
+ 'view_count': view_count,
+ 'uploader': uploader,
+ }
+ if total == n:
+ return
+ token = try_get(
+ slr_contents,
+ lambda x:
x[1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
+ compat_str)
+ if not token:
+ break
+ data['continuation'] = token
- if len(videos) > n:
- videos = videos[:n]
- return self.playlist_result(videos, query)
+ def _get_n_results(self, query, n):
+ """Get a specified number of results for a query"""
+ return self.playlist_result(self._entries(query, n), query)
class YoutubeSearchDateIE(YoutubeSearchIE):
IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
_SEARCH_KEY = 'ytsearchdate'
IE_DESC = 'YouTube.com searches, newest videos first'
- _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
+ _SEARCH_PARAMS = 'CAI%3D'
class YoutubeSearchURLIE(YoutubeSearchBaseInfoExtractor):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube_dl/utils.py
new/youtube-dl/youtube_dl/utils.py
--- old/youtube-dl/youtube_dl/utils.py 2020-09-20 07:29:46.000000000 +0200
+++ new/youtube-dl/youtube_dl/utils.py 2020-11-01 02:52:19.000000000 +0100
@@ -4088,12 +4088,12 @@
'\\\n': '',
'\\x': '\\u00',
}.get(m.group(0), m.group(0)), v[1:-1])
-
- for regex, base in INTEGER_TABLE:
- im = re.match(regex, v)
- if im:
- i = int(im.group(1), base)
- return '"%d":' % i if v.endswith(':') else '%d' % i
+ else:
+ for regex, base in INTEGER_TABLE:
+ im = re.match(regex, v)
+ if im:
+ i = int(im.group(1), base)
+ return '"%d":' % i if v.endswith(':') else '%d' % i
return '"%s"' % v
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/youtube-dl/youtube_dl/version.py
new/youtube-dl/youtube_dl/version.py
--- old/youtube-dl/youtube_dl/version.py 2020-09-20 07:30:42.000000000
+0200
+++ new/youtube-dl/youtube_dl/version.py 2020-11-01 02:58:37.000000000
+0100
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2020.09.20'
+__version__ = '2020.11.01.1'
