diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 6b931b3cf..dc303946e 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
- [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2019.05.11**
+- [ ] I've verified that I'm running youtube-dl version **2019.05.20**
- [ ] I've checked that all provided URLs are alive and playable in a browser
- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
- [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
[debug] User config: []
[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2019.05.11
+ [debug] youtube-dl version 2019.05.20
[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
[debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index f2dc784a2..46e143c8a 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
- [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2019.05.11**
+- [ ] I've verified that I'm running youtube-dl version **2019.05.20**
- [ ] I've checked that all provided URLs are alive and playable in a browser
- [ ] I've checked that none of provided URLs violate any copyrights
- [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 39a0af13f..bc6c4694b 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
- [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2019.05.11**
+- [ ] I've verified that I'm running youtube-dl version **2019.05.20**
- [ ] I've searched the bugtracker for similar site feature requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 139f36ab8..bcc51f986 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
- [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2019.05.11**
+- [ ] I've verified that I'm running youtube-dl version **2019.05.20**
- [ ] I've checked that all provided URLs are alive and playable in a browser
- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
- [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
[debug] User config: []
[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2019.05.11
+ [debug] youtube-dl version 2019.05.20
[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
[debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index d60da6db9..c8d16960e 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
- [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2019.05.11**
+- [ ] I've verified that I'm running youtube-dl version **2019.05.20**
- [ ] I've searched the bugtracker for similar feature requests including closed ones
diff --git a/.travis.yml b/.travis.yml
index 82e81d078..6d16c2955 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,7 @@ python:
- "3.6"
- "pypy"
- "pypy3"
+dist: trusty
env:
- YTDL_TEST_SET=core
- YTDL_TEST_SET=download
diff --git a/ChangeLog b/ChangeLog
index 13cb6288d..3babb6f48 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,22 @@
+version 2019.05.20
+
+Core
++ [extractor/common] Move workaround for applying first Set-Cookie header
+ into a separate _apply_first_set_cookie_header method
+
+Extractors
+* [safari] Fix authentication (#21090)
+* [vk] Use _apply_first_set_cookie_header
+* [vrt] Fix extraction (#20527)
++ [canvas] Add support for vrtnieuws and sporza site ids and extract
+ AES HLS formats
++ [vrv] Extract captions (#19238)
+* [tele5] Improve video id extraction
+* [tele5] Relax URL regular expression (#21020, #21063)
+* [svtplay] Update API URL (#21075)
++ [yahoo:gyao] Add X-User-Agent header to dam proxy requests (#21071)
+
+
version 2019.05.11
Core
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a8a9224cb..404a2f0a4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1071,7 +1071,7 @@
- **VoxMediaVolume**
- **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
- **Vrak**
- - **VRT**: deredactie.be, sporza.be, cobra.be and cobra.canvas.be
+ - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza
- **VrtNU**: VrtNU.be
- **vrv**
- **vrv:series**
diff --git a/test/test_utils.py b/test/test_utils.py
index 9ef0e422b..71980b3fc 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -73,6 +73,7 @@ from youtube_dl.utils import (
smuggle_url,
str_to_int,
strip_jsonp,
+ strip_or_none,
timeconvert,
unescapeHTML,
unified_strdate,
@@ -752,6 +753,18 @@ class TestUtil(unittest.TestCase):
d = json.loads(stripped)
self.assertEqual(d, {'status': 'success'})
+ def test_strip_or_none(self):
+ self.assertEqual(strip_or_none(' abc'), 'abc')
+ self.assertEqual(strip_or_none('abc '), 'abc')
+ self.assertEqual(strip_or_none(' abc '), 'abc')
+ self.assertEqual(strip_or_none('\tabc\t'), 'abc')
+ self.assertEqual(strip_or_none('\n\tabc\n\t'), 'abc')
+ self.assertEqual(strip_or_none('abc'), 'abc')
+ self.assertEqual(strip_or_none(''), '')
+ self.assertEqual(strip_or_none(None), None)
+ self.assertEqual(strip_or_none(42), None)
+ self.assertEqual(strip_or_none([]), None)
+
def test_uppercase_escape(self):
self.assertEqual(uppercase_escape('aä'), 'aä')
self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index 4f39424f5..1d69dafbd 100644
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -65,8 +65,9 @@ class BitChuteIE(InfoExtractor):
webpage, default=None) or self._html_search_meta(
'twitter:image:src', webpage, 'thumbnail')
uploader = self._html_search_regex(
- r'(?s)
]+\bclass=["\']video-author[^>]+>(.+?)
', webpage,
- 'uploader', fatal=False)
+ (r'(?s)]+\bclass=["\']name[^>]+>(.+?)',
+ r'(?s)
]+\bclass=["\']video-author[^>]+>(.+?)
'),
+ webpage, 'uploader', fatal=False)
return {
'id': video_id,
diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 174fd9e2b..c506bc5dd 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -17,7 +17,7 @@ from ..utils import (
class CanvasIE(InfoExtractor):
- _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P
canvas|een|ketnet|vrtvideo)/assets/(?P[^/?#&]+)'
+ _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?Pcanvas|een|ketnet|vrt(?:video|nieuws)|sporza)/assets/(?P[^/?#&]+)'
_TESTS = [{
'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
'md5': '90139b746a0a9bd7bb631283f6e2a64e',
@@ -35,6 +35,10 @@ class CanvasIE(InfoExtractor):
'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
'only_matching': True,
}]
+ _HLS_ENTRY_PROTOCOLS_MAP = {
+ 'HLS': 'm3u8_native',
+ 'HLS_AES': 'm3u8',
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
@@ -52,9 +56,9 @@ class CanvasIE(InfoExtractor):
format_url, format_type = target.get('url'), target.get('type')
if not format_url or not format_type:
continue
- if format_type == 'HLS':
+ if format_type in self._HLS_ENTRY_PROTOCOLS_MAP:
formats.extend(self._extract_m3u8_formats(
- format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+ format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type],
m3u8_id=format_type, fatal=False))
elif format_type == 'HDS':
formats.extend(self._extract_f4m_formats(
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 69c3bc755..9c3e9eec6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -67,6 +67,7 @@ from ..utils import (
sanitized_Request,
sanitize_filename,
str_or_none,
+ strip_or_none,
unescapeHTML,
unified_strdate,
unified_timestamp,
@@ -2480,7 +2481,7 @@ class InfoExtractor(object):
'subtitles': {},
}
media_attributes = extract_attributes(media_tag)
- src = media_attributes.get('src')
+ src = strip_or_none(media_attributes.get('src'))
if src:
_, formats = _media_formats(src, media_type)
media_info['formats'].extend(formats)
@@ -2490,7 +2491,7 @@ class InfoExtractor(object):
s_attr = extract_attributes(source_tag)
# data-video-src and data-src are non standard but seen
# several times in the wild
- src = dict_get(s_attr, ('src', 'data-video-src', 'data-src'))
+ src = strip_or_none(dict_get(s_attr, ('src', 'data-video-src', 'data-src')))
if not src:
continue
f = parse_content_type(s_attr.get('type'))
@@ -2533,7 +2534,7 @@ class InfoExtractor(object):
track_attributes = extract_attributes(track_tag)
kind = track_attributes.get('kind')
if not kind or kind in ('subtitles', 'captions'):
- src = track_attributes.get('src')
+ src = strip_or_none(track_attributes.get('src'))
if not src:
continue
lang = track_attributes.get('srclang') or track_attributes.get('lang') or track_attributes.get('label')
@@ -2817,6 +2818,33 @@ class InfoExtractor(object):
self._downloader.cookiejar.add_cookie_header(req)
return compat_cookies.SimpleCookie(req.get_header('Cookie'))
+ def _apply_first_set_cookie_header(self, url_handle, cookie):
+ """
+ Apply first Set-Cookie header instead of the last. Experimental.
+
+ Some sites (e.g. [1-3]) may serve two cookies under the same name
+ in Set-Cookie header and expect the first (old) one to be set rather
+ than second (new). However, as of RFC6265 the newer one cookie
+ should be set into cookie store what actually happens.
+ We will workaround this issue by resetting the cookie to
+ the first one manually.
+ 1. https://new.vk.com/
+ 2. https://github.com/ytdl-org/youtube-dl/issues/9841#issuecomment-227871201
+ 3. https://learning.oreilly.com/
+ """
+ for header, cookies in url_handle.headers.items():
+ if header.lower() != 'set-cookie':
+ continue
+ if sys.version_info[0] >= 3:
+ cookies = cookies.encode('iso-8859-1')
+ cookies = cookies.decode('utf-8')
+ cookie_value = re.search(
+ r'%s=(.+?);.*?\b[Dd]omain=(.+?)(?:[,;]|$)' % cookie, cookies)
+ if cookie_value:
+ value, domain = cookie_value.groups()
+ self._set_cookie(domain, cookie, value)
+ break
+
def get_testcases(self, include_onlymatching=False):
t = getattr(self, '_TEST', None)
if t:
diff --git a/youtube_dl/extractor/criterion.py b/youtube_dl/extractor/criterion.py
deleted file mode 100644
index f7815b905..000000000
--- a/youtube_dl/extractor/criterion.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class CriterionIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?criterion\.com/films/(?P[0-9]+)-.+'
- _TEST = {
- 'url': 'http://www.criterion.com/films/184-le-samourai',
- 'md5': 'bc51beba55685509883a9a7830919ec3',
- 'info_dict': {
- 'id': '184',
- 'ext': 'mp4',
- 'title': 'Le Samouraï',
- 'description': 'md5:a2b4b116326558149bef81f76dcbb93f',
- 'thumbnail': r're:^https?://.*\.jpg$',
- }
- }
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
-
- final_url = self._search_regex(
- r'so\.addVariable\("videoURL", "(.+?)"\)\;', webpage, 'video url')
- title = self._og_search_title(webpage)
- description = self._html_search_meta('description', webpage)
- thumbnail = self._search_regex(
- r'so\.addVariable\("thumbnailURL", "(.+?)"\)\;',
- webpage, 'thumbnail url')
-
- return {
- 'id': video_id,
- 'url': final_url,
- 'title': title,
- 'description': description,
- 'thumbnail': thumbnail,
- }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index dfdf77e3a..e14ffaacf 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -240,7 +240,6 @@ from .condenast import CondeNastIE
from .corus import CorusIE
from .cracked import CrackedIE
from .crackle import CrackleIE
-from .criterion import CriterionIE
from .crooksandliars import CrooksAndLiarsIE
from .crunchyroll import (
CrunchyrollIE,
@@ -772,13 +771,6 @@ from .nova import (
NovaEmbedIE,
NovaIE,
)
-from .novamov import (
- AuroraVidIE,
- CloudTimeIE,
- NowVideoIE,
- VideoWeedIE,
- WholeCloudIE,
-)
from .nowness import (
NownessIE,
NownessPlaylistIE,
@@ -896,7 +888,6 @@ from .polskieradio import (
from .popcorntv import PopcornTVIE
from .porn91 import Porn91IE
from .porncom import PornComIE
-from .pornflip import PornFlipIE
from .pornhd import PornHdIE
from .pornhub import (
PornHubIE,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3a13c62eb..eeb0d25f6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2583,19 +2583,6 @@ class GenericIE(InfoExtractor):
if mobj is not None:
return self.url_result(mobj.group(1), 'Mpora')
- # Look for embedded NovaMov-based player
- mobj = re.search(
- r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
- (?Phttp://(?:(?:embed|www)\.)?
- (?:novamov\.com|
- nowvideo\.(?:ch|sx|eu|at|ag|co)|
- videoweed\.(?:es|com)|
- movshare\.(?:net|sx|ag)|
- divxstage\.(?:eu|net|ch|co|at|ag))
- /embed\.php.+?)\1''', webpage)
- if mobj is not None:
- return self.url_result(mobj.group('url'))
-
# Look for embedded Facebook player
facebook_urls = FacebookIE._extract_urls(webpage)
if facebook_urls:
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 5df14bb41..4ac437c8b 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -82,6 +82,10 @@ class LiveLeakIE(InfoExtractor):
}, {
'url': 'https://www.liveleak.com/view?t=HvHi_1523016227',
'only_matching': True,
+ }, {
+ # No original video
+ 'url': 'https://www.liveleak.com/view?t=C26ZZ_1558612804',
+ 'only_matching': True,
}]
@staticmethod
@@ -134,11 +138,13 @@ class LiveLeakIE(InfoExtractor):
orig_url = re.sub(r'\.mp4\.[^.]+', '', a_format['url'])
if a_format['url'] != orig_url:
format_id = a_format.get('format_id')
- formats.append({
- 'format_id': 'original' + ('-' + format_id if format_id else ''),
- 'url': orig_url,
- 'preference': 1,
- })
+ format_id = 'original' + ('-' + format_id if format_id else '')
+ if self._is_valid_url(orig_url, video_id, format_id):
+ formats.append({
+ 'format_id': format_id,
+ 'url': orig_url,
+ 'preference': 1,
+ })
self._sort_formats(formats)
info_dict['formats'] = formats
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
deleted file mode 100644
index 829c71960..000000000
--- a/youtube_dl/extractor/novamov.py
+++ /dev/null
@@ -1,212 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
- ExtractorError,
- NO_DEFAULT,
- sanitized_Request,
- urlencode_postdata,
-)
-
-
-class NovaMovIE(InfoExtractor):
- IE_NAME = 'novamov'
- IE_DESC = 'NovaMov'
-
- _VALID_URL_TEMPLATE = r'''(?x)
- http://
- (?:
- (?:www\.)?%(host)s/(?:file|video|mobile/\#/videos)/|
- (?:(?:embed|www)\.)%(host)s/embed(?:\.php|/)?\?(?:.*?&)?\bv=
- )
- (?P[a-z\d]{13})
- '''
- _VALID_URL = _VALID_URL_TEMPLATE % {'host': r'novamov\.com'}
-
- _HOST = 'www.novamov.com'
-
- _FILE_DELETED_REGEX = r'This file no longer exists on our servers!'
- _FILEKEY_REGEX = r'flashvars\.filekey=(?P"?[^"]+"?);'
- _TITLE_REGEX = r'(?s)\s*
([^<]+)
'
- _DESCRIPTION_REGEX = r'(?s)
\s*
[^<]+
([^<]+)
'
- _URL_TEMPLATE = 'http://%s/video/%s'
-
- _TEST = None
-
- def _check_existence(self, webpage, video_id):
- if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
- raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
-
- url = self._URL_TEMPLATE % (self._HOST, video_id)
-
- webpage = self._download_webpage(
- url, video_id, 'Downloading video page')
-
- self._check_existence(webpage, video_id)
-
- def extract_filekey(default=NO_DEFAULT):
- filekey = self._search_regex(
- self._FILEKEY_REGEX, webpage, 'filekey', default=default)
- if filekey is not default and (filekey[0] != '"' or filekey[-1] != '"'):
- return self._search_regex(
- r'var\s+%s\s*=\s*"([^"]+)"' % re.escape(filekey), webpage, 'filekey', default=default)
- else:
- return filekey
-
- filekey = extract_filekey(default=None)
-
- if not filekey:
- fields = self._hidden_inputs(webpage)
- post_url = self._search_regex(
- r'