From d4e0cd69efe6240a188eac9cce296b57b06661ca Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Fri, 10 Jan 2020 05:06:45 +0100 Subject: [PATCH 01/17] [lego] fix extraction and extract subtitle(closes #23687) --- youtube_dl/extractor/lego.py | 183 +++++++++++++++++++---------------- 1 file changed, 102 insertions(+), 81 deletions(-) diff --git a/youtube_dl/extractor/lego.py b/youtube_dl/extractor/lego.py index b312e77f1..1e3c19dfd 100644 --- a/youtube_dl/extractor/lego.py +++ b/youtube_dl/extractor/lego.py @@ -2,23 +2,24 @@ from __future__ import unicode_literals import re +import uuid from .common import InfoExtractor -from ..compat import compat_str +from ..compat import compat_HTTPError from ..utils import ( - unescapeHTML, - parse_duration, - get_element_by_class, + ExtractorError, + int_or_none, + qualities, ) class LEGOIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?lego\.com/(?P[^/]+)/(?:[^/]+/)*videos/(?:[^/]+/)*[^/?#]+-(?P[0-9a-f]+)' + _VALID_URL = r'https?://(?:www\.)?lego\.com/(?P[a-z]{2}-[a-z]{2})/(?:[^/]+/)*videos/(?:[^/]+/)*[^/?#]+-(?P[0-9a-f]{32})' _TESTS = [{ 'url': 'http://www.lego.com/en-us/videos/themes/club/blocumentary-kawaguchi-55492d823b1b4d5e985787fa8c2973b1', 'md5': 'f34468f176cfd76488767fc162c405fa', 'info_dict': { - 'id': '55492d823b1b4d5e985787fa8c2973b1', + 'id': '55492d82-3b1b-4d5e-9857-87fa8c2973b1_en-US', 'ext': 'mp4', 'title': 'Blocumentary Great Creations: Akiyuki Kawaguchi', 'description': 'Blocumentary Great Creations: Akiyuki Kawaguchi', @@ -26,103 +27,123 @@ class LEGOIE(InfoExtractor): }, { # geo-restricted but the contentUrl contain a valid url 'url': 'http://www.lego.com/nl-nl/videos/themes/nexoknights/episode-20-kingdom-of-heroes-13bdc2299ab24d9685701a915b3d71e7##sp=399', - 'md5': '4c3fec48a12e40c6e5995abc3d36cc2e', + 'md5': 'c7420221f7ffd03ff056f9db7f8d807c', 'info_dict': { - 'id': '13bdc2299ab24d9685701a915b3d71e7', + 'id': '13bdc229-9ab2-4d96-8570-1a915b3d71e7_nl-NL', 'ext': 'mp4', - 'title': 'Aflevering 20 - Helden van het koninkrijk', + 'title': 'Aflevering 20: Helden van het koninkrijk', 'description': 'md5:8ee499aac26d7fa8bcb0cedb7f9c3941', + 'age_limit': 5, }, }, { - # special characters in title - 'url': 'http://www.lego.com/en-us/starwars/videos/lego-star-wars-force-surprise-9685ee9d12e84ff38e84b4e3d0db533d', + # with subtitle + 'url': 'https://www.lego.com/nl-nl/kids/videos/classic/creative-storytelling-the-little-puppy-aa24f27c7d5242bc86102ebdc0f24cba', 'info_dict': { - 'id': '9685ee9d12e84ff38e84b4e3d0db533d', + 'id': 'aa24f27c-7d52-42bc-8610-2ebdc0f24cba_nl-NL', 'ext': 'mp4', - 'title': 'Force Surprise – LEGO® Star Wars™ Microfighters', - 'description': 'md5:9c673c96ce6f6271b88563fe9dc56de3', + 'title': 'De kleine puppy', + 'description': 'md5:5b725471f849348ac73f2e12cfb4be06', + 'age_limit': 1, + 'subtitles': { + 'nl': [{ + 'ext': 'srt', + 'url': r're:^https://.+\.srt$', + }], + }, }, 'params': { 'skip_download': True, }, }] - _BITRATES = [256, 512, 1024, 1536, 2560] + _QUALITIES = { + 'Lowest': (64, 180, 320), + 'Low': (64, 270, 480), + 'Medium': (96, 360, 640), + 'High': (128, 540, 960), + 'Highest': (128, 720, 1280), + } def _real_extract(self, url): locale, video_id = re.match(self._VALID_URL, url).groups() - webpage = self._download_webpage(url, video_id) - title = get_element_by_class('video-header', webpage).strip() - progressive_base = 'https://lc-mediaplayerns-live-s.legocdn.com/' - streaming_base = 'http://legoprod-f.akamaihd.net/' - content_url = self._html_search_meta('contentUrl', webpage) - path = self._search_regex( - r'(?:https?:)?//[^/]+/(?:[iz]/s/)?public/(.+)_[0-9,]+\.(?:mp4|webm)', - content_url, 'video path', default=None) - if not path: - player_url = self._proto_relative_url(self._search_regex( - r']+src="((?:https?)?//(?:www\.)?lego\.com/[^/]+/mediaplayer/video/[^"]+)', - webpage, 'player url', default=None)) - if not player_url: - base_url = self._proto_relative_url(self._search_regex( - r'data-baseurl="([^"]+)"', webpage, 'base url', - default='http://www.lego.com/%s/mediaplayer/video/' % locale)) - player_url = base_url + video_id - player_webpage = self._download_webpage(player_url, video_id) - video_data = self._parse_json(unescapeHTML(self._search_regex( - r"video='([^']+)'", player_webpage, 'video data')), video_id) - progressive_base = self._search_regex( - r'data-video-progressive-url="([^"]+)"', - player_webpage, 'progressive base', default='https://lc-mediaplayerns-live-s.legocdn.com/') - streaming_base = self._search_regex( - r'data-video-streaming-url="([^"]+)"', - player_webpage, 'streaming base', default='http://legoprod-f.akamaihd.net/') - item_id = video_data['ItemId'] + countries = [locale.split('-')[1].upper()] + self._initialize_geo_bypass({ + 'countries': countries, + }) - net_storage_path = video_data.get('NetStoragePath') or '/'.join([item_id[:2], item_id[2:4]]) - base_path = '_'.join([item_id, video_data['VideoId'], video_data['Locale'], compat_str(video_data['VideoVersion'])]) - path = '/'.join([net_storage_path, base_path]) - streaming_path = ','.join(map(lambda bitrate: compat_str(bitrate), self._BITRATES)) + try: + item = self._download_json( + # https://contentfeed.services.lego.com/api/v2/item/[VIDEO_ID]?culture=[LOCALE]&contentType=Video + 'https://services.slingshot.lego.com/mediaplayer/v2', + video_id, query={ + 'videoId': '%s_%s' % (uuid.UUID(video_id), locale), + }, headers=self.geo_verification_headers()) + except ExtractorError as e: + if isinstance(e.cause, compat_HTTPError) and e.cause.code == 451: + self.raise_geo_restricted(countries=countries) + raise - formats = self._extract_akamai_formats( - '%si/s/public/%s_,%s,.mp4.csmil/master.m3u8' % (streaming_base, path, streaming_path), video_id) - m3u8_formats = list(filter( - lambda f: f.get('protocol') == 'm3u8_native' and f.get('vcodec') != 'none', - formats)) - if len(m3u8_formats) == len(self._BITRATES): - self._sort_formats(m3u8_formats) - for bitrate, m3u8_format in zip(self._BITRATES, m3u8_formats): - progressive_base_url = '%spublic/%s_%d.' % (progressive_base, path, bitrate) - mp4_f = m3u8_format.copy() - mp4_f.update({ - 'url': progressive_base_url + 'mp4', - 'format_id': m3u8_format['format_id'].replace('hls', 'mp4'), - 'protocol': 'http', - }) - web_f = { - 'url': progressive_base_url + 'webm', - 'format_id': m3u8_format['format_id'].replace('hls', 'webm'), - 'width': m3u8_format['width'], - 'height': m3u8_format['height'], - 'tbr': m3u8_format.get('tbr'), - 'ext': 'webm', + video = item['Video'] + video_id = video['Id'] + title = video['Title'] + + q = qualities(['Lowest', 'Low', 'Medium', 'High', 'Highest']) + formats = [] + for video_source in item.get('VideoFormats', []): + video_source_url = video_source.get('Url') + if not video_source_url: + continue + video_source_format = video_source.get('Format') + if video_source_format == 'F4M': + formats.extend(self._extract_f4m_formats( + video_source_url, video_id, + f4m_id=video_source_format, fatal=False)) + elif video_source_format == 'M3U8': + formats.extend(self._extract_m3u8_formats( + video_source_url, video_id, 'mp4', 'm3u8_native', + m3u8_id=video_source_format, fatal=False)) + else: + video_source_quality = video_source.get('Quality') + format_id = [] + for v in (video_source_format, video_source_quality): + if v: + format_id.append(v) + f = { + 'format_id': '-'.join(format_id), + 'quality': q(video_source_quality), + 'url': video_source_url, } - formats.extend([web_f, mp4_f]) - else: - for bitrate in self._BITRATES: - for ext in ('web', 'mp4'): - formats.append({ - 'format_id': '%s-%s' % (ext, bitrate), - 'url': '%spublic/%s_%d.%s' % (progressive_base, path, bitrate, ext), - 'tbr': bitrate, - 'ext': ext, - }) + quality = self._QUALITIES.get(video_source_quality) + if quality: + f.update({ + 'abr': quality[0], + 'height': quality[1], + 'width': quality[2], + }), + formats.append(f) self._sort_formats(formats) + subtitles = {} + sub_file_id = video.get('SubFileId') + if sub_file_id and sub_file_id != '00000000-0000-0000-0000-000000000000': + net_storage_path = video.get('NetstoragePath') + invariant_id = video.get('InvariantId') + video_file_id = video.get('VideoFileId') + video_version = video.get('VideoVersion') + if net_storage_path and invariant_id and video_file_id and video_version: + subtitles.setdefault(locale[:2], []).append({ + 'url': 'https://lc-mediaplayerns-live-s.legocdn.com/public/%s/%s_%s_%s_%s_sub.srt' % (net_storage_path, invariant_id, video_file_id, locale, video_version), + }) + return { 'id': video_id, 'title': title, - 'description': self._html_search_meta('description', webpage), - 'thumbnail': self._html_search_meta('thumbnail', webpage), - 'duration': parse_duration(self._html_search_meta('duration', webpage)), + 'description': video.get('Description'), + 'thumbnail': video.get('GeneratedCoverImage') or video.get('GeneratedThumbnail'), + 'duration': int_or_none(video.get('Length')), 'formats': formats, + 'subtitles': subtitles, + 'age_limit': int_or_none(video.get('AgeFrom')), + 'season': video.get('SeasonTitle'), + 'season_number': int_or_none(video.get('Season')) or None, + 'episode_number': int_or_none(video.get('Episode')) or None, } From 3fdf573148abaa4e3b0dc9a8ae7ee7d1bdab2289 Mon Sep 17 00:00:00 2001 From: cdarlint Date: Sat, 11 Jan 2020 02:34:26 +0800 Subject: [PATCH 02/17] [safari] Fix kaltura session extraction (closes #23679) (#23670) --- youtube_dl/extractor/safari.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py index bd9ee1647..4942437c7 100644 --- a/youtube_dl/extractor/safari.py +++ b/youtube_dl/extractor/safari.py @@ -165,7 +165,8 @@ class SafariIE(SafariBaseIE): kaltura_session = self._download_json( '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id), video_id, 'Downloading kaltura session JSON', - 'Unable to download kaltura session JSON', fatal=False) + 'Unable to download kaltura session JSON', fatal=False, + headers={'Accept': 'application/json'}) if kaltura_session: session = kaltura_session.get('session') if session: From 9ba179c1fabe8f1700f5cbd778ef2030fa246d6e Mon Sep 17 00:00:00 2001 From: Johannes N <31795504+jonolt@users.noreply.github.com> Date: Fri, 10 Jan 2020 19:51:15 +0100 Subject: [PATCH 03/17] [orf:fm4] Fix extraction (#23599) --- youtube_dl/extractor/orf.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py index 3425f7602..816bd212e 100644 --- a/youtube_dl/extractor/orf.py +++ b/youtube_dl/extractor/orf.py @@ -161,9 +161,6 @@ class ORFRadioIE(InfoExtractor): show_date = mobj.group('date') show_id = mobj.group('show') - if station == 'fm4': - show_id = '4%s' % show_id - data = self._download_json( 'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s' % (station, show_id, show_date), show_id @@ -195,10 +192,10 @@ class ORFRadioIE(InfoExtractor): class ORFFM4IE(ORFRadioIE): IE_NAME = 'orf:fm4' IE_DESC = 'radio FM4' - _VALID_URL = r'https?://(?Pfm4)\.orf\.at/player/(?P[0-9]+)/(?P\w+)' + _VALID_URL = r'https?://(?Pfm4)\.orf\.at/player/(?P[0-9]+)/(?P4\w+)' _TEST = { - 'url': 'http://fm4.orf.at/player/20170107/CC', + 'url': 'http://fm4.orf.at/player/20170107/4CC', 'md5': '2b0be47375432a7ef104453432a19212', 'info_dict': { 'id': '2017-01-07_2100_tl_54_7DaysSat18_31295', From aca2fd222fc951dd8e177d4c5584f5276e719825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 11 Jan 2020 02:18:36 +0700 Subject: [PATCH 04/17] [orf:radio] Clean description and improve extraction --- youtube_dl/extractor/orf.py | 46 ++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py index 816bd212e..45fc745a3 100644 --- a/youtube_dl/extractor/orf.py +++ b/youtube_dl/extractor/orf.py @@ -6,12 +6,14 @@ import re from .common import InfoExtractor from ..compat import compat_str from ..utils import ( + clean_html, determine_ext, float_or_none, HEADRequest, int_or_none, orderedSet, remove_end, + str_or_none, strip_jsonp, unescapeHTML, unified_strdate, @@ -162,30 +164,37 @@ class ORFRadioIE(InfoExtractor): show_id = mobj.group('show') data = self._download_json( - 'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s' % (station, show_id, show_date), - show_id - ) + 'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s' + % (station, show_id, show_date), show_id) - def extract_entry_dict(info, title, subtitle): - return { - 'id': info['loopStreamId'].replace('.mp3', ''), - 'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (station, info['loopStreamId']), + entries = [] + for info in data['streams']: + loop_stream_id = str_or_none(info.get('loopStreamId')) + if not loop_stream_id: + continue + title = str_or_none(data.get('title')) + if not title: + continue + start = int_or_none(info.get('start'), scale=1000) + end = int_or_none(info.get('end'), scale=1000) + duration = end - start if end and start else None + entries.append({ + 'id': loop_stream_id.replace('.mp3', ''), + 'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (station, loop_stream_id), 'title': title, - 'description': subtitle, - 'duration': (info['end'] - info['start']) / 1000, - 'timestamp': info['start'] / 1000, + 'description': clean_html(data.get('subtitle')), + 'duration': duration, + 'timestamp': start, 'ext': 'mp3', - 'series': data.get('programTitle') - } - - entries = [extract_entry_dict(t, data['title'], data['subtitle']) for t in data['streams']] + 'series': data.get('programTitle'), + }) return { '_type': 'playlist', 'id': show_id, - 'title': data['title'], - 'description': data['subtitle'], - 'entries': entries + 'title': data.get('title'), + 'description': clean_html(data.get('subtitle')), + 'entries': entries, } @@ -206,7 +215,8 @@ class ORFFM4IE(ORFRadioIE): 'timestamp': 1483819257, 'upload_date': '20170107', }, - 'skip': 'Shows from ORF radios are only available for 7 days.' + 'skip': 'Shows from ORF radios are only available for 7 days.', + 'only_matching': True, } From 10a5091e58077cc0fd916a21df4c702f8d519702 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 12 Jan 2020 11:55:11 +0100 Subject: [PATCH 05/17] [twitter] add support for promo_video_website cards(closes #23711) --- youtube_dl/extractor/twitter.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py index 5f8d90fb4..01468981c 100644 --- a/youtube_dl/extractor/twitter.py +++ b/youtube_dl/extractor/twitter.py @@ -251,10 +251,10 @@ class TwitterIE(TwitterBaseIE): 'info_dict': { 'id': '700207533655363584', 'ext': 'mp4', - 'title': 'Simon Vertugo - BEAT PROD: @suhmeduh #Damndaniel', + 'title': 'simon vetugo - BEAT PROD: @suhmeduh #Damndaniel', 'description': 'BEAT PROD: @suhmeduh https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ', 'thumbnail': r're:^https?://.*\.jpg', - 'uploader': 'Simon Vertugo', + 'uploader': 'simon vetugo', 'uploader_id': 'simonvertugo', 'duration': 30.0, 'timestamp': 1455777459, @@ -376,6 +376,10 @@ class TwitterIE(TwitterBaseIE): # Twitch Clip Embed 'url': 'https://twitter.com/GunB1g/status/1163218564784017422', 'only_matching': True, + }, { + # promo_video_website card + 'url': 'https://twitter.com/GunB1g/status/1163218564784017422', + 'only_matching': True, }] def _real_extract(self, url): @@ -458,10 +462,11 @@ class TwitterIE(TwitterBaseIE): return try_get(o, lambda x: x[x['type'].lower() + '_value']) card_name = card['name'].split(':')[-1] - if card_name == 'amplify': - formats = self._extract_formats_from_vmap_url( - get_binding_value('amplify_url_vmap'), - get_binding_value('amplify_content_id') or twid) + if card_name in ('amplify', 'promo_video_website'): + is_amplify = card_name == 'amplify' + vmap_url = get_binding_value('amplify_url_vmap') if is_amplify else get_binding_value('player_stream_url') + content_id = get_binding_value('%s_content_id' % (card_name if is_amplify else 'player')) + formats = self._extract_formats_from_vmap_url(vmap_url, content_id or twid) self._sort_formats(formats) thumbnails = [] From bd2c211fcc3b0390a1b404fa01409d455859caa8 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 12 Jan 2020 17:34:21 +0100 Subject: [PATCH 06/17] [vodplatform] add support for embed.kwikmotion.com domain --- youtube_dl/extractor/generic.py | 2 +- youtube_dl/extractor/vodplatform.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 743ef47db..a4aef106f 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -2960,7 +2960,7 @@ class GenericIE(InfoExtractor): # Look for VODPlatform embeds mobj = re.search( - r']+src=(["\'])(?P(?:https?:)?//(?:www\.)?vod-platform\.net/[eE]mbed/.+?)\1', + r']+src=(["\'])(?P(?:https?:)?//(?:(?:www\.)?vod-platform\.net|embed\.kwikmotion\.com)/[eE]mbed/.+?)\1', webpage) if mobj is not None: return self.url_result( diff --git a/youtube_dl/extractor/vodplatform.py b/youtube_dl/extractor/vodplatform.py index 239644340..74d2257e7 100644 --- a/youtube_dl/extractor/vodplatform.py +++ b/youtube_dl/extractor/vodplatform.py @@ -6,8 +6,8 @@ from ..utils import unescapeHTML class VODPlatformIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?vod-platform\.net/[eE]mbed/(?P[^/?#]+)' - _TEST = { + _VALID_URL = r'https?://(?:(?:www\.)?vod-platform\.net|embed\.kwikmotion\.com)/[eE]mbed/(?P[^/?#]+)' + _TESTS = [{ # from http://www.lbcgroup.tv/watch/chapter/29143/52844/%D8%A7%D9%84%D9%86%D8%B5%D8%B1%D8%A9-%D9%81%D9%8A-%D8%B6%D9%8A%D8%A7%D9%81%D8%A9-%D8%A7%D9%84%D9%80-cnn/ar 'url': 'http://vod-platform.net/embed/RufMcytHDolTH1MuKHY9Fw', 'md5': '1db2b7249ce383d6be96499006e951fc', @@ -16,7 +16,10 @@ class VODPlatformIE(InfoExtractor): 'ext': 'mp4', 'title': 'LBCi News_ النصرة في ضيافة الـ "سي.أن.أن"', } - } + }, { + 'url': 'http://embed.kwikmotion.com/embed/RufMcytHDolTH1MuKHY9Fw', + 'only_matching': True, + }] def _real_extract(self, url): video_id = self._match_id(url) From 3fc56635b7d375b262ac3c15aaae549227b5227e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 14 Jan 2020 21:46:56 +0700 Subject: [PATCH 07/17] [ndr:base:embed] Improve thumbnails extraction (closes #23731) --- youtube_dl/extractor/ndr.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py index aec2ea133..9c8bf05af 100644 --- a/youtube_dl/extractor/ndr.py +++ b/youtube_dl/extractor/ndr.py @@ -9,6 +9,8 @@ from ..utils import ( int_or_none, parse_iso8601, qualities, + try_get, + urljoin, ) @@ -220,11 +222,17 @@ class NDREmbedBaseIE(InfoExtractor): upload_date = ppjson.get('config', {}).get('publicationDate') duration = int_or_none(config.get('duration')) - thumbnails = [{ - 'id': thumbnail.get('quality') or thumbnail_id, - 'url': thumbnail['src'], - 'preference': quality_key(thumbnail.get('quality')), - } for thumbnail_id, thumbnail in config.get('poster', {}).items() if thumbnail.get('src')] + thumbnails = [] + poster = try_get(config, lambda x: x['poster'], dict) or {} + for thumbnail_id, thumbnail in poster.items(): + thumbnail_url = urljoin(url, thumbnail.get('src')) + if not thumbnail_url: + continue + thumbnails.append({ + 'id': thumbnail.get('quality') or thumbnail_id, + 'url': thumbnail_url, + 'preference': quality_key(thumbnail.get('quality')), + }) return { 'id': video_id, From 628e5bc0b715c239e5fe367bc538a1c1fa563787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 14 Jan 2020 23:48:36 +0700 Subject: [PATCH 08/17] [canvas] Add support for new API endpoint and update tests (closes #17680, closes #18629) --- youtube_dl/extractor/canvas.py | 83 +++++++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py index c506bc5dd..8667a0d04 100644 --- a/youtube_dl/extractor/canvas.py +++ b/youtube_dl/extractor/canvas.py @@ -13,6 +13,8 @@ from ..utils import ( int_or_none, merge_dicts, parse_iso8601, + str_or_none, + url_or_none, ) @@ -20,15 +22,15 @@ class CanvasIE(InfoExtractor): _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?Pcanvas|een|ketnet|vrt(?:video|nieuws)|sporza)/assets/(?P[^/?#&]+)' _TESTS = [{ 'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', - 'md5': '90139b746a0a9bd7bb631283f6e2a64e', + 'md5': '68993eda72ef62386a15ea2cf3c93107', 'info_dict': { 'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', 'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'Nachtwacht: De Greystook', - 'description': 'md5:1db3f5dc4c7109c821261e7512975be7', + 'description': 'Nachtwacht: De Greystook', 'thumbnail': r're:^https?://.*\.jpg$', - 'duration': 1468.03, + 'duration': 1468.04, }, 'expected_warnings': ['is not a supported codec', 'Unknown MIME type'], }, { @@ -39,23 +41,45 @@ class CanvasIE(InfoExtractor): 'HLS': 'm3u8_native', 'HLS_AES': 'm3u8', } + _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v1' def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) site_id, video_id = mobj.group('site_id'), mobj.group('id') + # Old API endpoint, serves more formats but may fail for some videos data = self._download_json( 'https://mediazone.vrt.be/api/v1/%s/assets/%s' - % (site_id, video_id), video_id) + % (site_id, video_id), video_id, 'Downloading asset JSON', + 'Unable to download asset JSON', fatal=False) + + # New API endpoint + if not data: + token = self._download_json( + '%s/tokens' % self._REST_API_BASE, video_id, + 'Downloading token', data=b'', + headers={'Content-Type': 'application/json'})['vrtPlayerToken'] + data = self._download_json( + '%s/videos/%s' % (self._REST_API_BASE, video_id), + video_id, 'Downloading video JSON', fatal=False, query={ + 'vrtPlayerToken': token, + 'client': '%s@PROD' % site_id, + }, expected_status=400) + message = data.get('message') + if message and not data.get('title'): + if data.get('code') == 'AUTHENTICATION_REQUIRED': + self.raise_login_required(message) + raise ExtractorError(message, expected=True) title = data['title'] description = data.get('description') formats = [] for target in data['targetUrls']: - format_url, format_type = target.get('url'), target.get('type') + format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type')) if not format_url or not format_type: continue + format_type = format_type.upper() if format_type in self._HLS_ENTRY_PROTOCOLS_MAP: formats.extend(self._extract_m3u8_formats( format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type], @@ -134,20 +158,20 @@ class CanvasEenIE(InfoExtractor): }, 'skip': 'Pagina niet gevonden', }, { - 'url': 'https://www.een.be/sorry-voor-alles/herbekijk-sorry-voor-alles', + 'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan', 'info_dict': { - 'id': 'mz-ast-11a587f8-b921-4266-82e2-0bce3e80d07f', - 'display_id': 'herbekijk-sorry-voor-alles', + 'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8', + 'display_id': 'emma-pakt-thilly-aan', 'ext': 'mp4', - 'title': 'Herbekijk Sorry voor alles', - 'description': 'md5:8bb2805df8164e5eb95d6a7a29dc0dd3', + 'title': 'Emma pakt Thilly aan', + 'description': 'md5:c5c9b572388a99b2690030afa3f3bad7', 'thumbnail': r're:^https?://.*\.jpg$', - 'duration': 3788.06, + 'duration': 118.24, }, 'params': { 'skip_download': True, }, - 'skip': 'Episode no longer available', + 'expected_warnings': ['is not a supported codec'], }, { 'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend', 'only_matching': True, @@ -183,19 +207,44 @@ class VrtNUIE(GigyaBaseIE): IE_DESC = 'VrtNU.be' _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?Pvrtnu)/(?:[^/]+/)*(?P[^/?#&]+)' _TESTS = [{ + # Available via old API endpoint 'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/', 'info_dict': { 'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'De zwarte weduwe', - 'description': 'md5:d90c21dced7db869a85db89a623998d4', + 'description': 'md5:db1227b0f318c849ba5eab1fef895ee4', 'duration': 1457.04, 'thumbnail': r're:^https?://.*\.jpg$', - 'season': '1', + 'season': 'Season 1', 'season_number': 1, 'episode_number': 1, }, - 'skip': 'This video is only available for registered users' + 'skip': 'This video is only available for registered users', + 'params': { + 'username': '', + 'password': '', + }, + 'expected_warnings': ['is not a supported codec'], + }, { + # Only available via new API endpoint + 'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/', + 'info_dict': { + 'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1', + 'ext': 'mp4', + 'title': 'Aflevering 5', + 'description': 'Wie valt door de mand tijdens een missie?', + 'duration': 2967.06, + 'season': 'Season 1', + 'season_number': 1, + 'episode_number': 5, + }, + 'skip': 'This video is only available for registered users', + 'params': { + 'username': '', + 'password': '', + }, + 'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'], }] _NETRC_MACHINE = 'vrtnu' _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy' From 14bb191634e2e8ab89a2e94e9e4d009b6406c8b2 Mon Sep 17 00:00:00 2001 From: jnozsc Date: Tue, 14 Jan 2020 10:09:08 -0800 Subject: [PATCH 09/17] [travis] Add flake8 job (#23720) --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 14d95fa84..51afd469a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ dist: trusty env: - YTDL_TEST_SET=core - YTDL_TEST_SET=download -matrix: +jobs: include: - python: 3.7 dist: xenial @@ -35,6 +35,11 @@ matrix: env: YTDL_TEST_SET=download - env: JYTHON=true; YTDL_TEST_SET=core - env: JYTHON=true; YTDL_TEST_SET=download + - name: flake8 + python: 3.8 + dist: xenial + install: pip install flake8 + script: flake8 . fast_finish: true allow_failures: - env: YTDL_TEST_SET=download From bfdc8340c90e0ce495d2927e7d555daa5ac05670 Mon Sep 17 00:00:00 2001 From: Moritz Patelscheck Date: Fri, 20 Dec 2019 00:02:39 +0100 Subject: [PATCH 10/17] [yourporn] Fix extraction (closes #21645, closes #22255, closes #23459) --- youtube_dl/extractor/yourporn.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/yourporn.py b/youtube_dl/extractor/yourporn.py index 8a2d5f63b..98347491e 100644 --- a/youtube_dl/extractor/yourporn.py +++ b/youtube_dl/extractor/yourporn.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from .common import InfoExtractor +from ..compat import compat_str from ..utils import ( parse_duration, urljoin, @@ -8,9 +9,9 @@ from ..utils import ( class YourPornIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?(?:yourporn\.sexy|sxyprn\.com)/post/(?P[^/?#&.]+)' + _VALID_URL = r'https?://(?:www\.)?sxyprn\.com/post/(?P[^/?#&.]+)' _TESTS = [{ - 'url': 'https://yourporn.sexy/post/57ffcb2e1179b.html', + 'url': 'https://sxyprn.com/post/57ffcb2e1179b.html', 'md5': '6f8682b6464033d87acaa7a8ff0c092e', 'info_dict': { 'id': '57ffcb2e1179b', @@ -33,11 +34,19 @@ class YourPornIE(InfoExtractor): webpage = self._download_webpage(url, video_id) - video_url = urljoin(url, self._parse_json( + parts = self._parse_json( self._search_regex( r'data-vnfo=(["\'])(?P{.+?})\1', webpage, 'data info', group='data'), - video_id)[video_id]).replace('/cdn/', '/cdn5/') + video_id)[video_id].split('/') + + num = 0 + for c in parts[6] + parts[7]: + if c.isnumeric(): + num += int(c) + parts[5] = compat_str(int(parts[5]) - num) + parts[1] += '8' + video_url = urljoin(url, '/'.join(parts)) title = (self._search_regex( r'<[^>]+\bclass=["\']PostEditTA[^>]+>([^<]+)', webpage, 'title', @@ -54,4 +63,5 @@ class YourPornIE(InfoExtractor): 'thumbnail': thumbnail, 'duration': duration, 'age_limit': 18, + 'ext': 'mp4', } From d7c55f226dd8fafb424eefb078f41b3fc410588b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 15 Jan 2020 01:34:01 +0700 Subject: [PATCH 11/17] [ChangeLog] Actualize [ci skip] --- ChangeLog | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/ChangeLog b/ChangeLog index c33169cd8..cdf4dbc96 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,35 @@ +version + +Extractors +* [yourporn] Fix extraction (#21645, #22255, #23459) ++ [canvas] Add support for new API endpoint (#17680, #18629) +* [ndr:base:embed] Improve thumbnails extraction (#23731) ++ [vodplatform] Add support for embed.kwikmotion.com domain ++ [twitter] Add support for promo_video_website cards (#23711) +* [orf:radio] Clean description and improve extraction +* [orf:fm4] Fix extraction (#23599) +* [safari] Fix kaltura session extraction (#23679, #23670) +* [lego] Fix extraction and extract subtitle (#23687) +* [cloudflarestream] Improve extraction + + Add support for bytehighway.net domain + + Add support for signed URLs + + Extract thumbnail +* [naver] Improve extraction + * Improve geo-restriction handling + + Extract automatic captions + + Extract uploader metadata + + Extract VLive HLS formats + * Improve metadata extraction +- [pandatv] Remove extractor (#23630) +* [dctp] Fix format extraction (#23656) ++ [scrippsnetworks] Add support for www.discovery.com videos +* [discovery] Fix anonymous token extraction (#23650) +* [nrktv:seriebase] Fix extraction (#23625, #23537) +* [wistia] Improve format extraction and extract subtitles (#22590) +* [vice] Improve extraction (#23631) +* [redtube] Detect private videos (#23518) + + version 2020.01.01 Extractors From e8cf0dbdd8aa4f2dcd521d0bf7e7798e87867b52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 15 Jan 2020 01:37:29 +0700 Subject: [PATCH 12/17] release 2020.01.15 --- .github/ISSUE_TEMPLATE/1_broken_site.md | 6 +++--- .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++-- .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++-- .github/ISSUE_TEMPLATE/4_bug_report.md | 6 +++--- .github/ISSUE_TEMPLATE/5_feature_request.md | 4 ++-- ChangeLog | 2 +- docs/supportedsites.md | 1 - youtube_dl/version.py | 2 +- 8 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md index 97b8afcf9..cf8e6e411 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dl version **2020.01.01** +- [ ] I've verified that I'm running youtube-dl version **2020.01.15** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2020.01.01 + [debug] youtube-dl version 2020.01.15 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md index de6c44a65..babbda464 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -19,7 +19,7 @@ labels: 'site-support-request' - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dl version **2020.01.01** +- [ ] I've verified that I'm running youtube-dl version **2020.01.15** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md index a9dd5ca52..5498983ff 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -18,13 +18,13 @@ title: '' - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dl version **2020.01.01** +- [ ] I've verified that I'm running youtube-dl version **2020.01.15** - [ ] I've searched the bugtracker for similar site feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md index 8347903ea..d46735951 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dl version **2020.01.01** +- [ ] I've verified that I'm running youtube-dl version **2020.01.15** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2020.01.01 + [debug] youtube-dl version 2020.01.15 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md index 92228513c..748b64756 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -19,13 +19,13 @@ labels: 'request' - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dl version **2020.01.01** +- [ ] I've verified that I'm running youtube-dl version **2020.01.15** - [ ] I've searched the bugtracker for similar feature requests including closed ones diff --git a/ChangeLog b/ChangeLog index cdf4dbc96..cc7fc4323 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -version +version 2020.01.15 Extractors * [yourporn] Fix extraction (#21645, #22255, #23459) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index e471aa79a..e9a8cc27a 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -628,7 +628,6 @@ - **OutsideTV** - **PacktPub** - **PacktPubCourse** - - **PandaTV**: 熊猫TV - **pandora.tv**: 판도라TV - **ParamountNetwork** - **parliamentlive.tv**: UK parliament videos diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 8ad2df674..932b138a9 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2020.01.01' +__version__ = '2020.01.15' From e4e5fa6e3c1c2ca2d48dfb5a8b1f734bd627b2dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 15 Jan 2020 04:13:10 +0700 Subject: [PATCH 13/17] [soundcloud] Restore previews extraction (closes #23739) --- youtube_dl/extractor/soundcloud.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 62e9d8643..a0b09f5b1 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -96,7 +96,7 @@ class SoundcloudIE(InfoExtractor): 'repost_count': int, } }, - # not streamable song + # not streamable song, preview { 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep', 'info_dict': { @@ -119,7 +119,6 @@ class SoundcloudIE(InfoExtractor): # rtmp 'skip_download': True, }, - 'skip': 'Preview', }, # private link { @@ -346,9 +345,9 @@ class SoundcloudIE(InfoExtractor): }) def invalid_url(url): - return not url or url in format_urls or re.search(r'/(?:preview|playlist)/0/30/', url) + return not url or url in format_urls - def add_format(f, protocol): + def add_format(f, protocol, is_preview=False): mobj = re.search(r'\.(?P\d+)\.(?P[0-9a-z]{3,4})(?=[/?])', stream_url) if mobj: for k, v in mobj.groupdict().items(): @@ -361,12 +360,16 @@ class SoundcloudIE(InfoExtractor): v = f.get(k) if v: format_id_list.append(v) + preview = is_preview or re.search(r'/(?:preview|playlist)/0/30/', f['url']) + if preview: + format_id_list.append('preview') abr = f.get('abr') if abr: f['abr'] = int(abr) f.update({ 'format_id': '_'.join(format_id_list), 'protocol': 'm3u8_native' if protocol == 'hls' else 'http', + 'preference': -10 if preview else None, }) formats.append(f) @@ -377,7 +380,7 @@ class SoundcloudIE(InfoExtractor): if not isinstance(t, dict): continue format_url = url_or_none(t.get('url')) - if not format_url or t.get('snipped') or '/preview/' in format_url: + if not format_url: continue stream = self._download_json( format_url, track_id, query=query, fatal=False) @@ -400,7 +403,8 @@ class SoundcloudIE(InfoExtractor): add_format({ 'url': stream_url, 'ext': ext, - }, 'http' if protocol == 'progressive' else protocol) + }, 'http' if protocol == 'progressive' else protocol, + t.get('snipped') or '/preview/' in format_url) if not formats: # Old API, does not work for some tracks (e.g. From 90ea83c64d904587992105fb4506e80f6abb28b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 15 Jan 2020 04:32:05 +0700 Subject: [PATCH 14/17] [orf:tvthek] Improve geo restricted videos detection (closes #23741) --- youtube_dl/extractor/orf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py index 45fc745a3..d54b8ace6 100644 --- a/youtube_dl/extractor/orf.py +++ b/youtube_dl/extractor/orf.py @@ -90,8 +90,11 @@ class ORFTVthekIE(InfoExtractor): format_id = '-'.join(format_id_list) ext = determine_ext(src) if ext == 'm3u8': - formats.extend(self._extract_m3u8_formats( - src, video_id, 'mp4', m3u8_id=format_id, fatal=False)) + m3u8_formats = self._extract_m3u8_formats( + src, video_id, 'mp4', m3u8_id=format_id, fatal=False) + if any('/geoprotection' in f['url'] for f in m3u8_formats): + self.raise_geo_restricted() + formats.extend(m3u8_formats) elif ext == 'f4m': formats.extend(self._extract_f4m_formats( src, video_id, f4m_id=format_id, fatal=False)) From a9866c0366bd6399b0f757527425466a3be4d128 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Wed, 15 Jan 2020 14:02:57 +0100 Subject: [PATCH 15/17] [zype] improve extraction - extract subtitles(closes #21258) - support URLs with alternative keys/tokens(#21258) - extract more metadata --- youtube_dl/extractor/generic.py | 3 + youtube_dl/extractor/trunews.py | 49 ++------------ youtube_dl/extractor/zype.py | 111 +++++++++++++++++++++++++++----- 3 files changed, 101 insertions(+), 62 deletions(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index a4aef106f..3c002472f 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -2098,6 +2098,9 @@ class GenericIE(InfoExtractor): 'ext': 'mp4', 'title': 'Smoky Barbecue Favorites', 'thumbnail': r're:^https?://.*\.jpe?g', + 'description': 'md5:5ff01e76316bd8d46508af26dc86023b', + 'upload_date': '20170909', + 'timestamp': 1504915200, }, 'add_ie': [ZypeIE.ie_key()], 'params': { diff --git a/youtube_dl/extractor/trunews.py b/youtube_dl/extractor/trunews.py index b0c7caabf..cca5b5ceb 100644 --- a/youtube_dl/extractor/trunews.py +++ b/youtube_dl/extractor/trunews.py @@ -1,21 +1,12 @@ from __future__ import unicode_literals from .common import InfoExtractor -from ..utils import ( - dict_get, - float_or_none, - int_or_none, - unified_timestamp, - update_url_query, - url_or_none, -) class TruNewsIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?trunews\.com/stream/(?P[^/?#&]+)' _TEST = { 'url': 'https://www.trunews.com/stream/will-democrats-stage-a-circus-during-president-trump-s-state-of-the-union-speech', - 'md5': 'a19c024c3906ff954fac9b96ce66bb08', 'info_dict': { 'id': '5c5a21e65d3c196e1c0020cc', 'display_id': 'will-democrats-stage-a-circus-during-president-trump-s-state-of-the-union-speech', @@ -28,48 +19,16 @@ class TruNewsIE(InfoExtractor): }, 'add_ie': ['Zype'], } + _ZYPE_TEMPL = 'https://player.zype.com/embed/%s.js?api_key=X5XnahkjCwJrT_l5zUqypnaLEObotyvtUKJWWlONxDoHVjP8vqxlArLV8llxMbyt' def _real_extract(self, url): display_id = self._match_id(url) - video = self._download_json( + zype_id = self._download_json( 'https://api.zype.com/videos', display_id, query={ 'app_key': 'PUVKp9WgGUb3-JUw6EqafLx8tFVP6VKZTWbUOR-HOm__g4fNDt1bCsm_LgYf_k9H', 'per_page': 1, 'active': 'true', 'friendly_title': display_id, - })['response'][0] - - zype_id = video['_id'] - - thumbnails = [] - thumbnails_list = video.get('thumbnails') - if isinstance(thumbnails_list, list): - for thumbnail in thumbnails_list: - if not isinstance(thumbnail, dict): - continue - thumbnail_url = url_or_none(thumbnail.get('url')) - if not thumbnail_url: - continue - thumbnails.append({ - 'url': thumbnail_url, - 'width': int_or_none(thumbnail.get('width')), - 'height': int_or_none(thumbnail.get('height')), - }) - - return { - '_type': 'url_transparent', - 'url': update_url_query( - 'https://player.zype.com/embed/%s.js' % zype_id, - {'api_key': 'X5XnahkjCwJrT_l5zUqypnaLEObotyvtUKJWWlONxDoHVjP8vqxlArLV8llxMbyt'}), - 'ie_key': 'Zype', - 'id': zype_id, - 'display_id': display_id, - 'title': video.get('title'), - 'description': dict_get(video, ('description', 'ott_description', 'short_description')), - 'duration': int_or_none(video.get('duration')), - 'timestamp': unified_timestamp(video.get('published_at')), - 'average_rating': float_or_none(video.get('rating')), - 'view_count': int_or_none(video.get('request_count')), - 'thumbnails': thumbnails, - } + })['response'][0]['_id'] + return self.url_result(self._ZYPE_TEMPL % zype_id, 'Zype', zype_id) diff --git a/youtube_dl/extractor/zype.py b/youtube_dl/extractor/zype.py index 3b16e703b..2e2e97a0c 100644 --- a/youtube_dl/extractor/zype.py +++ b/youtube_dl/extractor/zype.py @@ -4,10 +4,20 @@ from __future__ import unicode_literals import re from .common import InfoExtractor +from ..compat import compat_HTTPError +from ..utils import ( + dict_get, + ExtractorError, + int_or_none, + js_to_json, + parse_iso8601, +) class ZypeIE(InfoExtractor): - _VALID_URL = r'https?://player\.zype\.com/embed/(?P[\da-fA-F]+)\.js\?.*?api_key=[^&]+' + _ID_RE = r'[\da-fA-F]+' + _COMMON_RE = r'//player\.zype\.com/embed/%s\.(?:js|json|html)\?.*?(?:access_token|(?:ap[ip]|player)_key)=' + _VALID_URL = r'https?:%s[^&]+' % (_COMMON_RE % ('(?P%s)' % _ID_RE)) _TEST = { 'url': 'https://player.zype.com/embed/5b400b834b32992a310622b9.js?api_key=jZ9GUhRmxcPvX7M3SlfejB6Hle9jyHTdk2jVxG7wOHPLODgncEKVdPYBhuz9iWXQ&autoplay=false&controls=true&da=false', 'md5': 'eaee31d474c76a955bdaba02a505c595', @@ -16,6 +26,9 @@ class ZypeIE(InfoExtractor): 'ext': 'mp4', 'title': 'Smoky Barbecue Favorites', 'thumbnail': r're:^https?://.*\.jpe?g', + 'description': 'md5:5ff01e76316bd8d46508af26dc86023b', + 'timestamp': 1504915200, + 'upload_date': '20170909', }, } @@ -24,34 +37,98 @@ class ZypeIE(InfoExtractor): return [ mobj.group('url') for mobj in re.finditer( - r']+\bsrc=(["\'])(?P(?:https?:)?//player\.zype\.com/embed/[\da-fA-F]+\.js\?.*?api_key=.+?)\1', + r']+\bsrc=(["\'])(?P(?:https?:)?%s.+?)\1' % (ZypeIE._COMMON_RE % ZypeIE._ID_RE), webpage)] def _real_extract(self, url): video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) + try: + response = self._download_json(re.sub( + r'\.(?:js|html)\?', '.json?', url), video_id)['response'] + except ExtractorError as e: + if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 403): + raise ExtractorError(self._parse_json( + e.cause.read().decode(), video_id)['message'], expected=True) + raise - title = self._search_regex( - r'video_title\s*[:=]\s*(["\'])(?P(?:(?!\1).)+)\1', webpage, - 'title', group='value') + body = response['body'] + video = response['video'] + title = video['title'] - m3u8_url = self._search_regex( - r'(["\'])(?P(?:(?!\1).)+\.m3u8(?:(?!\1).)*)\1', webpage, - 'm3u8 url', group='url') - - formats = self._extract_m3u8_formats( - m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native', - m3u8_id='hls') + if isinstance(body, dict): + formats = [] + for output in body.get('outputs', []): + output_url = output.get('url') + if not output_url: + continue + name = output.get('name') + if name == 'm3u8': + formats = self._extract_m3u8_formats( + output_url, video_id, 'mp4', + 'm3u8_native', m3u8_id='hls', fatal=False) + else: + f = { + 'format_id': name, + 'tbr': int_or_none(output.get('bitrate')), + 'url': output_url, + } + if name in ('m4a', 'mp3'): + f['vcodec'] = 'none' + else: + f.update({ + 'height': int_or_none(output.get('height')), + 'width': int_or_none(output.get('width')), + }) + formats.append(f) + text_tracks = body.get('subtitles') or [] + else: + m3u8_url = self._search_regex( + r'(["\'])(?P(?:(?!\1).)+\.m3u8(?:(?!\1).)*)\1', + body, 'm3u8 url', group='url') + formats = self._extract_m3u8_formats( + m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls') + text_tracks = self._search_regex( + r'textTracks\s*:\s*(\[[^]]+\])', + body, 'text tracks', default=None) + if text_tracks: + text_tracks = self._parse_json( + text_tracks, video_id, js_to_json, False) self._sort_formats(formats) - thumbnail = self._search_regex( - r'poster\s*[:=]\s*(["\'])(?P(?:(?!\1).)+)\1', webpage, 'thumbnail', - default=False, group='url') + subtitles = {} + if text_tracks: + for text_track in text_tracks: + tt_url = dict_get(text_track, ('file', 'src')) + if not tt_url: + continue + subtitles.setdefault(text_track.get('label') or 'English', []).append({ + 'url': tt_url, + }) + + thumbnails = [] + for thumbnail in video.get('thumbnails', []): + thumbnail_url = thumbnail.get('url') + if not thumbnail_url: + continue + thumbnails.append({ + 'url': thumbnail_url, + 'width': int_or_none(thumbnail.get('width')), + 'height': int_or_none(thumbnail.get('height')), + }) return { 'id': video_id, + 'display_id': video.get('friendly_title'), 'title': title, - 'thumbnail': thumbnail, + 'thumbnails': thumbnails, + 'description': dict_get(video, ('description', 'ott_description', 'short_description')), + 'timestamp': parse_iso8601(video.get('published_at')), + 'duration': int_or_none(video.get('duration')), + 'view_count': int_or_none(video.get('request_count')), + 'average_rating': int_or_none(video.get('rating')), + 'season_number': int_or_none(video.get('season')), + 'episode_number': int_or_none(video.get('episode')), 'formats': formats, + 'subtitles': subtitles, } From 2c482bff7c91c364c55b74846a3ae416cf588df3 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Wed, 15 Jan 2020 14:16:58 +0100 Subject: [PATCH 16/17] [americastestkitchen] fix extraction --- youtube_dl/extractor/americastestkitchen.py | 42 ++++++++------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dl/extractor/americastestkitchen.py index 8b32aa886..9c9d77ae1 100644 --- a/youtube_dl/extractor/americastestkitchen.py +++ b/youtube_dl/extractor/americastestkitchen.py @@ -5,6 +5,7 @@ from .common import InfoExtractor from ..utils import ( clean_html, int_or_none, + js_to_json, try_get, unified_strdate, ) @@ -13,22 +14,21 @@ from ..utils import ( class AmericasTestKitchenIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?americastestkitchen\.com/(?:episode|videos)/(?P\d+)' _TESTS = [{ - 'url': 'https://www.americastestkitchen.com/episode/548-summer-dinner-party', + 'url': 'https://www.americastestkitchen.com/episode/582-weeknight-japanese-suppers', 'md5': 'b861c3e365ac38ad319cfd509c30577f', 'info_dict': { - 'id': '1_5g5zua6e', - 'title': 'Summer Dinner Party', + 'id': '5b400b9ee338f922cb06450c', + 'title': 'Weeknight Japanese Suppers', 'ext': 'mp4', - 'description': 'md5:858d986e73a4826979b6a5d9f8f6a1ec', - 'thumbnail': r're:^https?://.*\.jpg', - 'timestamp': 1497285541, - 'upload_date': '20170612', - 'uploader_id': 'roger.metcalf@americastestkitchen.com', - 'release_date': '20170617', + 'description': 'md5:3d0c1a44bb3b27607ce82652db25b4a8', + 'thumbnail': r're:^https?://', + 'timestamp': 1523664000, + 'upload_date': '20180414', + 'release_date': '20180414', 'series': "America's Test Kitchen", - 'season_number': 17, - 'episode': 'Summer Dinner Party', - 'episode_number': 24, + 'season_number': 18, + 'episode': 'Weeknight Japanese Suppers', + 'episode_number': 15, }, 'params': { 'skip_download': True, @@ -47,7 +47,7 @@ class AmericasTestKitchenIE(InfoExtractor): self._search_regex( r'window\.__INITIAL_STATE__\s*=\s*({.+?})\s*;\s*', webpage, 'initial context'), - video_id) + video_id, js_to_json) ep_data = try_get( video_data, @@ -55,17 +55,7 @@ class AmericasTestKitchenIE(InfoExtractor): lambda x: x['videoDetail']['content']['data']), dict) ep_meta = ep_data.get('full_video', {}) - zype_id = ep_meta.get('zype_id') - if zype_id: - embed_url = 'https://player.zype.com/embed/%s.js?api_key=jZ9GUhRmxcPvX7M3SlfejB6Hle9jyHTdk2jVxG7wOHPLODgncEKVdPYBhuz9iWXQ' % zype_id - ie_key = 'Zype' - else: - partner_id = self._search_regex( - r'src=["\'](?:https?:)?//(?:[^/]+\.)kaltura\.com/(?:[^/]+/)*(?:p|partner_id)/(\d+)', - webpage, 'kaltura partner id') - external_id = ep_data.get('external_id') or ep_meta['external_id'] - embed_url = 'kaltura:%s:%s' % (partner_id, external_id) - ie_key = 'Kaltura' + zype_id = ep_data.get('zype_id') or ep_meta['zype_id'] title = ep_data.get('title') or ep_meta.get('title') description = clean_html(ep_meta.get('episode_description') or ep_data.get( @@ -79,8 +69,8 @@ class AmericasTestKitchenIE(InfoExtractor): return { '_type': 'url_transparent', - 'url': embed_url, - 'ie_key': ie_key, + 'url': 'https://player.zype.com/embed/%s.js?api_key=jZ9GUhRmxcPvX7M3SlfejB6Hle9jyHTdk2jVxG7wOHPLODgncEKVdPYBhuz9iWXQ' % zype_id, + 'ie_key': 'Zype', 'title': title, 'description': description, 'thumbnail': thumbnail, From 48ff5590c160b89e4596b706f2b33c69557063a0 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Thu, 16 Jan 2020 15:37:16 +0100 Subject: [PATCH 17/17] [nbc] add support for nbc multi network URLs(closes #23049) --- youtube_dl/extractor/nbc.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py index 5bc39d002..6f3cb3003 100644 --- a/youtube_dl/extractor/nbc.py +++ b/youtube_dl/extractor/nbc.py @@ -87,11 +87,25 @@ class NBCIE(AdobePassIE): def _real_extract(self, url): permalink, video_id = re.match(self._VALID_URL, url).groups() permalink = 'http' + compat_urllib_parse_unquote(permalink) - response = self._download_json( + video_data = self._download_json( 'https://friendship.nbc.co/v2/graphql', video_id, query={ - 'query': '''{ - page(name: "%s", platform: web, type: VIDEO, userId: "0") { - data { + 'query': '''query bonanzaPage( + $app: NBCUBrands! = nbc + $name: String! + $oneApp: Boolean + $platform: SupportedPlatforms! = web + $type: EntityPageType! = VIDEO + $userId: String! +) { + bonanzaPage( + app: $app + name: $name + oneApp: $oneApp + platform: $platform + type: $type + userId: $userId + ) { + metadata { ... on VideoPageData { description episodeNumber @@ -100,15 +114,20 @@ class NBCIE(AdobePassIE): mpxAccountId mpxGuid rating + resourceId seasonNumber secondaryTitle seriesShortTitle } } } -}''' % permalink, - }) - video_data = response['data']['page']['data'] +}''', + 'variables': json.dumps({ + 'name': permalink, + 'oneApp': True, + 'userId': '0', + }), + })['data']['bonanzaPage']['metadata'] query = { 'mbr': 'true', 'manifest': 'm3u', @@ -117,8 +136,8 @@ class NBCIE(AdobePassIE): title = video_data['secondaryTitle'] if video_data.get('locked'): resource = self._get_mvpd_resource( - 'nbcentertainment', title, video_id, - video_data.get('rating')) + video_data.get('resourceId') or 'nbcentertainment', + title, video_id, video_data.get('rating')) query['auth'] = self._extract_mvpd_auth( url, video_id, 'nbcentertainment', resource) theplatform_url = smuggle_url(update_url_query(