From cf3d399727b87683aff25b5f57e026a4326244ef Mon Sep 17 00:00:00 2001 From: ealgase Date: Mon, 25 Mar 2019 12:04:31 -0400 Subject: [PATCH 01/17] [openload] add support for oladblock.services and oladblock.xyz domains --- youtube_dl/extractor/openload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index cc323e5e2..25b3bfdbd 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -243,7 +243,7 @@ class PhantomJSwrapper(object): class OpenloadIE(InfoExtractor): - _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space))' + _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space)|oladblock\.(?:services|xyz))' _VALID_URL = r'''(?x) https?:// (?P From bf6fb8b9dc921a30df24d9789d5bbb0ac5b370b0 Mon Sep 17 00:00:00 2001 From: ealgase Date: Sat, 6 Apr 2019 23:38:40 -0400 Subject: [PATCH 02/17] [openload] add tests --- youtube_dl/extractor/openload.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index 25b3bfdbd..130165b8c 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -350,6 +350,12 @@ class OpenloadIE(InfoExtractor): }, { 'url': 'https://oload.space/f/IY4eZSst3u8/', 'only_matching': True, + }, { + 'url': 'https://oladblock.services/f/b8NWEgkqNLI/', + 'only_matching': True, + }, { + 'url': 'https://oladblock.xyz/f/b8NWEgkqNLI/', + 'only_matching': True, }] _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36' From 4f1e02ad60d5a1b19c39be6665afc22726301685 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 14 Apr 2019 11:46:33 +0100 Subject: [PATCH 03/17] [aenetworks] fix history topic extraction and extract more formats --- youtube_dl/extractor/aenetworks.py | 137 +++++++++++++++-------------- 1 file changed, 71 insertions(+), 66 deletions(-) diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py index 85ec6392d..8304dba2f 100644 --- a/youtube_dl/extractor/aenetworks.py +++ b/youtube_dl/extractor/aenetworks.py @@ -4,11 +4,11 @@ import re from .theplatform import ThePlatformIE from ..utils import ( + extract_attributes, + ExtractorError, + int_or_none, smuggle_url, update_url_query, - unescapeHTML, - extract_attributes, - get_element_by_attribute, ) from ..compat import ( compat_urlparse, @@ -19,6 +19,43 @@ class AENetworksBaseIE(ThePlatformIE): _THEPLATFORM_KEY = 'crazyjava' _THEPLATFORM_SECRET = 's3cr3t' + def _extract_aen_smil(self, smil_url, video_id, auth=None): + query = {'mbr': 'true'} + if auth: + query['auth'] = auth + TP_SMIL_QUERY = [{ + 'assetTypes': 'high_video_ak', + 'switch': 'hls_high_ak' + }, { + 'assetTypes': 'high_video_s3' + }, { + 'assetTypes': 'high_video_s3', + 'switch': 'hls_ingest_fastly' + }] + formats = [] + subtitles = {} + last_e = None + for q in TP_SMIL_QUERY: + q.update(query) + m_url = update_url_query(smil_url, q) + m_url = self._sign_url(m_url, self._THEPLATFORM_KEY, self._THEPLATFORM_SECRET) + try: + tp_formats, tp_subtitles = self._extract_theplatform_smil( + m_url, video_id, 'Downloading %s SMIL data' % (q.get('switch') or q['assetTypes'])) + except ExtractorError as e: + last_e = e + continue + formats.extend(tp_formats) + subtitles = self._merge_subtitles(subtitles, tp_subtitles) + if last_e and not formats: + raise last_e + self._sort_formats(formats) + return { + 'id': video_id, + 'formats': formats, + 'subtitles': subtitles, + } + class AENetworksIE(AENetworksBaseIE): IE_NAME = 'aenetworks' @@ -33,22 +70,25 @@ class AENetworksIE(AENetworksBaseIE): (?: shows/(?P[^/]+(?:/[^/]+){0,2})| movies/(?P[^/]+)(?:/full-movie)?| - specials/(?P[^/]+)/full-special| + specials/(?P[^/]+)/(?:full-special|preview-)| collections/[^/]+/(?P[^/]+) ) ''' _TESTS = [{ 'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1', - 'md5': 'a97a65f7e823ae10e9244bc5433d5fe6', 'info_dict': { 'id': '22253814', 'ext': 'mp4', - 'title': 'Winter Is Coming', + 'title': 'Winter is Coming', 'description': 'md5:641f424b7a19d8e24f26dea22cf59d74', 'timestamp': 1338306241, 'upload_date': '20120529', 'uploader': 'AENE-NEW', }, + 'params': { + # m3u8 download + 'skip_download': True, + }, 'add_ie': ['ThePlatform'], }, { 'url': 'http://www.history.com/shows/ancient-aliens/season-1', @@ -84,6 +124,9 @@ class AENetworksIE(AENetworksBaseIE): }, { 'url': 'https://www.historyvault.com/collections/america-the-story-of-us/westward', 'only_matching': True + }, { + 'url': 'https://www.aetv.com/specials/hunting-jonbenets-killer-the-untold-story/preview-hunting-jonbenets-killer-the-untold-story', + 'only_matching': True }] _DOMAIN_TO_REQUESTOR_ID = { 'history.com': 'HISTORY', @@ -124,11 +167,6 @@ class AENetworksIE(AENetworksBaseIE): return self.playlist_result( entries, self._html_search_meta('aetn:SeasonId', webpage)) - query = { - 'mbr': 'true', - 'assetTypes': 'high_video_ak', - 'switch': 'hls_high_ak', - } video_id = self._html_search_meta('aetn:VideoID', webpage) media_url = self._search_regex( [r"media_url\s*=\s*'(?P[^']+)'", @@ -138,64 +176,39 @@ class AENetworksIE(AENetworksBaseIE): theplatform_metadata = self._download_theplatform_metadata(self._search_regex( r'https?://link\.theplatform\.com/s/([^?]+)', media_url, 'theplatform_path'), video_id) info = self._parse_theplatform_metadata(theplatform_metadata) + auth = None if theplatform_metadata.get('AETN$isBehindWall'): requestor_id = self._DOMAIN_TO_REQUESTOR_ID[domain] resource = self._get_mvpd_resource( requestor_id, theplatform_metadata['title'], theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'), theplatform_metadata['ratings'][0]['rating']) - query['auth'] = self._extract_mvpd_auth( + auth = self._extract_mvpd_auth( url, video_id, requestor_id, resource) info.update(self._search_json_ld(webpage, video_id, fatal=False)) - media_url = update_url_query(media_url, query) - media_url = self._sign_url(media_url, self._THEPLATFORM_KEY, self._THEPLATFORM_SECRET) - formats, subtitles = self._extract_theplatform_smil(media_url, video_id) - self._sort_formats(formats) - info.update({ - 'id': video_id, - 'formats': formats, - 'subtitles': subtitles, - }) + info.update(self._extract_aen_smil(media_url, video_id, auth)) return info class HistoryTopicIE(AENetworksBaseIE): IE_NAME = 'history:topic' IE_DESC = 'History.com Topic' - _VALID_URL = r'https?://(?:www\.)?history\.com/topics/(?:[^/]+/)?(?P[^/]+)(?:/[^/]+(?:/(?P[^/?#]+))?)?' + _VALID_URL = r'https?://(?:www\.)?history\.com/topics/[^/]+/(?P[\w+-]+?)-video' _TESTS = [{ - 'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false', + 'url': 'https://www.history.com/topics/valentines-day/history-of-valentines-day-video', 'info_dict': { 'id': '40700995724', 'ext': 'mp4', - 'title': "Bet You Didn't Know: Valentine's Day", + 'title': "History of Valentine’s Day", 'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7', 'timestamp': 1375819729, 'upload_date': '20130806', - 'uploader': 'AENE-NEW', }, 'params': { # m3u8 download 'skip_download': True, }, 'add_ie': ['ThePlatform'], - }, { - 'url': 'http://www.history.com/topics/world-war-i/world-war-i-history/videos', - 'info_dict': - { - 'id': 'world-war-i-history', - 'title': 'World War I History', - }, - 'playlist_mincount': 23, - }, { - 'url': 'http://www.history.com/topics/world-war-i-history/videos', - 'only_matching': True, - }, { - 'url': 'http://www.history.com/topics/world-war-i/world-war-i-history', - 'only_matching': True, - }, { - 'url': 'http://www.history.com/topics/world-war-i/world-war-i-history/speeches', - 'only_matching': True, }] def theplatform_url_result(self, theplatform_url, video_id, query): @@ -215,27 +228,19 @@ class HistoryTopicIE(AENetworksBaseIE): } def _real_extract(self, url): - topic_id, video_display_id = re.match(self._VALID_URL, url).groups() - if video_display_id: - webpage = self._download_webpage(url, video_display_id) - release_url, video_id = re.search(r"_videoPlayer.play\('([^']+)'\s*,\s*'[^']+'\s*,\s*'(\d+)'\)", webpage).groups() - release_url = unescapeHTML(release_url) - - return self.theplatform_url_result( - release_url, video_id, { - 'mbr': 'true', - 'switch': 'hls', - 'assetTypes': 'high_video_ak', - }) - else: - webpage = self._download_webpage(url, topic_id) - entries = [] - for episode_item in re.findall(r']*>', webpage): - video_attributes = extract_attributes(episode_item) - entries.append(self.theplatform_url_result( - video_attributes['data-release-url'], video_attributes['data-id'], { - 'mbr': 'true', - 'switch': 'hls', - 'assetTypes': 'high_video_ak', - })) - return self.playlist_result(entries, topic_id, get_element_by_attribute('class', 'show-title', webpage)) + display_id = self._match_id(url) + webpage = self._download_webpage(url, display_id) + video_id = self._search_regex( + r']+src="[^"]+\btpid=(\d+)', webpage, 'tpid') + result = self._download_json( + 'https://feeds.video.aetnd.com/api/v2/history/videos', + video_id, query={'filter[id]': video_id})['results'][0] + title = result['title'] + info = self._extract_aen_smil(result['publicUrl'], video_id) + info.update({ + 'title': title, + 'description': result.get('description'), + 'duration': int_or_none(result.get('duration')), + 'timestamp': int_or_none(result.get('added'), 1000), + }) + return info From 10385322137eaf36b66b9e78931ea0a39e914c33 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 14 Apr 2019 13:18:16 +0100 Subject: [PATCH 04/17] [aenetworks] add encoding declaration --- youtube_dl/extractor/aenetworks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py index 8304dba2f..611b948f5 100644 --- a/youtube_dl/extractor/aenetworks.py +++ b/youtube_dl/extractor/aenetworks.py @@ -1,3 +1,4 @@ +# coding: utf-8 from __future__ import unicode_literals import re From 174f62992dc78c1f8464ce8945cb5af60ae72c1d Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 14 Apr 2019 14:29:04 +0100 Subject: [PATCH 05/17] [yahoo] add support for gyao.yahoo.co.jp --- youtube_dl/extractor/extractors.py | 2 + youtube_dl/extractor/yahoo.py | 63 ++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index cc19af5c4..86ecc0b66 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1452,6 +1452,8 @@ from .xxxymovies import XXXYMoviesIE from .yahoo import ( YahooIE, YahooSearchIE, + YahooGyaOPlayerIE, + YahooGyaOIE, ) from .yandexdisk import YandexDiskIE from .yandexmusic import ( diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py index 552013a74..955e12d2f 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dl/extractor/yahoo.py @@ -477,3 +477,66 @@ class YahooSearchIE(SearchInfoExtractor): 'id': query, 'entries': entries, } + + +class YahooGyaOPlayerIE(InfoExtractor): + IE_NAME = 'yahoo:gyao:player' + _VALID_URL = r'https?://gyao\.yahoo\.co\.jp/player/(?P\d+/v\d+/v\d+)' + _TEST = { + 'url': 'https://gyao.yahoo.co.jp/player/00998/v00818/v0000000000000008564/', + 'info_dict': { + 'id': '5993125228001', + 'ext': 'mp4', + 'title': 'フューリー　【字幕版】', + 'description': 'md5:21e691c798a15330eda4db17a8fe45a5', + 'uploader_id': '4235717419001', + 'upload_date': '20190124', + 'timestamp': 1548294365, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + } + + def _real_extract(self, url): + video_id = self._match_id(url).replace('/', ':') + video = self._download_json( + 'https://gyao.yahoo.co.jp/dam/v1/videos/' + video_id, + video_id, query={ + 'fields': 'longDescription,title,videoId', + }) + return { + '_type': 'url_transparent', + 'id': video_id, + 'title': video['title'], + 'url': 'http://players.brightcove.net/4235717419001/default_default/index.html?videoId=' + video['videoId'], + 'description': video.get('longDescription'), + 'ie_key': BrightcoveNewIE.ie_key(), + } + + +class YahooGyaOIE(InfoExtractor): + IE_NAME = 'yahoo:gyao' + _VALID_URL = r'https?://gyao\.yahoo\.co\.jp/p/(?P\d+/v\d+)' + _TEST = { + 'url': 'https://gyao.yahoo.co.jp/p/00449/v03102/', + 'info_dict': { + 'id': '00449:v03102', + }, + 'playlist_count': 2, + } + + def _real_extract(self, url): + program_id = self._match_id(url).replace('/', ':') + videos = self._download_json( + 'https://gyao.yahoo.co.jp/api/programs/%s/videos' % program_id, program_id)['videos'] + entries = [] + for video in videos: + video_id = video.get('id') + if not video_id: + continue + entries.append(self.url_result( + 'https://gyao.yahoo.co.jp/player/%s/' % '/'.join(video_id.split(':')), + YahooGyaOPlayerIE.ie_key(), video_id)) + return self.playlist_result(entries, program_id) From 3534b6329aebaad0f27d389d96fed2b5738585e2 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 14 Apr 2019 14:39:20 +0100 Subject: [PATCH 06/17] [yahoo] add support for streaming.yahoo.co.jp(closes #5811)(closes #7098) --- youtube_dl/extractor/yahoo.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py index 955e12d2f..730f69b09 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dl/extractor/yahoo.py @@ -481,8 +481,8 @@ class YahooSearchIE(SearchInfoExtractor): class YahooGyaOPlayerIE(InfoExtractor): IE_NAME = 'yahoo:gyao:player' - _VALID_URL = r'https?://gyao\.yahoo\.co\.jp/player/(?P\d+/v\d+/v\d+)' - _TEST = { + _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/player|streaming\.yahoo\.co\.jp/c/y)/(?P\d+/v\d+/v\d+)' + _TESTS = [{ 'url': 'https://gyao.yahoo.co.jp/player/00998/v00818/v0000000000000008564/', 'info_dict': { 'id': '5993125228001', @@ -497,7 +497,10 @@ class YahooGyaOPlayerIE(InfoExtractor): # m3u8 download 'skip_download': True, }, - } + }, { + 'url': 'https://streaming.yahoo.co.jp/c/y/01034/v00133/v0000000000000000706/', + 'only_matching': True, + }] def _real_extract(self, url): video_id = self._match_id(url).replace('/', ':') @@ -518,14 +521,17 @@ class YahooGyaOPlayerIE(InfoExtractor): class YahooGyaOIE(InfoExtractor): IE_NAME = 'yahoo:gyao' - _VALID_URL = r'https?://gyao\.yahoo\.co\.jp/p/(?P\d+/v\d+)' - _TEST = { + _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/p|streaming\.yahoo\.co\.jp/p/y)/(?P\d+/v\d+)' + _TESTS = [{ 'url': 'https://gyao.yahoo.co.jp/p/00449/v03102/', 'info_dict': { 'id': '00449:v03102', }, 'playlist_count': 2, - } + }, { + 'url': 'https://streaming.yahoo.co.jp/p/y/01034/v00133/', + 'only_matching': True, + }] def _real_extract(self, url): program_id = self._match_id(url).replace('/', ':') From 0b758fea1ccb98c38c17bc3dcbb851515a7f070d Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 14 Apr 2019 15:01:01 +0100 Subject: [PATCH 07/17] [yahoo] add support GYAO episode URLs --- youtube_dl/extractor/yahoo.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py index 730f69b09..86ba7d3c9 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dl/extractor/yahoo.py @@ -481,7 +481,7 @@ class YahooSearchIE(SearchInfoExtractor): class YahooGyaOPlayerIE(InfoExtractor): IE_NAME = 'yahoo:gyao:player' - _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/player|streaming\.yahoo\.co\.jp/c/y)/(?P\d+/v\d+/v\d+)' + _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/(?:player|episode/[^/]+)|streaming\.yahoo\.co\.jp/c/y)/(?P\d+/v\d+/v\d+|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})' _TESTS = [{ 'url': 'https://gyao.yahoo.co.jp/player/00998/v00818/v0000000000000008564/', 'info_dict': { @@ -500,6 +500,9 @@ class YahooGyaOPlayerIE(InfoExtractor): }, { 'url': 'https://streaming.yahoo.co.jp/c/y/01034/v00133/v0000000000000000706/', 'only_matching': True, + }, { + 'url': 'https://gyao.yahoo.co.jp/episode/%E3%81%8D%E3%81%AE%E3%81%86%E4%BD%95%E9%A3%9F%E3%81%B9%E3%81%9F%EF%BC%9F%20%E7%AC%AC2%E8%A9%B1%202019%2F4%2F12%E6%94%BE%E9%80%81%E5%88%86/5cb02352-b725-409e-9f8d-88f947a9f682', + 'only_matching': True, }] def _real_extract(self, url): @@ -513,7 +516,9 @@ class YahooGyaOPlayerIE(InfoExtractor): '_type': 'url_transparent', 'id': video_id, 'title': video['title'], - 'url': 'http://players.brightcove.net/4235717419001/default_default/index.html?videoId=' + video['videoId'], + 'url': smuggle_url( + 'http://players.brightcove.net/4235717419001/default_default/index.html?videoId=' + video['videoId'], + {'geo_countries': ['JP']}), 'description': video.get('longDescription'), 'ie_key': BrightcoveNewIE.ie_key(), } @@ -543,6 +548,6 @@ class YahooGyaOIE(InfoExtractor): if not video_id: continue entries.append(self.url_result( - 'https://gyao.yahoo.co.jp/player/%s/' % '/'.join(video_id.split(':')), + 'https://gyao.yahoo.co.jp/player/%s/' % video_id.replace(':', '/'), YahooGyaOPlayerIE.ie_key(), video_id)) return self.playlist_result(entries, program_id) From cb6cd76f7bc4c964e148d3a592db256547a79bae Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 14 Apr 2019 23:18:36 +0100 Subject: [PATCH 08/17] [aol] add support for localized websites --- youtube_dl/extractor/aol.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py index dffa9733d..e87994a6a 100644 --- a/youtube_dl/extractor/aol.py +++ b/youtube_dl/extractor/aol.py @@ -17,7 +17,7 @@ from ..utils import ( class AolIE(InfoExtractor): IE_NAME = 'aol.com' - _VALID_URL = r'(?:aol-video:|https?://(?:www\.)?aol\.com/video/(?:[^/]+/)*)(?P[0-9a-f]+)' + _VALID_URL = r'(?:aol-video:|https?://(?:www\.)?aol\.(?:com|ca|co\.uk|de|jp)/video/(?:[^/]+/)*)(?P[0-9a-f]+)' _TESTS = [{ # video with 5min ID @@ -64,6 +64,18 @@ class AolIE(InfoExtractor): }, { 'url': 'https://www.aol.com/video/playlist/PL8245/5ca79d19d21f1a04035db606/', 'only_matching': True, + }, { + 'url': 'https://www.aol.ca/video/view/u-s-woman-s-family-arrested-for-murder-first-pinned-on-panhandler-police/5c7ccf45bc03931fa04b2fe1/', + 'only_matching': True, + }, { + 'url': 'https://www.aol.co.uk/video/view/-one-dead-and-22-hurt-in-bus-crash-/5cb3a6f3d21f1a072b457347/', + 'only_matching': True, + }, { + 'url': 'https://www.aol.de/video/view/eva-braun-privataufnahmen-von-hitlers-geliebter-werden-digitalisiert/5cb2d49de98ab54c113d3d5d/', + 'only_matching': True, + }, { + 'url': 'https://www.aol.jp/video/playlist/5a28e936a1334d000137da0c/5a28f3151e642219fde19831/', + 'only_matching': True, }] def _real_extract(self, url): From f114e43d3845b13227d4a1251bfb7d2750e76032 Mon Sep 17 00:00:00 2001 From: ealgase Date: Mon, 15 Apr 2019 23:08:28 -0400 Subject: [PATCH 09/17] [openload] add oladblock.me domain --- youtube_dl/extractor/openload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index 130165b8c..b441bd6ba 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -243,7 +243,7 @@ class PhantomJSwrapper(object): class OpenloadIE(InfoExtractor): - _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space)|oladblock\.(?:services|xyz))' + _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space)|oladblock\.(?:services|xyz|me))' _VALID_URL = r'''(?x) https?:// (?P From 6104cc15915abfe113152d24b18c84178e027265 Mon Sep 17 00:00:00 2001 From: ealgase Date: Mon, 15 Apr 2019 23:09:05 -0400 Subject: [PATCH 10/17] [openload] add test for oladblock.me --- youtube_dl/extractor/openload.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index b441bd6ba..da9c721d2 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -356,6 +356,9 @@ class OpenloadIE(InfoExtractor): }, { 'url': 'https://oladblock.xyz/f/b8NWEgkqNLI/', 'only_matching': True, + }, { + 'url': 'https://oladblock.me/f/b8NWEgkqNLI/', + 'only_matching': True, }] _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36' From e6c9ae31df1196dc12607b83be01141eb52b80b6 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Tue, 16 Apr 2019 13:04:13 +0100 Subject: [PATCH 11/17] [adn] fix subtitle extraction(#12724) --- youtube_dl/extractor/adn.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py index 1e04a55a6..923c351e4 100644 --- a/youtube_dl/extractor/adn.py +++ b/youtube_dl/extractor/adn.py @@ -60,14 +60,19 @@ class ADNIE(InfoExtractor): enc_subtitles = self._download_webpage( urljoin(self._BASE_URL, sub_path), - video_id, 'Downloading subtitles data', fatal=False) + video_id, 'Downloading subtitles location', fatal=False) or '{}' + subtitle_location = (self._parse_json(enc_subtitles, video_id, fatal=False) or {}).get('location') + if subtitle_location: + enc_subtitles = self._download_webpage( + urljoin(self._BASE_URL, subtitle_location), + video_id, 'Downloading subtitles data', fatal=False) if not enc_subtitles: return None # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js dec_subtitles = intlist_to_bytes(aes_cbc_decrypt( bytes_to_intlist(compat_b64decode(enc_subtitles[24:])), - bytes_to_intlist(binascii.unhexlify(self._K + '083db5aebd9353b4')), + bytes_to_intlist(binascii.unhexlify(self._K + '4421de0a5f0814ba')), bytes_to_intlist(compat_b64decode(enc_subtitles[:24])) )) subtitles_json = self._parse_json( From 7fc3b68ad32e4ab436cdcc99ed1a410e361e7b12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 17 Apr 2019 00:08:50 +0700 Subject: [PATCH 12/17] [openload] Randomize User-Agent (closes #20688) --- youtube_dl/extractor/openload.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index da9c721d2..5f5425fbc 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import json import os +import random import re import subprocess import tempfile @@ -361,7 +362,7 @@ class OpenloadIE(InfoExtractor): 'only_matching': True, }] - _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36' + _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{major}.0.{build}.{patch} Safari/537.36' @staticmethod def _extract_urls(webpage): @@ -376,7 +377,11 @@ class OpenloadIE(InfoExtractor): url_pattern = 'https://%s/%%s/%s/' % (host, video_id) headers = { - 'User-Agent': self._USER_AGENT, + 'User-Agent': self._USER_AGENT_TPL % { + 'major': random.randint(63, 73), + 'build': random.randint(3239, 3683), + 'patch': random.randint(0, 100), + }, } for path in ('embed', 'f'): From 9846935256a4e6398e6a16ec9ab18e45cba8f262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 17 Apr 2019 00:15:48 +0700 Subject: [PATCH 13/17] [ChangeLog] Actualize [ci skip] --- ChangeLog | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/ChangeLog b/ChangeLog index 421f247fd..58e2cf62b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,28 @@ +version + +Extractors +* [openload] Randomize User-Agent (closes #20688) ++ [openload] Add support for oladblock domains (#20471) +* [adn] Fix subtitle extraction (#12724) ++ [aol] Add support for localized websites ++ [yahoo] Add support GYAO episode URLs ++ [yahoo] Add support for streaming.yahoo.co.jp (#5811, #7098) ++ [yahoo] Add support for gyao.yahoo.co.jp +* [aenetworks] Fix history topic extraction and extract more formats ++ [cbs] Extract smpte and vtt subtitles ++ [streamango] Add support for streamcherry.com (#20592) ++ [yourporn] Add support for sxyprn.com (#20646) +* [mgtv] Fix extraction (#20650) +* [linkedin:learning] Use urljoin for form action URL (#20431) ++ [gdc] Add support for kaltura embeds (#20575) +* [dispeak] Improve mp4 bitrate extraction +* [kaltura] Sanitize embed URLs +* [jwplatfom] Do not match manifest URLs (#20596) +* [aol] Restrict URL regular expression and improve format extraction ++ [tiktok] Add support for new URL schema (#20573) ++ [stv:player] Add support for player.stv.tv (#20586) + + version 2019.04.07 Core From cd6c75b05f64797d5ce092485c8c092ceaf9d8c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 17 Apr 2019 00:20:09 +0700 Subject: [PATCH 14/17] release 2019.04.17 --- .github/ISSUE_TEMPLATE.md | 6 +++--- ChangeLog | 2 +- docs/supportedsites.md | 5 ++++- youtube_dl/version.py | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 5469c73cf..d63aaad06 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -6,8 +6,8 @@ --- -### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.04.07*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected. -- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.04.07** +### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.04.17*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected. +- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.04.17** ### Before submitting an *issue* make sure you have: - [ ] At least skimmed through the [README](https://github.com/ytdl-org/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/ytdl-org/youtube-dl#faq) and [BUGS](https://github.com/ytdl-org/youtube-dl#bugs) sections @@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 -[debug] youtube-dl version 2019.04.07 +[debug] youtube-dl version 2019.04.17 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/ChangeLog b/ChangeLog index 58e2cf62b..8365ea6de 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -version +version 2019.04.17 Extractors * [openload] Randomize User-Agent (closes #20688) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index df272c479..c5419fd42 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -46,6 +46,7 @@ - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl - **AnimeOnDemand** - **Anvato** + - **aol.com** - **APA** - **Aparat** - **AppleConnect** @@ -632,7 +633,6 @@ - **OdaTV** - **Odnoklassniki** - **OktoberfestTV** - - **on.aol.com** - **OnDemandKorea** - **onet.pl** - **onet.tv** @@ -853,6 +853,7 @@ - **StreamCZ** - **StreetVoice** - **StretchInternet** + - **stv:player** - **SunPorno** - **SVT** - **SVTPage** @@ -1124,6 +1125,8 @@ - **XVideos** - **XXXYMovies** - **Yahoo**: Yahoo screen and movies + - **yahoo:gyao** + - **yahoo:gyao:player** - **YandexDisk** - **yandexmusic:album**: Яндекс.Музыка - Альбом - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 5c7d550f5..4f2ba8e47 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2019.04.07' +__version__ = '2019.04.17' From 81d989c21ea3670815b78003eb0457a3b3e10428 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Tue, 16 Apr 2019 19:06:18 +0100 Subject: [PATCH 15/17] [bravotv] fix extraction(closes #19213) --- youtube_dl/extractor/bravotv.py | 40 ++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/youtube_dl/extractor/bravotv.py b/youtube_dl/extractor/bravotv.py index a25d500e4..b9715df00 100644 --- a/youtube_dl/extractor/bravotv.py +++ b/youtube_dl/extractor/bravotv.py @@ -1,6 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .adobepass import AdobePassIE from ..utils import ( smuggle_url, @@ -12,16 +14,16 @@ from ..utils import ( class BravoTVIE(AdobePassIE): _VALID_URL = r'https?://(?:www\.)?bravotv\.com/(?:[^/]+/)+(?P[^/?#]+)' _TESTS = [{ - 'url': 'http://www.bravotv.com/last-chance-kitchen/season-5/videos/lck-ep-12-fishy-finale', - 'md5': '9086d0b7ef0ea2aabc4781d75f4e5863', + 'url': 'https://www.bravotv.com/top-chef/season-16/episode-15/videos/the-top-chef-season-16-winner-is', + 'md5': 'e34684cfea2a96cd2ee1ef3a60909de9', 'info_dict': { - 'id': 'zHyk1_HU_mPy', + 'id': 'epL0pmK1kQlT', 'ext': 'mp4', - 'title': 'LCK Ep 12: Fishy Finale', - 'description': 'S13/E12: Two eliminated chefs have just 12 minutes to cook up a delicious fish dish.', + 'title': 'The Top Chef Season 16 Winner Is...', + 'description': 'Find out who takes the title of Top Chef!', 'uploader': 'NBCU-BRAV', - 'upload_date': '20160302', - 'timestamp': 1456945320, + 'upload_date': '20190314', + 'timestamp': 1552591860, } }, { 'url': 'http://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1', @@ -32,30 +34,38 @@ class BravoTVIE(AdobePassIE): display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) settings = self._parse_json(self._search_regex( - r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);', webpage, 'drupal settings'), + r']+data-drupal-selector="drupal-settings-json"[^>]*>({.+?})', webpage, 'drupal settings'), display_id) info = {} query = { 'mbr': 'true', } account_pid, release_pid = [None] * 2 - tve = settings.get('sharedTVE') + tve = settings.get('ls_tve') if tve: query['manifest'] = 'm3u' - account_pid = 'HNK2IC' - release_pid = tve['release_pid'] + mobj = re.search(r'<[^>]+id="pdk-player"[^>]+data-url=["\']?(?:https?:)?//player\.theplatform\.com/p/([^/]+)/(?:[^/]+/)*select/([^?#&"\']+)', webpage) + if mobj: + account_pid, tp_path = mobj.groups() + release_pid = tp_path.strip('/').split('/')[-1] + else: + account_pid = 'HNK2IC' + tp_path = release_pid = tve['release_pid'] if tve.get('entitlement') == 'auth': - adobe_pass = settings.get('adobePass', {}) + adobe_pass = settings.get('tve_adobe_auth', {}) resource = self._get_mvpd_resource( adobe_pass.get('adobePassResourceId', 'bravo'), tve['title'], release_pid, tve.get('rating')) query['auth'] = self._extract_mvpd_auth( url, release_pid, adobe_pass.get('adobePassRequestorId', 'bravo'), resource) else: - shared_playlist = settings['shared_playlist'] + shared_playlist = settings['ls_playlist'] account_pid = shared_playlist['account_pid'] metadata = shared_playlist['video_metadata'][shared_playlist['default_clip']] - release_pid = metadata['release_pid'] + tp_path = release_pid = metadata.get('release_pid') + if not release_pid: + release_pid = metadata['guid'] + tp_path = 'media/guid/2140479951/' + release_pid info.update({ 'title': metadata['title'], 'description': metadata.get('description'), @@ -67,7 +77,7 @@ class BravoTVIE(AdobePassIE): '_type': 'url_transparent', 'id': release_pid, 'url': smuggle_url(update_url_query( - 'http://link.theplatform.com/s/%s/%s' % (account_pid, release_pid), + 'http://link.theplatform.com/s/%s/%s' % (account_pid, tp_path), query), {'force_smil_url': True}), 'ie_key': 'ThePlatform', }) From f3914b06a0eb0f90b0a2326468e792f107968884 Mon Sep 17 00:00:00 2001 From: ealgase Date: Tue, 16 Apr 2019 18:33:50 -0400 Subject: [PATCH 16/17] [openload] Add support for openloed.co (closes #20691) While the .co could be captured directly, I anticipate that there will be more TLD's for openloed in the future. --- youtube_dl/extractor/openload.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index 5f5425fbc..0d5c992fc 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -244,7 +244,7 @@ class PhantomJSwrapper(object): class OpenloadIE(InfoExtractor): - _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space)|oladblock\.(?:services|xyz|me))' + _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space)|oladblock\.(?:services|xyz|me)|openloed\.co)' _VALID_URL = r'''(?x) https?:// (?P @@ -360,6 +360,9 @@ class OpenloadIE(InfoExtractor): }, { 'url': 'https://oladblock.me/f/b8NWEgkqNLI/', 'only_matching': True, + }, { + 'url': 'https://openloed.co/f/b8NWEgkqNLI/', + 'only_matching': True, }] _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{major}.0.{build}.{patch} Safari/537.36' From 9abeefd5275e4b468d4ba3a46446ae6cf8365171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 18 Apr 2019 23:56:20 +0700 Subject: [PATCH 17/17] [openload] Add support for oload.services (closes #20691) --- youtube_dl/extractor/openload.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index 0d5c992fc..43cdedb1e 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -244,7 +244,7 @@ class PhantomJSwrapper(object): class OpenloadIE(InfoExtractor): - _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space)|oladblock\.(?:services|xyz|me)|openloed\.co)' + _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|pw|live|space|services)|oladblock\.(?:services|xyz|me)|openloed\.co)' _VALID_URL = r'''(?x) https?:// (?P @@ -351,6 +351,9 @@ class OpenloadIE(InfoExtractor): }, { 'url': 'https://oload.space/f/IY4eZSst3u8/', 'only_matching': True, + }, { + 'url': 'https://oload.services/embed/bs1NWj1dCag/', + 'only_matching': True, }, { 'url': 'https://oladblock.services/f/b8NWEgkqNLI/', 'only_matching': True,