From 5de538787df3f13476489b4a7dfae442c046fab7 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Fri, 19 Apr 2019 20:44:31 +0100 Subject: [PATCH 01/45] [udemy] add another course id extraction pattern(closes #20491) --- youtube_dl/extractor/udemy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index 66ea6fb15..2a4faecef 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -76,7 +76,10 @@ class UdemyIE(InfoExtractor): webpage, 'course', default='{}')), video_id, fatal=False) or {} course_id = course.get('id') or self._search_regex( - r'data-course-id=["\'](\d+)', webpage, 'course id') + [ + r'data-course-id=["\'](\d+)', + r'"courseId"\s*:\s*(\d+)' + ], webpage, 'course id') return course_id, course.get('title') def _enroll_course(self, base_url, webpage, course_id): From 061d1cd9486d1b31cb37e000e8181f7684024798 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 21 Apr 2019 13:17:22 +0100 Subject: [PATCH 02/45] [nhk] add support for audio URLs --- youtube_dl/extractor/nhk.py | 96 +++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py index d4acbcc3e..727302560 100644 --- a/youtube_dl/extractor/nhk.py +++ b/youtube_dl/extractor/nhk.py @@ -1,54 +1,80 @@ from __future__ import unicode_literals +import re + from .common import InfoExtractor -from ..utils import ExtractorError class NhkVodIE(InfoExtractor): - _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/en/(?:vod|ondemand)/(?P[^/]+/[^/?#&]+)' + _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/(?P[a-z]{2})/ondemand/(?Pvideo|audio)/(?P\d{7}|[a-z]+-\d{8}-\d+)' + # Content available only for a limited period of time. Visit + # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples. _TESTS = [{ - # Videos available only for a limited period of time. Visit - # http://www3.nhk.or.jp/nhkworld/en/vod/ for working samples. - 'url': 'http://www3.nhk.or.jp/nhkworld/en/vod/tokyofashion/20160815', - 'info_dict': { - 'id': 'A1bnNiNTE6nY3jLllS-BIISfcC_PpvF5', - 'ext': 'flv', - 'title': 'TOKYO FASHION EXPRESS - The Kimono as Global Fashion', - 'description': 'md5:db338ee6ce8204f415b754782f819824', - 'series': 'TOKYO FASHION EXPRESS', - 'episode': 'The Kimono as Global Fashion', - }, - 'skip': 'Videos available only for a limited period of time', - }, { 'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/', 'only_matching': True, + }, { + 'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/plugin-20190404-1/', + 'only_matching': True, + }, { + 'url': 'https://www3.nhk.or.jp/nhkworld/fr/ondemand/audio/plugin-20190404-1/', + 'only_matching': True, }] - _API_URL = 'http://api.nhk.or.jp/nhkworld/vodesdlist/v1/all/all/all.json?apikey=EJfK8jdS57GqlupFgAfAAwr573q01y6k' + _API_URL_TEMPLATE = 'https://api.nhk.or.jp/nhkworld/%sodesdlist/v7/episode/%s/%s/all%s.json' def _real_extract(self, url): - video_id = self._match_id(url) - - data = self._download_json(self._API_URL, video_id) - - try: - episode = next( - e for e in data['data']['episodes'] - if e.get('url') and video_id in e['url']) - except StopIteration: - raise ExtractorError('Unable to find episode') - - embed_code = episode['vod_id'] + lang, m_type, episode_id = re.match(self._VALID_URL, url).groups() + if episode_id.isdigit(): + episode_id = episode_id[:4] + '-' + episode_id[4:] + is_video = m_type == 'video' + episode = self._download_json( + self._API_URL_TEMPLATE % ('v' if is_video else 'r', episode_id, lang, '/all' if is_video else ''), + episode_id, query={'apikey': 'EJfK8jdS57GqlupFgAfAAwr573q01y6k'})['data']['episodes'][0] title = episode.get('sub_title_clean') or episode['sub_title'] - description = episode.get('description_clean') or episode.get('description') - series = episode.get('title_clean') or episode.get('title') - return { - '_type': 'url_transparent', - 'ie_key': 'Ooyala', - 'url': 'ooyala:%s' % embed_code, + def get_clean_field(key): + return episode.get(key + '_clean') or episode.get(key) + + series = get_clean_field('title') + + thumbnails = [] + for s, w, h in [('', 640, 360), ('_l', 1280, 720)]: + img_path = episode.get('image' + s) + if not img_path: + continue + thumbnails.append({ + 'id': '%dp' % h, + 'height': h, + 'width': w, + 'url': 'https://www3.nhk.or.jp' + img_path, + }) + + info = { + 'id': episode_id + '-' + lang, 'title': '%s - %s' % (series, title) if series and title else title, - 'description': description, + 'description': get_clean_field('description'), + 'thumbnails': thumbnails, 'series': series, 'episode': title, } + if is_video: + info.update({ + '_type': 'url_transparent', + 'ie_key': 'Ooyala', + 'url': 'ooyala:' + episode['vod_id'], + }) + else: + audio = episode['audio'] + audio_path = audio['audio'] + info['formats'] = self._extract_m3u8_formats( + 'https://nhks-vh.akamaihd.net/i%s/master.m3u8' % audio_path, + episode_id, 'm4a', m3u8_id='hls', fatal=False) + info['formats'].append({ + 'ext': 'flv', + 'format_id': 'flv', + 'url': 'rtmp://flv.nhk.or.jp/ondemand/mp4:flv' + audio_path, + 'vcodec': 'none', + }) + for f in info['formats']: + f['language'] = lang + return info From 47cfa0051641d65894da02d64484055b04f767e0 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 21 Apr 2019 13:25:04 +0100 Subject: [PATCH 03/45] [nhk] extract rtmpt format --- youtube_dl/extractor/nhk.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py index 727302560..241412f98 100644 --- a/youtube_dl/extractor/nhk.py +++ b/youtube_dl/extractor/nhk.py @@ -69,12 +69,13 @@ class NhkVodIE(InfoExtractor): info['formats'] = self._extract_m3u8_formats( 'https://nhks-vh.akamaihd.net/i%s/master.m3u8' % audio_path, episode_id, 'm4a', m3u8_id='hls', fatal=False) - info['formats'].append({ - 'ext': 'flv', - 'format_id': 'flv', - 'url': 'rtmp://flv.nhk.or.jp/ondemand/mp4:flv' + audio_path, - 'vcodec': 'none', - }) + for proto in ('rtmpt', 'rtmp'): + info['formats'].append({ + 'ext': 'flv', + 'format_id': proto, + 'url': '%s://flv.nhk.or.jp/ondemand/mp4:flv%s' % (proto, audio_path), + 'vcodec': 'none', + }) for f in info['formats']: f['language'] = lang return info From c9b19d7a55549baa8b931390d94bdefb12a76d1d Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 21 Apr 2019 14:51:26 +0100 Subject: [PATCH 04/45] [ntvcojp] Add new extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/ntvcojp.py | 49 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 youtube_dl/extractor/ntvcojp.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 86ecc0b66..bac90f277 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -808,6 +808,7 @@ from .nrk import ( NRKTVSeasonIE, NRKTVSeriesIE, ) +from .ntvcojp import NTVCoJpCUIE from .ntvde import NTVDeIE from .ntvru import NTVRuIE from .nytimes import ( diff --git a/youtube_dl/extractor/ntvcojp.py b/youtube_dl/extractor/ntvcojp.py new file mode 100644 index 000000000..0c8221b22 --- /dev/null +++ b/youtube_dl/extractor/ntvcojp.py @@ -0,0 +1,49 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import ( + js_to_json, + smuggle_url, +) + + +class NTVCoJpCUIE(InfoExtractor): + IE_NAME = 'cu.ntv.co.jp' + IE_DESC = 'Nippon Television Network' + _VALID_URL = r'https?://cu\.ntv\.co\.jp/(?!program)(?P[^/?&#]+)' + _TEST = { + 'url': 'https://cu.ntv.co.jp/televiva-chill-gohan_181031/', + 'info_dict': { + 'id': '5978891207001', + 'ext': 'mp4', + 'title': '桜エビと炒り卵がポイント！ 「中華風 エビチリおにぎり」──『美虎』五十嵐美幸', + 'upload_date': '20181213', + 'description': 'md5:211b52f4fd60f3e0e72b68b0c6ba52a9', + 'uploader_id': '3855502814001', + 'timestamp': 1544669941, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + } + BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' + + def _real_extract(self, url): + display_id = self._match_id(url) + webpage = self._download_webpage(url, display_id) + player_config = self._parse_json(self._search_regex( + r'(?s)PLAYER_CONFIG\s*=\s*({.+?})', + webpage, 'player config'), display_id, js_to_json) + video_id = player_config['videoId'] + account_id = player_config.get('account') or '3855502814001' + return { + '_type': 'url_transparent', + 'id': video_id, + 'display_id': display_id, + 'title': self._search_regex(r']+class="title"[^>]*>([^<]+)', webpage, 'title').strip(), + 'description': self._html_search_meta(['description', 'og:description'], webpage), + 'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account_id, video_id), {'geo_countries': ['JP']}), + 'ie_key': 'BrightcoveNew', + } From c25720ef6ab7e100d107df64efb3a1e1776fd66a Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 21 Apr 2019 17:20:28 +0100 Subject: [PATCH 05/45] [vimeo] add support live streams and improve info extraction(closes #19144) --- youtube_dl/extractor/common.py | 2 ++ youtube_dl/extractor/vimeo.py | 51 +++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 0889288f0..59ad455c1 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -2019,6 +2019,8 @@ class InfoExtractor(object): if res is False: return [] mpd_doc, urlh = res + if mpd_doc is None: + return [] mpd_base_url = base_url(urlh.geturl()) return self._parse_mpd_formats( diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 9aec8a2ab..d404498aa 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -109,23 +109,8 @@ class VimeoBaseInfoExtractor(InfoExtractor): def _parse_config(self, config, video_id): video_data = config['video'] - # Extract title video_title = video_data['title'] - - # Extract uploader, uploader_url and uploader_id - video_uploader = video_data.get('owner', {}).get('name') - video_uploader_url = video_data.get('owner', {}).get('url') - video_uploader_id = video_uploader_url.split('/')[-1] if video_uploader_url else None - - # Extract video thumbnail - video_thumbnail = video_data.get('thumbnail') - if video_thumbnail is None: - video_thumbs = video_data.get('thumbs') - if video_thumbs and isinstance(video_thumbs, dict): - _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1] - - # Extract video duration - video_duration = int_or_none(video_data.get('duration')) + is_live = try_get(video_data, lambda x: x['live_event']['status']) == 'started' formats = [] config_files = video_data.get('files') or config['request'].get('files', {}) @@ -151,7 +136,7 @@ class VimeoBaseInfoExtractor(InfoExtractor): if files_type == 'hls': formats.extend(self._extract_m3u8_formats( manifest_url, video_id, 'mp4', - 'm3u8_native', m3u8_id=format_id, + 'm3u8' if is_live else 'm3u8_native', m3u8_id=format_id, note='Downloading %s m3u8 information' % cdn_name, fatal=False)) elif files_type == 'dash': @@ -164,6 +149,10 @@ class VimeoBaseInfoExtractor(InfoExtractor): else: mpd_manifest_urls = [(format_id, manifest_url)] for f_id, m_url in mpd_manifest_urls: + if 'json=1' in m_url: + real_m_url = (self._download_json(m_url, video_id, fatal=False) or {}).get('url') + if real_m_url: + m_url = real_m_url mpd_formats = self._extract_mpd_formats( m_url.replace('/master.json', '/master.mpd'), video_id, f_id, 'Downloading %s MPD information' % cdn_name, @@ -184,15 +173,33 @@ class VimeoBaseInfoExtractor(InfoExtractor): 'url': 'https://vimeo.com' + tt['url'], }] + thumbnails = [] + if not is_live: + for key, thumb in video_data.get('thumbs', {}).items(): + thumbnails.append({ + 'id': key, + 'width': int_or_none(key), + 'url': thumb, + }) + thumbnail = video_data.get('thumbnail') + if thumbnail: + thumbnails.append({ + 'url': thumbnail, + }) + + owner = video_data.get('owner') or {} + video_uploader_url = owner.get('url') + return { - 'title': video_title, - 'uploader': video_uploader, - 'uploader_id': video_uploader_id, + 'title': self._live_title(video_title) if is_live else video_title, + 'uploader': owner.get('name'), + 'uploader_id': video_uploader_url.split('/')[-1] if video_uploader_url else None, 'uploader_url': video_uploader_url, - 'thumbnail': video_thumbnail, - 'duration': video_duration, + 'thumbnails': thumbnails, + 'duration': int_or_none(video_data.get('duration')), 'formats': formats, 'subtitles': subtitles, + 'is_live': is_live, } def _extract_original_format(self, url, video_id): From 85b6335d55c7b0ed7f6815f7b8b9a365b0a28c37 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 21 Apr 2019 21:05:58 +0100 Subject: [PATCH 06/45] [vimeo] extract live archive source format(#19144) --- youtube_dl/extractor/vimeo.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index d404498aa..a41178bab 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -110,7 +110,8 @@ class VimeoBaseInfoExtractor(InfoExtractor): def _parse_config(self, config, video_id): video_data = config['video'] video_title = video_data['title'] - is_live = try_get(video_data, lambda x: x['live_event']['status']) == 'started' + live_event = video_data.get('live_event') or {} + is_live = live_event.get('status') == 'started' formats = [] config_files = video_data.get('files') or config['request'].get('files', {}) @@ -127,6 +128,7 @@ class VimeoBaseInfoExtractor(InfoExtractor): 'tbr': int_or_none(f.get('bitrate')), }) + # TODO: fix handling of 308 status code returned for live archive manifest requests for files_type in ('hls', 'dash'): for cdn_name, cdn_data in config_files.get(files_type, {}).get('cdns', {}).items(): manifest_url = cdn_data.get('url') @@ -164,6 +166,15 @@ class VimeoBaseInfoExtractor(InfoExtractor): f['preference'] = -40 formats.extend(mpd_formats) + live_archive = live_event.get('archive') or {} + live_archive_source_url = live_archive.get('source_url') + if live_archive_source_url and live_archive.get('status') == 'done': + formats.append({ + 'format_id': 'live-archive-source', + 'url': live_archive_source_url, + 'preference': 1, + }) + subtitles = {} text_tracks = config['request'].get('text_tracks') if text_tracks: From fdc2183650a1aed22266bb59d83a1198525d4111 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Mon, 22 Apr 2019 10:04:00 +0100 Subject: [PATCH 07/45] [nrl] Add new extractor(closes #15991) --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/nrl.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 youtube_dl/extractor/nrl.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index bac90f277..0e3ccb82d 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -808,6 +808,7 @@ from .nrk import ( NRKTVSeasonIE, NRKTVSeriesIE, ) +from .nrl import NRLTVIE from .ntvcojp import NTVCoJpCUIE from .ntvde import NTVDeIE from .ntvru import NTVRuIE diff --git a/youtube_dl/extractor/nrl.py b/youtube_dl/extractor/nrl.py new file mode 100644 index 000000000..798b91e04 --- /dev/null +++ b/youtube_dl/extractor/nrl.py @@ -0,0 +1,30 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + + +class NRLTVIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?nrl\.com/tv(/[^/]+)*/(?P[^/?&#]+)' + _TEST = { + 'url': 'https://www.nrl.com/tv/news/match-highlights-titans-v-knights-862805/', + 'info_dict': { + 'id': 'YyNnFuaDE6kPJqlDhG4CGQ_w89mKTau4', + 'ext': 'mp4', + 'title': 'Match Highlights: Titans v Knights', + }, + 'params': { + # m3u8 download + 'skip_download': True, + 'format': 'bestvideo', + }, + } + + def _real_extract(self, url): + display_id = self._match_id(url) + webpage = self._download_webpage(url, display_id) + q_data = self._parse_json(self._search_regex( + r"(?s)q-data='({.+?})'", webpage, 'player data'), display_id) + ooyala_id = q_data['videoId'] + return self.url_result( + 'ooyala:' + ooyala_id, 'Ooyala', ooyala_id, q_data.get('title')) From 3fd86cfe13c2ca83c81cc43ed106152a07dcf012 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Mon, 22 Apr 2019 10:04:56 +0100 Subject: [PATCH 08/45] [ooyala] add support for geo verification proxy --- youtube_dl/extractor/ooyala.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py index ad8bf03f8..e42d67df9 100644 --- a/youtube_dl/extractor/ooyala.py +++ b/youtube_dl/extractor/ooyala.py @@ -36,7 +36,7 @@ class OoyalaBaseIE(InfoExtractor): 'domain': domain, 'supportedFormats': supportedformats or 'mp4,rtmp,m3u8,hds,dash,smooth', 'embedToken': embed_token, - }), video_id) + }), video_id, headers=self.geo_verification_headers()) cur_auth_data = auth_data['authorization_data'][embed_code] From e09965d550d8d76ea0c616cbb58800ee2249f15c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 23 Apr 2019 00:39:16 +0700 Subject: [PATCH 09/45] [soundcloud] Add support for new rendition and improve extraction (closes #20699) --- youtube_dl/extractor/soundcloud.py | 204 +++++++++++++++++++++-------- 1 file changed, 149 insertions(+), 55 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 15da3496e..277c3c7b4 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -15,7 +15,12 @@ from ..compat import ( ) from ..utils import ( ExtractorError, + float_or_none, int_or_none, + KNOWN_EXTENSIONS, + merge_dicts, + mimetype2ext, + str_or_none, try_get, unified_timestamp, update_url_query, @@ -57,7 +62,7 @@ class SoundcloudIE(InfoExtractor): 'uploader': 'E.T. ExTerrestrial Music', 'timestamp': 1349920598, 'upload_date': '20121011', - 'duration': 143, + 'duration': 143.216, 'license': 'all-rights-reserved', 'view_count': int, 'like_count': int, @@ -100,7 +105,7 @@ class SoundcloudIE(InfoExtractor): 'uploader': 'jaimeMF', 'timestamp': 1386604920, 'upload_date': '20131209', - 'duration': 9, + 'duration': 9.927, 'license': 'all-rights-reserved', 'view_count': int, 'like_count': int, @@ -120,7 +125,7 @@ class SoundcloudIE(InfoExtractor): 'uploader': 'jaimeMF', 'timestamp': 1386604920, 'upload_date': '20131209', - 'duration': 9, + 'duration': 9.927, 'license': 'all-rights-reserved', 'view_count': int, 'like_count': int, @@ -140,7 +145,7 @@ class SoundcloudIE(InfoExtractor): 'uploader': 'oddsamples', 'timestamp': 1389232924, 'upload_date': '20140109', - 'duration': 17, + 'duration': 17.346, 'license': 'cc-by-sa', 'view_count': int, 'like_count': int, @@ -160,7 +165,7 @@ class SoundcloudIE(InfoExtractor): 'uploader': 'Ori Uplift Music', 'timestamp': 1504206263, 'upload_date': '20170831', - 'duration': 7449, + 'duration': 7449.096, 'license': 'all-rights-reserved', 'view_count': int, 'like_count': int, @@ -180,7 +185,7 @@ class SoundcloudIE(InfoExtractor): 'uploader': 'garyvee', 'timestamp': 1488152409, 'upload_date': '20170226', - 'duration': 207, + 'duration': 207.012, 'thumbnail': r're:https?://.*\.jpg', 'license': 'all-rights-reserved', 'view_count': int, @@ -192,9 +197,31 @@ class SoundcloudIE(InfoExtractor): 'skip_download': True, }, }, + # not avaialble via api.soundcloud.com/i1/tracks/id/streams + { + 'url': 'https://soundcloud.com/giovannisarani/mezzo-valzer', + 'md5': 'e22aecd2bc88e0e4e432d7dcc0a1abf7', + 'info_dict': { + 'id': '583011102', + 'ext': 'mp3', + 'title': 'Mezzo Valzer', + 'description': 'md5:4138d582f81866a530317bae316e8b61', + 'uploader': 'Giovanni Sarani', + 'timestamp': 1551394171, + 'upload_date': '20190228', + 'duration': 180.157, + 'thumbnail': r're:https?://.*\.jpg', + 'license': 'all-rights-reserved', + 'view_count': int, + 'like_count': int, + 'comment_count': int, + 'repost_count': int, + }, + 'expected_warnings': ['Unable to download JSON metadata'], + } ] - _CLIENT_ID = 'NmW1FlPaiL94ueEu7oziOWjYEzZzQDcK' + _CLIENT_ID = 'FweeGBOOEOYJWLJN3oEyToGLKhmSz0I7' @staticmethod def _extract_urls(webpage): @@ -202,10 +229,6 @@ class SoundcloudIE(InfoExtractor): r']+src=(["\'])(?P(?:https?://)?(?:w\.)?soundcloud\.com/player.+?)\1', webpage)] - def report_resolve(self, video_id): - """Report information extraction.""" - self.to_screen('%s: Resolving id' % video_id) - @classmethod def _resolv_url(cls, url): return 'https://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID @@ -224,6 +247,10 @@ class SoundcloudIE(InfoExtractor): def extract_count(key): return int_or_none(info.get('%s_count' % key)) + like_count = extract_count('favoritings') + if like_count is None: + like_count = extract_count('likes') + result = { 'id': track_id, 'uploader': username, @@ -231,15 +258,17 @@ class SoundcloudIE(InfoExtractor): 'title': title, 'description': info.get('description'), 'thumbnail': thumbnail, - 'duration': int_or_none(info.get('duration'), 1000), + 'duration': float_or_none(info.get('duration'), 1000), 'webpage_url': info.get('permalink_url'), 'license': info.get('license'), 'view_count': extract_count('playback'), - 'like_count': extract_count('favoritings'), + 'like_count': like_count, 'comment_count': extract_count('comment'), 'repost_count': extract_count('reposts'), 'genre': info.get('genre'), } + + format_urls = set() formats = [] query = {'client_id': self._CLIENT_ID} if secret_token is not None: @@ -248,6 +277,7 @@ class SoundcloudIE(InfoExtractor): # We can build a direct link to the song format_url = update_url_query( 'https://api.soundcloud.com/tracks/%s/download' % track_id, query) + format_urls.add(format_url) formats.append({ 'format_id': 'download', 'ext': info.get('original_format', 'mp3'), @@ -256,44 +286,91 @@ class SoundcloudIE(InfoExtractor): 'preference': 10, }) - # We have to retrieve the url + # Old API, does not work for some tracks (e.g. + # https://soundcloud.com/giovannisarani/mezzo-valzer) format_dict = self._download_json( 'https://api.soundcloud.com/i1/tracks/%s/streams' % track_id, - track_id, 'Downloading track url', query=query) + track_id, 'Downloading track url', query=query, fatal=False) - for key, stream_url in format_dict.items(): - ext, abr = 'mp3', None - mobj = re.search(r'_([^_]+)_(\d+)_url', key) - if mobj: - ext, abr = mobj.groups() - abr = int(abr) - if key.startswith('http'): - stream_formats = [{ - 'format_id': key, - 'ext': ext, - 'url': stream_url, - }] - elif key.startswith('rtmp'): - # The url doesn't have an rtmp app, we have to extract the playpath - url, path = stream_url.split('mp3:', 1) - stream_formats = [{ - 'format_id': key, - 'url': url, - 'play_path': 'mp3:' + path, - 'ext': 'flv', - }] - elif key.startswith('hls'): - stream_formats = self._extract_m3u8_formats( - stream_url, track_id, ext, entry_protocol='m3u8_native', - m3u8_id=key, fatal=False) - else: + if format_dict: + for key, stream_url in format_dict.items(): + if stream_url in format_urls: + continue + format_urls.add(stream_url) + ext, abr = 'mp3', None + mobj = re.search(r'_([^_]+)_(\d+)_url', key) + if mobj: + ext, abr = mobj.groups() + abr = int(abr) + if key.startswith('http'): + stream_formats = [{ + 'format_id': key, + 'ext': ext, + 'url': stream_url, + }] + elif key.startswith('rtmp'): + # The url doesn't have an rtmp app, we have to extract the playpath + url, path = stream_url.split('mp3:', 1) + stream_formats = [{ + 'format_id': key, + 'url': url, + 'play_path': 'mp3:' + path, + 'ext': 'flv', + }] + elif key.startswith('hls'): + stream_formats = self._extract_m3u8_formats( + stream_url, track_id, ext, entry_protocol='m3u8_native', + m3u8_id=key, fatal=False) + else: + continue + + if abr: + for f in stream_formats: + f['abr'] = abr + + formats.extend(stream_formats) + + # New API + transcodings = try_get( + info, lambda x: x['media']['transcodings'], list) or [] + for t in transcodings: + if not isinstance(t, dict): continue - - if abr: - for f in stream_formats: - f['abr'] = abr - - formats.extend(stream_formats) + format_url = url_or_none(t.get('url')) + if not format_url: + continue + stream = self._download_json( + update_url_query(format_url, query), track_id, fatal=False) + if not isinstance(stream, dict): + continue + stream_url = url_or_none(stream.get('url')) + if not stream_url: + continue + if stream_url in format_urls: + continue + format_urls.add(stream_url) + protocol = try_get(t, lambda x: x['format']['protocol'], compat_str) + if protocol != 'hls' and '/hls' in format_url: + protocol = 'hls' + ext = None + preset = str_or_none(t.get('preset')) + if preset: + ext = preset.split('_')[0] + if ext not in KNOWN_EXTENSIONS: + mimetype = try_get( + t, lambda x: x['format']['mime_type'], compat_str) + ext = mimetype2ext(mimetype) or 'mp3' + format_id_list = [] + if protocol: + format_id_list.append(protocol) + format_id_list.append(ext) + format_id = '_'.join(format_id_list) + formats.append({ + 'url': stream_url, + 'format_id': format_id, + 'ext': ext, + 'protocol': 'm3u8_native' if protocol == 'hls' else 'http', + }) if not formats: # We fallback to the stream_url in the original info, this @@ -303,11 +380,11 @@ class SoundcloudIE(InfoExtractor): 'url': update_url_query(info['stream_url'], query), 'ext': 'mp3', }) + self._check_formats(formats, track_id) for f in formats: f['vcodec'] = 'none' - self._check_formats(formats, track_id) self._sort_formats(formats) result['formats'] = formats @@ -319,6 +396,7 @@ class SoundcloudIE(InfoExtractor): raise ExtractorError('Invalid URL: %s' % url) track_id = mobj.group('track_id') + new_info = {} if track_id is not None: info_json_url = 'https://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID @@ -344,13 +422,31 @@ class SoundcloudIE(InfoExtractor): if token: resolve_title += '/%s' % token - self.report_resolve(full_title) + webpage = self._download_webpage(url, full_title, fatal=False) + if webpage: + entries = self._parse_json( + self._search_regex( + r'var\s+c\s*=\s*(\[.+?\])\s*,\s*o\s*=Date\b', webpage, + 'data', default='[]'), full_title, fatal=False) + if entries: + for e in entries: + if not isinstance(e, dict): + continue + if e.get('id') != 67: + continue + data = try_get(e, lambda x: x['data'][0], dict) + if data: + new_info = data + break + info_json_url = self._resolv_url( + 'https://soundcloud.com/%s' % resolve_title) - url = 'https://soundcloud.com/%s' % resolve_title - info_json_url = self._resolv_url(url) - info = self._download_json(info_json_url, full_title, 'Downloading info JSON') + # Contains some additional info missing from new_info + info = self._download_json( + info_json_url, full_title, 'Downloading info JSON') - return self._extract_info_dict(info, full_title, secret_token=token) + return self._extract_info_dict( + merge_dicts(info, new_info), full_title, secret_token=token) class SoundcloudPlaylistBaseIE(SoundcloudIE): @@ -396,8 +492,6 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE): full_title += '/' + token url += '/' + token - self.report_resolve(full_title) - resolv_url = self._resolv_url(url) info = self._download_json(resolv_url, full_title) From 15be3eb5e526c232a36f5f73fd5e586572b06fc5 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Mon, 22 Apr 2019 20:31:09 +0100 Subject: [PATCH 10/45] [youtube] raise ExtractorError when no data available(#20737) --- youtube_dl/extractor/youtube.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 132572c88..67eceb5c4 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1671,6 +1671,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor): r'(?s)]+id="unavailable-message"[^>]*>(.+?)', video_webpage, 'unavailable message', default=None) + if not video_info: + unavailable_message = extract_unavailable_message() + if not unavailable_message: + unavailable_message = 'Unable to extract video data' + raise ExtractorError( + 'YouTube said: %s' % unavailable_message, expected=True, video_id=video_id) + if 'token' not in video_info: if 'reason' in video_info: if 'The uploader has not made this video available in your country.' in video_info['reason']: From 1fa8893734154cd2144a966c89a9f3801103c6f0 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Mon, 22 Apr 2019 23:50:37 +0100 Subject: [PATCH 11/45] [vrv] add support for movie listings(closes #19229) --- youtube_dl/extractor/vrv.py | 38 +++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py index c11da97de..33530fe8a 100644 --- a/youtube_dl/extractor/vrv.py +++ b/youtube_dl/extractor/vrv.py @@ -102,6 +102,15 @@ class VRVIE(VRVBaseIE): # m3u8 download 'skip_download': True, }, + }, { + # movie listing + 'url': 'https://vrv.co/watch/G6NQXZ1J6/Lily-CAT', + 'info_dict': { + 'id': 'G6NQXZ1J6', + 'title': 'Lily C.A.T', + 'description': 'md5:988b031e7809a6aeb60968be4af7db07', + }, + 'playlist_count': 2, }] _NETRC_MACHINE = 'vrv' @@ -123,23 +132,23 @@ class VRVIE(VRVBaseIE): def _extract_vrv_formats(self, url, video_id, stream_format, audio_lang, hardsub_lang): if not url or stream_format not in ('hls', 'dash'): return [] - assert audio_lang or hardsub_lang stream_id_list = [] if audio_lang: stream_id_list.append('audio-%s' % audio_lang) if hardsub_lang: stream_id_list.append('hardsub-%s' % hardsub_lang) - stream_id = '-'.join(stream_id_list) - format_id = '%s-%s' % (stream_format, stream_id) + format_id = stream_format + if stream_id_list: + format_id += '-' + '-'.join(stream_id_list) if stream_format == 'hls': adaptive_formats = self._extract_m3u8_formats( url, video_id, 'mp4', m3u8_id=format_id, - note='Downloading %s m3u8 information' % stream_id, + note='Downloading %s information' % format_id, fatal=False) elif stream_format == 'dash': adaptive_formats = self._extract_mpd_formats( url, video_id, mpd_id=format_id, - note='Downloading %s MPD information' % stream_id, + note='Downloading %s information' % format_id, fatal=False) if audio_lang: for f in adaptive_formats: @@ -155,6 +164,23 @@ class VRVIE(VRVBaseIE): resource_path = object_data['__links__']['resource']['href'] video_data = self._call_cms(resource_path, video_id, 'video') title = video_data['title'] + description = video_data.get('description') + + if video_data.get('__class__') == 'movie_listing': + items = self._call_cms( + video_data['__links__']['movie_listing/movies']['href'], + video_id, 'movie listing').get('items') or [] + if len(items) != 1: + entries = [] + for item in items: + item_id = item.get('id') + if not item_id: + continue + entries.append(self.url_result( + 'https://vrv.co/watch/' + item_id, + self.ie_key(), item_id, item.get('title'))) + return self.playlist_result(entries, video_id, title, description) + video_data = items[0] streams_path = video_data['__links__'].get('streams', {}).get('href') if not streams_path: @@ -198,7 +224,7 @@ class VRVIE(VRVBaseIE): 'formats': formats, 'subtitles': subtitles, 'thumbnails': thumbnails, - 'description': video_data.get('description'), + 'description': description, 'duration': float_or_none(video_data.get('duration_ms'), 1000), 'uploader_id': video_data.get('channel_id'), 'series': video_data.get('series_title'), From 50d660479d03c0c4f1dd6b094d976c21751b8549 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Wed, 24 Apr 2019 00:28:00 +0100 Subject: [PATCH 12/45] [toutv] fix extraction and extract series info(closes #20757) --- youtube_dl/extractor/toutv.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py index 8cc57b919..00f58a087 100644 --- a/youtube_dl/extractor/toutv.py +++ b/youtube_dl/extractor/toutv.py @@ -66,7 +66,12 @@ class TouTvIE(RadioCanadaIE): def _real_extract(self, url): path = self._match_id(url) - metadata = self._download_json('http://ici.tou.tv/presentation/%s' % path, path) + metadata = self._download_json( + 'https://services.radio-canada.ca/toutv/presentation/%s' % path, path, query={ + 'client_key': self._CLIENT_KEY, + 'device': 'web', + 'version': 4, + }) # IsDrm does not necessarily mean the video is DRM protected (see # https://github.com/ytdl-org/youtube-dl/issues/13994). if metadata.get('IsDrm'): @@ -77,6 +82,12 @@ class TouTvIE(RadioCanadaIE): return merge_dicts({ 'id': video_id, 'title': details.get('OriginalTitle'), + 'description': details.get('Description'), 'thumbnail': details.get('ImageUrl'), 'duration': int_or_none(details.get('LengthInSeconds')), + 'series': metadata.get('ProgramTitle'), + 'season_number': int_or_none(metadata.get('SeasonNumber')), + 'season': metadata.get('SeasonTitle'), + 'episode_number': int_or_none(metadata.get('EpisodeNumber')), + 'episode': metadata.get('EpisodeTitle'), }, self._extract_info(metadata.get('AppCode', 'toutv'), video_id)) From 56667d622c3f6e7594a04f8cd5f4371875940725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 24 Apr 2019 09:58:00 +0700 Subject: [PATCH 13/45] [youtube] Fix extraction (closes #20758, closes #20759, closes #20761, closes #20762, closes #20764, closes #20766, closes #20767, closes #20769, closes #20771, closes #20768, closes #20770) --- youtube_dl/extractor/youtube.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 67eceb5c4..1bc2c27ad 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1652,7 +1652,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): view_count = extract_view_count(get_video_info) if not video_info: video_info = get_video_info - if 'token' in get_video_info: + get_token = get_video_info.get('token') or get_video_info.get('account_playback_token') + if get_token: # Different get_video_info requests may report different results, e.g. # some may report video unavailability, but some may serve it without # any complaint (see https://github.com/ytdl-org/youtube-dl/issues/7362, @@ -1662,7 +1663,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # due to YouTube measures against IP ranges of hosting providers. # Working around by preferring the first succeeded video_info containing # the token if no such video_info yet was found. - if 'token' not in video_info: + token = video_info.get('token') or video_info.get('account_playback_token') + if not token: video_info = get_video_info break @@ -1678,7 +1680,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): raise ExtractorError( 'YouTube said: %s' % unavailable_message, expected=True, video_id=video_id) - if 'token' not in video_info: + token = video_info.get('token') or video_info.get('account_playback_token') + if not token: if 'reason' in video_info: if 'The uploader has not made this video available in your country.' in video_info['reason']: regions_allowed = self._html_search_meta( From 98933c14e1d950b8b55a2a6278e3e002484ef56d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 24 Apr 2019 10:05:08 +0700 Subject: [PATCH 14/45] [ChangeLog] Actualize [ci skip] --- ChangeLog | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8365ea6de..9467c5e6f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,25 @@ +version + +Extractors +* [youtube] Fix extraction (#20758, #20759, #20761, #20762, #20764, #20766, + #20767, #20769, #20771, #20768, #20770) +* [toutv] Fix extraction and extract series info (#20757) ++ [vrv] Add support for movie listings (#19229) ++ [youtube] Print error when no data is available (#20737) ++ [soundcloud] Add support for new rendition and improve extraction (#20699) ++ [ooyala] Add support for geo verification proxy ++ [nrl] Add support for nrl.com (#15991) ++ [vimeo] Extract live archive source format (#19144) ++ [vimeo] Add support for live streams and improve info extraction (#19144) ++ [ntvcojp] Add support for cu.ntv.co.jp ++ [nhk] Extract RTMPT format ++ [nhk] Add support for audio URLs ++ [udemy] Add another course id extraction pattern (#20491) ++ [openload] Add support for oload.services (#20691) ++ [openload] Add support for openloed.co (#20691, #20693) +* [bravotv] Fix extraction (#19213) + + version 2019.04.17 Extractors From 3e7ec5330acf29845a1c3851c3cbfd8777ad0ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 24 Apr 2019 10:05:54 +0700 Subject: [PATCH 15/45] release 2019.04.24 --- .github/ISSUE_TEMPLATE.md | 6 +++--- ChangeLog | 2 +- docs/supportedsites.md | 2 ++ youtube_dl/version.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index d63aaad06..7123c7b50 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -6,8 +6,8 @@ --- -### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.04.17*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected. -- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.04.17** +### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.04.24*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected. +- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.04.24** ### Before submitting an *issue* make sure you have: - [ ] At least skimmed through the [README](https://github.com/ytdl-org/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/ytdl-org/youtube-dl#faq) and [BUGS](https://github.com/ytdl-org/youtube-dl#bugs) sections @@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 -[debug] youtube-dl version 2019.04.17 +[debug] youtube-dl version 2019.04.24 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/ChangeLog b/ChangeLog index 9467c5e6f..1a71e2fff 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -version +version 2019.04.24 Extractors * [youtube] Fix extraction (#20758, #20759, #20761, #20762, #20764, #20766, diff --git a/docs/supportedsites.md b/docs/supportedsites.md index c5419fd42..6a320306b 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -201,6 +201,7 @@ - **CSpan**: C-SPAN - **CtsNews**: 華視新聞 - **CTVNews** + - **cu.ntv.co.jp**: Nippon Television Network - **Culturebox** - **CultureUnplugged** - **curiositystream** @@ -624,6 +625,7 @@ - **NRKTVEpisodes** - **NRKTVSeason** - **NRKTVSeries** + - **NRLTV** - **ntv.ru** - **Nuvid** - **NYTimes** diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 4f2ba8e47..ddd3b692a 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2019.04.17' +__version__ = '2019.04.24' From 58ef5e788100dc338390e345a2744600e02e0f5c Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Thu, 25 Apr 2019 11:36:44 +0100 Subject: [PATCH 16/45] [yandexmusic] fix track url extraction(closes #20820) --- youtube_dl/extractor/yandexmusic.py | 51 +++++++++++++---------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py index 456f95f69..1dfee59e9 100644 --- a/youtube_dl/extractor/yandexmusic.py +++ b/youtube_dl/extractor/yandexmusic.py @@ -69,25 +69,28 @@ class YandexMusicTrackIE(YandexMusicBaseIE): 'skip': 'Travis CI servers blocked by YandexMusic', } - def _get_track_url(self, storage_dir, track_id): - data = self._download_json( - 'http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?action=getTrackSrc&p=download-info/%s' - % storage_dir, - track_id, 'Downloading track location JSON') + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + album_id, track_id = mobj.group('album_id'), mobj.group('id') - # Each string is now wrapped in a list, this is probably only temporarily thus - # supporting both scenarios (see https://github.com/ytdl-org/youtube-dl/issues/10193) - for k, v in data.items(): - if v and isinstance(v, list): - data[k] = v[0] + track = self._download_json( + 'http://music.yandex.ru/handlers/track.jsx?track=%s:%s' % (track_id, album_id), + track_id, 'Downloading track JSON')['track'] + track_title = track['title'] - key = hashlib.md5(('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s']).encode('utf-8')).hexdigest() - storage = storage_dir.split('.') + download_data = self._download_json( + 'https://music.yandex.ru/api/v2.1/handlers/track/%s:%s/web-album_track-track-track-main/download/m' % (track_id, album_id), + track_id, 'Downloading track location url JSON', + headers={'X-Retpath-Y': url}) - return ('http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default' - % (data['host'], key, data['ts'] + data['path'], storage[1])) + fd_data = self._download_json( + download_data['src'], track_id, + 'Downloading track location JSON', + query={'format': 'json'}) + key = hashlib.md5(('XGRlBW9FXlekgbPrRHuSiA' + fd_data['path'][1:] + fd_data['s']).encode('utf-8')).hexdigest() + storage = track['storageDir'].split('.') + f_url = 'http://%s/get-mp3/%s/%s?track-id=%s ' % (fd_data['host'], key, fd_data['ts'] + fd_data['path'], storage[1]) - def _get_track_info(self, track): thumbnail = None cover_uri = track.get('albums', [{}])[0].get('coverUri') if cover_uri: @@ -95,15 +98,16 @@ class YandexMusicTrackIE(YandexMusicBaseIE): if not thumbnail.startswith('http'): thumbnail = 'http://' + thumbnail - track_title = track['title'] track_info = { - 'id': track['id'], + 'id': track_id, 'ext': 'mp3', - 'url': self._get_track_url(track['storageDir'], track['id']), + 'url': f_url, 'filesize': int_or_none(track.get('fileSize')), 'duration': float_or_none(track.get('durationMs'), 1000), 'thumbnail': thumbnail, 'track': track_title, + 'acodec': download_data.get('codec'), + 'abr': int_or_none(download_data.get('bitrate')), } def extract_artist(artist_list): @@ -131,18 +135,9 @@ class YandexMusicTrackIE(YandexMusicBaseIE): }) else: track_info['title'] = track_title + return track_info - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - album_id, track_id = mobj.group('album_id'), mobj.group('id') - - track = self._download_json( - 'http://music.yandex.ru/handlers/track.jsx?track=%s:%s' % (track_id, album_id), - track_id, 'Downloading track JSON')['track'] - - return self._get_track_info(track) - class YandexMusicPlaylistBaseIE(YandexMusicBaseIE): def _build_playlist(self, tracks): From da668a23bdc24dbd4bd289497fb7a258d9b8b2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 26 Apr 2019 00:46:41 +0700 Subject: [PATCH 17/45] [ISSUE_TEMPLATE.md] Add entry on argument escaping in make-sure checklist [ci skip] --- .github/ISSUE_TEMPLATE.md | 1 + .github/ISSUE_TEMPLATE_tmpl.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 7123c7b50..6911e2d5c 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -13,6 +13,7 @@ - [ ] At least skimmed through the [README](https://github.com/ytdl-org/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/ytdl-org/youtube-dl#faq) and [BUGS](https://github.com/ytdl-org/youtube-dl#bugs) sections - [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones - [ ] Checked that provided video/audio/playlist URLs (if any) are alive and playable in a browser +- [ ] Checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) ### What is the purpose of your *issue*? - [ ] Bug report (encountered problems with youtube-dl) diff --git a/.github/ISSUE_TEMPLATE_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl.md index 8b7e73417..bbd79afb0 100644 --- a/.github/ISSUE_TEMPLATE_tmpl.md +++ b/.github/ISSUE_TEMPLATE_tmpl.md @@ -13,6 +13,7 @@ - [ ] At least skimmed through the [README](https://github.com/ytdl-org/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/ytdl-org/youtube-dl#faq) and [BUGS](https://github.com/ytdl-org/youtube-dl#bugs) sections - [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones - [ ] Checked that provided video/audio/playlist URLs (if any) are alive and playable in a browser +- [ ] Checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) ### What is the purpose of your *issue*? - [ ] Bug report (encountered problems with youtube-dl) From 97abf05ad305a5c06e72a5d368c00722d867433b Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Fri, 26 Apr 2019 10:26:51 +0100 Subject: [PATCH 18/45] [reddit] check thumbnail URL(closes #20030) --- youtube_dl/extractor/reddit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/reddit.py b/youtube_dl/extractor/reddit.py index 7b0aa6232..663f622b3 100644 --- a/youtube_dl/extractor/reddit.py +++ b/youtube_dl/extractor/reddit.py @@ -7,6 +7,7 @@ from ..utils import ( ExtractorError, int_or_none, float_or_none, + url_or_none, ) @@ -119,7 +120,7 @@ class RedditRIE(InfoExtractor): '_type': 'url_transparent', 'url': video_url, 'title': data.get('title'), - 'thumbnail': data.get('thumbnail'), + 'thumbnail': url_or_none(data.get('thumbnail')), 'timestamp': float_or_none(data.get('created_utc')), 'uploader': data.get('author'), 'like_count': int_or_none(data.get('ups')), From 00a9a25cf9ff72a83aaef7dae6e4e16296f74d89 Mon Sep 17 00:00:00 2001 From: Mao Zedong <42905588+maozed@users.noreply.github.com> Date: Fri, 26 Apr 2019 18:34:23 +0900 Subject: [PATCH 19/45] [twitcasting] Fix test: video title (#20840) --- youtube_dl/extractor/twitcasting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/twitcasting.py b/youtube_dl/extractor/twitcasting.py index 05f8aa9ce..44cda6f51 100644 --- a/youtube_dl/extractor/twitcasting.py +++ b/youtube_dl/extractor/twitcasting.py @@ -14,7 +14,7 @@ class TwitCastingIE(InfoExtractor): 'info_dict': { 'id': '2357609', 'ext': 'mp4', - 'title': 'Recorded Live #2357609', + 'title': 'Live #2357609', 'uploader_id': 'ivetesangalo', 'description': "Moi! I'm live on TwitCasting from my iPhone.", 'thumbnail': r're:^https?://.*\.jpg$', From 88b547492f5d9440072b0037494402f46aabc810 Mon Sep 17 00:00:00 2001 From: Mao Zedong <42905588+maozed@users.noreply.github.com> Date: Sat, 27 Apr 2019 01:17:40 +0900 Subject: [PATCH 20/45] [twitcasting] Add support for private videos (#20843) --- youtube_dl/extractor/twitcasting.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/twitcasting.py b/youtube_dl/extractor/twitcasting.py index 44cda6f51..2dbe89f5b 100644 --- a/youtube_dl/extractor/twitcasting.py +++ b/youtube_dl/extractor/twitcasting.py @@ -2,13 +2,14 @@ from __future__ import unicode_literals from .common import InfoExtractor +from ..utils import urlencode_postdata import re class TwitCastingIE(InfoExtractor): _VALID_URL = r'https?://(?:[^/]+\.)?twitcasting\.tv/(?P[^/]+)/movie/(?P\d+)' - _TEST = { + _TESTS = [{ 'url': 'https://twitcasting.tv/ivetesangalo/movie/2357609', 'md5': '745243cad58c4681dc752490f7540d7f', 'info_dict': { @@ -22,14 +23,34 @@ class TwitCastingIE(InfoExtractor): 'params': { 'skip_download': True, }, - } + }, { + 'url': 'https://twitcasting.tv/mttbernardini/movie/3689740', + 'info_dict': { + 'id': '3689740', + 'ext': 'mp4', + 'title': 'Live playing something #3689740', + 'uploader_id': 'mttbernardini', + 'description': "I'm live on TwitCasting from my iPad. password: abc (Santa Marinella/Lazio, Italia)", + 'thumbnail': r're:^https?://.*\.jpg$', + }, + 'params': { + 'skip_download': True, + 'videopassword': 'abc', + }, + }] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') uploader_id = mobj.group('uploader_id') - webpage = self._download_webpage(url, video_id) + video_password = self._downloader.params.get('videopassword') + request_data = None + if video_password: + request_data = urlencode_postdata({ + 'password': video_password, + }) + webpage = self._download_webpage(url, video_id, data=request_data) title = self._html_search_regex( r'(?s)<[^>]+id=["\']movietitle[^>]+>(.+?) Date: Sat, 27 Apr 2019 04:22:35 +0700 Subject: [PATCH 21/45] Issue template overhaul --- .github/ISSUE_TEMPLATE.md | 62 ------------------ .github/ISSUE_TEMPLATE/1_broken_site.md | 63 ++++++++++++++++++ .github/ISSUE_TEMPLATE/1_broken_site_tmpl.md | 63 ++++++++++++++++++ .../ISSUE_TEMPLATE/2_site_support_request.md | 54 +++++++++++++++ .../2_site_support_request_tmpl.md | 54 +++++++++++++++ .../ISSUE_TEMPLATE/3_site_feature_request.md | 37 +++++++++++ .../3_site_feature_request_tmpl.md | 37 +++++++++++ .github/ISSUE_TEMPLATE/4_bug_report.md | 65 +++++++++++++++++++ .github/ISSUE_TEMPLATE/4_bug_report_tmpl.md | 65 +++++++++++++++++++ .github/ISSUE_TEMPLATE/5_feature_request.md | 38 +++++++++++ .../ISSUE_TEMPLATE/5_feature_request_tmpl.md | 38 +++++++++++ .github/ISSUE_TEMPLATE/6_question.md | 38 +++++++++++ .github/ISSUE_TEMPLATE_tmpl.md | 62 ------------------ Makefile | 10 ++- devscripts/release.sh | 4 +- 15 files changed, 561 insertions(+), 129 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/1_broken_site.md create mode 100644 .github/ISSUE_TEMPLATE/1_broken_site_tmpl.md create mode 100644 .github/ISSUE_TEMPLATE/2_site_support_request.md create mode 100644 .github/ISSUE_TEMPLATE/2_site_support_request_tmpl.md create mode 100644 .github/ISSUE_TEMPLATE/3_site_feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/3_site_feature_request_tmpl.md create mode 100644 .github/ISSUE_TEMPLATE/4_bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/4_bug_report_tmpl.md create mode 100644 .github/ISSUE_TEMPLATE/5_feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/5_feature_request_tmpl.md create mode 100644 .github/ISSUE_TEMPLATE/6_question.md delete mode 100644 .github/ISSUE_TEMPLATE_tmpl.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 6911e2d5c..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,62 +0,0 @@ -## Please follow the guide below - -- You will be asked some questions and requested to provide some information, please read them **carefully** and answer honestly -- Put an `x` into all the boxes [ ] relevant to your *issue* (like this: `[x]`) -- Use the *Preview* tab to see what your issue will actually look like - ---- - -### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2019.04.24*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected. -- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2019.04.24** - -### Before submitting an *issue* make sure you have: -- [ ] At least skimmed through the [README](https://github.com/ytdl-org/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/ytdl-org/youtube-dl#faq) and [BUGS](https://github.com/ytdl-org/youtube-dl#bugs) sections -- [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones -- [ ] Checked that provided video/audio/playlist URLs (if any) are alive and playable in a browser -- [ ] Checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) - -### What is the purpose of your *issue*? -- [ ] Bug report (encountered problems with youtube-dl) -- [ ] Site support request (request for adding support for a new site) -- [ ] Feature request (request for a new functionality) -- [ ] Question -- [ ] Other - ---- - -### The following sections concretize particular purposed issues, you can erase any section (the contents between triple ---) not applicable to your *issue* - ---- - -### If the purpose of this *issue* is a *bug report*, *site support request* or you are not completely sure provide the full verbose output as follows: - -Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl -v `), copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```): - -``` -[debug] System config: [] -[debug] User config: [] -[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] -[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 -[debug] youtube-dl version 2019.04.24 -[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 -[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 -[debug] Proxy map: {} -... - -``` - ---- - -### If the purpose of this *issue* is a *site support request* please provide all kinds of example URLs support for which should be included (replace following example URLs by **yours**): -- Single video: https://www.youtube.com/watch?v=BaW_jenozKc -- Single video: https://youtu.be/BaW_jenozKc -- Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc - -Note that **youtube-dl does not support sites dedicated to [copyright infringement](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. In order for site support request to be accepted all provided example URLs should not violate any copyrights. - ---- - -### Description of your *issue*, suggested solution and other information - -Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible. -If work on your *issue* requires account credentials please provide them or explain how one can obtain them. diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md new file mode 100644 index 000000000..bab917400 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -0,0 +1,63 @@ +--- +name: Broken site support +about: Report broken or misfunctioning site +title: '' +--- + + + + +## Checklist + + + +- [ ] I'm reporting a broken site support +- [ ] I've verified that I'm running youtube-dl version **2019.04.24** +- [ ] I've checked that all provided URLs are alive and playable in a browser +- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped +- [ ] I've searched the bugtracker for similar issues including closed ones + + +## Verbose log + + + +``` +PASTE VERBOSE LOG HERE +``` + + +## Description + + + +WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/1_broken_site_tmpl.md b/.github/ISSUE_TEMPLATE/1_broken_site_tmpl.md new file mode 100644 index 000000000..c7600d5b5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1_broken_site_tmpl.md @@ -0,0 +1,63 @@ +--- +name: Broken site support +about: Report broken or misfunctioning site +title: '' +--- + + + + +## Checklist + + + +- [ ] I'm reporting a broken site support +- [ ] I've verified that I'm running youtube-dl version **%(version)s** +- [ ] I've checked that all provided URLs are alive and playable in a browser +- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped +- [ ] I've searched the bugtracker for similar issues including closed ones + + +## Verbose log + + + +``` +PASTE VERBOSE LOG HERE +``` + + +## Description + + + +WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md new file mode 100644 index 000000000..7d78921ee --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -0,0 +1,54 @@ +--- +name: Site support request +about: Request support for a new site +title: '' +labels: 'site-support-request' +--- + + + + +## Checklist + + + +- [ ] I'm reporting a new site support request +- [ ] I've verified that I'm running youtube-dl version **2019.04.24** +- [ ] I've checked that all provided URLs are alive and playable in a browser +- [ ] I've checked that none of provided URLs violate any copyrights +- [ ] I've searched the bugtracker for similar site support requests including closed ones + + +## Example URLs + + + +- Single video: https://www.youtube.com/watch?v=BaW_jenozKc +- Single video: https://youtu.be/BaW_jenozKc +- Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc + + +## Description + + + +WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request_tmpl.md b/.github/ISSUE_TEMPLATE/2_site_support_request_tmpl.md new file mode 100644 index 000000000..d4988e639 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2_site_support_request_tmpl.md @@ -0,0 +1,54 @@ +--- +name: Site support request +about: Request support for a new site +title: '' +labels: 'site-support-request' +--- + + + + +## Checklist + + + +- [ ] I'm reporting a new site support request +- [ ] I've verified that I'm running youtube-dl version **%(version)s** +- [ ] I've checked that all provided URLs are alive and playable in a browser +- [ ] I've checked that none of provided URLs violate any copyrights +- [ ] I've searched the bugtracker for similar site support requests including closed ones + + +## Example URLs + + + +- Single video: https://www.youtube.com/watch?v=BaW_jenozKc +- Single video: https://youtu.be/BaW_jenozKc +- Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc + + +## Description + + + +WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md new file mode 100644 index 000000000..0ed4d1d6a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -0,0 +1,37 @@ +--- +name: Site feature request +about: Request a new functionality for a site +title: '' +--- + + + + +## Checklist + + + +- [ ] I'm reporting a site feature request +- [ ] I've verified that I'm running youtube-dl version **2019.04.24** +- [ ] I've searched the bugtracker for similar site feature requests including closed ones + + +## Description + + + +WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request_tmpl.md b/.github/ISSUE_TEMPLATE/3_site_feature_request_tmpl.md new file mode 100644 index 000000000..65f0a32f3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request_tmpl.md @@ -0,0 +1,37 @@ +--- +name: Site feature request +about: Request a new functionality for a site +title: '' +--- + + + + +## Checklist + + + +- [ ] I'm reporting a site feature request +- [ ] I've verified that I'm running youtube-dl version **%(version)s** +- [ ] I've searched the bugtracker for similar site feature requests including closed ones + + +## Description + + + +WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md new file mode 100644 index 000000000..fd9b09d6f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -0,0 +1,65 @@ +--- +name: Bug report +about: Report a bug unrelated to any particular site or extractor +title: '' +--- + + + + +## Checklist + + + +- [ ] I'm reporting a broken site support issue +- [ ] I've verified that I'm running youtube-dl version **2019.04.24** +- [ ] I've checked that all provided URLs are alive and playable in a browser +- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped +- [ ] I've searched the bugtracker for similar bug reports including closed ones +- [ ] I've read bugs section in FAQ + + +## Verbose log + + + +``` +PASTE VERBOSE LOG HERE +``` + + +## Description + + + +WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/4_bug_report_tmpl.md b/.github/ISSUE_TEMPLATE/4_bug_report_tmpl.md new file mode 100644 index 000000000..41fb14b72 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/4_bug_report_tmpl.md @@ -0,0 +1,65 @@ +--- +name: Bug report +about: Report a bug unrelated to any particular site or extractor +title: '' +--- + + + + +## Checklist + + + +- [ ] I'm reporting a broken site support issue +- [ ] I've verified that I'm running youtube-dl version **%(version)s** +- [ ] I've checked that all provided URLs are alive and playable in a browser +- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped +- [ ] I've searched the bugtracker for similar bug reports including closed ones +- [ ] I've read bugs section in FAQ + + +## Verbose log + + + +``` +PASTE VERBOSE LOG HERE +``` + + +## Description + + + +WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md new file mode 100644 index 000000000..94f373d4e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -0,0 +1,38 @@ +--- +name: Feature request +about: Request a new functionality unrelated to any particular site or extractor +title: '' +labels: 'request' +--- + + + + +## Checklist + + + +- [ ] I'm reporting a feature request +- [ ] I've verified that I'm running youtube-dl version **2019.04.24** +- [ ] I've searched the bugtracker for similar feature requests including closed ones + + +## Description + + + +WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/5_feature_request_tmpl.md b/.github/ISSUE_TEMPLATE/5_feature_request_tmpl.md new file mode 100644 index 000000000..b3431a7f0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/5_feature_request_tmpl.md @@ -0,0 +1,38 @@ +--- +name: Feature request +about: Request a new functionality unrelated to any particular site or extractor +title: '' +labels: 'request' +--- + + + + +## Checklist + + + +- [ ] I'm reporting a feature request +- [ ] I've verified that I'm running youtube-dl version **%(version)s** +- [ ] I've searched the bugtracker for similar feature requests including closed ones + + +## Description + + + +WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/6_question.md b/.github/ISSUE_TEMPLATE/6_question.md new file mode 100644 index 000000000..1fd7cd5dc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/6_question.md @@ -0,0 +1,38 @@ +--- +name: Ask question +about: Ask youtube-dl related question +title: '' +labels: 'question' +--- + + + + +## Checklist + + + +- [ ] I'm asking a question +- [ ] I've looked through the README and FAQ for similar questions +- [ ] I've searched the bugtracker for similar questions including closed ones + + +## Question + + + +WRITE QUESTION HERE diff --git a/.github/ISSUE_TEMPLATE_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl.md deleted file mode 100644 index bbd79afb0..000000000 --- a/.github/ISSUE_TEMPLATE_tmpl.md +++ /dev/null @@ -1,62 +0,0 @@ -## Please follow the guide below - -- You will be asked some questions and requested to provide some information, please read them **carefully** and answer honestly -- Put an `x` into all the boxes [ ] relevant to your *issue* (like this: `[x]`) -- Use the *Preview* tab to see what your issue will actually look like - ---- - -### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *%(version)s*. If it's not, read [this FAQ entry](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected. -- [ ] I've **verified** and **I assure** that I'm running youtube-dl **%(version)s** - -### Before submitting an *issue* make sure you have: -- [ ] At least skimmed through the [README](https://github.com/ytdl-org/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/ytdl-org/youtube-dl#faq) and [BUGS](https://github.com/ytdl-org/youtube-dl#bugs) sections -- [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones -- [ ] Checked that provided video/audio/playlist URLs (if any) are alive and playable in a browser -- [ ] Checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) - -### What is the purpose of your *issue*? -- [ ] Bug report (encountered problems with youtube-dl) -- [ ] Site support request (request for adding support for a new site) -- [ ] Feature request (request for a new functionality) -- [ ] Question -- [ ] Other - ---- - -### The following sections concretize particular purposed issues, you can erase any section (the contents between triple ---) not applicable to your *issue* - ---- - -### If the purpose of this *issue* is a *bug report*, *site support request* or you are not completely sure provide the full verbose output as follows: - -Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl -v `), copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```): - -``` -[debug] System config: [] -[debug] User config: [] -[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] -[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 -[debug] youtube-dl version %(version)s -[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 -[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 -[debug] Proxy map: {} -... - -``` - ---- - -### If the purpose of this *issue* is a *site support request* please provide all kinds of example URLs support for which should be included (replace following example URLs by **yours**): -- Single video: https://www.youtube.com/watch?v=BaW_jenozKc -- Single video: https://youtu.be/BaW_jenozKc -- Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc - -Note that **youtube-dl does not support sites dedicated to [copyright infringement](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. In order for site support request to be accepted all provided example URLs should not violate any copyrights. - ---- - -### Description of your *issue*, suggested solution and other information - -Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible. -If work on your *issue* requires account credentials please provide them or explain how one can obtain them. diff --git a/Makefile b/Makefile index 4a62f44bc..8658a2af1 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites clean: - rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe + rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe find . -name "*.pyc" -delete find . -name "*.class" -delete @@ -78,8 +78,12 @@ README.md: youtube_dl/*.py youtube_dl/*/*.py CONTRIBUTING.md: README.md $(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md -.github/ISSUE_TEMPLATE.md: devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl.md youtube_dl/version.py - $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl.md .github/ISSUE_TEMPLATE.md +issuetemplates: devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/1_broken_site_tmpl.md .github/ISSUE_TEMPLATE/2_site_support_request_tmpl.md .github/ISSUE_TEMPLATE/3_site_feature_request_tmpl.md .github/ISSUE_TEMPLATE/4_bug_report_tmpl.md .github/ISSUE_TEMPLATE/5_feature_request_tmpl.md youtube_dl/version.py + $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/1_broken_site_tmpl.md .github/ISSUE_TEMPLATE/1_broken_site.md + $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/2_site_support_request_tmpl.md .github/ISSUE_TEMPLATE/2_site_support_request.md + $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/3_site_feature_request_tmpl.md .github/ISSUE_TEMPLATE/3_site_feature_request.md + $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/4_bug_report_tmpl.md .github/ISSUE_TEMPLATE/4_bug_report.md + $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/5_feature_request_tmpl.md .github/ISSUE_TEMPLATE/5_feature_request.md supportedsites: $(PYTHON) devscripts/make_supportedsites.py docs/supportedsites.md diff --git a/devscripts/release.sh b/devscripts/release.sh index 4c413bf6d..f2411c927 100755 --- a/devscripts/release.sh +++ b/devscripts/release.sh @@ -78,8 +78,8 @@ sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py sed -i "s//$version/" ChangeLog /bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..." -make README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md supportedsites -git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md docs/supportedsites.md youtube_dl/version.py ChangeLog +make README.md CONTRIBUTING.md issuetemplates supportedsites +git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE/1_broken_site.md .github/ISSUE_TEMPLATE/2_site_support_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md .github/ISSUE_TEMPLATE/4_bug_report.md .github/ISSUE_TEMPLATE/5_feature_request.md .github/ISSUE_TEMPLATE/6_question.md docs/supportedsites.md youtube_dl/version.py ChangeLog git commit $gpg_sign_commits -m "release $version" /bin/echo -e "\n### Now tagging, signing and pushing..." From eefa0f21573fe6c21996f7d4d90806dfed3adb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 27 Apr 2019 04:55:30 +0700 Subject: [PATCH 22/45] Move issue template templates into separate folder --- .../1_broken_site.md} | 0 .../2_site_support_request.md} | 0 .../3_site_feature_request.md} | 0 .../4_bug_report.md} | 0 .../5_feature_request.md} | 0 Makefile | 12 ++++++------ 6 files changed, 6 insertions(+), 6 deletions(-) rename .github/{ISSUE_TEMPLATE/1_broken_site_tmpl.md => ISSUE_TEMPLATE_tmpl/1_broken_site.md} (100%) rename .github/{ISSUE_TEMPLATE/2_site_support_request_tmpl.md => ISSUE_TEMPLATE_tmpl/2_site_support_request.md} (100%) rename .github/{ISSUE_TEMPLATE/3_site_feature_request_tmpl.md => ISSUE_TEMPLATE_tmpl/3_site_feature_request.md} (100%) rename .github/{ISSUE_TEMPLATE/4_bug_report_tmpl.md => ISSUE_TEMPLATE_tmpl/4_bug_report.md} (100%) rename .github/{ISSUE_TEMPLATE/5_feature_request_tmpl.md => ISSUE_TEMPLATE_tmpl/5_feature_request.md} (100%) diff --git a/.github/ISSUE_TEMPLATE/1_broken_site_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.md similarity index 100% rename from .github/ISSUE_TEMPLATE/1_broken_site_tmpl.md rename to .github/ISSUE_TEMPLATE_tmpl/1_broken_site.md diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/2_site_support_request_tmpl.md rename to .github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/3_site_feature_request_tmpl.md rename to .github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md diff --git a/.github/ISSUE_TEMPLATE/4_bug_report_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/4_bug_report_tmpl.md rename to .github/ISSUE_TEMPLATE_tmpl/4_bug_report.md diff --git a/.github/ISSUE_TEMPLATE/5_feature_request_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/5_feature_request_tmpl.md rename to .github/ISSUE_TEMPLATE_tmpl/5_feature_request.md diff --git a/Makefile b/Makefile index 8658a2af1..3e17365b8 100644 --- a/Makefile +++ b/Makefile @@ -78,12 +78,12 @@ README.md: youtube_dl/*.py youtube_dl/*/*.py CONTRIBUTING.md: README.md $(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md -issuetemplates: devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/1_broken_site_tmpl.md .github/ISSUE_TEMPLATE/2_site_support_request_tmpl.md .github/ISSUE_TEMPLATE/3_site_feature_request_tmpl.md .github/ISSUE_TEMPLATE/4_bug_report_tmpl.md .github/ISSUE_TEMPLATE/5_feature_request_tmpl.md youtube_dl/version.py - $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/1_broken_site_tmpl.md .github/ISSUE_TEMPLATE/1_broken_site.md - $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/2_site_support_request_tmpl.md .github/ISSUE_TEMPLATE/2_site_support_request.md - $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/3_site_feature_request_tmpl.md .github/ISSUE_TEMPLATE/3_site_feature_request.md - $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/4_bug_report_tmpl.md .github/ISSUE_TEMPLATE/4_bug_report.md - $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE/5_feature_request_tmpl.md .github/ISSUE_TEMPLATE/5_feature_request.md +issuetemplates: devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/1_broken_site.md .github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md .github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md .github/ISSUE_TEMPLATE_tmpl/4_bug_report.md .github/ISSUE_TEMPLATE_tmpl/5_feature_request.md youtube_dl/version.py + $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/1_broken_site.md .github/ISSUE_TEMPLATE/1_broken_site.md + $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md .github/ISSUE_TEMPLATE/2_site_support_request.md + $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md + $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/4_bug_report.md .github/ISSUE_TEMPLATE/4_bug_report.md + $(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl/5_feature_request.md .github/ISSUE_TEMPLATE/5_feature_request.md supportedsites: $(PYTHON) devscripts/make_supportedsites.py docs/supportedsites.md From 60e67c5b2c60fdda41db316282d2ed5a8d9ffacc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 27 Apr 2019 05:08:27 +0700 Subject: [PATCH 23/45] [twitch] Prefer source format (closes #20850) --- youtube_dl/extractor/twitch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py index 8c87f6dd3..dc5ff29c3 100644 --- a/youtube_dl/extractor/twitch.py +++ b/youtube_dl/extractor/twitch.py @@ -134,12 +134,12 @@ class TwitchBaseIE(InfoExtractor): def _prefer_source(self, formats): try: source = next(f for f in formats if f['format_id'] == 'Source') - source['preference'] = 10 + source['quality'] = 10 except StopIteration: for f in formats: if '/chunked/' in f['url']: f.update({ - 'source_preference': 10, + 'quality': 10, 'format_note': 'Source', }) self._sort_formats(formats) From aa05a093bb94e97670e44e051d103614e8f4cd9f Mon Sep 17 00:00:00 2001 From: Jakub Wilk Date: Sat, 27 Apr 2019 00:12:15 +0200 Subject: [PATCH 24/45] [wrzuta] Remove extractor (closes #20684) (#20801) Wrzuta.pl was shut down in 2017. --- youtube_dl/extractor/extractors.py | 4 - youtube_dl/extractor/wrzuta.py | 158 ----------------------------- 2 files changed, 162 deletions(-) delete mode 100644 youtube_dl/extractor/wrzuta.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 0e3ccb82d..676ad3f7d 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1418,10 +1418,6 @@ from .weiqitv import WeiqiTVIE from .wimp import WimpIE from .wistia import WistiaIE from .worldstarhiphop import WorldStarHipHopIE -from .wrzuta import ( - WrzutaIE, - WrzutaPlaylistIE, -) from .wsj import ( WSJIE, WSJArticleIE, diff --git a/youtube_dl/extractor/wrzuta.py b/youtube_dl/extractor/wrzuta.py deleted file mode 100644 index 0f53f1bcb..000000000 --- a/youtube_dl/extractor/wrzuta.py +++ /dev/null @@ -1,158 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - int_or_none, - qualities, - remove_start, -) - - -class WrzutaIE(InfoExtractor): - IE_NAME = 'wrzuta.pl' - - _VALID_URL = r'https?://(?P[0-9a-zA-Z]+)\.wrzuta\.pl/(?Pfilm|audio)/(?P[0-9a-zA-Z]+)' - - _TESTS = [{ - 'url': 'http://laboratoriumdextera.wrzuta.pl/film/aq4hIZWrkBu/nike_football_the_last_game', - 'md5': '9e67e05bed7c03b82488d87233a9efe7', - 'info_dict': { - 'id': 'aq4hIZWrkBu', - 'ext': 'mp4', - 'title': 'Nike Football: The Last Game', - 'duration': 307, - 'uploader_id': 'laboratoriumdextera', - 'description': 'md5:7fb5ef3c21c5893375fda51d9b15d9cd', - }, - 'skip': 'Redirected to wrzuta.pl', - }, { - 'url': 'http://vexling.wrzuta.pl/audio/01xBFabGXu6/james_horner_-_into_the_na_39_vi_world_bonus', - 'md5': 'f80564fb5a2ec6ec59705ae2bf2ba56d', - 'info_dict': { - 'id': '01xBFabGXu6', - 'ext': 'mp3', - 'title': 'James Horner - Into The Na\'vi World [Bonus]', - 'description': 'md5:30a70718b2cd9df3120fce4445b0263b', - 'duration': 95, - 'uploader_id': 'vexling', - }, - }] - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - typ = mobj.group('typ') - uploader = mobj.group('uploader') - - webpage, urlh = self._download_webpage_handle(url, video_id) - - if urlh.geturl() == 'http://www.wrzuta.pl/': - raise ExtractorError('Video removed', expected=True) - - quality = qualities(['SD', 'MQ', 'HQ', 'HD']) - - audio_table = {'flv': 'mp3', 'webm': 'ogg', '???': 'mp3'} - - embedpage = self._download_json('http://www.wrzuta.pl/npp/embed/%s/%s' % (uploader, video_id), video_id) - - formats = [] - for media in embedpage['url']: - fmt = media['type'].split('@')[0] - if typ == 'audio': - ext = audio_table.get(fmt, fmt) - else: - ext = fmt - - formats.append({ - 'format_id': '%s_%s' % (ext, media['quality'].lower()), - 'url': media['url'], - 'ext': ext, - 'quality': quality(media['quality']), - }) - - self._sort_formats(formats) - - return { - 'id': video_id, - 'title': self._og_search_title(webpage), - 'thumbnail': self._og_search_thumbnail(webpage), - 'formats': formats, - 'duration': int_or_none(embedpage['duration']), - 'uploader_id': uploader, - 'description': self._og_search_description(webpage), - 'age_limit': embedpage.get('minimalAge', 0), - } - - -class WrzutaPlaylistIE(InfoExtractor): - """ - this class covers extraction of wrzuta playlist entries - the extraction process bases on following steps: - * collect information of playlist size - * download all entries provided on - the playlist webpage (the playlist is split - on two pages: first directly reached from webpage - second: downloaded on demand by ajax call and rendered - using the ajax call response) - * in case size of extracted entries not reached total number of entries - use the ajax call to collect the remaining entries - """ - - IE_NAME = 'wrzuta.pl:playlist' - _VALID_URL = r'https?://(?P[0-9a-zA-Z]+)\.wrzuta\.pl/playlista/(?P[0-9a-zA-Z]+)' - _TESTS = [{ - 'url': 'http://miromak71.wrzuta.pl/playlista/7XfO4vE84iR/moja_muza', - 'playlist_mincount': 14, - 'info_dict': { - 'id': '7XfO4vE84iR', - 'title': 'Moja muza', - }, - }, { - 'url': 'http://heroesf70.wrzuta.pl/playlista/6Nj3wQHx756/lipiec_-_lato_2015_muzyka_swiata', - 'playlist_mincount': 144, - 'info_dict': { - 'id': '6Nj3wQHx756', - 'title': 'Lipiec - Lato 2015 Muzyka Świata', - }, - }, { - 'url': 'http://miromak71.wrzuta.pl/playlista/7XfO4vE84iR', - 'only_matching': True, - }] - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - playlist_id = mobj.group('id') - uploader = mobj.group('uploader') - - webpage = self._download_webpage(url, playlist_id) - - playlist_size = int_or_none(self._html_search_regex( - (r']+class=["\']playlist-counter["\'][^>]*>\d+/(\d+)', - r']+class=["\']all-counter["\'][^>]*>(.+?)'), - webpage, 'playlist size', default=None)) - - playlist_title = remove_start( - self._og_search_title(webpage), 'Playlista: ') - - entries = [] - if playlist_size: - entries = [ - self.url_result(entry_url) - for _, entry_url in re.findall( - r']+href=(["\'])(http.+?)\1[^>]+class=["\']playlist-file-page', - webpage)] - if playlist_size > len(entries): - playlist_content = self._download_json( - 'http://%s.wrzuta.pl/xhr/get_playlist_offset/%s' % (uploader, playlist_id), - playlist_id, - 'Downloading playlist JSON', - 'Unable to download playlist JSON') - entries.extend([ - self.url_result(entry['filelink']) - for entry in playlist_content.get('files', []) if entry.get('filelink')]) - - return self.playlist_result(entries, playlist_id, playlist_title) From 5caabd3c701a484271d197f7006ecf831e38136b Mon Sep 17 00:00:00 2001 From: quinlander Date: Mon, 22 Apr 2019 00:26:48 -0400 Subject: [PATCH 25/45] [youtube] Extract additional meta data from video description on youtube music videos YouTube music videos often have auto-generated video descriptions that can be utilized to extract additional information about the video. This is desirable in order to provide the user with as much meta data as possible. This commit adds extraction methods for the following fields for youtube music videos: - artist (fallback extraction methods added) - track (fallback extraction methods added) - album (new in this commit) - release_date (new in this commit) - release_year (new in this commit) 4 tests have been added to test this new functionality: - YoutubeIE tests 27, 28, 29, and 30 Resolves: #20599 --- youtube_dl/extractor/youtube.py | 123 +++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 1bc2c27ad..438eb5aa7 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1086,7 +1086,95 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'skip_download': True, 'youtube_include_dash_manifest': False, }, - } + }, + { + # artist and track fields should return non-null, per issue #20599 + 'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs', + 'info_dict': { + 'id': 'MgNrAu2pzNs', + 'ext': 'mp4', + 'title': 'Voyeur Girl', + 'description': 'md5:7ae382a65843d6df2685993e90a8628f', + 'upload_date': '20190312', + 'uploader': 'Various Artists - Topic', + 'uploader_id': 'UCVWKBi1ELZn0QX2CBLSkiyw', + 'artist': 'Stephen', + 'track': 'Voyeur Girl', + 'album': 'it\'s too much love to know my dear', + 'release_date': '20190313', + 'release_year': 2019, + }, + 'params': { + 'skip_download': True, + }, + }, + { + # Retrieve 'artist' field from 'Artist:' in video description + # when it is present on youtube music video + # Some videos have release_date and no release_year - + # (release_year should be extracted from release_date) + # https://github.com/ytdl-org/youtube-dl/pull/20742#issuecomment-485740932 + 'url': 'https://www.youtube.com/watch?v=k0jLE7tTwjY', + 'info_dict': { + 'id': 'k0jLE7tTwjY', + 'ext': 'mp4', + 'title': 'Latch Feat. Sam Smith', + 'description': 'md5:3cb1e8101a7c85fcba9b4fb41b951335', + 'upload_date': '20150110', + 'uploader': 'Various Artists - Topic', + 'uploader_id': 'UCNkEcmYdjrH4RqtNgh7BZ9w', + 'artist': 'Disclosure', + 'track': 'Latch Feat. Sam Smith', + 'album': 'Latch Featuring Sam Smith', + 'release_date': '20121008', + 'release_year': 2012, + }, + 'params': { + 'skip_download': True, + }, + }, + { + # handle multiple artists on youtube music video + 'url': 'https://www.youtube.com/watch?v=74qn0eJSjpA', + 'info_dict': { + 'id': '74qn0eJSjpA', + 'ext': 'mp4', + 'title': 'Eastside', + 'description': 'md5:290516bb73dcbfab0dcc4efe6c3de5f2', + 'upload_date': '20180710', + 'uploader': 'Benny Blanco - Topic', + 'uploader_id': 'UCzqz_ksRu_WkIzmivMdIS7A', + 'artist': 'benny blanco, Halsey, Khalid', + 'track': 'Eastside', + 'album': 'Eastside', + 'release_date': '20180713', + 'release_year': 2018, + }, + 'params': { + 'skip_download': True, + }, + }, + { + # handle youtube music video with release_year and no release_date + 'url': 'https://www.youtube.com/watch?v=-hcAI0g-f5M', + 'info_dict': { + 'id': '-hcAI0g-f5M', + 'ext': 'mp4', + 'title': 'Put It On Me', + 'description': 'md5:93c55acc682ae7b0c668f2e34e1c069e', + 'upload_date': '20180426', + 'uploader': 'Matt Maeson - Topic', + 'uploader_id': 'UCnEkIGqtGcQMLk73Kp-Q5LQ', + 'artist': 'Matt Maeson', + 'track': 'Put It On Me', + 'album': 'The Hearse', + 'release_date': None, + 'release_year': 2018, + }, + 'params': { + 'skip_download': True, + }, + }, ] def __init__(self, *args, **kwargs): @@ -2073,6 +2161,36 @@ class YoutubeIE(YoutubeBaseInfoExtractor): track = extract_meta('Song') artist = extract_meta('Artist') + album = None + release_date = None + release_year = None + + description_info = video_description.split('\n\n') + # If the description of the video has the youtube music auto-generated format, extract additional info + if len(description_info) >= 5 and description_info[-1] == 'Auto-generated by YouTube.': + track_artist = description_info[1].split(' · ') + if len(track_artist) >= 2: + if track is None: + track = track_artist[0] + if artist is None: + artist = re.search(r'Artist: ([^\n]+)', description_info[-2]) + if artist: + artist = artist.group(1) + if artist is None: + artist = track_artist[1] + # handle multiple artists + if len(track_artist) > 2: + for i in range(2, len(track_artist)): + artist += ', %s' % track_artist[i] + release_year = re.search(r'℗ ([0-9]+)', video_description) + if release_year: + release_year = int_or_none(release_year.group(1)) + album = description_info[2] + if description_info[4].startswith('Released on: '): + release_date = description_info[4].split(': ')[1].replace('-', '') + # extract release_year from release_date if necessary + if release_year is None: + release_year = int_or_none(release_date[0:4]) m_episode = re.search( r']+id="watch7-headline"[^>]*>\s*]*>.*?>(?P[^<]+)\s*S(?P\d+)\s*•\s*E(?P\d+)', @@ -2226,6 +2344,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'episode_number': episode_number, 'track': track, 'artist': artist, + 'album': album, + 'release_date': release_date, + 'release_year': release_year, } From 822b9d9cb09429645582791dba31f4cbed7583cf Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sat, 27 Apr 2019 09:16:17 +0100 Subject: [PATCH 26/45] [youtube] improve Youtube Music Auto-generated description parsing(closes #20742) --- youtube_dl/extractor/youtube.py | 55 +++++++++++++-------------------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 438eb5aa7..55eafb866 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1088,7 +1088,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): }, }, { - # artist and track fields should return non-null, per issue #20599 + # Youtube Music Auto-generated description 'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs', 'info_dict': { 'id': 'MgNrAu2pzNs', @@ -1109,11 +1109,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): }, }, { + # Youtube Music Auto-generated description # Retrieve 'artist' field from 'Artist:' in video description # when it is present on youtube music video - # Some videos have release_date and no release_year - - # (release_year should be extracted from release_date) - # https://github.com/ytdl-org/youtube-dl/pull/20742#issuecomment-485740932 'url': 'https://www.youtube.com/watch?v=k0jLE7tTwjY', 'info_dict': { 'id': 'k0jLE7tTwjY', @@ -1134,6 +1132,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): }, }, { + # Youtube Music Auto-generated description # handle multiple artists on youtube music video 'url': 'https://www.youtube.com/watch?v=74qn0eJSjpA', 'info_dict': { @@ -1155,6 +1154,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): }, }, { + # Youtube Music Auto-generated description # handle youtube music video with release_year and no release_date 'url': 'https://www.youtube.com/watch?v=-hcAI0g-f5M', 'info_dict': { @@ -2161,36 +2161,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor): track = extract_meta('Song') artist = extract_meta('Artist') - album = None - release_date = None - release_year = None - description_info = video_description.split('\n\n') - # If the description of the video has the youtube music auto-generated format, extract additional info - if len(description_info) >= 5 and description_info[-1] == 'Auto-generated by YouTube.': - track_artist = description_info[1].split(' · ') - if len(track_artist) >= 2: - if track is None: - track = track_artist[0] - if artist is None: - artist = re.search(r'Artist: ([^\n]+)', description_info[-2]) - if artist: - artist = artist.group(1) - if artist is None: - artist = track_artist[1] - # handle multiple artists - if len(track_artist) > 2: - for i in range(2, len(track_artist)): - artist += ', %s' % track_artist[i] - release_year = re.search(r'℗ ([0-9]+)', video_description) - if release_year: - release_year = int_or_none(release_year.group(1)) - album = description_info[2] - if description_info[4].startswith('Released on: '): - release_date = description_info[4].split(': ')[1].replace('-', '') - # extract release_year from release_date if necessary - if release_year is None: - release_year = int_or_none(release_date[0:4]) + # Youtube Music Auto-generated description + album = release_date = release_year = None + if video_description: + mobj = re.search(r'(?s)Provided to YouTube by [^\n]+\n+(?P[^·]+)·(?P[^\n]+)\n+(?P[^\n]+)(?:.+?℗\s*(?P\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P[^\n]+))?', video_description) + if mobj: + if not track: + track = mobj.group('track').strip() + if not artist: + artist = mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')) + album = mobj.group('album'.strip()) + release_year = mobj.group('release_year') + release_date = mobj.group('release_date') + if release_date: + release_date = release_date.replace('-', '') + if not release_year: + release_year = int(release_date[:4]) + if release_year: + release_year = int(release_year) m_episode = re.search( r']+id="watch7-headline"[^>]*>\s*]*>.*?>(?P[^<]+)\s*S(?P\d+)\s*•\s*E(?P\d+)', From 2309d6bf92435e2c9a0a9a3ebca7e15f68651225 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sat, 27 Apr 2019 10:17:26 +0100 Subject: [PATCH 27/45] [sixplay] try to extract non drm protected manifests(closes #20849) --- youtube_dl/extractor/sixplay.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py index 35bc9fa50..ce21797a7 100644 --- a/youtube_dl/extractor/sixplay.py +++ b/youtube_dl/extractor/sixplay.py @@ -65,7 +65,7 @@ class SixPlayIE(InfoExtractor): for asset in assets: asset_url = asset.get('full_physical_path') protocol = asset.get('protocol') - if not asset_url or protocol == 'primetime' or asset.get('type') == 'usp_hlsfp_h264' or asset_url in urls: + if not asset_url or (protocol == 'primetime' or asset.get('type') == 'usp_hlsfp_h264') and not ('_drmnp.ism/' in asset_url or '_unpnp.ism/' in asset_url) or asset_url in urls: continue urls.append(asset_url) container = asset.get('video_container') @@ -82,6 +82,7 @@ class SixPlayIE(InfoExtractor): if not urlh: continue asset_url = urlh.geturl() + asset_url = asset_url.replace('_drmnp.ism/', '_unpnp.ism/') for i in range(3, 0, -1): asset_url = asset_url = asset_url.replace('_sd1/', '_sd%d/' % i) m3u8_formats = self._extract_m3u8_formats( From 3545d38bfba40ab1ccc5f73b08b18709c580a180 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sat, 27 Apr 2019 10:32:53 +0100 Subject: [PATCH 28/45] [sixplay] add missing parenthesis --- youtube_dl/extractor/sixplay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py index ce21797a7..2a72af11b 100644 --- a/youtube_dl/extractor/sixplay.py +++ b/youtube_dl/extractor/sixplay.py @@ -65,7 +65,7 @@ class SixPlayIE(InfoExtractor): for asset in assets: asset_url = asset.get('full_physical_path') protocol = asset.get('protocol') - if not asset_url or (protocol == 'primetime' or asset.get('type') == 'usp_hlsfp_h264') and not ('_drmnp.ism/' in asset_url or '_unpnp.ism/' in asset_url) or asset_url in urls: + if not asset_url or ((protocol == 'primetime' or asset.get('type') == 'usp_hlsfp_h264') and not ('_drmnp.ism/' in asset_url or '_unpnp.ism/' in asset_url)) or asset_url in urls: continue urls.append(asset_url) container = asset.get('video_container') From 4e4db743e76387991b9f20613caec2c5ec38473a Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 28 Apr 2019 00:42:55 +0100 Subject: [PATCH 29/45] [cinemax] Add new extractor --- youtube_dl/extractor/cinemax.py | 29 ++++++++++++++++++++ youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/hbo.py | 44 +++++++++++++++++------------- 3 files changed, 55 insertions(+), 19 deletions(-) create mode 100644 youtube_dl/extractor/cinemax.py diff --git a/youtube_dl/extractor/cinemax.py b/youtube_dl/extractor/cinemax.py new file mode 100644 index 000000000..7f89d33de --- /dev/null +++ b/youtube_dl/extractor/cinemax.py @@ -0,0 +1,29 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .hbo import HBOBaseIE + + +class CinemaxIE(HBOBaseIE): + _VALID_URL = r'https?://(?:www\.)?cinemax\.com/(?P[^/]+/video/[0-9a-z-]+-(?P\d+))' + _TESTS = [{ + 'url': 'https://www.cinemax.com/warrior/video/s1-ep-1-recap-20126903', + 'md5': '82e0734bba8aa7ef526c9dd00cf35a05', + 'info_dict': { + 'id': '20126903', + 'ext': 'mp4', + 'title': 'S1 Ep 1: Recap', + }, + 'expected_warnings': ['Unknown MIME type application/mp4 in DASH manifest'], + }, { + 'url': 'https://www.cinemax.com/warrior/video/s1-ep-1-recap-20126903.embed', + 'only_matching': True, + }] + + def _real_extract(self, url): + path, video_id = re.match(self._VALID_URL, url).groups() + info = self._extract_info('https://www.cinemax.com/%s.xml' % path, video_id) + info['id'] = video_id + return info diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 676ad3f7d..487ef2778 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -194,6 +194,7 @@ from .chirbit import ( ChirbitProfileIE, ) from .cinchcast import CinchcastIE +from .cinemax import CinemaxIE from .ciscolive import ( CiscoLiveSessionIE, CiscoLiveSearchIE, diff --git a/youtube_dl/extractor/hbo.py b/youtube_dl/extractor/hbo.py index 44440233d..68df748f5 100644 --- a/youtube_dl/extractor/hbo.py +++ b/youtube_dl/extractor/hbo.py @@ -13,19 +13,7 @@ from ..utils import ( ) -class HBOIE(InfoExtractor): - IE_NAME = 'hbo' - _VALID_URL = r'https?://(?:www\.)?hbo\.com/(?:video|embed)(?:/[^/]+)*/(?P[^/?#]+)' - _TEST = { - 'url': 'https://www.hbo.com/video/game-of-thrones/seasons/season-8/videos/trailer', - 'md5': '8126210656f433c452a21367f9ad85b3', - 'info_dict': { - 'id': '22113301', - 'ext': 'mp4', - 'title': 'Game of Thrones - Trailer', - }, - 'expected_warnings': ['Unknown MIME type application/mp4 in DASH manifest'], - } +class HBOBaseIE(InfoExtractor): _FORMATS_INFO = { 'pro7': { 'width': 1280, @@ -65,12 +53,8 @@ class HBOIE(InfoExtractor): }, } - def _real_extract(self, url): - display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) - location_path = self._parse_json(self._html_search_regex( - r'data-state="({.+?})"', webpage, 'state'), display_id)['video']['locationUrl'] - video_data = self._download_xml(urljoin(url, location_path), display_id) + def _extract_info(self, url, display_id): + video_data = self._download_xml(url, display_id) video_id = xpath_text(video_data, 'id', fatal=True) episode_title = title = xpath_text(video_data, 'title', fatal=True) series = xpath_text(video_data, 'program') @@ -167,3 +151,25 @@ class HBOIE(InfoExtractor): 'thumbnails': thumbnails, 'subtitles': subtitles, } + + +class HBOIE(HBOBaseIE): + IE_NAME = 'hbo' + _VALID_URL = r'https?://(?:www\.)?hbo\.com/(?:video|embed)(?:/[^/]+)*/(?P[^/?#]+)' + _TEST = { + 'url': 'https://www.hbo.com/video/game-of-thrones/seasons/season-8/videos/trailer', + 'md5': '8126210656f433c452a21367f9ad85b3', + 'info_dict': { + 'id': '22113301', + 'ext': 'mp4', + 'title': 'Game of Thrones - Trailer', + }, + 'expected_warnings': ['Unknown MIME type application/mp4 in DASH manifest'], + } + + def _real_extract(self, url): + display_id = self._match_id(url) + webpage = self._download_webpage(url, display_id) + location_path = self._parse_json(self._html_search_regex( + r'data-state="({.+?})"', webpage, 'state'), display_id)['video']['locationUrl'] + return self._extract_info(urljoin(url, location_path), display_id) From 7ff8ad80f1442fc213a6463fa824a70d397b0745 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Tue, 25 Dec 2018 14:29:48 +0100 Subject: [PATCH 30/45] [sverigesradio] Add extractor --- youtube_dl/extractor/extractors.py | 4 + youtube_dl/extractor/sverigesradio.py | 105 ++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 youtube_dl/extractor/sverigesradio.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 487ef2778..26a30b4c3 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1098,6 +1098,10 @@ from .streetvoice import StreetVoiceIE from .stretchinternet import StretchInternetIE from .stv import STVPlayerIE from .sunporno import SunPornoIE +from .sverigesradio import ( + SverigesRadioEpisodeIE, + SverigesRadioPublicationIE, +) from .svt import ( SVTIE, SVTPageIE, diff --git a/youtube_dl/extractor/sverigesradio.py b/youtube_dl/extractor/sverigesradio.py new file mode 100644 index 000000000..05de31a79 --- /dev/null +++ b/youtube_dl/extractor/sverigesradio.py @@ -0,0 +1,105 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import int_or_none + + +class SverigesRadioBaseIE(InfoExtractor): + _BASE_URL = 'https://sverigesradio.se/sida/playerajax' + _QUALITIES = ['high', 'medium', 'low'] + _CODING_FORMATS = { + 5: {'acodec': 'mp3', 'abr': 128}, + 11: {'acodec': 'aac', 'abr': 192}, + 12: {'acodec': 'aac', 'abr': 32}, + 13: {'acodec': 'aac', 'abr': 96}, + } + + def _extract_formats(self, query, audio_id, audio_type): + audiourls = {} + for quality in self._QUALITIES: + audiourl = self._download_json( + self._BASE_URL + '/getaudiourl', audio_id, + fatal=True, + query=dict(query, type=audio_type, quality=quality, format='iis')) + if audiourl is None: + continue + + # for some reason url can be empty, skip if so + # also skip if url has already been seen (quality parameter is ignored?) + url = audiourl.get('audioUrl') + if url is None or url == "" or url in audiourls: + continue + + audioformat = {'vcodec': 'none', 'url': url} + # add codec and bitrate if known coding format + codingformat = audiourl.get('codingFormat') + if codingformat: + audioformat.update(self._CODING_FORMATS.get(codingformat, {})) + + audiourls[url] = audioformat + + return audiourls.values() + + def _extract_audio(self, audio_type, url): + audio_id = self._match_id(url) + query = {'id': audio_id, 'type': audio_type} + + metadata = self._download_json(self._BASE_URL + '/audiometadata', audio_id, query=query) + item = metadata['items'][0] + + formats = self._extract_formats(query, audio_id, audio_type) + self._sort_formats(formats) + + return { + 'id': audio_id, + 'title': item['subtitle'], + 'formats': formats, + 'series': item.get('title'), + 'duration': int_or_none(item.get('duration')), + 'thumbnail': item.get('displayimageurl'), + 'description': item.get('description'), + } + + +class SverigesRadioPublicationIE(SverigesRadioBaseIE): + _VALID_URL = r'https?://(?:www\.)?sverigesradio\.se/sida/(?:artikel|gruppsida)\.aspx\?.*artikel=(?P[0-9]+)' + _TESTS = [{ + 'url': 'https://sverigesradio.se/sida/artikel.aspx?programid=83&artikel=7038546', + 'md5': '6a4917e1923fccb080e5a206a5afa542', + 'info_dict': { + 'id': '7038546', + 'ext': 'm4a', + 'duration': 132, + 'series': 'Nyheter (Ekot)', + 'title': 'Esa Teittinen: Sanningen har inte kommit fram', + 'description': 'md5:daf7ce66a8f0a53d5465a5984d3839df', + 'thumbnail': 're:^https://static-cdn.sr.se/sida/images/', + }, + }, { + 'url': 'https://sverigesradio.se/sida/gruppsida.aspx?programid=3304&grupp=6247&artikel=7146887', + 'only_matching': True, + }] + + def _real_extract(self, url): + return self._extract_audio('publication', url) + + +class SverigesRadioEpisodeIE(SverigesRadioBaseIE): + _VALID_URL = r'https?://(?:www\.)?sverigesradio\.se/(?:sida/)?avsnitt/(?P[0-9]+)' + _TEST = { + 'url': 'https://sverigesradio.se/avsnitt/1140922?programid=1300', + 'md5': '20dc4d8db24228f846be390b0c59a07c', + 'info_dict': { + 'id': '1140922', + 'ext': 'mp3', + 'duration': 3307, + 'series': 'Konflikt', + 'title': 'Metoo och valen', + 'description': 'md5:fcb5c1f667f00badcc702b196f10a27e', + 'thumbnail': 're:^https://static-cdn.sr.se/sida/images/' + } + } + + def _real_extract(self, url): + return self._extract_audio('episode', url) From 280913800dff225d7171ccdbe09d7ce01fdf5d3f Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 28 Apr 2019 12:03:39 +0100 Subject: [PATCH 31/45] [sverigesradio] improve extraction(closes #18635) --- youtube_dl/extractor/sverigesradio.py | 108 ++++++++++++++------------ 1 file changed, 59 insertions(+), 49 deletions(-) diff --git a/youtube_dl/extractor/sverigesradio.py b/youtube_dl/extractor/sverigesradio.py index 05de31a79..aa0691f0d 100644 --- a/youtube_dl/extractor/sverigesradio.py +++ b/youtube_dl/extractor/sverigesradio.py @@ -2,58 +2,70 @@ from __future__ import unicode_literals from .common import InfoExtractor -from ..utils import int_or_none +from ..utils import ( + determine_ext, + int_or_none, + str_or_none, +) class SverigesRadioBaseIE(InfoExtractor): - _BASE_URL = 'https://sverigesradio.se/sida/playerajax' - _QUALITIES = ['high', 'medium', 'low'] - _CODING_FORMATS = { - 5: {'acodec': 'mp3', 'abr': 128}, - 11: {'acodec': 'aac', 'abr': 192}, - 12: {'acodec': 'aac', 'abr': 32}, - 13: {'acodec': 'aac', 'abr': 96}, + _BASE_URL = 'https://sverigesradio.se/sida/playerajax/' + _QUALITIES = ['low', 'medium', 'high'] + _EXT_TO_CODEC_MAP = { + 'mp3': 'mp3', + 'm4a': 'aac', + } + _CODING_FORMAT_TO_ABR_MAP = { + 5: 128, + 11: 192, + 12: 32, + 13: 96, } - def _extract_formats(self, query, audio_id, audio_type): - audiourls = {} - for quality in self._QUALITIES: - audiourl = self._download_json( - self._BASE_URL + '/getaudiourl', audio_id, - fatal=True, - query=dict(query, type=audio_type, quality=quality, format='iis')) - if audiourl is None: - continue - - # for some reason url can be empty, skip if so - # also skip if url has already been seen (quality parameter is ignored?) - url = audiourl.get('audioUrl') - if url is None or url == "" or url in audiourls: - continue - - audioformat = {'vcodec': 'none', 'url': url} - # add codec and bitrate if known coding format - codingformat = audiourl.get('codingFormat') - if codingformat: - audioformat.update(self._CODING_FORMATS.get(codingformat, {})) - - audiourls[url] = audioformat - - return audiourls.values() - - def _extract_audio(self, audio_type, url): + def _real_extract(self, url): audio_id = self._match_id(url) - query = {'id': audio_id, 'type': audio_type} + query = { + 'id': audio_id, + 'type': self._AUDIO_TYPE, + } - metadata = self._download_json(self._BASE_URL + '/audiometadata', audio_id, query=query) - item = metadata['items'][0] + item = self._download_json( + self._BASE_URL + 'audiometadata', audio_id, + 'Downloading audio JSON metadata', query=query)['items'][0] + title = item['subtitle'] - formats = self._extract_formats(query, audio_id, audio_type) + query['format'] = 'iis' + urls = [] + formats = [] + for quality in self._QUALITIES: + query['quality'] = quality + audio_url_data = self._download_json( + self._BASE_URL + 'getaudiourl', audio_id, + 'Downloading %s format JSON metadata' % quality, + fatal=False, query=query) or {} + audio_url = audio_url_data.get('audioUrl') + if not audio_url or audio_url in urls: + continue + urls.append(audio_url) + ext = determine_ext(audio_url) + coding_format = audio_url_data.get('codingFormat') + abr = int_or_none(self._search_regex( + r'_a(\d+)\.m4a', audio_url, 'audio bitrate', + default=None)) or self._CODING_FORMAT_TO_ABR_MAP.get(coding_format) + formats.append({ + 'abr': abr, + 'acodec': self._EXT_TO_CODEC_MAP.get(ext), + 'ext': ext, + 'format_id': str_or_none(coding_format), + 'vcodec': 'none', + 'url': audio_url, + }) self._sort_formats(formats) return { 'id': audio_id, - 'title': item['subtitle'], + 'title': title, 'formats': formats, 'series': item.get('title'), 'duration': int_or_none(item.get('duration')), @@ -63,7 +75,8 @@ class SverigesRadioBaseIE(InfoExtractor): class SverigesRadioPublicationIE(SverigesRadioBaseIE): - _VALID_URL = r'https?://(?:www\.)?sverigesradio\.se/sida/(?:artikel|gruppsida)\.aspx\?.*artikel=(?P[0-9]+)' + IE_NAME = 'sverigesradio:publication' + _VALID_URL = r'https?://(?:www\.)?sverigesradio\.se/sida/(?:artikel|gruppsida)\.aspx\?.*?\bartikel=(?P[0-9]+)' _TESTS = [{ 'url': 'https://sverigesradio.se/sida/artikel.aspx?programid=83&artikel=7038546', 'md5': '6a4917e1923fccb080e5a206a5afa542', @@ -74,18 +87,17 @@ class SverigesRadioPublicationIE(SverigesRadioBaseIE): 'series': 'Nyheter (Ekot)', 'title': 'Esa Teittinen: Sanningen har inte kommit fram', 'description': 'md5:daf7ce66a8f0a53d5465a5984d3839df', - 'thumbnail': 're:^https://static-cdn.sr.se/sida/images/', + 'thumbnail': r're:^https?://.*\.jpg', }, }, { 'url': 'https://sverigesradio.se/sida/gruppsida.aspx?programid=3304&grupp=6247&artikel=7146887', 'only_matching': True, }] - - def _real_extract(self, url): - return self._extract_audio('publication', url) + _AUDIO_TYPE = 'publication' class SverigesRadioEpisodeIE(SverigesRadioBaseIE): + IE_NAME = 'sverigesradio:episode' _VALID_URL = r'https?://(?:www\.)?sverigesradio\.se/(?:sida/)?avsnitt/(?P[0-9]+)' _TEST = { 'url': 'https://sverigesradio.se/avsnitt/1140922?programid=1300', @@ -97,9 +109,7 @@ class SverigesRadioEpisodeIE(SverigesRadioBaseIE): 'series': 'Konflikt', 'title': 'Metoo och valen', 'description': 'md5:fcb5c1f667f00badcc702b196f10a27e', - 'thumbnail': 're:^https://static-cdn.sr.se/sida/images/' + 'thumbnail': r're:^https?://.*\.jpg', } } - - def _real_extract(self, url): - return self._extract_audio('episode', url) + _AUDIO_TYPE = 'episode' From ae8c13565eaed326179b26a91a1b0c3179eb3d07 Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Fri, 27 Oct 2017 13:22:13 +0200 Subject: [PATCH 32/45] [ccc:playlist] Add extractor --- youtube_dl/extractor/ccc.py | 24 ++++++++++++++++++++++++ youtube_dl/extractor/extractors.py | 5 ++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py index 734702144..a4fdf74e8 100644 --- a/youtube_dl/extractor/ccc.py +++ b/youtube_dl/extractor/ccc.py @@ -75,3 +75,27 @@ class CCCIE(InfoExtractor): 'tags': event_data.get('tags'), 'formats': formats, } + + +class CCCPlaylistIE(InfoExtractor): + IE_NAME = 'media.ccc.de:lists' + _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/c/(?P[^/?#&]+)' + _TESTS = [{ + 'url': 'https://media.ccc.de/c/30c3', + 'info_dict': { + 'title': '30C3', + 'id': '30c3', + }, + 'playlist_count': 135, + }] + + def _real_extract(self, url): + acronym = self._match_id(url).lower() + + conf = self._download_json('https://media.ccc.de/public/conferences/' + acronym, acronym) + + return self.playlist_result( + [self.url_result(event['frontend_link']) for event in conf['events']], + acronym, + conf['title'], + ) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 26a30b4c3..c011cf981 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -177,7 +177,10 @@ from .cbsnews import ( CBSNewsLiveVideoIE, ) from .cbssports import CBSSportsIE -from .ccc import CCCIE +from .ccc import ( + CCCIE, + CCCPlaylistIE, +) from .ccma import CCMAIE from .cctv import CCTVIE from .cda import CDAIE From 24510bdcfab3e450f9e1a1b82cf7fca4183c3333 Mon Sep 17 00:00:00 2001 From: Tobias Gruetzmacher Date: Thu, 14 Mar 2019 21:30:01 +0100 Subject: [PATCH 33/45] [ccc] Extract creator --- youtube_dl/extractor/ccc.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py index a4fdf74e8..5fc473fce 100644 --- a/youtube_dl/extractor/ccc.py +++ b/youtube_dl/extractor/ccc.py @@ -1,9 +1,11 @@ +# coding: utf-8 from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( int_or_none, parse_iso8601, + try_get, ) @@ -18,15 +20,26 @@ class CCCIE(InfoExtractor): 'id': '1839', 'ext': 'mp4', 'title': 'Introduction to Processor Design', + 'creator': 'byterazor', 'description': 'md5:df55f6d073d4ceae55aae6f2fd98a0ac', 'thumbnail': r're:^https?://.*\.jpg$', 'upload_date': '20131228', 'timestamp': 1388188800, 'duration': 3710, + 'tags': list, } }, { 'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download', - 'only_matching': True, + 'info_dict': { + 'id': '2835', + 'ext': 'mp4', + 'title': 'Shopshifting', + 'creator': 'Karsten Nohl, Fabian Bräunlein, dexter', + 'description': 'md5:0fade0535e9dc3076d0cbda4958a18eb', + 'upload_date': '20151227', + 'timestamp': 1451249100, + 'tags': list, + } }] def _real_extract(self, url): @@ -68,6 +81,7 @@ class CCCIE(InfoExtractor): 'id': event_id, 'display_id': display_id, 'title': event_data['title'], + 'creator': try_get(event_data, lambda x: ', '.join(x['persons'])), 'description': event_data.get('description'), 'thumbnail': event_data.get('thumb_url'), 'timestamp': parse_iso8601(event_data.get('date')), From f916abc0ac4d1dc7f3a243d791d0f57fd3848a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 28 Apr 2019 23:05:36 +0700 Subject: [PATCH 34/45] [ccc] Improve extraction (closes #14601, closes #20355) --- youtube_dl/extractor/ccc.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py index 5fc473fce..36e6dff72 100644 --- a/youtube_dl/extractor/ccc.py +++ b/youtube_dl/extractor/ccc.py @@ -6,6 +6,7 @@ from ..utils import ( int_or_none, parse_iso8601, try_get, + url_or_none, ) @@ -30,16 +31,7 @@ class CCCIE(InfoExtractor): } }, { 'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download', - 'info_dict': { - 'id': '2835', - 'ext': 'mp4', - 'title': 'Shopshifting', - 'creator': 'Karsten Nohl, Fabian Bräunlein, dexter', - 'description': 'md5:0fade0535e9dc3076d0cbda4958a18eb', - 'upload_date': '20151227', - 'timestamp': 1451249100, - 'tags': list, - } + 'only_matching': True, }] def _real_extract(self, url): @@ -104,12 +96,16 @@ class CCCPlaylistIE(InfoExtractor): }] def _real_extract(self, url): - acronym = self._match_id(url).lower() + playlist_id = self._match_id(url).lower() - conf = self._download_json('https://media.ccc.de/public/conferences/' + acronym, acronym) + conf = self._download_json( + 'https://media.ccc.de/public/conferences/' + playlist_id, + playlist_id) - return self.playlist_result( - [self.url_result(event['frontend_link']) for event in conf['events']], - acronym, - conf['title'], - ) + entries = [] + for e in conf['events']: + event_url = url_or_none(e.get('frontend_link')) + if event_url: + entries.append(self.url_result(event_url, ie=CCCIE.ie_key())) + + return self.playlist_result(entries, playlist_id, conf.get('title')) From 92bc97d398cb66e4968070f9d73f02a367193c2b Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 28 Apr 2019 17:37:46 +0100 Subject: [PATCH 35/45] [youtube] extract album from Music in this video section(#20301) --- youtube_dl/extractor/youtube.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 55eafb866..5e0a9e10c 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -908,6 +908,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'creator': 'Todd Haberman, Daniel Law Heath and Aaron Kaplan', 'track': 'Dark Walk - Position Music', 'artist': 'Todd Haberman, Daniel Law Heath and Aaron Kaplan', + 'album': 'Position Music - Production Music Vol. 143 - Dark Walk', }, 'params': { 'skip_download': True, @@ -2161,9 +2162,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor): track = extract_meta('Song') artist = extract_meta('Artist') + album = extract_meta('Album') # Youtube Music Auto-generated description - album = release_date = release_year = None + release_date = release_year = None if video_description: mobj = re.search(r'(?s)Provided to YouTube by [^\n]+\n+(?P[^·]+)·(?P[^\n]+)\n+(?P[^\n]+)(?:.+?℗\s*(?P\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P[^\n]+))?', video_description) if mobj: @@ -2171,7 +2173,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): track = mobj.group('track').strip() if not artist: artist = mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')) - album = mobj.group('album'.strip()) + if not album: + album = mobj.group('album'.strip()) release_year = mobj.group('release_year') release_date = mobj.group('release_date') if release_date: From c464e1df2c3769969b447e80b126140880d00e67 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 28 Apr 2019 17:50:47 +0100 Subject: [PATCH 36/45] [adn] fix subtitle extraction(#12724) --- youtube_dl/extractor/adn.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py index 923c351e4..c95ad2173 100644 --- a/youtube_dl/extractor/adn.py +++ b/youtube_dl/extractor/adn.py @@ -65,14 +65,15 @@ class ADNIE(InfoExtractor): if subtitle_location: enc_subtitles = self._download_webpage( urljoin(self._BASE_URL, subtitle_location), - video_id, 'Downloading subtitles data', fatal=False) + video_id, 'Downloading subtitles data', fatal=False, + headers={'Origin': 'https://animedigitalnetwork.fr'}) if not enc_subtitles: return None # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js dec_subtitles = intlist_to_bytes(aes_cbc_decrypt( bytes_to_intlist(compat_b64decode(enc_subtitles[24:])), - bytes_to_intlist(binascii.unhexlify(self._K + '4421de0a5f0814ba')), + bytes_to_intlist(binascii.unhexlify(self._K + '4b8ef13ec1872730')), bytes_to_intlist(compat_b64decode(enc_subtitles[:24])) )) subtitles_json = self._parse_json( From 6e07b5a6d53f8ab8a628177e8b40f06ec5897777 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 28 Apr 2019 18:02:41 +0100 Subject: [PATCH 37/45] [dramafever] Remove extractor(closes #20868) --- youtube_dl/extractor/dramafever.py | 266 ----------------------------- youtube_dl/extractor/extractors.py | 4 - 2 files changed, 270 deletions(-) delete mode 100644 youtube_dl/extractor/dramafever.py diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py deleted file mode 100644 index db1de699f..000000000 --- a/youtube_dl/extractor/dramafever.py +++ /dev/null @@ -1,266 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import itertools -import json - -from .common import InfoExtractor -from ..compat import ( - compat_HTTPError, - compat_urlparse, -) -from ..utils import ( - clean_html, - ExtractorError, - int_or_none, - parse_age_limit, - parse_duration, - unified_timestamp, - url_or_none, -) - - -class DramaFeverBaseIE(InfoExtractor): - _NETRC_MACHINE = 'dramafever' - - _CONSUMER_SECRET = 'DA59dtVXYLxajktV' - - _consumer_secret = None - - def _get_consumer_secret(self): - mainjs = self._download_webpage( - 'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js', - None, 'Downloading main.js', fatal=False) - if not mainjs: - return self._CONSUMER_SECRET - return self._search_regex( - r"var\s+cs\s*=\s*'([^']+)'", mainjs, - 'consumer secret', default=self._CONSUMER_SECRET) - - def _real_initialize(self): - self._consumer_secret = self._get_consumer_secret() - self._login() - - def _login(self): - username, password = self._get_login_info() - if username is None: - return - - login_form = { - 'username': username, - 'password': password, - } - - try: - response = self._download_json( - 'https://www.dramafever.com/api/users/login', None, 'Logging in', - data=json.dumps(login_form).encode('utf-8'), headers={ - 'x-consumer-key': self._consumer_secret, - }) - except ExtractorError as e: - if isinstance(e.cause, compat_HTTPError) and e.cause.code in (403, 404): - response = self._parse_json( - e.cause.read().decode('utf-8'), None) - else: - raise - - # Successful login - if response.get('result') or response.get('guid') or response.get('user_guid'): - return - - errors = response.get('errors') - if errors and isinstance(errors, list): - error = errors[0] - message = error.get('message') or error['reason'] - raise ExtractorError('Unable to login: %s' % message, expected=True) - raise ExtractorError('Unable to log in') - - -class DramaFeverIE(DramaFeverBaseIE): - IE_NAME = 'dramafever' - _VALID_URL = r'https?://(?:www\.)?dramafever\.com/(?:[^/]+/)?drama/(?P[0-9]+/[0-9]+)(?:/|$)' - _TESTS = [{ - 'url': 'https://www.dramafever.com/drama/4274/1/Heirs/', - 'info_dict': { - 'id': '4274.1', - 'ext': 'wvm', - 'title': 'Heirs - Episode 1', - 'description': 'md5:362a24ba18209f6276e032a651c50bc2', - 'thumbnail': r're:^https?://.*\.jpg', - 'duration': 3783, - 'timestamp': 1381354993, - 'upload_date': '20131009', - 'series': 'Heirs', - 'season_number': 1, - 'episode': 'Episode 1', - 'episode_number': 1, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, { - 'url': 'http://www.dramafever.com/drama/4826/4/Mnet_Asian_Music_Awards_2015/?ap=1', - 'info_dict': { - 'id': '4826.4', - 'ext': 'flv', - 'title': 'Mnet Asian Music Awards 2015', - 'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91', - 'episode': 'Mnet Asian Music Awards 2015 - Part 3', - 'episode_number': 4, - 'thumbnail': r're:^https?://.*\.jpg', - 'timestamp': 1450213200, - 'upload_date': '20151215', - 'duration': 5359, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, { - 'url': 'https://www.dramafever.com/zh-cn/drama/4972/15/Doctor_Romantic/', - 'only_matching': True, - }] - - def _call_api(self, path, video_id, note, fatal=False): - return self._download_json( - 'https://www.dramafever.com/api/5/' + path, - video_id, note=note, headers={ - 'x-consumer-key': self._consumer_secret, - }, fatal=fatal) - - def _get_subtitles(self, video_id): - subtitles = {} - subs = self._call_api( - 'video/%s/subtitles/webvtt/' % video_id, video_id, - 'Downloading subtitles JSON', fatal=False) - if not subs or not isinstance(subs, list): - return subtitles - for sub in subs: - if not isinstance(sub, dict): - continue - sub_url = url_or_none(sub.get('url')) - if not sub_url: - continue - subtitles.setdefault( - sub.get('code') or sub.get('language') or 'en', []).append({ - 'url': sub_url - }) - return subtitles - - def _real_extract(self, url): - video_id = self._match_id(url).replace('/', '.') - - series_id, episode_number = video_id.split('.') - - video = self._call_api( - 'series/%s/episodes/%s/' % (series_id, episode_number), video_id, - 'Downloading video JSON') - - formats = [] - download_assets = video.get('download_assets') - if download_assets and isinstance(download_assets, dict): - for format_id, format_dict in download_assets.items(): - if not isinstance(format_dict, dict): - continue - format_url = url_or_none(format_dict.get('url')) - if not format_url: - continue - formats.append({ - 'url': format_url, - 'format_id': format_id, - 'filesize': int_or_none(video.get('filesize')), - }) - - stream = self._call_api( - 'video/%s/stream/' % video_id, video_id, 'Downloading stream JSON', - fatal=False) - if stream: - stream_url = stream.get('stream_url') - if stream_url: - formats.extend(self._extract_m3u8_formats( - stream_url, video_id, 'mp4', entry_protocol='m3u8_native', - m3u8_id='hls', fatal=False)) - self._sort_formats(formats) - - title = video.get('title') or 'Episode %s' % episode_number - description = video.get('description') - thumbnail = video.get('thumbnail') - timestamp = unified_timestamp(video.get('release_date')) - duration = parse_duration(video.get('duration')) - age_limit = parse_age_limit(video.get('tv_rating')) - series = video.get('series_title') - season_number = int_or_none(video.get('season')) - - if series: - title = '%s - %s' % (series, title) - - subtitles = self.extract_subtitles(video_id) - - return { - 'id': video_id, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'duration': duration, - 'timestamp': timestamp, - 'age_limit': age_limit, - 'series': series, - 'season_number': season_number, - 'episode_number': int_or_none(episode_number), - 'formats': formats, - 'subtitles': subtitles, - } - - -class DramaFeverSeriesIE(DramaFeverBaseIE): - IE_NAME = 'dramafever:series' - _VALID_URL = r'https?://(?:www\.)?dramafever\.com/(?:[^/]+/)?drama/(?P[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$' - _TESTS = [{ - 'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/', - 'info_dict': { - 'id': '4512', - 'title': 'Cooking with Shin', - 'description': 'md5:84a3f26e3cdc3fb7f500211b3593b5c1', - }, - 'playlist_count': 4, - }, { - 'url': 'http://www.dramafever.com/drama/124/IRIS/', - 'info_dict': { - 'id': '124', - 'title': 'IRIS', - 'description': 'md5:b3a30e587cf20c59bd1c01ec0ee1b862', - }, - 'playlist_count': 20, - }] - - _PAGE_SIZE = 60 # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-) - - def _real_extract(self, url): - series_id = self._match_id(url) - - series = self._download_json( - 'http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s' - % (self._consumer_secret, series_id), - series_id, 'Downloading series JSON')['series'][series_id] - - title = clean_html(series['name']) - description = clean_html(series.get('description') or series.get('description_short')) - - entries = [] - for page_num in itertools.count(1): - episodes = self._download_json( - 'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d' - % (self._consumer_secret, series_id, self._PAGE_SIZE, page_num), - series_id, 'Downloading episodes JSON page #%d' % page_num) - for episode in episodes.get('value', []): - episode_url = episode.get('episode_url') - if not episode_url: - continue - entries.append(self.url_result( - compat_urlparse.urljoin(url, episode_url), - 'DramaFever', episode.get('guid'))) - if page_num == episodes['num_pages']: - break - - return self.playlist_result(entries, series_id, title, description) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index c011cf981..392b1f92b 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -287,10 +287,6 @@ from .dplay import ( DPlayIE, DPlayItIE, ) -from .dramafever import ( - DramaFeverIE, - DramaFeverSeriesIE, -) from .dreisat import DreiSatIE from .drbonanza import DRBonanzaIE from .drtuber import DrTuberIE From 026fbedc855fa2870664798e03f58447b3a61a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 30 Apr 2019 04:32:55 +0700 Subject: [PATCH 38/45] [youtube] Improve extraction robustness Fail on missing token only when no formats found --- youtube_dl/extractor/youtube.py | 53 ++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 5e0a9e10c..88dba1353 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -27,6 +27,7 @@ from ..compat import ( ) from ..utils import ( clean_html, + dict_get, error_to_compat_str, ExtractorError, float_or_none, @@ -1652,6 +1653,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def extract_view_count(v_info): return int_or_none(try_get(v_info, lambda x: x['view_count'][0])) + def extract_token(v_info): + return dict_get(v_info, ('account_playback_token', 'accountPlaybackToken', 'token')) + player_response = {} # Get video info @@ -1741,7 +1745,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): view_count = extract_view_count(get_video_info) if not video_info: video_info = get_video_info - get_token = get_video_info.get('token') or get_video_info.get('account_playback_token') + get_token = extract_token(get_video_info) if get_token: # Different get_video_info requests may report different results, e.g. # some may report video unavailability, but some may serve it without @@ -1752,7 +1756,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # due to YouTube measures against IP ranges of hosting providers. # Working around by preferring the first succeeded video_info containing # the token if no such video_info yet was found. - token = video_info.get('token') or video_info.get('account_playback_token') + token = extract_token(video_info) if not token: video_info = get_video_info break @@ -1769,28 +1773,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): raise ExtractorError( 'YouTube said: %s' % unavailable_message, expected=True, video_id=video_id) - token = video_info.get('token') or video_info.get('account_playback_token') - if not token: - if 'reason' in video_info: - if 'The uploader has not made this video available in your country.' in video_info['reason']: - regions_allowed = self._html_search_meta( - 'regionsAllowed', video_webpage, default=None) - countries = regions_allowed.split(',') if regions_allowed else None - self.raise_geo_restricted( - msg=video_info['reason'][0], countries=countries) - reason = video_info['reason'][0] - if 'Invalid parameters' in reason: - unavailable_message = extract_unavailable_message() - if unavailable_message: - reason = unavailable_message - raise ExtractorError( - 'YouTube said: %s' % reason, - expected=True, video_id=video_id) - else: - raise ExtractorError( - '"token" parameter not in video info for unknown reason', - video_id=video_id) - if video_info.get('license_info'): raise ExtractorError('This video is DRM protected.', expected=True) @@ -2296,6 +2278,29 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if f.get('vcodec') != 'none': f['stretched_ratio'] = ratio + if not formats: + token = extract_token(video_info) + if not token: + if 'reason' in video_info: + if 'The uploader has not made this video available in your country.' in video_info['reason']: + regions_allowed = self._html_search_meta( + 'regionsAllowed', video_webpage, default=None) + countries = regions_allowed.split(',') if regions_allowed else None + self.raise_geo_restricted( + msg=video_info['reason'][0], countries=countries) + reason = video_info['reason'][0] + if 'Invalid parameters' in reason: + unavailable_message = extract_unavailable_message() + if unavailable_message: + reason = unavailable_message + raise ExtractorError( + 'YouTube said: %s' % reason, + expected=True, video_id=video_id) + else: + raise ExtractorError( + '"token" parameter not in video info for unknown reason', + video_id=video_id) + self._sort_formats(formats) self.mark_watched(video_id, video_info, player_response) From a61ce71468cb222338ccd8039dc631f3619dc585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 30 Apr 2019 04:49:12 +0700 Subject: [PATCH 39/45] [youtube] Remove info el for get_video_info request Since it does not work for quite a long time --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 88dba1353..9d542f893 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1715,7 +1715,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # The general idea is to take a union of itags of both DASH manifests (for example # video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093) self.report_video_info_webpage_download(video_id) - for el in ('info', 'embedded', 'detailpage', 'vevo', ''): + for el in ('embedded', 'detailpage', 'vevo', ''): query = { 'video_id': video_id, 'ps': 'default', From 54f3b612169cdff5da49cc7cf794dcca65d1be7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 30 Apr 2019 05:59:12 +0700 Subject: [PATCH 40/45] [openload] Use real Chrome versions (closes #20902) --- youtube_dl/extractor/openload.py | 1586 +++++++++++++++++++++++++++++- 1 file changed, 1580 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index 43cdedb1e..6a8ef67bd 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -368,7 +368,1585 @@ class OpenloadIE(InfoExtractor): 'only_matching': True, }] - _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{major}.0.{build}.{patch} Safari/537.36' + _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36' + _CHROME_VERSIONS = ( + '74.0.3729.129', + '76.0.3780.3', + '76.0.3780.2', + '74.0.3729.128', + '76.0.3780.1', + '76.0.3780.0', + '75.0.3770.15', + '74.0.3729.127', + '74.0.3729.126', + '76.0.3779.1', + '76.0.3779.0', + '75.0.3770.14', + '74.0.3729.125', + '76.0.3778.1', + '76.0.3778.0', + '75.0.3770.13', + '74.0.3729.124', + '74.0.3729.123', + '73.0.3683.121', + '76.0.3777.1', + '76.0.3777.0', + '75.0.3770.12', + '74.0.3729.122', + '76.0.3776.4', + '75.0.3770.11', + '74.0.3729.121', + '76.0.3776.3', + '76.0.3776.2', + '73.0.3683.120', + '74.0.3729.120', + '74.0.3729.119', + '74.0.3729.118', + '76.0.3776.1', + '76.0.3776.0', + '76.0.3775.5', + '75.0.3770.10', + '74.0.3729.117', + '76.0.3775.4', + '76.0.3775.3', + '74.0.3729.116', + '75.0.3770.9', + '76.0.3775.2', + '76.0.3775.1', + '76.0.3775.0', + '75.0.3770.8', + '74.0.3729.115', + '74.0.3729.114', + '76.0.3774.1', + '76.0.3774.0', + '75.0.3770.7', + '74.0.3729.113', + '74.0.3729.112', + '74.0.3729.111', + '76.0.3773.1', + '76.0.3773.0', + '75.0.3770.6', + '74.0.3729.110', + '74.0.3729.109', + '76.0.3772.1', + '76.0.3772.0', + '75.0.3770.5', + '74.0.3729.108', + '74.0.3729.107', + '76.0.3771.1', + '76.0.3771.0', + '75.0.3770.4', + '74.0.3729.106', + '74.0.3729.105', + '75.0.3770.3', + '74.0.3729.104', + '74.0.3729.103', + '74.0.3729.102', + '75.0.3770.2', + '74.0.3729.101', + '75.0.3770.1', + '75.0.3770.0', + '74.0.3729.100', + '75.0.3769.5', + '75.0.3769.4', + '74.0.3729.99', + '75.0.3769.3', + '75.0.3769.2', + '75.0.3768.6', + '74.0.3729.98', + '75.0.3769.1', + '75.0.3769.0', + '74.0.3729.97', + '73.0.3683.119', + '73.0.3683.118', + '74.0.3729.96', + '75.0.3768.5', + '75.0.3768.4', + '75.0.3768.3', + '75.0.3768.2', + '74.0.3729.95', + '74.0.3729.94', + '75.0.3768.1', + '75.0.3768.0', + '74.0.3729.93', + '74.0.3729.92', + '73.0.3683.117', + '74.0.3729.91', + '75.0.3766.3', + '74.0.3729.90', + '75.0.3767.2', + '75.0.3767.1', + '75.0.3767.0', + '74.0.3729.89', + '73.0.3683.116', + '75.0.3766.2', + '74.0.3729.88', + '75.0.3766.1', + '75.0.3766.0', + '74.0.3729.87', + '73.0.3683.115', + '74.0.3729.86', + '75.0.3765.1', + '75.0.3765.0', + '74.0.3729.85', + '73.0.3683.114', + '74.0.3729.84', + '75.0.3764.1', + '75.0.3764.0', + '74.0.3729.83', + '73.0.3683.113', + '75.0.3763.2', + '75.0.3761.4', + '74.0.3729.82', + '75.0.3763.1', + '75.0.3763.0', + '74.0.3729.81', + '73.0.3683.112', + '75.0.3762.1', + '75.0.3762.0', + '74.0.3729.80', + '75.0.3761.3', + '74.0.3729.79', + '73.0.3683.111', + '75.0.3761.2', + '74.0.3729.78', + '74.0.3729.77', + '75.0.3761.1', + '75.0.3761.0', + '73.0.3683.110', + '74.0.3729.76', + '74.0.3729.75', + '75.0.3760.0', + '74.0.3729.74', + '75.0.3759.8', + '75.0.3759.7', + '75.0.3759.6', + '74.0.3729.73', + '75.0.3759.5', + '74.0.3729.72', + '73.0.3683.109', + '75.0.3759.4', + '75.0.3759.3', + '74.0.3729.71', + '75.0.3759.2', + '74.0.3729.70', + '73.0.3683.108', + '74.0.3729.69', + '75.0.3759.1', + '75.0.3759.0', + '74.0.3729.68', + '73.0.3683.107', + '74.0.3729.67', + '75.0.3758.1', + '75.0.3758.0', + '74.0.3729.66', + '73.0.3683.106', + '74.0.3729.65', + '75.0.3757.1', + '75.0.3757.0', + '74.0.3729.64', + '73.0.3683.105', + '74.0.3729.63', + '75.0.3756.1', + '75.0.3756.0', + '74.0.3729.62', + '73.0.3683.104', + '75.0.3755.3', + '75.0.3755.2', + '73.0.3683.103', + '75.0.3755.1', + '75.0.3755.0', + '74.0.3729.61', + '73.0.3683.102', + '74.0.3729.60', + '75.0.3754.2', + '74.0.3729.59', + '75.0.3753.4', + '74.0.3729.58', + '75.0.3754.1', + '75.0.3754.0', + '74.0.3729.57', + '73.0.3683.101', + '75.0.3753.3', + '75.0.3752.2', + '75.0.3753.2', + '74.0.3729.56', + '75.0.3753.1', + '75.0.3753.0', + '74.0.3729.55', + '73.0.3683.100', + '74.0.3729.54', + '75.0.3752.1', + '75.0.3752.0', + '74.0.3729.53', + '73.0.3683.99', + '74.0.3729.52', + '75.0.3751.1', + '75.0.3751.0', + '74.0.3729.51', + '73.0.3683.98', + '74.0.3729.50', + '75.0.3750.0', + '74.0.3729.49', + '74.0.3729.48', + '74.0.3729.47', + '75.0.3749.3', + '74.0.3729.46', + '73.0.3683.97', + '75.0.3749.2', + '74.0.3729.45', + '75.0.3749.1', + '75.0.3749.0', + '74.0.3729.44', + '73.0.3683.96', + '74.0.3729.43', + '74.0.3729.42', + '75.0.3748.1', + '75.0.3748.0', + '74.0.3729.41', + '75.0.3747.1', + '73.0.3683.95', + '75.0.3746.4', + '74.0.3729.40', + '74.0.3729.39', + '75.0.3747.0', + '75.0.3746.3', + '75.0.3746.2', + '74.0.3729.38', + '75.0.3746.1', + '75.0.3746.0', + '74.0.3729.37', + '73.0.3683.94', + '75.0.3745.5', + '75.0.3745.4', + '75.0.3745.3', + '75.0.3745.2', + '74.0.3729.36', + '75.0.3745.1', + '75.0.3745.0', + '75.0.3744.2', + '74.0.3729.35', + '73.0.3683.93', + '74.0.3729.34', + '75.0.3744.1', + '75.0.3744.0', + '74.0.3729.33', + '73.0.3683.92', + '74.0.3729.32', + '74.0.3729.31', + '73.0.3683.91', + '75.0.3741.2', + '75.0.3740.5', + '74.0.3729.30', + '75.0.3741.1', + '75.0.3741.0', + '74.0.3729.29', + '75.0.3740.4', + '73.0.3683.90', + '74.0.3729.28', + '75.0.3740.3', + '73.0.3683.89', + '75.0.3740.2', + '74.0.3729.27', + '75.0.3740.1', + '75.0.3740.0', + '74.0.3729.26', + '73.0.3683.88', + '73.0.3683.87', + '74.0.3729.25', + '75.0.3739.1', + '75.0.3739.0', + '73.0.3683.86', + '74.0.3729.24', + '73.0.3683.85', + '75.0.3738.4', + '75.0.3738.3', + '75.0.3738.2', + '75.0.3738.1', + '75.0.3738.0', + '74.0.3729.23', + '73.0.3683.84', + '74.0.3729.22', + '74.0.3729.21', + '75.0.3737.1', + '75.0.3737.0', + '74.0.3729.20', + '73.0.3683.83', + '74.0.3729.19', + '75.0.3736.1', + '75.0.3736.0', + '74.0.3729.18', + '73.0.3683.82', + '74.0.3729.17', + '75.0.3735.1', + '75.0.3735.0', + '74.0.3729.16', + '73.0.3683.81', + '75.0.3734.1', + '75.0.3734.0', + '74.0.3729.15', + '73.0.3683.80', + '74.0.3729.14', + '75.0.3733.1', + '75.0.3733.0', + '75.0.3732.1', + '74.0.3729.13', + '74.0.3729.12', + '73.0.3683.79', + '74.0.3729.11', + '75.0.3732.0', + '74.0.3729.10', + '73.0.3683.78', + '74.0.3729.9', + '74.0.3729.8', + '74.0.3729.7', + '75.0.3731.3', + '75.0.3731.2', + '75.0.3731.0', + '74.0.3729.6', + '73.0.3683.77', + '73.0.3683.76', + '75.0.3730.5', + '75.0.3730.4', + '73.0.3683.75', + '74.0.3729.5', + '73.0.3683.74', + '75.0.3730.3', + '75.0.3730.2', + '74.0.3729.4', + '73.0.3683.73', + '73.0.3683.72', + '75.0.3730.1', + '75.0.3730.0', + '74.0.3729.3', + '73.0.3683.71', + '74.0.3729.2', + '73.0.3683.70', + '74.0.3729.1', + '74.0.3729.0', + '74.0.3726.4', + '73.0.3683.69', + '74.0.3726.3', + '74.0.3728.0', + '74.0.3726.2', + '73.0.3683.68', + '74.0.3726.1', + '74.0.3726.0', + '74.0.3725.4', + '73.0.3683.67', + '73.0.3683.66', + '74.0.3725.3', + '74.0.3725.2', + '74.0.3725.1', + '74.0.3724.8', + '74.0.3725.0', + '73.0.3683.65', + '74.0.3724.7', + '74.0.3724.6', + '74.0.3724.5', + '74.0.3724.4', + '74.0.3724.3', + '74.0.3724.2', + '74.0.3724.1', + '74.0.3724.0', + '73.0.3683.64', + '74.0.3723.1', + '74.0.3723.0', + '73.0.3683.63', + '74.0.3722.1', + '74.0.3722.0', + '73.0.3683.62', + '74.0.3718.9', + '74.0.3702.3', + '74.0.3721.3', + '74.0.3721.2', + '74.0.3721.1', + '74.0.3721.0', + '74.0.3720.6', + '73.0.3683.61', + '72.0.3626.122', + '73.0.3683.60', + '74.0.3720.5', + '72.0.3626.121', + '74.0.3718.8', + '74.0.3720.4', + '74.0.3720.3', + '74.0.3718.7', + '74.0.3720.2', + '74.0.3720.1', + '74.0.3720.0', + '74.0.3718.6', + '74.0.3719.5', + '73.0.3683.59', + '74.0.3718.5', + '74.0.3718.4', + '74.0.3719.4', + '74.0.3719.3', + '74.0.3719.2', + '74.0.3719.1', + '73.0.3683.58', + '74.0.3719.0', + '73.0.3683.57', + '73.0.3683.56', + '74.0.3718.3', + '73.0.3683.55', + '74.0.3718.2', + '74.0.3718.1', + '74.0.3718.0', + '73.0.3683.54', + '74.0.3717.2', + '73.0.3683.53', + '74.0.3717.1', + '74.0.3717.0', + '73.0.3683.52', + '74.0.3716.1', + '74.0.3716.0', + '73.0.3683.51', + '74.0.3715.1', + '74.0.3715.0', + '73.0.3683.50', + '74.0.3711.2', + '74.0.3714.2', + '74.0.3713.3', + '74.0.3714.1', + '74.0.3714.0', + '73.0.3683.49', + '74.0.3713.1', + '74.0.3713.0', + '72.0.3626.120', + '73.0.3683.48', + '74.0.3712.2', + '74.0.3712.1', + '74.0.3712.0', + '73.0.3683.47', + '72.0.3626.119', + '73.0.3683.46', + '74.0.3710.2', + '72.0.3626.118', + '74.0.3711.1', + '74.0.3711.0', + '73.0.3683.45', + '72.0.3626.117', + '74.0.3710.1', + '74.0.3710.0', + '73.0.3683.44', + '72.0.3626.116', + '74.0.3709.1', + '74.0.3709.0', + '74.0.3704.9', + '73.0.3683.43', + '72.0.3626.115', + '74.0.3704.8', + '74.0.3704.7', + '74.0.3708.0', + '74.0.3706.7', + '74.0.3704.6', + '73.0.3683.42', + '72.0.3626.114', + '74.0.3706.6', + '72.0.3626.113', + '74.0.3704.5', + '74.0.3706.5', + '74.0.3706.4', + '74.0.3706.3', + '74.0.3706.2', + '74.0.3706.1', + '74.0.3706.0', + '73.0.3683.41', + '72.0.3626.112', + '74.0.3705.1', + '74.0.3705.0', + '73.0.3683.40', + '72.0.3626.111', + '73.0.3683.39', + '74.0.3704.4', + '73.0.3683.38', + '74.0.3704.3', + '74.0.3704.2', + '74.0.3704.1', + '74.0.3704.0', + '73.0.3683.37', + '72.0.3626.110', + '72.0.3626.109', + '74.0.3703.3', + '74.0.3703.2', + '73.0.3683.36', + '74.0.3703.1', + '74.0.3703.0', + '73.0.3683.35', + '72.0.3626.108', + '74.0.3702.2', + '74.0.3699.3', + '74.0.3702.1', + '74.0.3702.0', + '73.0.3683.34', + '72.0.3626.107', + '73.0.3683.33', + '74.0.3701.1', + '74.0.3701.0', + '73.0.3683.32', + '73.0.3683.31', + '72.0.3626.105', + '74.0.3700.1', + '74.0.3700.0', + '73.0.3683.29', + '72.0.3626.103', + '74.0.3699.2', + '74.0.3699.1', + '74.0.3699.0', + '73.0.3683.28', + '72.0.3626.102', + '73.0.3683.27', + '73.0.3683.26', + '74.0.3698.0', + '74.0.3696.2', + '72.0.3626.101', + '73.0.3683.25', + '74.0.3696.1', + '74.0.3696.0', + '74.0.3694.8', + '72.0.3626.100', + '74.0.3694.7', + '74.0.3694.6', + '74.0.3694.5', + '74.0.3694.4', + '72.0.3626.99', + '72.0.3626.98', + '74.0.3694.3', + '73.0.3683.24', + '72.0.3626.97', + '72.0.3626.96', + '72.0.3626.95', + '73.0.3683.23', + '72.0.3626.94', + '73.0.3683.22', + '73.0.3683.21', + '72.0.3626.93', + '74.0.3694.2', + '72.0.3626.92', + '74.0.3694.1', + '74.0.3694.0', + '74.0.3693.6', + '73.0.3683.20', + '72.0.3626.91', + '74.0.3693.5', + '74.0.3693.4', + '74.0.3693.3', + '74.0.3693.2', + '73.0.3683.19', + '74.0.3693.1', + '74.0.3693.0', + '73.0.3683.18', + '72.0.3626.90', + '74.0.3692.1', + '74.0.3692.0', + '73.0.3683.17', + '72.0.3626.89', + '74.0.3687.3', + '74.0.3691.1', + '74.0.3691.0', + '73.0.3683.16', + '72.0.3626.88', + '72.0.3626.87', + '73.0.3683.15', + '74.0.3690.1', + '74.0.3690.0', + '73.0.3683.14', + '72.0.3626.86', + '73.0.3683.13', + '73.0.3683.12', + '74.0.3689.1', + '74.0.3689.0', + '73.0.3683.11', + '72.0.3626.85', + '73.0.3683.10', + '72.0.3626.84', + '73.0.3683.9', + '74.0.3688.1', + '74.0.3688.0', + '73.0.3683.8', + '72.0.3626.83', + '74.0.3687.2', + '74.0.3687.1', + '74.0.3687.0', + '73.0.3683.7', + '72.0.3626.82', + '74.0.3686.4', + '72.0.3626.81', + '74.0.3686.3', + '74.0.3686.2', + '74.0.3686.1', + '74.0.3686.0', + '73.0.3683.6', + '72.0.3626.80', + '74.0.3685.1', + '74.0.3685.0', + '73.0.3683.5', + '72.0.3626.79', + '74.0.3684.1', + '74.0.3684.0', + '73.0.3683.4', + '72.0.3626.78', + '72.0.3626.77', + '73.0.3683.3', + '73.0.3683.2', + '72.0.3626.76', + '73.0.3683.1', + '73.0.3683.0', + '72.0.3626.75', + '71.0.3578.141', + '73.0.3682.1', + '73.0.3682.0', + '72.0.3626.74', + '71.0.3578.140', + '73.0.3681.4', + '73.0.3681.3', + '73.0.3681.2', + '73.0.3681.1', + '73.0.3681.0', + '72.0.3626.73', + '71.0.3578.139', + '72.0.3626.72', + '72.0.3626.71', + '73.0.3680.1', + '73.0.3680.0', + '72.0.3626.70', + '71.0.3578.138', + '73.0.3678.2', + '73.0.3679.1', + '73.0.3679.0', + '72.0.3626.69', + '71.0.3578.137', + '73.0.3678.1', + '73.0.3678.0', + '71.0.3578.136', + '73.0.3677.1', + '73.0.3677.0', + '72.0.3626.68', + '72.0.3626.67', + '71.0.3578.135', + '73.0.3676.1', + '73.0.3676.0', + '73.0.3674.2', + '72.0.3626.66', + '71.0.3578.134', + '73.0.3674.1', + '73.0.3674.0', + '72.0.3626.65', + '71.0.3578.133', + '73.0.3673.2', + '73.0.3673.1', + '73.0.3673.0', + '72.0.3626.64', + '71.0.3578.132', + '72.0.3626.63', + '72.0.3626.62', + '72.0.3626.61', + '72.0.3626.60', + '73.0.3672.1', + '73.0.3672.0', + '72.0.3626.59', + '71.0.3578.131', + '73.0.3671.3', + '73.0.3671.2', + '73.0.3671.1', + '73.0.3671.0', + '72.0.3626.58', + '71.0.3578.130', + '73.0.3670.1', + '73.0.3670.0', + '72.0.3626.57', + '71.0.3578.129', + '73.0.3669.1', + '73.0.3669.0', + '72.0.3626.56', + '71.0.3578.128', + '73.0.3668.2', + '73.0.3668.1', + '73.0.3668.0', + '72.0.3626.55', + '71.0.3578.127', + '73.0.3667.2', + '73.0.3667.1', + '73.0.3667.0', + '72.0.3626.54', + '71.0.3578.126', + '73.0.3666.1', + '73.0.3666.0', + '72.0.3626.53', + '71.0.3578.125', + '73.0.3665.4', + '73.0.3665.3', + '72.0.3626.52', + '73.0.3665.2', + '73.0.3664.4', + '73.0.3665.1', + '73.0.3665.0', + '72.0.3626.51', + '71.0.3578.124', + '72.0.3626.50', + '73.0.3664.3', + '73.0.3664.2', + '73.0.3664.1', + '73.0.3664.0', + '73.0.3663.2', + '72.0.3626.49', + '71.0.3578.123', + '73.0.3663.1', + '73.0.3663.0', + '72.0.3626.48', + '71.0.3578.122', + '73.0.3662.1', + '73.0.3662.0', + '72.0.3626.47', + '71.0.3578.121', + '73.0.3661.1', + '72.0.3626.46', + '73.0.3661.0', + '72.0.3626.45', + '71.0.3578.120', + '73.0.3660.2', + '73.0.3660.1', + '73.0.3660.0', + '72.0.3626.44', + '71.0.3578.119', + '73.0.3659.1', + '73.0.3659.0', + '72.0.3626.43', + '71.0.3578.118', + '73.0.3658.1', + '73.0.3658.0', + '72.0.3626.42', + '71.0.3578.117', + '73.0.3657.1', + '73.0.3657.0', + '72.0.3626.41', + '71.0.3578.116', + '73.0.3656.1', + '73.0.3656.0', + '72.0.3626.40', + '71.0.3578.115', + '73.0.3655.1', + '73.0.3655.0', + '72.0.3626.39', + '71.0.3578.114', + '73.0.3654.1', + '73.0.3654.0', + '72.0.3626.38', + '71.0.3578.113', + '73.0.3653.1', + '73.0.3653.0', + '72.0.3626.37', + '71.0.3578.112', + '73.0.3652.1', + '73.0.3652.0', + '72.0.3626.36', + '71.0.3578.111', + '73.0.3651.1', + '73.0.3651.0', + '72.0.3626.35', + '71.0.3578.110', + '73.0.3650.1', + '73.0.3650.0', + '72.0.3626.34', + '71.0.3578.109', + '73.0.3649.1', + '73.0.3649.0', + '72.0.3626.33', + '71.0.3578.108', + '73.0.3648.2', + '73.0.3648.1', + '73.0.3648.0', + '72.0.3626.32', + '71.0.3578.107', + '73.0.3647.2', + '73.0.3647.1', + '73.0.3647.0', + '72.0.3626.31', + '71.0.3578.106', + '73.0.3635.3', + '73.0.3646.2', + '73.0.3646.1', + '73.0.3646.0', + '72.0.3626.30', + '71.0.3578.105', + '72.0.3626.29', + '73.0.3645.2', + '73.0.3645.1', + '73.0.3645.0', + '72.0.3626.28', + '71.0.3578.104', + '72.0.3626.27', + '72.0.3626.26', + '72.0.3626.25', + '72.0.3626.24', + '73.0.3644.0', + '73.0.3643.2', + '72.0.3626.23', + '71.0.3578.103', + '73.0.3643.1', + '73.0.3643.0', + '72.0.3626.22', + '71.0.3578.102', + '73.0.3642.1', + '73.0.3642.0', + '72.0.3626.21', + '71.0.3578.101', + '73.0.3641.1', + '73.0.3641.0', + '72.0.3626.20', + '71.0.3578.100', + '72.0.3626.19', + '73.0.3640.1', + '73.0.3640.0', + '72.0.3626.18', + '73.0.3639.1', + '71.0.3578.99', + '73.0.3639.0', + '72.0.3626.17', + '73.0.3638.2', + '72.0.3626.16', + '73.0.3638.1', + '73.0.3638.0', + '72.0.3626.15', + '71.0.3578.98', + '73.0.3635.2', + '71.0.3578.97', + '73.0.3637.1', + '73.0.3637.0', + '72.0.3626.14', + '71.0.3578.96', + '71.0.3578.95', + '72.0.3626.13', + '71.0.3578.94', + '73.0.3636.2', + '71.0.3578.93', + '73.0.3636.1', + '73.0.3636.0', + '72.0.3626.12', + '71.0.3578.92', + '73.0.3635.1', + '73.0.3635.0', + '72.0.3626.11', + '71.0.3578.91', + '73.0.3634.2', + '73.0.3634.1', + '73.0.3634.0', + '72.0.3626.10', + '71.0.3578.90', + '71.0.3578.89', + '73.0.3633.2', + '73.0.3633.1', + '73.0.3633.0', + '72.0.3610.4', + '72.0.3626.9', + '71.0.3578.88', + '73.0.3632.5', + '73.0.3632.4', + '73.0.3632.3', + '73.0.3632.2', + '73.0.3632.1', + '73.0.3632.0', + '72.0.3626.8', + '71.0.3578.87', + '73.0.3631.2', + '73.0.3631.1', + '73.0.3631.0', + '72.0.3626.7', + '71.0.3578.86', + '72.0.3626.6', + '73.0.3630.1', + '73.0.3630.0', + '72.0.3626.5', + '71.0.3578.85', + '72.0.3626.4', + '73.0.3628.3', + '73.0.3628.2', + '73.0.3629.1', + '73.0.3629.0', + '72.0.3626.3', + '71.0.3578.84', + '73.0.3628.1', + '73.0.3628.0', + '71.0.3578.83', + '73.0.3627.1', + '73.0.3627.0', + '72.0.3626.2', + '71.0.3578.82', + '71.0.3578.81', + '71.0.3578.80', + '72.0.3626.1', + '72.0.3626.0', + '71.0.3578.79', + '70.0.3538.124', + '71.0.3578.78', + '72.0.3623.4', + '72.0.3625.2', + '72.0.3625.1', + '72.0.3625.0', + '71.0.3578.77', + '70.0.3538.123', + '72.0.3624.4', + '72.0.3624.3', + '72.0.3624.2', + '71.0.3578.76', + '72.0.3624.1', + '72.0.3624.0', + '72.0.3623.3', + '71.0.3578.75', + '70.0.3538.122', + '71.0.3578.74', + '72.0.3623.2', + '72.0.3610.3', + '72.0.3623.1', + '72.0.3623.0', + '72.0.3622.3', + '72.0.3622.2', + '71.0.3578.73', + '70.0.3538.121', + '72.0.3622.1', + '72.0.3622.0', + '71.0.3578.72', + '70.0.3538.120', + '72.0.3621.1', + '72.0.3621.0', + '71.0.3578.71', + '70.0.3538.119', + '72.0.3620.1', + '72.0.3620.0', + '71.0.3578.70', + '70.0.3538.118', + '71.0.3578.69', + '72.0.3619.1', + '72.0.3619.0', + '71.0.3578.68', + '70.0.3538.117', + '71.0.3578.67', + '72.0.3618.1', + '72.0.3618.0', + '71.0.3578.66', + '70.0.3538.116', + '72.0.3617.1', + '72.0.3617.0', + '71.0.3578.65', + '70.0.3538.115', + '72.0.3602.3', + '71.0.3578.64', + '72.0.3616.1', + '72.0.3616.0', + '71.0.3578.63', + '70.0.3538.114', + '71.0.3578.62', + '72.0.3615.1', + '72.0.3615.0', + '71.0.3578.61', + '70.0.3538.113', + '72.0.3614.1', + '72.0.3614.0', + '71.0.3578.60', + '70.0.3538.112', + '72.0.3613.1', + '72.0.3613.0', + '71.0.3578.59', + '70.0.3538.111', + '72.0.3612.2', + '72.0.3612.1', + '72.0.3612.0', + '70.0.3538.110', + '71.0.3578.58', + '70.0.3538.109', + '72.0.3611.2', + '72.0.3611.1', + '72.0.3611.0', + '71.0.3578.57', + '70.0.3538.108', + '72.0.3610.2', + '71.0.3578.56', + '71.0.3578.55', + '72.0.3610.1', + '72.0.3610.0', + '71.0.3578.54', + '70.0.3538.107', + '71.0.3578.53', + '72.0.3609.3', + '71.0.3578.52', + '72.0.3609.2', + '71.0.3578.51', + '72.0.3608.5', + '72.0.3609.1', + '72.0.3609.0', + '71.0.3578.50', + '70.0.3538.106', + '72.0.3608.4', + '72.0.3608.3', + '72.0.3608.2', + '71.0.3578.49', + '72.0.3608.1', + '72.0.3608.0', + '70.0.3538.105', + '71.0.3578.48', + '72.0.3607.1', + '72.0.3607.0', + '71.0.3578.47', + '70.0.3538.104', + '72.0.3606.2', + '72.0.3606.1', + '72.0.3606.0', + '71.0.3578.46', + '70.0.3538.103', + '70.0.3538.102', + '72.0.3605.3', + '72.0.3605.2', + '72.0.3605.1', + '72.0.3605.0', + '71.0.3578.45', + '70.0.3538.101', + '71.0.3578.44', + '71.0.3578.43', + '70.0.3538.100', + '70.0.3538.99', + '71.0.3578.42', + '72.0.3604.1', + '72.0.3604.0', + '71.0.3578.41', + '70.0.3538.98', + '71.0.3578.40', + '72.0.3603.2', + '72.0.3603.1', + '72.0.3603.0', + '71.0.3578.39', + '70.0.3538.97', + '72.0.3602.2', + '71.0.3578.38', + '71.0.3578.37', + '72.0.3602.1', + '72.0.3602.0', + '71.0.3578.36', + '70.0.3538.96', + '72.0.3601.1', + '72.0.3601.0', + '71.0.3578.35', + '70.0.3538.95', + '72.0.3600.1', + '72.0.3600.0', + '71.0.3578.34', + '70.0.3538.94', + '72.0.3599.3', + '72.0.3599.2', + '72.0.3599.1', + '72.0.3599.0', + '71.0.3578.33', + '70.0.3538.93', + '72.0.3598.1', + '72.0.3598.0', + '71.0.3578.32', + '70.0.3538.87', + '72.0.3597.1', + '72.0.3597.0', + '72.0.3596.2', + '71.0.3578.31', + '70.0.3538.86', + '71.0.3578.30', + '71.0.3578.29', + '72.0.3596.1', + '72.0.3596.0', + '71.0.3578.28', + '70.0.3538.85', + '72.0.3595.2', + '72.0.3591.3', + '72.0.3595.1', + '72.0.3595.0', + '71.0.3578.27', + '70.0.3538.84', + '72.0.3594.1', + '72.0.3594.0', + '71.0.3578.26', + '70.0.3538.83', + '72.0.3593.2', + '72.0.3593.1', + '72.0.3593.0', + '71.0.3578.25', + '70.0.3538.82', + '72.0.3589.3', + '72.0.3592.2', + '72.0.3592.1', + '72.0.3592.0', + '71.0.3578.24', + '72.0.3589.2', + '70.0.3538.81', + '70.0.3538.80', + '72.0.3591.2', + '72.0.3591.1', + '72.0.3591.0', + '71.0.3578.23', + '70.0.3538.79', + '71.0.3578.22', + '72.0.3590.1', + '72.0.3590.0', + '71.0.3578.21', + '70.0.3538.78', + '70.0.3538.77', + '72.0.3589.1', + '72.0.3589.0', + '71.0.3578.20', + '70.0.3538.76', + '71.0.3578.19', + '70.0.3538.75', + '72.0.3588.1', + '72.0.3588.0', + '71.0.3578.18', + '70.0.3538.74', + '72.0.3586.2', + '72.0.3587.0', + '71.0.3578.17', + '70.0.3538.73', + '72.0.3586.1', + '72.0.3586.0', + '71.0.3578.16', + '70.0.3538.72', + '72.0.3585.1', + '72.0.3585.0', + '71.0.3578.15', + '70.0.3538.71', + '71.0.3578.14', + '72.0.3584.1', + '72.0.3584.0', + '71.0.3578.13', + '70.0.3538.70', + '72.0.3583.2', + '71.0.3578.12', + '72.0.3583.1', + '72.0.3583.0', + '71.0.3578.11', + '70.0.3538.69', + '71.0.3578.10', + '72.0.3582.0', + '72.0.3581.4', + '71.0.3578.9', + '70.0.3538.67', + '72.0.3581.3', + '72.0.3581.2', + '72.0.3581.1', + '72.0.3581.0', + '71.0.3578.8', + '70.0.3538.66', + '72.0.3580.1', + '72.0.3580.0', + '71.0.3578.7', + '70.0.3538.65', + '71.0.3578.6', + '72.0.3579.1', + '72.0.3579.0', + '71.0.3578.5', + '70.0.3538.64', + '71.0.3578.4', + '71.0.3578.3', + '71.0.3578.2', + '71.0.3578.1', + '71.0.3578.0', + '70.0.3538.63', + '69.0.3497.128', + '70.0.3538.62', + '70.0.3538.61', + '70.0.3538.60', + '70.0.3538.59', + '71.0.3577.1', + '71.0.3577.0', + '70.0.3538.58', + '69.0.3497.127', + '71.0.3576.2', + '71.0.3576.1', + '71.0.3576.0', + '70.0.3538.57', + '70.0.3538.56', + '71.0.3575.2', + '70.0.3538.55', + '69.0.3497.126', + '70.0.3538.54', + '71.0.3575.1', + '71.0.3575.0', + '71.0.3574.1', + '71.0.3574.0', + '70.0.3538.53', + '69.0.3497.125', + '70.0.3538.52', + '71.0.3573.1', + '71.0.3573.0', + '70.0.3538.51', + '69.0.3497.124', + '71.0.3572.1', + '71.0.3572.0', + '70.0.3538.50', + '69.0.3497.123', + '71.0.3571.2', + '70.0.3538.49', + '69.0.3497.122', + '71.0.3571.1', + '71.0.3571.0', + '70.0.3538.48', + '69.0.3497.121', + '71.0.3570.1', + '71.0.3570.0', + '70.0.3538.47', + '69.0.3497.120', + '71.0.3568.2', + '71.0.3569.1', + '71.0.3569.0', + '70.0.3538.46', + '69.0.3497.119', + '70.0.3538.45', + '71.0.3568.1', + '71.0.3568.0', + '70.0.3538.44', + '69.0.3497.118', + '70.0.3538.43', + '70.0.3538.42', + '71.0.3567.1', + '71.0.3567.0', + '70.0.3538.41', + '69.0.3497.117', + '71.0.3566.1', + '71.0.3566.0', + '70.0.3538.40', + '69.0.3497.116', + '71.0.3565.1', + '71.0.3565.0', + '70.0.3538.39', + '69.0.3497.115', + '71.0.3564.1', + '71.0.3564.0', + '70.0.3538.38', + '69.0.3497.114', + '71.0.3563.0', + '71.0.3562.2', + '70.0.3538.37', + '69.0.3497.113', + '70.0.3538.36', + '70.0.3538.35', + '71.0.3562.1', + '71.0.3562.0', + '70.0.3538.34', + '69.0.3497.112', + '70.0.3538.33', + '71.0.3561.1', + '71.0.3561.0', + '70.0.3538.32', + '69.0.3497.111', + '71.0.3559.6', + '71.0.3560.1', + '71.0.3560.0', + '71.0.3559.5', + '71.0.3559.4', + '70.0.3538.31', + '69.0.3497.110', + '71.0.3559.3', + '70.0.3538.30', + '69.0.3497.109', + '71.0.3559.2', + '71.0.3559.1', + '71.0.3559.0', + '70.0.3538.29', + '69.0.3497.108', + '71.0.3558.2', + '71.0.3558.1', + '71.0.3558.0', + '70.0.3538.28', + '69.0.3497.107', + '71.0.3557.2', + '71.0.3557.1', + '71.0.3557.0', + '70.0.3538.27', + '69.0.3497.106', + '71.0.3554.4', + '70.0.3538.26', + '71.0.3556.1', + '71.0.3556.0', + '70.0.3538.25', + '71.0.3554.3', + '69.0.3497.105', + '71.0.3554.2', + '70.0.3538.24', + '69.0.3497.104', + '71.0.3555.2', + '70.0.3538.23', + '71.0.3555.1', + '71.0.3555.0', + '70.0.3538.22', + '69.0.3497.103', + '71.0.3554.1', + '71.0.3554.0', + '70.0.3538.21', + '69.0.3497.102', + '71.0.3553.3', + '70.0.3538.20', + '69.0.3497.101', + '71.0.3553.2', + '69.0.3497.100', + '71.0.3553.1', + '71.0.3553.0', + '70.0.3538.19', + '69.0.3497.99', + '69.0.3497.98', + '69.0.3497.97', + '71.0.3552.6', + '71.0.3552.5', + '71.0.3552.4', + '71.0.3552.3', + '71.0.3552.2', + '71.0.3552.1', + '71.0.3552.0', + '70.0.3538.18', + '69.0.3497.96', + '71.0.3551.3', + '71.0.3551.2', + '71.0.3551.1', + '71.0.3551.0', + '70.0.3538.17', + '69.0.3497.95', + '71.0.3550.3', + '71.0.3550.2', + '71.0.3550.1', + '71.0.3550.0', + '70.0.3538.16', + '69.0.3497.94', + '71.0.3549.1', + '71.0.3549.0', + '70.0.3538.15', + '69.0.3497.93', + '69.0.3497.92', + '71.0.3548.1', + '71.0.3548.0', + '70.0.3538.14', + '69.0.3497.91', + '71.0.3547.1', + '71.0.3547.0', + '70.0.3538.13', + '69.0.3497.90', + '71.0.3546.2', + '69.0.3497.89', + '71.0.3546.1', + '71.0.3546.0', + '70.0.3538.12', + '69.0.3497.88', + '71.0.3545.4', + '71.0.3545.3', + '71.0.3545.2', + '71.0.3545.1', + '71.0.3545.0', + '70.0.3538.11', + '69.0.3497.87', + '71.0.3544.5', + '71.0.3544.4', + '71.0.3544.3', + '71.0.3544.2', + '71.0.3544.1', + '71.0.3544.0', + '69.0.3497.86', + '70.0.3538.10', + '69.0.3497.85', + '70.0.3538.9', + '69.0.3497.84', + '71.0.3543.4', + '70.0.3538.8', + '71.0.3543.3', + '71.0.3543.2', + '71.0.3543.1', + '71.0.3543.0', + '70.0.3538.7', + '69.0.3497.83', + '71.0.3542.2', + '71.0.3542.1', + '71.0.3542.0', + '70.0.3538.6', + '69.0.3497.82', + '69.0.3497.81', + '71.0.3541.1', + '71.0.3541.0', + '70.0.3538.5', + '69.0.3497.80', + '71.0.3540.1', + '71.0.3540.0', + '70.0.3538.4', + '69.0.3497.79', + '70.0.3538.3', + '71.0.3539.1', + '71.0.3539.0', + '69.0.3497.78', + '68.0.3440.134', + '69.0.3497.77', + '70.0.3538.2', + '70.0.3538.1', + '70.0.3538.0', + '69.0.3497.76', + '68.0.3440.133', + '69.0.3497.75', + '70.0.3537.2', + '70.0.3537.1', + '70.0.3537.0', + '69.0.3497.74', + '68.0.3440.132', + '70.0.3536.0', + '70.0.3535.5', + '70.0.3535.4', + '70.0.3535.3', + '69.0.3497.73', + '68.0.3440.131', + '70.0.3532.8', + '70.0.3532.7', + '69.0.3497.72', + '69.0.3497.71', + '70.0.3535.2', + '70.0.3535.1', + '70.0.3535.0', + '69.0.3497.70', + '68.0.3440.130', + '69.0.3497.69', + '68.0.3440.129', + '70.0.3534.4', + '70.0.3534.3', + '70.0.3534.2', + '70.0.3534.1', + '70.0.3534.0', + '69.0.3497.68', + '68.0.3440.128', + '70.0.3533.2', + '70.0.3533.1', + '70.0.3533.0', + '69.0.3497.67', + '68.0.3440.127', + '70.0.3532.6', + '70.0.3532.5', + '70.0.3532.4', + '69.0.3497.66', + '68.0.3440.126', + '70.0.3532.3', + '70.0.3532.2', + '70.0.3532.1', + '69.0.3497.60', + '69.0.3497.65', + '69.0.3497.64', + '70.0.3532.0', + '70.0.3531.0', + '70.0.3530.4', + '70.0.3530.3', + '70.0.3530.2', + '69.0.3497.58', + '68.0.3440.125', + '69.0.3497.57', + '69.0.3497.56', + '69.0.3497.55', + '69.0.3497.54', + '70.0.3530.1', + '70.0.3530.0', + '69.0.3497.53', + '68.0.3440.124', + '69.0.3497.52', + '70.0.3529.3', + '70.0.3529.2', + '70.0.3529.1', + '70.0.3529.0', + '69.0.3497.51', + '70.0.3528.4', + '68.0.3440.123', + '70.0.3528.3', + '70.0.3528.2', + '70.0.3528.1', + '70.0.3528.0', + '69.0.3497.50', + '68.0.3440.122', + '70.0.3527.1', + '70.0.3527.0', + '69.0.3497.49', + '68.0.3440.121', + '70.0.3526.1', + '70.0.3526.0', + '68.0.3440.120', + '69.0.3497.48', + '69.0.3497.47', + '68.0.3440.119', + '68.0.3440.118', + '70.0.3525.5', + '70.0.3525.4', + '70.0.3525.3', + '68.0.3440.117', + '69.0.3497.46', + '70.0.3525.2', + '70.0.3525.1', + '70.0.3525.0', + '69.0.3497.45', + '68.0.3440.116', + '70.0.3524.4', + '70.0.3524.3', + '69.0.3497.44', + '70.0.3524.2', + '70.0.3524.1', + '70.0.3524.0', + '70.0.3523.2', + '69.0.3497.43', + '68.0.3440.115', + '70.0.3505.9', + '69.0.3497.42', + '70.0.3505.8', + '70.0.3523.1', + '70.0.3523.0', + '69.0.3497.41', + '68.0.3440.114', + '70.0.3505.7', + '69.0.3497.40', + '70.0.3522.1', + '70.0.3522.0', + '70.0.3521.2', + '69.0.3497.39', + '68.0.3440.113', + '70.0.3505.6', + '70.0.3521.1', + '70.0.3521.0', + '69.0.3497.38', + '68.0.3440.112', + '70.0.3520.1', + '70.0.3520.0', + '69.0.3497.37', + '68.0.3440.111', + '70.0.3519.3', + '70.0.3519.2', + '70.0.3519.1', + '70.0.3519.0', + '69.0.3497.36', + '68.0.3440.110', + '70.0.3518.1', + '70.0.3518.0', + '69.0.3497.35', + '69.0.3497.34', + '68.0.3440.109', + '70.0.3517.1', + '70.0.3517.0', + '69.0.3497.33', + '68.0.3440.108', + '69.0.3497.32', + '70.0.3516.3', + '70.0.3516.2', + '70.0.3516.1', + '70.0.3516.0', + '69.0.3497.31', + '68.0.3440.107', + '70.0.3515.4', + '68.0.3440.106', + '70.0.3515.3', + '70.0.3515.2', + '70.0.3515.1', + '70.0.3515.0', + '69.0.3497.30', + '68.0.3440.105', + '68.0.3440.104', + '70.0.3514.2', + '70.0.3514.1', + '70.0.3514.0', + '69.0.3497.29', + '68.0.3440.103', + '70.0.3513.1', + '70.0.3513.0', + '69.0.3497.28', + ) @staticmethod def _extract_urls(webpage): @@ -383,11 +1961,7 @@ class OpenloadIE(InfoExtractor): url_pattern = 'https://%s/%%s/%s/' % (host, video_id) headers = { - 'User-Agent': self._USER_AGENT_TPL % { - 'major': random.randint(63, 73), - 'build': random.randint(3239, 3683), - 'patch': random.randint(0, 100), - }, + 'User-Agent': self._USER_AGENT_TPL % random.choice(self._CHROME_VERSIONS), } for path in ('embed', 'f'): From 67bfbe49429f11c54de7f24767a807845b00e054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 30 Apr 2019 06:08:12 +0700 Subject: [PATCH 41/45] [ChangeLog] Actualize [ci skip] --- ChangeLog | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ChangeLog b/ChangeLog index 1a71e2fff..db7e24a43 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,24 @@ +version + +Extractors +* [openload] Use real Chrome versions (#20902) +- [youtube] Remove info el for get_video_info request +* [youtube] Improve extraction robustness +- [dramafever] Remove extractor (#20868) +* [adn] Fix subtitle extraction (#12724) ++ [ccc] Extract creator (#20355) ++ [ccc:playlist] Add support for media.ccc.de playlists (#14601, #20355) ++ [sverigesradio] Add support for sverigesradio.se (#18635) ++ [cinemax] Add support for cinemax.com +* [sixplay] Try extracting non-DRM protected manifests (#20849) ++ [youtube] Extract Youtube Music Auto-generated metadata (#20599, #20742) +- [wrzuta] Remove extractor (#20684, #20801) +* [twitch] Prefer source format (#20850) ++ [twitcasting] Add support for private videos (#20843) +* [reddit] Validate thumbnail URL (#20030) +* [yandexmusic] Fix track URL extraction (#20820) + + version 2019.04.24 Extractors From 091200c368af6416a658a3d605c67230becb81e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 30 Apr 2019 06:11:50 +0700 Subject: [PATCH 42/45] release 2019.04.30 --- .github/ISSUE_TEMPLATE/1_broken_site.md | 6 +++--- .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++-- .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++-- .github/ISSUE_TEMPLATE/4_bug_report.md | 6 +++--- .github/ISSUE_TEMPLATE/5_feature_request.md | 4 ++-- ChangeLog | 2 +- docs/supportedsites.md | 8 ++++---- youtube_dl/version.py | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md index bab917400..0b8a124e6 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dl version **2019.04.24** +- [ ] I've verified that I'm running youtube-dl version **2019.04.30** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2019.04.24 + [debug] youtube-dl version 2019.04.30 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md index 7d78921ee..79ed338cd 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -19,7 +19,7 @@ labels: 'site-support-request' - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dl version **2019.04.24** +- [ ] I've verified that I'm running youtube-dl version **2019.04.30** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md index 0ed4d1d6a..f0a035c01 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -18,13 +18,13 @@ title: '' - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dl version **2019.04.24** +- [ ] I've verified that I'm running youtube-dl version **2019.04.30** - [ ] I've searched the bugtracker for similar site feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md index fd9b09d6f..55f91d86f 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dl version **2019.04.24** +- [ ] I've verified that I'm running youtube-dl version **2019.04.30** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2019.04.24 + [debug] youtube-dl version 2019.04.30 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md index 94f373d4e..a521646b4 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -19,13 +19,13 @@ labels: 'request' - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dl version **2019.04.24** +- [ ] I've verified that I'm running youtube-dl version **2019.04.30** - [ ] I've searched the bugtracker for similar feature requests including closed ones diff --git a/ChangeLog b/ChangeLog index db7e24a43..3de97cc78 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -version +version 2019.04.30 Extractors * [openload] Use real Chrome versions (#20902) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 6a320306b..cc95ea981 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -164,6 +164,7 @@ - **chirbit** - **chirbit:profile** - **Cinchcast** + - **Cinemax** - **CiscoLiveSearch** - **CiscoLiveSession** - **CJSW** @@ -237,8 +238,6 @@ - **DouyuTV**: 斗鱼 - **DPlay** - **DPlayIt** - - **dramafever** - - **dramafever:series** - **DRBonanza** - **Dropbox** - **DrTuber** @@ -488,6 +487,7 @@ - **MatchTV** - **MDR**: MDR.DE and KiKA - **media.ccc.de** + - **media.ccc.de:lists** - **Medialaan** - **Mediaset** - **Mediasite** @@ -857,6 +857,8 @@ - **StretchInternet** - **stv:player** - **SunPorno** + - **sverigesradio:episode** + - **sverigesradio:publication** - **SVT** - **SVTPage** - **SVTPlay**: SVT Play and Öppet arkiv @@ -1102,8 +1104,6 @@ - **Wistia** - **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl - **WorldStarHipHop** - - **wrzuta.pl** - - **wrzuta.pl:playlist** - **WSJ**: Wall Street Journal - **WSJArticle** - **WWE** diff --git a/youtube_dl/version.py b/youtube_dl/version.py index ddd3b692a..33c68e44a 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2019.04.24' +__version__ = '2019.04.30' From 274519dd08a312e3677c5bc9dd81bec743571261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 1 May 2019 21:36:19 +0700 Subject: [PATCH 43/45] [redbulltv] Extend _VALID_URL (closes #20922) --- youtube_dl/extractor/redbulltv.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py index 7e8d58f38..5a03e8e11 100644 --- a/youtube_dl/extractor/redbulltv.py +++ b/youtube_dl/extractor/redbulltv.py @@ -10,7 +10,7 @@ from ..utils import ( class RedBullTVIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?redbull(?:\.tv|\.com/(?:[^/]+/)?tv)/video/(?PAP-\w+)' + _VALID_URL = r'https?://(?:www\.)?redbull(?:\.tv|\.com(?:/[^/]+)?(?:/tv)?)(?:/events/[^/]+)?/(?:videos?|live)/(?PAP-\w+)' _TESTS = [{ # film 'url': 'https://www.redbull.tv/video/AP-1Q6XCDTAN1W11', @@ -38,6 +38,12 @@ class RedBullTVIE(InfoExtractor): }, { 'url': 'https://www.redbull.com/int-en/tv/video/AP-1UWHCAR9S1W11/rob-meets-sam-gaze?playlist=playlists::3f81040a-2f31-4832-8e2e-545b1d39d173', 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/us-en/videos/AP-1YM9QCYE52111', + 'only_matching': True, + }, { + 'url': 'https://www.redbull.com/us-en/events/AP-1XV2K61Q51W11/live/AP-1XUJ86FDH1W11', + 'only_matching': True, }] def _real_extract(self, url): From 62d10f0d325cf925855f5720163799298fe688dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 2 May 2019 00:36:52 +0700 Subject: [PATCH 44/45] [fox] Fix API error handling under python 2 (closes #20925) --- youtube_dl/extractor/fox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py index f30d3cba8..f1fbaa0fc 100644 --- a/youtube_dl/extractor/fox.py +++ b/youtube_dl/extractor/fox.py @@ -66,7 +66,7 @@ class FOXIE(AdobePassIE): 'https://api2.fox.com/v2.0/' + path, video_id, data=data, headers=headers) except ExtractorError as e: - if isinstance(e.cause, compat_HTTPError) and e.cause.status == 403: + if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403: entitlement_issues = self._parse_json( e.cause.read().decode(), video_id)['entitlementIssues'] for e in entitlement_issues: From e0dde1d8e28cee673e4362a4141a21326937999d Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Thu, 2 May 2019 10:46:29 +0100 Subject: [PATCH 45/45] [fox] fix Uplynk PrePlay error handling under python 2(#20925) --- youtube_dl/extractor/fox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py index f1fbaa0fc..04f4bdba6 100644 --- a/youtube_dl/extractor/fox.py +++ b/youtube_dl/extractor/fox.py @@ -100,7 +100,7 @@ class FOXIE(AdobePassIE): try: m3u8_url = self._download_json(release_url, video_id)['playURL'] except ExtractorError as e: - if isinstance(e.cause, compat_HTTPError) and e.cause.status == 403: + if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403: error = self._parse_json(e.cause.read().decode(), video_id) if error.get('exception') == 'GeoLocationBlocked': self.raise_geo_restricted(countries=['US'])