From baa16c603776716db00711d75a012880649d6559 Mon Sep 17 00:00:00 2001 From: Daniel Vogt Date: Sun, 18 Oct 2020 22:27:40 +0200 Subject: [PATCH 1/6] [opencast] Add new extractor. Currently only single videos. --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/opencast.py | 129 +++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 youtube_dl/extractor/opencast.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index ae7079a6a..ce19b168c 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -798,6 +798,7 @@ from .ooyala import ( OoyalaIE, OoyalaExternalIE, ) +from .opencast import OpencastIE from .ora import OraTVIE from .orf import ( ORFTVthekIE, diff --git a/youtube_dl/extractor/opencast.py b/youtube_dl/extractor/opencast.py new file mode 100644 index 000000000..81fbd6a7b --- /dev/null +++ b/youtube_dl/extractor/opencast.py @@ -0,0 +1,129 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import parse_iso8601, parse_resolution, int_or_none + + +class OpencastIE(InfoExtractor): + IE_NAME = 'Opencast' + _INSTANCES_RE = r'''(?: + opencast\.informatik\.kit\.edu| + electures\.uni-muenster\.de| + oc-presentation\.ltcc\.tuwien\.ac\.at| + medien\.ph-noe\.ac\.at| + oc-video\.ruhr-uni-bochum\.de| + oc-video1\.ruhr-uni-bochum\.de| + opencast\.informatik\.uni-goettingen\.de| + heicast\.uni-heidelberg\.de| + opencast\.hawk\.de:8080| + opencast\.hs-osnabrueck\.de| + opencast\.uni-koeln\.de| + media\.opencast\.hochschule-rhein-waal\.de| + matterhorn\.dce\.harvard\.edu| + hs-harz\.opencast\.uni-halle\.de| + videocampus\.urz\.uni-leipzig\.de| + media\.uct\.ac\.za| + vid\.igb\.illinois\.edu| + cursosabertos\.c3sl\.ufpr\.br| + mcmedia\.missioncollege\.org| + clases\.odon\.edu\.uy + )''' + + _UUID_RE = r'[\da-fA-F]{8}-[\da-fA-F]{4}-[\da-fA-F]{4}-[\da-fA-F]{4}-[\da-fA-F]{12}' + + _VALID_URL = r'''(?x) + https?://(?P%s)/paella/ui/watch.html\?.*? + id=(?P%s) + ''' % ( + _INSTANCES_RE, + _UUID_RE, + ) + + _API_BASE = 'https://%s/search/episode.json?id=%s' + + _TEST = { + 'url': 'https://oc-video1.ruhr-uni-bochum.de/paella/ui/watch.html?id=ed063cd5-72c8-46b5-a60a-569243edcea8', + 'md5': '554c8e99a90f7be7e874619fcf2a3bc9', + 'info_dict': { + 'id': 'ed063cd5-72c8-46b5-a60a-569243edcea8', + 'ext': 'mp4', + 'title': '11 - Kryptographie - 24.11.2015', + 'thumbnail': r're:^https?://.*\.jpg$', + 'timestamp': 1606208400, + 'upload_date': '20201124' + }, + } + + def _call_api(self, host, video_id, path, note=None, errnote=None, fatal=True): + return self._download_json(self._API_BASE % (host, video_id), video_id, note=note, errnote=errnote, fatal=fatal) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + host = mobj.group('host') + video_id = mobj.group('id') + + api_json = self._call_api(host, video_id, '', note='Downloading video JSON') + + video = api_json.get('search-results', {}).get('result', {}).get('mediapackage', {}) + + # webpage = self._download_webpage(url, video_id) + + title = video.get('title', '') + series = video.get('seriestitle', '') + season_id = video.get('series', '') + creator = video.get('creators', {}).get('creator', '') + timestamp = parse_iso8601(video.get('start', '')) + + tracks = video.get('media', {}).get('track', []) + formats = [] + + for track in tracks: + track_obj = {'url': track['url']} + + audio_info = track.get('audio') + if audio_info is not None: + if 'bitrate' in audio_info: + track_obj.update({'abr': int_or_none(audio_info.get('bitrate'), 1000)}) + if 'samplingrate' in audio_info: + track_obj.update({'asr': int_or_none(audio_info.get('samplingrate'))}) + audio_encoder = audio_info.get('encoder', {}) + if 'type' in audio_encoder: + track_obj.update({'acodec': audio_encoder.get('type')}) + + video_info = track.get('video') + if video_info is not None: + if 'resolution' in video_info: + track_obj.update({'resolution': video_info.get('resolution')}) + resolution = parse_resolution(video_info.get('resolution')) + track_obj.update(resolution) + if 'framerate' in video_info: + track_obj.update({'fps': int_or_none(video_info.get('framerate'))}) + if 'bitrate' in video_info: + track_obj.update({'vbr': int_or_none(video_info.get('bitrate'), 1000)}) + video_encoder = video_info.get('encoder', {}) + if 'type' in video_encoder: + track_obj.update({'vcodec': video_encoder.get('type')}) + + formats.append(track_obj) + + self._sort_formats(formats) + + result_obj = { + 'id': video_id, + 'title': title, + 'creator': creator, + 'series': series, + 'season_id': season_id, + 'formats': formats, + 'timestamp': timestamp, + } + + attachments = video.get('attachments', {}).get('attachment', []) + if len(attachments) > 0: + thumbnail = attachments[0].get('url') + result_obj.update({'thumbnail': thumbnail}) + + return result_obj From 36a34bc4ba33e1f43d532dd6ec7e909c20580803 Mon Sep 17 00:00:00 2001 From: Daniel Vogt Date: Mon, 19 Oct 2020 15:59:10 +0200 Subject: [PATCH 2/6] [opencast] Add support for opencast playlists --- youtube_dl/extractor/extractors.py | 5 +- youtube_dl/extractor/opencast.py | 181 ++++++++++++++++++++--------- 2 files changed, 132 insertions(+), 54 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index ce19b168c..6b56590b9 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -798,7 +798,10 @@ from .ooyala import ( OoyalaIE, OoyalaExternalIE, ) -from .opencast import OpencastIE +from .opencast import ( + OpencastIE, + OpencastPlaylistIE, +) from .ora import OraTVIE from .orf import ( ORFTVthekIE, diff --git a/youtube_dl/extractor/opencast.py b/youtube_dl/extractor/opencast.py index 81fbd6a7b..c12f2a834 100644 --- a/youtube_dl/extractor/opencast.py +++ b/youtube_dl/extractor/opencast.py @@ -4,11 +4,15 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..utils import parse_iso8601, parse_resolution, int_or_none +from ..utils import ( + parse_iso8601, + parse_resolution, + int_or_none, + ExtractorError, +) -class OpencastIE(InfoExtractor): - IE_NAME = 'Opencast' +class OpencastBaseIE(InfoExtractor): _INSTANCES_RE = r'''(?: opencast\.informatik\.kit\.edu| electures\.uni-muenster\.de| @@ -31,52 +35,12 @@ class OpencastIE(InfoExtractor): mcmedia\.missioncollege\.org| clases\.odon\.edu\.uy )''' - _UUID_RE = r'[\da-fA-F]{8}-[\da-fA-F]{4}-[\da-fA-F]{4}-[\da-fA-F]{4}-[\da-fA-F]{12}' - _VALID_URL = r'''(?x) - https?://(?P%s)/paella/ui/watch.html\?.*? - id=(?P%s) - ''' % ( - _INSTANCES_RE, - _UUID_RE, - ) - - _API_BASE = 'https://%s/search/episode.json?id=%s' - - _TEST = { - 'url': 'https://oc-video1.ruhr-uni-bochum.de/paella/ui/watch.html?id=ed063cd5-72c8-46b5-a60a-569243edcea8', - 'md5': '554c8e99a90f7be7e874619fcf2a3bc9', - 'info_dict': { - 'id': 'ed063cd5-72c8-46b5-a60a-569243edcea8', - 'ext': 'mp4', - 'title': '11 - Kryptographie - 24.11.2015', - 'thumbnail': r're:^https?://.*\.jpg$', - 'timestamp': 1606208400, - 'upload_date': '20201124' - }, - } - def _call_api(self, host, video_id, path, note=None, errnote=None, fatal=True): return self._download_json(self._API_BASE % (host, video_id), video_id, note=note, errnote=errnote, fatal=fatal) - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - host = mobj.group('host') - video_id = mobj.group('id') - - api_json = self._call_api(host, video_id, '', note='Downloading video JSON') - - video = api_json.get('search-results', {}).get('result', {}).get('mediapackage', {}) - - # webpage = self._download_webpage(url, video_id) - - title = video.get('title', '') - series = video.get('seriestitle', '') - season_id = video.get('series', '') - creator = video.get('creators', {}).get('creator', '') - timestamp = parse_iso8601(video.get('start', '')) - + def _parse_mediapackage(self, video): tracks = video.get('media', {}).get('track', []) formats = [] @@ -111,15 +75,31 @@ class OpencastIE(InfoExtractor): self._sort_formats(formats) - result_obj = { - 'id': video_id, - 'title': title, - 'creator': creator, - 'series': series, - 'season_id': season_id, - 'formats': formats, - 'timestamp': timestamp, - } + result_obj = {'formats': formats} + + video_id = video.get('id') + if video_id is not None: + result_obj.update({'id': video_id}) + + title = video.get('title') + if title is not None: + result_obj.update({'title': title}) + + series = video.get('seriestitle') + if series is not None: + result_obj.update({'series': series}) + + season_id = video.get('series') + if season_id is not None: + result_obj.update({'season_id': season_id}) + + creator = video.get('creators', {}).get('creator') + if creator is not None: + result_obj.update({'creator': creator}) + + timestamp = parse_iso8601(video.get('start')) + if timestamp is not None: + result_obj.update({'timestamp': timestamp}) attachments = video.get('attachments', {}).get('attachment', []) if len(attachments) > 0: @@ -127,3 +107,98 @@ class OpencastIE(InfoExtractor): result_obj.update({'thumbnail': thumbnail}) return result_obj + + +class OpencastIE(OpencastBaseIE): + _VALID_URL = r'''(?x) + https?://(?P%s)/paella/ui/watch.html\?.*? + id=(?P%s) + ''' % ( + OpencastBaseIE._INSTANCES_RE, + OpencastBaseIE._UUID_RE, + ) + + _API_BASE = 'https://%s/search/episode.json?id=%s' + + _TEST = { + 'url': 'https://oc-video1.ruhr-uni-bochum.de/paella/ui/watch.html?id=ed063cd5-72c8-46b5-a60a-569243edcea8', + 'md5': '554c8e99a90f7be7e874619fcf2a3bc9', + 'info_dict': { + 'id': 'ed063cd5-72c8-46b5-a60a-569243edcea8', + 'ext': 'mp4', + 'title': '11 - Kryptographie - 24.11.2015', + 'thumbnail': r're:^https?://.*\.jpg$', + 'timestamp': 1606208400, + 'upload_date': '20201124', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + host = mobj.group('host') + video_id = mobj.group('id') + + api_json = self._call_api(host, video_id, '', note='Downloading video JSON') + + search_results = api_json.get('search-results', {}) + if 'result' not in search_results: + raise ExtractorError('Video was not found') + + result_dict = search_results.get('result', {}) + if not isinstance(result_dict, dict): + raise ExtractorError('More than one video was unexpectedly returned.') + + video = result_dict.get('mediapackage', {}) + + result_obj = self._parse_mediapackage(video) + return result_obj + + +class OpencastPlaylistIE(OpencastBaseIE): + _VALID_URL = r'''(?x) + https?://(?P%s)/engage/ui/index.html\?.*? + epFrom=(?P%s) + ''' % ( + OpencastBaseIE._INSTANCES_RE, + OpencastBaseIE._UUID_RE, + ) + + _API_BASE = 'https://%s/search/episode.json?sid=%s' + + _TEST = { + 'url': 'https://oc-video1.ruhr-uni-bochum.de/engage/ui/index.html?epFrom=cf68a4a1-36b1-4a53-a6ba-61af5705a0d0', + 'md5': '554c8e99a90f7be7e874619fcf2a3bc9', + 'info_dict': { + 'id': 'cf68a4a1-36b1-4a53-a6ba-61af5705a0d0', + 'title': 'Kryptographie - WiSe 15/16', + }, + 'playlist_mincount': 28, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + host = mobj.group('host') + video_id = mobj.group('id') + + api_json = self._call_api(host, video_id, '', note='Downloading video JSON') + + search_results = api_json.get('search-results', {}) + if 'result' not in search_results: + raise ExtractorError('Playlist was not found') + + result_list = search_results.get('result', {}) + if isinstance(result_list, dict): + result_list = [result_list] + + entries = [] + for episode in result_list: + video = episode.get('mediapackage', {}) + entries.append(self._parse_mediapackage(video)) + + if len(entries) == 0: + raise ExtractorError('Playlist has no entries') + + playlist_title = entries[0].get('series') + + result_obj = self.playlist_result(entries, playlist_id=video_id, playlist_title=playlist_title) + return result_obj From b49ec4214e110e2c7c6560b45ff7ec6fbc2ce31f Mon Sep 17 00:00:00 2001 From: Daniel Vogt Date: Wed, 21 Oct 2020 19:31:09 +0200 Subject: [PATCH 3/6] [opencast] Support all other formats not only raw download --- youtube_dl/extractor/opencast.py | 111 +++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 34 deletions(-) diff --git a/youtube_dl/extractor/opencast.py b/youtube_dl/extractor/opencast.py index c12f2a834..4390b1741 100644 --- a/youtube_dl/extractor/opencast.py +++ b/youtube_dl/extractor/opencast.py @@ -5,6 +5,7 @@ import re from .common import InfoExtractor from ..utils import ( + determine_ext, parse_iso8601, parse_resolution, int_or_none, @@ -42,42 +43,75 @@ class OpencastBaseIE(InfoExtractor): def _parse_mediapackage(self, video): tracks = video.get('media', {}).get('track', []) + + video_id = video.get('id') + formats = [] - for track in tracks: - track_obj = {'url': track['url']} + href = track['url'] + ext = determine_ext(href, None) + track_obj = {'url': href} - audio_info = track.get('audio') - if audio_info is not None: - if 'bitrate' in audio_info: - track_obj.update({'abr': int_or_none(audio_info.get('bitrate'), 1000)}) - if 'samplingrate' in audio_info: - track_obj.update({'asr': int_or_none(audio_info.get('samplingrate'))}) - audio_encoder = audio_info.get('encoder', {}) - if 'type' in audio_encoder: - track_obj.update({'acodec': audio_encoder.get('type')}) + transport = track.get('transport') - video_info = track.get('video') - if video_info is not None: - if 'resolution' in video_info: - track_obj.update({'resolution': video_info.get('resolution')}) - resolution = parse_resolution(video_info.get('resolution')) - track_obj.update(resolution) - if 'framerate' in video_info: - track_obj.update({'fps': int_or_none(video_info.get('framerate'))}) - if 'bitrate' in video_info: - track_obj.update({'vbr': int_or_none(video_info.get('bitrate'), 1000)}) - video_encoder = video_info.get('encoder', {}) - if 'type' in video_encoder: - track_obj.update({'vcodec': video_encoder.get('type')}) + if transport == 'DASH' or ext == 'mpd': + formats.extend(self._extract_mpd_formats(href, video_id, mpd_id='dash', fatal=False)) + elif transport == 'HLS' or ext == 'm3u8': + formats.extend( + self._extract_m3u8_formats( + href, video_id, ext='mp4', m3u8_id='hls', entry_protocol='m3u8_native', fatal=False + ) + ) + elif transport == 'HDS' or ext == 'f4m': + formats.extend(self._extract_f4m_formats(href, video_id, f4m_id='hds', fatal=False)) + elif transport == 'SMOOTH': + formats.extend(self._extract_ism_formats(href, video_id, ism_id='smooth', fatal=False)) + elif ext == 'smil': + formats.extend(self._extract_smil_formats(href, video_id, fatal=False)) + else: + if transport is not None: + track_obj.update({'format_note': track.get('transport')}) + if transport == 'RTMP': + m_obj = re.search(r'^(?:rtmp://[^/]+/(?P[^/]+))/(?P.+)$', href) + if not m_obj: + continue + track_obj.update( + { + 'app': m_obj.group('app'), + 'play_path': m_obj.group('playpath'), + } + ) - formats.append(track_obj) + audio_info = track.get('audio') + if audio_info is not None: + if 'bitrate' in audio_info: + track_obj.update({'abr': int_or_none(audio_info.get('bitrate'), 1000)}) + if 'samplingrate' in audio_info: + track_obj.update({'asr': int_or_none(audio_info.get('samplingrate'))}) + audio_encoder = audio_info.get('encoder', {}) + if 'type' in audio_encoder: + track_obj.update({'acodec': audio_encoder.get('type')}) + + video_info = track.get('video') + if video_info is not None: + if 'resolution' in video_info: + track_obj.update({'resolution': video_info.get('resolution')}) + resolution = parse_resolution(video_info.get('resolution')) + track_obj.update(resolution) + if 'framerate' in video_info: + track_obj.update({'fps': int_or_none(video_info.get('framerate'))}) + if 'bitrate' in video_info: + track_obj.update({'vbr': int_or_none(video_info.get('bitrate'), 1000)}) + video_encoder = video_info.get('encoder', {}) + if 'type' in video_encoder: + track_obj.update({'vcodec': video_encoder.get('type')}) + + formats.append(track_obj) self._sort_formats(formats) result_obj = {'formats': formats} - video_id = video.get('id') if video_id is not None: result_obj.update({'id': video_id}) @@ -165,15 +199,24 @@ class OpencastPlaylistIE(OpencastBaseIE): _API_BASE = 'https://%s/search/episode.json?sid=%s' - _TEST = { - 'url': 'https://oc-video1.ruhr-uni-bochum.de/engage/ui/index.html?epFrom=cf68a4a1-36b1-4a53-a6ba-61af5705a0d0', - 'md5': '554c8e99a90f7be7e874619fcf2a3bc9', - 'info_dict': { - 'id': 'cf68a4a1-36b1-4a53-a6ba-61af5705a0d0', - 'title': 'Kryptographie - WiSe 15/16', + _TESTS = [ + { + 'url': 'https://oc-video1.ruhr-uni-bochum.de/engage/ui/index.html?epFrom=cf68a4a1-36b1-4a53-a6ba-61af5705a0d0', + 'info_dict': { + 'id': 'cf68a4a1-36b1-4a53-a6ba-61af5705a0d0', + 'title': 'Kryptographie - WiSe 15/16', + }, + 'playlist_mincount': 28, }, - 'playlist_mincount': 28, - } + { + 'url': 'https://oc-video.ruhr-uni-bochum.de/engage/ui/index.html?e=1&p=1&epFrom=b1a54262-3684-403f-9731-8e77c3766f9a', + 'info_dict': { + 'id': 'b1a54262-3684-403f-9731-8e77c3766f9a', + 'title': 'inSTUDIES-Social movements and prefigurative politics in a global perspective', + }, + 'playlist_mincount': 8, + }, + ] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) From 16b4448ae0703cc6ac3a144a3d24a7e527474237 Mon Sep 17 00:00:00 2001 From: Daniel Vogt Date: Wed, 21 Oct 2020 20:39:36 +0200 Subject: [PATCH 4/6] [opencast] Fix RTMP --- youtube_dl/extractor/opencast.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/opencast.py b/youtube_dl/extractor/opencast.py index 4390b1741..9d2802a06 100644 --- a/youtube_dl/extractor/opencast.py +++ b/youtube_dl/extractor/opencast.py @@ -59,7 +59,7 @@ class OpencastBaseIE(InfoExtractor): elif transport == 'HLS' or ext == 'm3u8': formats.extend( self._extract_m3u8_formats( - href, video_id, ext='mp4', m3u8_id='hls', entry_protocol='m3u8_native', fatal=False + href, video_id, m3u8_id='hls', entry_protocol='m3u8_native', fatal=False ) ) elif transport == 'HDS' or ext == 'f4m': @@ -79,8 +79,12 @@ class OpencastBaseIE(InfoExtractor): { 'app': m_obj.group('app'), 'play_path': m_obj.group('playpath'), + 'rtmp_live': True, } ) + extention = m_obj.group('playpath').split(':') + if len(extention) > 1: + track_obj.update({'ext': extention[0]}) audio_info = track.get('audio') if audio_info is not None: From 8508e6962e160a7a22caabdb2a022c9ce29bf621 Mon Sep 17 00:00:00 2001 From: Daniel Vogt Date: Wed, 21 Oct 2020 20:44:04 +0200 Subject: [PATCH 5/6] [opencast] Set RTMP preference to -2. It is very slow --- youtube_dl/extractor/opencast.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/opencast.py b/youtube_dl/extractor/opencast.py index 9d2802a06..5252d1cea 100644 --- a/youtube_dl/extractor/opencast.py +++ b/youtube_dl/extractor/opencast.py @@ -80,6 +80,7 @@ class OpencastBaseIE(InfoExtractor): 'app': m_obj.group('app'), 'play_path': m_obj.group('playpath'), 'rtmp_live': True, + 'preference': -2 } ) extention = m_obj.group('playpath').split(':') From 0eb21dcea3de118a79c6cde1fe2e14e32f7f9839 Mon Sep 17 00:00:00 2001 From: Daniel Vogt Date: Wed, 21 Oct 2020 20:47:24 +0200 Subject: [PATCH 6/6] [opencast] formatting code --- youtube_dl/extractor/opencast.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/opencast.py b/youtube_dl/extractor/opencast.py index 5252d1cea..3df53d204 100644 --- a/youtube_dl/extractor/opencast.py +++ b/youtube_dl/extractor/opencast.py @@ -58,9 +58,7 @@ class OpencastBaseIE(InfoExtractor): formats.extend(self._extract_mpd_formats(href, video_id, mpd_id='dash', fatal=False)) elif transport == 'HLS' or ext == 'm3u8': formats.extend( - self._extract_m3u8_formats( - href, video_id, m3u8_id='hls', entry_protocol='m3u8_native', fatal=False - ) + self._extract_m3u8_formats(href, video_id, m3u8_id='hls', entry_protocol='m3u8_native', fatal=False) ) elif transport == 'HDS' or ext == 'f4m': formats.extend(self._extract_f4m_formats(href, video_id, f4m_id='hds', fatal=False)) @@ -80,7 +78,7 @@ class OpencastBaseIE(InfoExtractor): 'app': m_obj.group('app'), 'play_path': m_obj.group('playpath'), 'rtmp_live': True, - 'preference': -2 + 'preference': -2, } ) extention = m_obj.group('playpath').split(':')