Merge remote-tracking branch 'upstream/master' into porntrex

youtube_dl/extractor/extractors.py

@@ -808,6 +808,7 @@ from .nrk import (
     NRKTVSeasonIE,
     NRKTVSeriesIE,
 )
+from .ntvcojp import NTVCoJpCUIE
 from .ntvde import NTVDeIE
 from .ntvru import NTVRuIE
 from .nytimes import (

youtube_dl/extractor/nhk.py

@@ -1,54 +1,81 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import ExtractorError
 
 
 class NhkVodIE(InfoExtractor):
-    _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/en/(?:vod|ondemand)/(?P<id>[^/]+/[^/?#&]+)'
+    _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/(?P<lang>[a-z]{2})/ondemand/(?P<type>video|audio)/(?P<id>\d{7}|[a-z]+-\d{8}-\d+)'
+    # Content available only for a limited period of time. Visit
+    # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
     _TESTS = [{
-        # Videos available only for a limited period of time. Visit
-        # http://www3.nhk.or.jp/nhkworld/en/vod/ for working samples.
-        'url': 'http://www3.nhk.or.jp/nhkworld/en/vod/tokyofashion/20160815',
-        'info_dict': {
-            'id': 'A1bnNiNTE6nY3jLllS-BIISfcC_PpvF5',
-            'ext': 'flv',
-            'title': 'TOKYO FASHION EXPRESS - The Kimono as Global Fashion',
-            'description': 'md5:db338ee6ce8204f415b754782f819824',
-            'series': 'TOKYO FASHION EXPRESS',
-            'episode': 'The Kimono as Global Fashion',
-        },
-        'skip': 'Videos available only for a limited period of time',
-    }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/',
         'only_matching': True,
+    }, {
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/plugin-20190404-1/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www3.nhk.or.jp/nhkworld/fr/ondemand/audio/plugin-20190404-1/',
+        'only_matching': True,
     }]
-    _API_URL = 'http://api.nhk.or.jp/nhkworld/vodesdlist/v1/all/all/all.json?apikey=EJfK8jdS57GqlupFgAfAAwr573q01y6k'
+    _API_URL_TEMPLATE = 'https://api.nhk.or.jp/nhkworld/%sodesdlist/v7/episode/%s/%s/all%s.json'
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        lang, m_type, episode_id = re.match(self._VALID_URL, url).groups()
-
+        if episode_id.isdigit():
-        data = self._download_json(self._API_URL, video_id)
+            episode_id = episode_id[:4] + '-' + episode_id[4:]
-
-        try:
-            episode = next(
-                e for e in data['data']['episodes']
-                if e.get('url') and video_id in e['url'])
-        except StopIteration:
-            raise ExtractorError('Unable to find episode')
-
-        embed_code = episode['vod_id']
 
+        is_video = m_type == 'video'
+        episode = self._download_json(
+            self._API_URL_TEMPLATE % ('v' if is_video else 'r', episode_id, lang, '/all' if is_video else ''),
+            episode_id, query={'apikey': 'EJfK8jdS57GqlupFgAfAAwr573q01y6k'})['data']['episodes'][0]
         title = episode.get('sub_title_clean') or episode['sub_title']
-        description = episode.get('description_clean') or episode.get('description')
-        series = episode.get('title_clean') or episode.get('title')
 
-        return {
+        def get_clean_field(key):
-            '_type': 'url_transparent',
+            return episode.get(key + '_clean') or episode.get(key)
-            'ie_key': 'Ooyala',
+
-            'url': 'ooyala:%s' % embed_code,
+        series = get_clean_field('title')
+
+        thumbnails = []
+        for s, w, h in [('', 640, 360), ('_l', 1280, 720)]:
+            img_path = episode.get('image' + s)
+            if not img_path:
+                continue
+            thumbnails.append({
+                'id': '%dp' % h,
+                'height': h,
+                'width': w,
+                'url': 'https://www3.nhk.or.jp' + img_path,
+            })
+
+        info = {
+            'id': episode_id + '-' + lang,
             'title': '%s - %s' % (series, title) if series and title else title,
-            'description': description,
+            'description': get_clean_field('description'),
+            'thumbnails': thumbnails,
             'series': series,
             'episode': title,
         }
+        if is_video:
+            info.update({
+                '_type': 'url_transparent',
+                'ie_key': 'Ooyala',
+                'url': 'ooyala:' + episode['vod_id'],
+            })
+        else:
+            audio = episode['audio']
+            audio_path = audio['audio']
+            info['formats'] = self._extract_m3u8_formats(
+                'https://nhks-vh.akamaihd.net/i%s/master.m3u8' % audio_path,
+                episode_id, 'm4a', m3u8_id='hls', fatal=False)
+            for proto in ('rtmpt', 'rtmp'):
+                info['formats'].append({
+                    'ext': 'flv',
+                    'format_id': proto,
+                    'url': '%s://flv.nhk.or.jp/ondemand/mp4:flv%s' % (proto, audio_path),
+                    'vcodec': 'none',
+                })
+            for f in info['formats']:
+                f['language'] = lang
+        return info

youtube_dl/extractor/ntvcojp.py

@@ -0,0 +1,49 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    smuggle_url,
+)
+
+
+class NTVCoJpCUIE(InfoExtractor):
+    IE_NAME = 'cu.ntv.co.jp'
+    IE_DESC = 'Nippon Television Network'
+    _VALID_URL = r'https?://cu\.ntv\.co\.jp/(?!program)(?P<id>[^/?&#]+)'
+    _TEST = {
+        'url': 'https://cu.ntv.co.jp/televiva-chill-gohan_181031/',
+        'info_dict': {
+            'id': '5978891207001',
+            'ext': 'mp4',
+            'title': '桜エビと炒り卵がポイント！ 「中華風 エビチリおにぎり」──『美虎』五十嵐美幸',
+            'upload_date': '20181213',
+            'description': 'md5:211b52f4fd60f3e0e72b68b0c6ba52a9',
+            'uploader_id': '3855502814001',
+            'timestamp': 1544669941,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        player_config = self._parse_json(self._search_regex(
+            r'(?s)PLAYER_CONFIG\s*=\s*({.+?})',
+            webpage, 'player config'), display_id, js_to_json)
+        video_id = player_config['videoId']
+        account_id = player_config.get('account') or '3855502814001'
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'display_id': display_id,
+            'title': self._search_regex(r'<h1[^>]+class="title"[^>]*>([^<]+)', webpage, 'title').strip(),
+            'description': self._html_search_meta(['description', 'og:description'], webpage),
+            'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account_id, video_id), {'geo_countries': ['JP']}),
+            'ie_key': 'BrightcoveNew',
+        }