Merge branch 'master' of https://github.com/speakerender/youtube-dl into directvnow-auth-fix

README.md

@@ -700,7 +700,7 @@ Note that on Windows you may need to use double quotes instead of single.
 # Download best mp4 format available or any other best if no mp4 available
 $ youtube-dl -f 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best'
 
-# Download best format available but not better that 480p
+# Download best format available but no better than 480p
 $ youtube-dl -f 'bestvideo[height<=480]+bestaudio/best[height<=480]'
 
 # Download best video only format but no bigger than 50 MB

youtube_dl/extractor/cbs.py

@@ -13,13 +13,17 @@ from ..utils import (
 
 class CBSBaseIE(ThePlatformFeedIE):
     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
-        closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
+        subtitles = {}
-        return {
+        for k, ext in [('sMPTE-TTCCURL', 'tt'), ('ClosedCaptionURL', 'ttml'), ('webVTTCaptionURL', 'vtt')]:
-            'en': [{
+            cc_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', k)
-                'ext': 'ttml',
+            if cc_e is not None:
-                'url': closed_caption_e.attrib['value'],
+                cc_url = cc_e.get('value')
-            }]
+                if cc_url:
-        } if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
+                    subtitles.setdefault(subtitles_lang, []).append({
+                        'ext': ext,
+                        'url': cc_url,
+                    })
+        return subtitles
 
 
 class CBSIE(CBSBaseIE):

youtube_dl/extractor/dispeak.py

@@ -58,10 +58,17 @@ class DigitallySpeakingIE(InfoExtractor):
             stream_name = xpath_text(a_format, 'streamName', fatal=True)
             video_path = re.match(r'mp4\:(?P<path>.*)', stream_name).group('path')
             url = video_root + video_path
-            vbr = xpath_text(a_format, 'bitrate')
+            bitrate = xpath_text(a_format, 'bitrate')
+            tbr = int_or_none(bitrate)
+            vbr = int_or_none(self._search_regex(
+                r'-(\d+)\.mp4', video_path, 'vbr', default=None))
+            abr = tbr - vbr if tbr and vbr else None
             video_formats.append({
+                'format_id': bitrate,
                 'url': url,
-                'vbr': int_or_none(vbr),
+                'tbr': tbr,
+                'vbr': vbr,
+                'abr': abr,
             })
         return video_formats
 

youtube_dl/extractor/gdcvault.py

@@ -3,22 +3,24 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .kaltura import KalturaIE
 from ..utils import (
     HEADRequest,
     sanitized_Request,
+    smuggle_url,
     urlencode_postdata,
 )
 
 
 class GDCVaultIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)/(?P<name>(\w|-)+)?'
+    _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)(?:/(?P<name>[\w-]+))?'
     _NETRC_MACHINE = 'gdcvault'
     _TESTS = [
         {
             'url': 'http://www.gdcvault.com/play/1019721/Doki-Doki-Universe-Sweet-Simple',
             'md5': '7ce8388f544c88b7ac11c7ab1b593704',
             'info_dict': {
-                'id': '1019721',
+                'id': '201311826596_AWNY',
                 'display_id': 'Doki-Doki-Universe-Sweet-Simple',
                 'ext': 'mp4',
                 'title': 'Doki-Doki Universe: Sweet, Simple and Genuine (GDC Next 10)'
@@ -27,7 +29,7 @@ class GDCVaultIE(InfoExtractor):
         {
             'url': 'http://www.gdcvault.com/play/1015683/Embracing-the-Dark-Art-of',
             'info_dict': {
-                'id': '1015683',
+                'id': '201203272_1330951438328RSXR',
                 'display_id': 'Embracing-the-Dark-Art-of',
                 'ext': 'flv',
                 'title': 'Embracing the Dark Art of Mathematical Modeling in AI'
@@ -56,7 +58,7 @@ class GDCVaultIE(InfoExtractor):
             'url': 'http://gdcvault.com/play/1023460/Tenacious-Design-and-The-Interface',
             'md5': 'a8efb6c31ed06ca8739294960b2dbabd',
             'info_dict': {
-                'id': '1023460',
+                'id': '840376_BQRC',
                 'ext': 'mp4',
                 'display_id': 'Tenacious-Design-and-The-Interface',
                 'title': 'Tenacious Design and The Interface of \'Destiny\'',
@@ -66,26 +68,38 @@ class GDCVaultIE(InfoExtractor):
             # Multiple audios
             'url': 'http://www.gdcvault.com/play/1014631/Classic-Game-Postmortem-PAC',
             'info_dict': {
-                'id': '1014631',
+                'id': '12396_1299111843500GMPX',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'How to Create a Good Game - From My Experience of Designing Pac-Man',
             },
-            'params': {
+            # 'params': {
-                'skip_download': True,  # Requires rtmpdump
+            #     'skip_download': True,  # Requires rtmpdump
-                'format': 'jp',  # The japanese audio
+            #     'format': 'jp',  # The japanese audio
-            }
+            # }
         },
         {
             # gdc-player.html
             'url': 'http://www.gdcvault.com/play/1435/An-American-engine-in-Tokyo',
             'info_dict': {
-                'id': '1435',
+                'id': '9350_1238021887562UHXB',
                 'display_id': 'An-American-engine-in-Tokyo',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'An American Engine in Tokyo:/nThe collaboration of Epic Games and Square Enix/nFor THE LAST REMINANT',
             },
+        },
+        {
+            # Kaltura Embed
+            'url': 'https://www.gdcvault.com/play/1026180/Mastering-the-Apex-of-Scaling',
+            'info_dict': {
+                'id': '0_h1fg8j3p',
+                'ext': 'mp4',
+                'title': 'Mastering the Apex of Scaling Game Servers (Presented by Multiplay)',
+                'timestamp': 1554401811,
+                'upload_date': '20190404',
+                'uploader_id': 'joe@blazestreaming.com',
+            },
             'params': {
-                'skip_download': True,  # Requires rtmpdump
+                'format': 'mp4-408',
             },
         },
     ]
@@ -114,10 +128,8 @@ class GDCVaultIE(InfoExtractor):
         return start_page
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        video_id, name = re.match(self._VALID_URL, url).groups()
-
+        display_id = name or video_id
-        video_id = mobj.group('id')
-        display_id = mobj.group('name') or video_id
 
         webpage_url = 'http://www.gdcvault.com/play/' + video_id
         start_page = self._download_webpage(webpage_url, display_id)
@@ -127,12 +139,12 @@ class GDCVaultIE(InfoExtractor):
             start_page, 'url', default=None)
         if direct_url:
             title = self._html_search_regex(
-                r'<td><strong>Session Name</strong></td>\s*<td>(.*?)</td>',
+                r'<td><strong>Session Name:?</strong></td>\s*<td>(.*?)</td>',
                 start_page, 'title')
             video_url = 'http://www.gdcvault.com' + direct_url
             # resolve the url so that we can detect the correct extension
-            head = self._request_webpage(HEADRequest(video_url), video_id)
+            video_url = self._request_webpage(
-            video_url = head.geturl()
+                HEADRequest(video_url), video_id).geturl()
 
             return {
                 'id': video_id,
@@ -141,34 +153,36 @@ class GDCVaultIE(InfoExtractor):
                 'title': title,
             }
 
-        PLAYER_REGEX = r'<iframe src="(?P<xml_root>.+?)/(?:gdc-)?player.*?\.html.*?".*?</iframe>'
+        embed_url = KalturaIE._extract_url(start_page)
+        if embed_url:
+            embed_url = smuggle_url(embed_url, {'source_url': url})
+            ie_key = 'Kaltura'
+        else:
+            PLAYER_REGEX = r'<iframe src="(?P<xml_root>.+?)/(?:gdc-)?player.*?\.html.*?".*?</iframe>'
 
-        xml_root = self._html_search_regex(
+            xml_root = self._html_search_regex(
-            PLAYER_REGEX, start_page, 'xml root', default=None)
+                PLAYER_REGEX, start_page, 'xml root', default=None)
-        if xml_root is None:
+            if xml_root is None:
-            # Probably need to authenticate
+                # Probably need to authenticate
-            login_res = self._login(webpage_url, display_id)
+                login_res = self._login(webpage_url, display_id)
-            if login_res is None:
+                if login_res is None:
-                self.report_warning('Could not login.')
+                    self.report_warning('Could not login.')
-            else:
+                else:
-                start_page = login_res
+                    start_page = login_res
-                # Grab the url from the authenticated page
+                    # Grab the url from the authenticated page
-                xml_root = self._html_search_regex(
+                    xml_root = self._html_search_regex(
-                    PLAYER_REGEX, start_page, 'xml root')
+                        PLAYER_REGEX, start_page, 'xml root')
 
-        xml_name = self._html_search_regex(
-            r'<iframe src=".*?\?xml=(.+?\.xml).*?".*?</iframe>',
-            start_page, 'xml filename', default=None)
-        if xml_name is None:
-            # Fallback to the older format
             xml_name = self._html_search_regex(
-                r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>',
+                r'<iframe src=".*?\?xml(?:=|URL=xml/)(.+?\.xml).*?".*?</iframe>',
                 start_page, 'xml filename')
+            embed_url = '%s/xml/%s' % (xml_root, xml_name)
+            ie_key = 'DigitallySpeaking'
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
             'display_id': display_id,
-            'url': '%s/xml/%s' % (xml_root, xml_name),
+            'url': embed_url,
-            'ie_key': 'DigitallySpeaking',
+            'ie_key': ie_key,
         }

youtube_dl/extractor/jwplatform.py

@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class JWPlatformIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://(?:content\.jwplatform|cdn\.jwplayer)\.com/(?:(?:feed|player|thumb|preview|video|manifest)s|jw6|v2/media)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
+    _VALID_URL = r'(?:https?://(?:content\.jwplatform|cdn\.jwplayer)\.com/(?:(?:feed|player|thumb|preview|video)s|jw6|v2/media)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
     _TESTS = [{
         'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
         'md5': 'fa8899fa601eb7c83a64e9d568bdf325',

youtube_dl/extractor/kaltura.py

@@ -145,6 +145,8 @@ class KalturaIE(InfoExtractor):
         )
         if mobj:
             embed_info = mobj.groupdict()
+            for k, v in embed_info.items():
+                embed_info[k] = v.strip()
             url = 'kaltura:%(partner_id)s:%(id)s' % embed_info
             escaped_pid = re.escape(embed_info['partner_id'])
             service_url = re.search(

youtube_dl/extractor/linkedin.py

@@ -9,11 +9,13 @@ from ..utils import (
     float_or_none,
     int_or_none,
     urlencode_postdata,
+    urljoin,
 )
 
 
 class LinkedInLearningBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'linkedin'
+    _LOGIN_URL = 'https://www.linkedin.com/uas/login?trk=learning'
 
     def _call_api(self, course_slug, fields, video_slug=None, resolution=None):
         query = {
@@ -50,11 +52,10 @@ class LinkedInLearningBaseIE(InfoExtractor):
             return
 
         login_page = self._download_webpage(
-            'https://www.linkedin.com/uas/login?trk=learning',
+            self._LOGIN_URL, None, 'Downloading login page')
-            None, 'Downloading login page')
+        action_url = urljoin(self._LOGIN_URL, self._search_regex(
-        action_url = self._search_regex(
             r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page, 'post url',
-            default='https://www.linkedin.com/uas/login-submit', group='url')
+            default='https://www.linkedin.com/uas/login-submit', group='url'))
         data = self._hidden_inputs(login_page)
         data.update({
             'session_key': email,

youtube_dl/extractor/mgtv.py

@@ -1,22 +1,32 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import base64
+import time
+import uuid
+
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
-from ..utils import int_or_none
+    compat_HTTPError,
+    compat_str,
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
 
 
 class MGTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?mgtv\.com/(v|b)/(?:[^/]+/)*(?P<id>\d+)\.html'
     IE_DESC = '芒果TV'
+    _GEO_COUNTRIES = ['CN']
 
     _TESTS = [{
         'url': 'http://www.mgtv.com/v/1/290525/f/3116640.html',
-        'md5': 'b1ffc0fc163152acf6beaa81832c9ee7',
         'info_dict': {
             'id': '3116640',
             'ext': 'mp4',
-            'title': '我是歌手第四季双年巅峰会：韩红李玟“双王”领军对抗',
+            'title': '我是歌手 第四季',
             'description': '我是歌手第四季双年巅峰会',
             'duration': 7461,
             'thumbnail': r're:^https?://.*\.jpg$',
@@ -28,16 +38,30 @@ class MGTVIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        api_data = self._download_json(
+        try:
-            'http://pcweb.api.mgtv.com/player/video', video_id,
+            api_data = self._download_json(
-            query={'video_id': video_id},
+                'https://pcweb.api.mgtv.com/player/video', video_id, query={
-            headers=self.geo_verification_headers())['data']
+                    'tk2': base64.urlsafe_b64encode(b'did=%s|pno=1030|ver=0.3.0301|clit=%d' % (compat_str(uuid.uuid4()).encode(), time.time()))[::-1],
+                    'video_id': video_id,
+                }, headers=self.geo_verification_headers())['data']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                error = self._parse_json(e.cause.read().decode(), None)
+                if error.get('code') == 40005:
+                    self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+                raise ExtractorError(error['msg'], expected=True)
+            raise
         info = api_data['info']
         title = info['title'].strip()
-        stream_domain = api_data['stream_domain'][0]
+        stream_data = self._download_json(
+            'https://pcweb.api.mgtv.com/player/getSource', video_id, query={
+                'pm2': api_data['atc']['pm2'],
+                'video_id': video_id,
+            }, headers=self.geo_verification_headers())['data']
+        stream_domain = stream_data['stream_domain'][0]
 
         formats = []
-        for idx, stream in enumerate(api_data['stream']):
+        for idx, stream in enumerate(stream_data['stream']):
             stream_path = stream.get('url')
             if not stream_path:
                 continue
@@ -47,7 +71,7 @@ class MGTVIE(InfoExtractor):
             format_url = format_data.get('info')
             if not format_url:
                 continue
-            tbr = int_or_none(self._search_regex(
+            tbr = int_or_none(stream.get('filebitrate') or self._search_regex(
                 r'_(\d+)_mp4/', format_url, 'tbr', default=None))
             formats.append({
                 'format_id': compat_str(tbr or idx),

youtube_dl/extractor/streamango.py

@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class StreamangoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:streamango\.com|fruithosts\.net)/(?:f|embed)/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:streamango\.com|fruithosts\.net|streamcherry\.com)/(?:f|embed)/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://streamango.com/f/clapasobsptpkdfe/20170315_150006_mp4',
         'md5': 'e992787515a182f55e38fc97588d802a',
@@ -41,6 +41,9 @@ class StreamangoIE(InfoExtractor):
     }, {
         'url': 'https://fruithosts.net/f/mreodparcdcmspsm/w1f1_r4lph_2018_brrs_720p_latino_mp4',
         'only_matching': True,
+    }, {
+        'url': 'https://streamcherry.com/f/clapasobsptpkdfe/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

youtube_dl/extractor/yourporn.py

@@ -8,8 +8,8 @@ from ..utils import (
 
 
 class YourPornIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?yourporn\.sexy/post/(?P<id>[^/?#&.]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:yourporn\.sexy|sxyprn\.com)/post/(?P<id>[^/?#&.]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://yourporn.sexy/post/57ffcb2e1179b.html',
         'md5': '6f8682b6464033d87acaa7a8ff0c092e',
         'info_dict': {
@@ -23,7 +23,10 @@ class YourPornIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'https://sxyprn.com/post/57ffcb2e1179b.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)