]+\bclass=["\']bottom[^>]+>\s*[^<]*
\s*([^<]+)',
- webpage, 'description', fatal=False)
- thumbnail = self._og_search_thumbnail(webpage)
- uploader = self._search_regex(
- r'class="user"[^>]*>]+>([^<]+)',
+ webpage, 'description', default=None)
+ thumbnail = self._og_search_thumbnail(webpage, default=None)
+ uploader = self._html_search_regex(
+ (r'(?s)]+class=["\']profile[^>]+>(.+?)',
+ r'class="user"[^>]*>]+>([^<]+)'),
webpage, 'uploader', default=None)
duration = parse_duration(self._search_regex(
r']+\bclass=["\']right_side[^>]+>\s*([^<]+)',
- webpage, 'duration', fatal=False))
+ webpage, 'duration', default=None))
view_count = str_to_int(self._search_regex(
- r'([\d,.]+)\s+plays', webpage, 'view count', fatal=False))
+ r'([\d,.]+)\s+plays', webpage, 'view count', default=None))
age_limit = self._rta_search(webpage)
- return {
+ return merge_dicts({
'id': video_id,
- 'title': title,
+ 'title': title or video_id,
'description': description,
'thumbnail': thumbnail,
'uploader': uploader,
@@ -138,7 +146,8 @@ class SpankBangIE(InfoExtractor):
'view_count': view_count,
'formats': formats,
'age_limit': age_limit,
- }
+ }, info
+ )
class SpankBangPlaylistIE(InfoExtractor):
diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index 21b93a5b3..7c11ea7aa 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -22,7 +22,7 @@ class BellatorIE(MTVServicesInfoExtractor):
'only_matching': True,
}]
- _FEED_URL = 'http://www.spike.com/feeds/mrss/'
+ _FEED_URL = 'http://www.bellator.com/feeds/mrss/'
_GEO_COUNTRIES = ['US']
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 9b60cc462..db5a4f44e 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -133,7 +133,7 @@ class TEDIE(InfoExtractor):
def _extract_info(self, webpage):
info_json = self._search_regex(
- r'(?s)q\(\s*"\w+.init"\s*,\s*({.+})\)\s*',
+ r'(?s)q\(\s*"\w+.init"\s*,\s*({.+?})\)\s*',
webpage, 'info json')
return json.loads(info_json)
diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index 51923e44a..a819d048c 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -72,8 +72,13 @@ class TV4IE(InfoExtractor):
video_id = self._match_id(url)
info = self._download_json(
- 'http://www.tv4play.se/player/assets/%s.json' % video_id,
- video_id, 'Downloading video info JSON')
+ 'https://playback-api.b17g.net/asset/%s' % video_id,
+ video_id, 'Downloading video info JSON', query={
+ 'service': 'tv4',
+ 'device': 'browser',
+ 'protocol': 'hls,dash',
+ 'drm': 'widevine',
+ })['metadata']
title = info['title']
@@ -111,5 +116,9 @@ class TV4IE(InfoExtractor):
'timestamp': parse_iso8601(info.get('broadcast_date_time')),
'duration': int_or_none(info.get('duration')),
'thumbnail': info.get('image'),
- 'is_live': info.get('is_live') is True,
+ 'is_live': info.get('isLive') is True,
+ 'series': info.get('seriesTitle'),
+ 'season_number': int_or_none(info.get('seasonNumber')),
+ 'episode': info.get('episodeTitle'),
+ 'episode_number': int_or_none(info.get('episodeNumber')),
}
diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index 3475ef4c3..180259aba 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -9,6 +9,8 @@ from ..utils import (
float_or_none,
int_or_none,
parse_age_limit,
+ try_get,
+ url_or_none,
)
@@ -23,11 +25,10 @@ class TvigleIE(InfoExtractor):
_TESTS = [
{
'url': 'http://www.tvigle.ru/video/sokrat/',
- 'md5': '36514aed3657d4f70b4b2cef8eb520cd',
'info_dict': {
'id': '1848932',
'display_id': 'sokrat',
- 'ext': 'flv',
+ 'ext': 'mp4',
'title': 'Сократ',
'description': 'md5:d6b92ffb7217b4b8ebad2e7665253c17',
'duration': 6586,
@@ -37,7 +38,6 @@ class TvigleIE(InfoExtractor):
},
{
'url': 'http://www.tvigle.ru/video/vladimir-vysotskii/vedushchii-teleprogrammy-60-minut-ssha-o-vladimire-vysotskom/',
- 'md5': 'e7efe5350dd5011d0de6550b53c3ba7b',
'info_dict': {
'id': '5142516',
'ext': 'flv',
@@ -62,7 +62,7 @@ class TvigleIE(InfoExtractor):
webpage = self._download_webpage(url, display_id)
video_id = self._html_search_regex(
(r']+class=["\']player["\'][^>]+id=["\'](\d+)',
- r'var\s+cloudId\s*=\s*["\'](\d+)',
+ r'cloudId\s*=\s*["\'](\d+)',
r'class="video-preview current_playing" id="(\d+)"'),
webpage, 'video id')
@@ -90,21 +90,40 @@ class TvigleIE(InfoExtractor):
age_limit = parse_age_limit(item.get('ageRestrictions'))
formats = []
- for vcodec, fmts in item['videos'].items():
+ for vcodec, url_or_fmts in item['videos'].items():
if vcodec == 'hls':
- continue
- for format_id, video_url in fmts.items():
- if format_id == 'm3u8':
+ m3u8_url = url_or_none(url_or_fmts)
+ if not m3u8_url:
continue
- height = self._search_regex(
- r'^(\d+)[pP]$', format_id, 'height', default=None)
- formats.append({
- 'url': video_url,
- 'format_id': '%s-%s' % (vcodec, format_id),
- 'vcodec': vcodec,
- 'height': int_or_none(height),
- 'filesize': int_or_none(item.get('video_files_size', {}).get(vcodec, {}).get(format_id)),
- })
+ formats.extend(self._extract_m3u8_formats(
+ m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+ m3u8_id='hls', fatal=False))
+ elif vcodec == 'dash':
+ mpd_url = url_or_none(url_or_fmts)
+ if not mpd_url:
+ continue
+ formats.extend(self._extract_mpd_formats(
+ mpd_url, video_id, mpd_id='dash', fatal=False))
+ else:
+ if not isinstance(url_or_fmts, dict):
+ continue
+ for format_id, video_url in url_or_fmts.items():
+ if format_id == 'm3u8':
+ continue
+ video_url = url_or_none(video_url)
+ if not video_url:
+ continue
+ height = self._search_regex(
+ r'^(\d+)[pP]$', format_id, 'height', default=None)
+ filesize = int_or_none(try_get(
+ item, lambda x: x['video_files_size'][vcodec][format_id]))
+ formats.append({
+ 'url': video_url,
+ 'format_id': '%s-%s' % (vcodec, format_id),
+ 'vcodec': vcodec,
+ 'height': int_or_none(height),
+ 'filesize': filesize,
+ })
self._sort_formats(formats)
return {
diff --git a/youtube_dl/extractor/tvland.py b/youtube_dl/extractor/tvland.py
index 957cf1ea2..791144128 100644
--- a/youtube_dl/extractor/tvland.py
+++ b/youtube_dl/extractor/tvland.py
@@ -1,32 +1,35 @@
# coding: utf-8
from __future__ import unicode_literals
-from .mtv import MTVServicesInfoExtractor
+from .spike import ParamountNetworkIE
-class TVLandIE(MTVServicesInfoExtractor):
+class TVLandIE(ParamountNetworkIE):
IE_NAME = 'tvland.com'
_VALID_URL = r'https?://(?:www\.)?tvland\.com/(?:video-clips|(?:full-)?episodes)/(?P[^/?#.]+)'
_FEED_URL = 'http://www.tvland.com/feeds/mrss/'
_TESTS = [{
# Geo-restricted. Without a proxy metadata are still there. With a
# proxy it redirects to http://m.tvland.com/app/
- 'url': 'http://www.tvland.com/episodes/hqhps2/everybody-loves-raymond-the-invasion-ep-048',
+ 'url': 'https://www.tvland.com/episodes/s04pzf/everybody-loves-raymond-the-dog-season-1-ep-19',
'info_dict': {
- 'description': 'md5:80973e81b916a324e05c14a3fb506d29',
- 'title': 'The Invasion',
+ 'description': 'md5:84928e7a8ad6649371fbf5da5e1ad75a',
+ 'title': 'The Dog',
},
- 'playlist': [],
+ 'playlist_mincount': 5,
}, {
- 'url': 'http://www.tvland.com/video-clips/zea2ev/younger-younger--hilary-duff---little-lies',
+ 'url': 'https://www.tvland.com/video-clips/4n87f2/younger-a-first-look-at-younger-season-6',
'md5': 'e2c6389401cf485df26c79c247b08713',
'info_dict': {
- 'id': 'b8697515-4bbe-4e01-83d5-fa705ce5fa88',
+ 'id': '891f7d3c-5b5b-4753-b879-b7ba1a601757',
'ext': 'mp4',
- 'title': 'Younger|December 28, 2015|2|NO-EPISODE#|Younger: Hilary Duff - Little Lies',
- 'description': 'md5:7d192f56ca8d958645c83f0de8ef0269',
- 'upload_date': '20151228',
- 'timestamp': 1451289600,
+ 'title': 'Younger|April 30, 2019|6|NO-EPISODE#|A First Look at Younger Season 6',
+ 'description': 'md5:595ea74578d3a888ae878dfd1c7d4ab2',
+ 'upload_date': '20190430',
+ 'timestamp': 1556658000,
+ },
+ 'params': {
+ 'skip_download': True,
},
}, {
'url': 'http://www.tvland.com/full-episodes/iu0hz6/younger-a-kiss-is-just-a-kiss-season-3-ep-301',
diff --git a/youtube_dl/extractor/tvn24.py b/youtube_dl/extractor/tvn24.py
index 6590e1fd0..de0fb5063 100644
--- a/youtube_dl/extractor/tvn24.py
+++ b/youtube_dl/extractor/tvn24.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
from .common import InfoExtractor
from ..utils import (
int_or_none,
+ NO_DEFAULT,
unescapeHTML,
)
@@ -17,9 +18,21 @@ class TVN24IE(InfoExtractor):
'id': '1584444',
'ext': 'mp4',
'title': '"Święta mają być wesołe, dlatego, ludziska, wszyscy pod jemiołę"',
- 'description': 'Wyjątkowe orędzie Artura Andrusa, jednego z gości "Szkła kontaktowego".',
+ 'description': 'Wyjątkowe orędzie Artura Andrusa, jednego z gości Szkła kontaktowego.',
'thumbnail': 're:https?://.*[.]jpeg',
}
+ }, {
+ # different layout
+ 'url': 'https://tvnmeteo.tvn24.pl/magazyny/maja-w-ogrodzie,13/odcinki-online,1,4,1,0/pnacza-ptaki-i-iglaki-odc-691-hgtv-odc-29,1771763.html',
+ 'info_dict': {
+ 'id': '1771763',
+ 'ext': 'mp4',
+ 'title': 'Pnącza, ptaki i iglaki (odc. 691 /HGTV odc. 29)',
+ 'thumbnail': 're:https?://.*',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
}, {
'url': 'http://fakty.tvn24.pl/ogladaj-online,60/53-konferencja-bezpieczenstwa-w-monachium,716431.html',
'only_matching': True,
@@ -35,18 +48,21 @@ class TVN24IE(InfoExtractor):
}]
def _real_extract(self, url):
- video_id = self._match_id(url)
+ display_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
+ webpage = self._download_webpage(url, display_id)
- title = self._og_search_title(webpage)
+ title = self._og_search_title(
+ webpage, default=None) or self._search_regex(
+ r']+class=["\']magazineItemHeader[^>]+>(.+?)(?!\1).+?)\1' % attr, webpage,
- name, group='json', fatal=fatal) or '{}',
- video_id, transform_source=unescapeHTML, fatal=fatal)
+ name, group='json', default=default, fatal=fatal) or '{}',
+ display_id, transform_source=unescapeHTML, fatal=fatal)
quality_data = extract_json('data-quality', 'formats')
@@ -59,16 +75,24 @@ class TVN24IE(InfoExtractor):
})
self._sort_formats(formats)
- description = self._og_search_description(webpage)
+ description = self._og_search_description(webpage, default=None)
thumbnail = self._og_search_thumbnail(
webpage, default=None) or self._html_search_regex(
r'\bdata-poster=(["\'])(?P(?!\1).+?)\1', webpage,
'thumbnail', group='url')
+ video_id = None
+
share_params = extract_json(
- 'data-share-params', 'share params', fatal=False)
+ 'data-share-params', 'share params', default=None)
if isinstance(share_params, dict):
- video_id = share_params.get('id') or video_id
+ video_id = share_params.get('id')
+
+ if not video_id:
+ video_id = self._search_regex(
+ r'data-vid-id=["\'](\d+)', webpage, 'video id',
+ default=None) or self._search_regex(
+ r',(\d+)\.html', url, 'video id', default=display_id)
return {
'id': video_id,
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index dc5ff29c3..0500e33a6 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -317,7 +317,7 @@ class TwitchVodIE(TwitchItemBaseIE):
'Downloading %s access token' % self._ITEM_TYPE)
formats = self._extract_m3u8_formats(
- '%s/vod/%s?%s' % (
+ '%s/vod/%s.m3u8?%s' % (
self._USHER_BASE, item_id,
compat_urllib_parse_urlencode({
'allow_source': 'true',
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 41d0b6be8..cebb6238c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -428,11 +428,22 @@ class TwitterIE(InfoExtractor):
'params': {
'skip_download': True, # requires ffmpeg
},
+ }, {
+ 'url': 'https://twitter.com/foobar/status/1087791357756956680',
+ 'info_dict': {
+ 'id': '1087791357756956680',
+ 'ext': 'mp4',
+ 'title': 'Twitter - A new is coming. Some of you got an opt-in to try it now. Check out the emoji button, quick keyboard shortcuts, upgraded trends, advanced search, and more. Let us know your thoughts!',
+ 'thumbnail': r're:^https?://.*\.jpg',
+ 'description': 'md5:66d493500c013e3e2d434195746a7f78',
+ 'uploader': 'Twitter',
+ 'uploader_id': 'Twitter',
+ 'duration': 61.567,
+ },
}]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
- user_id = mobj.group('user_id')
twid = mobj.group('id')
webpage, urlh = self._download_webpage_handle(
@@ -441,8 +452,13 @@ class TwitterIE(InfoExtractor):
if 'twitter.com/account/suspended' in urlh.geturl():
raise ExtractorError('Account suspended by Twitter.', expected=True)
- if user_id is None:
- mobj = re.match(self._VALID_URL, urlh.geturl())
+ user_id = None
+
+ redirect_mobj = re.match(self._VALID_URL, urlh.geturl())
+ if redirect_mobj:
+ user_id = redirect_mobj.group('user_id')
+
+ if not user_id:
user_id = mobj.group('user_id')
username = remove_end(self._og_search_title(webpage), ' on Twitter')
diff --git a/youtube_dl/extractor/usanetwork.py b/youtube_dl/extractor/usanetwork.py
index 823340776..54c7495cc 100644
--- a/youtube_dl/extractor/usanetwork.py
+++ b/youtube_dl/extractor/usanetwork.py
@@ -1,11 +1,9 @@
# coding: utf-8
from __future__ import unicode_literals
-import re
-
from .adobepass import AdobePassIE
from ..utils import (
- extract_attributes,
+ NO_DEFAULT,
smuggle_url,
update_url_query,
)
@@ -31,22 +29,22 @@ class USANetworkIE(AdobePassIE):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
- player_params = extract_attributes(self._search_regex(
- r'(]+data-usa-tve-player-container[^>]*>)', webpage, 'player params'))
- video_id = player_params['data-mpx-guid']
- title = player_params['data-episode-title']
+ def _x(name, default=NO_DEFAULT):
+ return self._search_regex(
+ r'data-%s\s*=\s*(["\'])(?P(?:(?!\1).)+)\1' % name,
+ webpage, name, default=default, group='value')
- account_pid, path = re.search(
- r'data-src="(?:https?)?//player\.theplatform\.com/p/([^/]+)/.*?/(media/guid/\d+/\d+)',
- webpage).groups()
+ video_id = _x('mpx-guid')
+ title = _x('episode-title')
+ mpx_account_id = _x('mpx-account-id', '2304992029')
query = {
'mbr': 'true',
}
- if player_params.get('data-is-full-episode') == '1':
+ if _x('is-full-episode', None) == '1':
query['manifest'] = 'm3u'
- if player_params.get('data-entitlement') == 'auth':
+ if _x('is-entitlement', None) == '1':
adobe_pass = {}
drupal_settings = self._search_regex(
r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
@@ -57,7 +55,7 @@ class USANetworkIE(AdobePassIE):
adobe_pass = drupal_settings.get('adobePass', {})
resource = self._get_mvpd_resource(
adobe_pass.get('adobePassResourceId', 'usa'),
- title, video_id, player_params.get('data-episode-rating', 'TV-14'))
+ title, video_id, _x('episode-rating', 'TV-14'))
query['auth'] = self._extract_mvpd_auth(
url, video_id, adobe_pass.get('adobePassRequestorId', 'usa'), resource)
@@ -65,11 +63,11 @@ class USANetworkIE(AdobePassIE):
info.update({
'_type': 'url_transparent',
'url': smuggle_url(update_url_query(
- 'http://link.theplatform.com/s/%s/%s' % (account_pid, path),
+ 'http://link.theplatform.com/s/HNK2IC/media/guid/%s/%s' % (mpx_account_id, video_id),
query), {'force_smil_url': True}),
'id': video_id,
'title': title,
- 'series': player_params.get('data-show-title'),
+ 'series': _x('show-title', None),
'episode': title,
'ie_key': 'ThePlatform',
})
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 232e05816..4ea9f1b4b 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -34,6 +34,7 @@ class VevoIE(VevoBaseIE):
(?:https?://(?:www\.)?vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?|
https?://cache\.vevo\.com/m/html/embed\.html\?video=|
https?://videoplayer\.vevo\.com/embed/embedded\?videoId=|
+ https?://embed\.vevo\.com/.*?[?&]isrc=|
vevo:)
(?P[^&?#]+)'''
@@ -144,6 +145,9 @@ class VevoIE(VevoBaseIE):
# Geo-restricted to Netherlands/Germany
'url': 'http://www.vevo.com/watch/boostee/pop-corn-clip-officiel/FR1A91600909',
'only_matching': True,
+ }, {
+ 'url': 'https://embed.vevo.com/?isrc=USH5V1923499&partnerId=4d61b777-8023-4191-9ede-497ed6c24647&partnerAdCode=',
+ 'only_matching': True,
}]
_VERSIONS = {
0: 'youtube', # only in AuthenticateVideo videoVersions
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b5b44a79a..ddf375c6c 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -2,12 +2,14 @@
from __future__ import unicode_literals
import base64
+import functools
import json
import re
import itertools
from .common import InfoExtractor
from ..compat import (
+ compat_kwargs,
compat_HTTPError,
compat_str,
compat_urlparse,
@@ -19,6 +21,7 @@ from ..utils import (
int_or_none,
merge_dicts,
NO_DEFAULT,
+ OnDemandPagedList,
parse_filesize,
qualities,
RegexNotFoundError,
@@ -98,6 +101,13 @@ class VimeoBaseInfoExtractor(InfoExtractor):
webpage, 'vuid', group='vuid')
return xsrft, vuid
+ def _extract_vimeo_config(self, webpage, video_id, *args, **kwargs):
+ vimeo_config = self._search_regex(
+ r'vimeo\.config\s*=\s*(?:({.+?})|_extend\([^,]+,\s+({.+?})\));',
+ webpage, 'vimeo config', *args, **compat_kwargs(kwargs))
+ if vimeo_config:
+ return self._parse_json(vimeo_config, video_id)
+
def _set_vimeo_cookie(self, name, value):
self._set_cookie('vimeo.com', name, value)
@@ -253,7 +263,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
\.
)?
vimeo(?Ppro)?\.com/
- (?!(?:channels|album)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
+ (?!(?:channels|album|showcase)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
(?:.*?/)?
(?:
(?:
@@ -580,11 +590,9 @@ class VimeoIE(VimeoBaseInfoExtractor):
# and latter we extract those that are Vimeo specific.
self.report_extraction(video_id)
- vimeo_config = self._search_regex(
- r'vimeo\.config\s*=\s*(?:({.+?})|_extend\([^,]+,\s+({.+?})\));', webpage,
- 'vimeo config', default=None)
+ vimeo_config = self._extract_vimeo_config(webpage, video_id, default=None)
if vimeo_config:
- seed_status = self._parse_json(vimeo_config, video_id).get('seed_status', {})
+ seed_status = vimeo_config.get('seed_status', {})
if seed_status.get('state') == 'failed':
raise ExtractorError(
'%s said: %s' % (self.IE_NAME, seed_status['title']),
@@ -905,7 +913,7 @@ class VimeoUserIE(VimeoChannelIE):
class VimeoAlbumIE(VimeoChannelIE):
IE_NAME = 'vimeo:album'
- _VALID_URL = r'https://vimeo\.com/album/(?P\d+)(?:$|[?#]|/(?!video))'
+ _VALID_URL = r'https://vimeo\.com/(?:album|showcase)/(?P\d+)(?:$|[?#]|/(?!video))'
_TITLE_RE = r'