From 8b55cadc83f198e0fa6bac7158f9b05826f39257 Mon Sep 17 00:00:00 2001 From: remitamine Date: Mon, 7 Sep 2015 16:39:01 +0100 Subject: [PATCH 1/3] [canal13cl] fix info extraction --- youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/canal13cl.py | 48 ------------------- youtube_dl/extractor/tele13.py | 77 +++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 49 deletions(-) delete mode 100644 youtube_dl/extractor/canal13cl.py create mode 100644 youtube_dl/extractor/tele13.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 5d2ea39d0..661b53e63 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -67,7 +67,6 @@ from .camdemy import ( CamdemyIE, CamdemyFolderIE ) -from .canal13cl import Canal13clIE from .canalplus import CanalplusIE from .canalc2 import Canalc2IE from .cbs import CBSIE @@ -612,6 +611,7 @@ from .teachingchannel import TeachingChannelIE from .teamcoco import TeamcocoIE from .techtalks import TechTalksIE from .ted import TEDIE +from .tele13 import Tele13IE from .telebruxelles import TeleBruxellesIE from .telecinco import TelecincoIE from .telegraaf import TelegraafIE diff --git a/youtube_dl/extractor/canal13cl.py b/youtube_dl/extractor/canal13cl.py deleted file mode 100644 index 93241fefe..000000000 --- a/youtube_dl/extractor/canal13cl.py +++ /dev/null @@ -1,48 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor - - -class Canal13clIE(InfoExtractor): - _VALID_URL = r'^http://(?:www\.)?13\.cl/(?:[^/?#]+/)*(?P[^/?#]+)' - _TEST = { - 'url': 'http://www.13.cl/t13/nacional/el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda', - 'md5': '4cb1fa38adcad8fea88487a078831755', - 'info_dict': { - 'id': '1403022125', - 'display_id': 'el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda', - 'ext': 'mp4', - 'title': 'El "círculo de hierro" de Michelle Bachelet en su regreso a La Moneda', - 'description': '(Foto: Agencia Uno) En nueve días más, Michelle Bachelet va a asumir por segunda vez como presidenta de la República. Entre aquellos que la acompañarán hay caras que se repiten y otras que se consolidan en su entorno de colaboradores más cercanos.', - } - } - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - display_id = mobj.group('id') - - webpage = self._download_webpage(url, display_id) - - title = self._html_search_meta( - 'twitter:title', webpage, 'title', fatal=True) - description = self._html_search_meta( - 'twitter:description', webpage, 'description') - url = self._html_search_regex( - r'articuloVideo = \"(.*?)\"', webpage, 'url') - real_id = self._search_regex( - r'[^0-9]([0-9]{7,})[^0-9]', url, 'id', default=display_id) - thumbnail = self._html_search_regex( - r'articuloImagen = \"(.*?)\"', webpage, 'thumbnail') - - return { - 'id': real_id, - 'display_id': display_id, - 'url': url, - 'title': title, - 'description': description, - 'ext': 'mp4', - 'thumbnail': thumbnail, - } diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py new file mode 100644 index 000000000..5d89e757f --- /dev/null +++ b/youtube_dl/extractor/tele13.py @@ -0,0 +1,77 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import js_to_json + + +class Tele13IE(InfoExtractor): + _VALID_URL = r'^http://(?:www\.)?t13\.cl/videos(?:/[^/]+)+/(?P[\w-]+)' + _TESTS = [ + { + 'url': 'http://www.t13.cl/videos/actualidad/el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda', + 'md5': '4cb1fa38adcad8fea88487a078831755', + 'info_dict': { + 'id': 'el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda', + 'ext': 'mp4', + 'title': 'El c\u00edrculo de hierro de Michelle Bachelet en su regreso a La Moneda', + } + }, + { + 'url': 'http://www.t13.cl/videos/mundo/tendencias/video-captan-misteriosa-bola-fuego-cielos-bangkok', + 'md5': '65d1ae54812c96f4b345dd21d3bb1adc', + 'info_dict': { + 'id': 'rOoKv2OMpOw', + 'ext': 'mp4', + 'title': 'Shooting star seen on 7-Sep-2015', + 'description': 'md5:a1cd2e74f6ee6851552c9cf5851d6b06', + 'uploader': 'Porjai Jaturongkhakun', + 'upload_date': '20150906', + 'uploader_id': 'UCnLY_3ezwNcDSC_Wc6suZxw', + }, + 'add_ie': ['Youtube'], + } + ] + + def _real_extract(self, url): + display_id = self._match_id(url) + + webpage = self._download_webpage(url, display_id) + + setup_js = self._parse_json( + js_to_json( + self._search_regex( + r"jwplayer\('player-vivo'\).setup\((\{.*?\})\)", + webpage, + 'setup code', + flags=re.DOTALL + ).replace('\n//', '') + ), + display_id + ) + title = setup_js['title'] + thumbnail = setup_js.get('image') or setup_js['playlist'][0].get('image') + description = self._html_search_meta( + 'description', webpage, 'description') + + formats = [] + for f in setup_js['playlist'][0]['sources']: + format_url = f['file'] + if format_url != '': + if '.m3u8' in format_url: + formats.extend(self._extract_m3u8_formats(format_url, display_id)) + else: + if 'youtube.com' in format_url: + return self.url_result(format_url, 'Youtube') + else: + formats.append({'url': format_url, 'format_id': f.get('label')}) + + return { + 'id': display_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'formats': formats, + } From 436416afe2ea70dd6b55f8c9d699ddb0bdc1ec5f Mon Sep 17 00:00:00 2001 From: remitamine Date: Mon, 7 Sep 2015 21:13:49 +0100 Subject: [PATCH 2/3] [tele13] skip test --- youtube_dl/extractor/tele13.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py index 5d89e757f..f1764eb2f 100644 --- a/youtube_dl/extractor/tele13.py +++ b/youtube_dl/extractor/tele13.py @@ -16,8 +16,12 @@ class Tele13IE(InfoExtractor): 'info_dict': { 'id': 'el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda', 'ext': 'mp4', - 'title': 'El c\u00edrculo de hierro de Michelle Bachelet en su regreso a La Moneda', - } + 'title': 'El círculo de hierro de Michelle Bachelet en su regreso a La Moneda', + }, + 'params': { + # HTTP Error 404: Not Found + 'skip_download': True, + }, }, { 'url': 'http://www.t13.cl/videos/mundo/tendencias/video-captan-misteriosa-bola-fuego-cielos-bangkok', From 6882c0870eb89a17eb81d3d273a9f9e42a8bea5e Mon Sep 17 00:00:00 2001 From: remitamine Date: Sun, 20 Dec 2015 15:48:19 +0100 Subject: [PATCH 3/3] [tele13] improve extraction - improve jwplayer setup regex - sort formats - remove duplicate formats - update youtube test --- youtube_dl/extractor/tele13.py | 64 +++++++++++++++++----------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py index f1764eb2f..a363b4d40 100644 --- a/youtube_dl/extractor/tele13.py +++ b/youtube_dl/extractor/tele13.py @@ -1,10 +1,13 @@ # coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor -from ..utils import js_to_json +from .youtube import YoutubeIE +from ..utils import ( + js_to_json, + qualities, + determine_ext, +) class Tele13IE(InfoExtractor): @@ -25,12 +28,12 @@ class Tele13IE(InfoExtractor): }, { 'url': 'http://www.t13.cl/videos/mundo/tendencias/video-captan-misteriosa-bola-fuego-cielos-bangkok', - 'md5': '65d1ae54812c96f4b345dd21d3bb1adc', + 'md5': '867adf6a3b3fef932c68a71d70b70946', 'info_dict': { 'id': 'rOoKv2OMpOw', 'ext': 'mp4', 'title': 'Shooting star seen on 7-Sep-2015', - 'description': 'md5:a1cd2e74f6ee6851552c9cf5851d6b06', + 'description': 'md5:7292ff2a34b2f673da77da222ae77e1e', 'uploader': 'Porjai Jaturongkhakun', 'upload_date': '20150906', 'uploader_id': 'UCnLY_3ezwNcDSC_Wc6suZxw', @@ -41,41 +44,38 @@ class Tele13IE(InfoExtractor): def _real_extract(self, url): display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) - setup_js = self._parse_json( - js_to_json( - self._search_regex( - r"jwplayer\('player-vivo'\).setup\((\{.*?\})\)", - webpage, - 'setup code', - flags=re.DOTALL - ).replace('\n//', '') - ), - display_id - ) - title = setup_js['title'] - thumbnail = setup_js.get('image') or setup_js['playlist'][0].get('image') - description = self._html_search_meta( - 'description', webpage, 'description') + setup_js = self._search_regex(r"(?s)jwplayer\('player-vivo'\).setup\((\{.*?\})\)", webpage, 'setup code') + sources = self._parse_json(self._search_regex(r'sources\s*:\s*(\[[^\]]+\])', setup_js, 'sources'), display_id, js_to_json) + preference = qualities(['Móvil', 'SD', 'HD']) formats = [] - for f in setup_js['playlist'][0]['sources']: + urls = [] + for f in sources: format_url = f['file'] - if format_url != '': - if '.m3u8' in format_url: - formats.extend(self._extract_m3u8_formats(format_url, display_id)) + if format_url and format_url not in urls: + ext = determine_ext(format_url) + if ext == 'm3u8': + m3u8_formats = self._extract_m3u8_formats(format_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False) + if m3u8_formats: + formats.extend(m3u8_formats) + elif YoutubeIE.suitable(format_url): + return self.url_result(format_url, 'Youtube') else: - if 'youtube.com' in format_url: - return self.url_result(format_url, 'Youtube') - else: - formats.append({'url': format_url, 'format_id': f.get('label')}) + formats.append({ + 'url': format_url, + 'format_id': f.get('label'), + 'preference': preference(f.get('label')), + 'ext': ext, + }) + urls.append(format_url) + self._sort_formats(formats) return { 'id': display_id, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, + 'title': self._search_regex(r'title\s*:\s*"([^"]+)"', setup_js, 'title'), + 'description': self._html_search_meta('description', webpage, 'description'), + 'thumbnail': self._search_regex(r'image\s*:\s*"([^"]+)"', setup_js, 'thumbnail', default=None), 'formats': formats, }