2013-07-10 17:49:11 +02:00
# encoding: utf-8
2014-01-06 01:47:52 +01:00
from __future__ import unicode_literals
2013-06-23 20:31:45 +02:00
import os
import re
from . common import InfoExtractor
2014-01-06 01:42:58 +01:00
from . youtube import YoutubeIE
2013-06-23 20:31:45 +02:00
from . . utils import (
compat_urllib_parse ,
2013-08-28 12:47:27 +02:00
compat_urlparse ,
2014-02-21 16:59:10 +01:00
compat_xml_parse_error ,
2013-06-23 20:31:45 +02:00
2014-08-28 01:00:59 +02:00
determine_ext ,
2013-06-23 20:31:45 +02:00
ExtractorError ,
2014-08-24 02:02:17 +02:00
float_or_none ,
2013-12-20 17:05:28 +01:00
HEADRequest ,
2014-08-22 18:19:56 +02:00
orderedSet ,
2014-03-10 17:31:32 +01:00
parse_xml ,
2013-10-15 12:05:13 +02:00
smuggle_url ,
unescapeHTML ,
2013-12-17 12:33:55 +01:00
unified_strdate ,
2014-08-24 04:47:18 +02:00
unsmuggle_url ,
2013-12-17 12:33:55 +01:00
url_basename ,
2013-06-23 20:31:45 +02:00
)
2013-07-10 17:49:11 +02:00
from . brightcove import BrightcoveIE
2013-12-19 20:28:52 +01:00
from . ooyala import OoyalaIE
2014-03-16 20:00:31 +01:00
from . rutv import RUTVIE
2014-03-28 13:58:49 +01:00
from . smotri import SmotriIE
2014-10-13 14:59:35 +02:00
from . condenast import CondeNastIE
2013-06-23 20:31:45 +02:00
2013-08-24 22:49:52 +02:00
2013-06-23 20:31:45 +02:00
class GenericIE ( InfoExtractor ) :
2014-01-06 01:47:52 +01:00
IE_DESC = ' Generic downloader that works on some sites '
2013-06-23 20:31:45 +02:00
_VALID_URL = r ' .* '
2014-01-06 01:47:52 +01:00
IE_NAME = ' generic '
2013-07-10 17:49:11 +02:00
_TESTS = [
{
2014-01-06 01:47:52 +01:00
' url ' : ' http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html ' ,
2014-04-14 13:51:46 +02:00
' md5 ' : ' 85b90ccc9d73b4acd9138d3af4c27f89 ' ,
2014-01-06 01:47:52 +01:00
' info_dict ' : {
2014-04-14 13:51:46 +02:00
' id ' : ' 13601338388002 ' ,
' ext ' : ' mp4 ' ,
2014-01-06 01:47:52 +01:00
' uploader ' : ' www.hodiho.fr ' ,
' title ' : ' R \u00e9 gis plante sa Jeep ' ,
2013-07-10 17:49:11 +02:00
}
} ,
2013-10-27 14:40:25 +01:00
# bandcamp page with custom domain
{
2014-01-06 01:47:52 +01:00
' add_ie ' : [ ' Bandcamp ' ] ,
' url ' : ' http://bronyrock.com/track/the-pony-mash ' ,
' info_dict ' : {
2014-04-14 13:56:29 +02:00
' id ' : ' 3235767654 ' ,
' ext ' : ' mp3 ' ,
2014-01-06 01:47:52 +01:00
' title ' : ' The Pony Mash ' ,
' uploader ' : ' M_Pallante ' ,
2013-10-27 14:40:25 +01:00
} ,
2014-01-06 01:47:52 +01:00
' skip ' : ' There is a limit of 200 free downloads / month for the test song ' ,
2013-10-27 14:40:25 +01:00
} ,
2013-11-06 16:40:24 +01:00
# embedded brightcove video
2013-11-07 21:06:48 +01:00
# it also tests brightcove videos that need to set the 'Referer' in the
# http requests
2013-11-06 16:40:24 +01:00
{
2014-01-06 01:47:52 +01:00
' add_ie ' : [ ' Brightcove ' ] ,
' url ' : ' http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/ ' ,
' info_dict ' : {
' id ' : ' 2765128793001 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Le cours de bourse : l’ analyse technique ' ,
' description ' : ' md5:7e9ad046e968cb2d1114004aba466fd9 ' ,
' uploader ' : ' BFM BUSINESS ' ,
2013-11-06 16:40:24 +01:00
} ,
2014-01-06 01:47:52 +01:00
' params ' : {
' skip_download ' : True ,
2013-11-06 16:40:24 +01:00
} ,
} ,
2014-01-28 03:35:32 +01:00
{
# https://github.com/rg3/youtube-dl/issues/2253
' url ' : ' http://bcove.me/i6nfkrc3 ' ,
' md5 ' : ' 0ba9446db037002366bab3b3eb30c88c ' ,
' info_dict ' : {
2014-04-14 13:56:29 +02:00
' id ' : ' 3101154703001 ' ,
' ext ' : ' mp4 ' ,
2014-01-28 03:35:32 +01:00
' title ' : ' Still no power ' ,
' uploader ' : ' thestar.com ' ,
' description ' : ' Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs. ' ,
} ,
' add_ie ' : [ ' Brightcove ' ] ,
} ,
2014-04-01 15:17:35 +02:00
{
' url ' : ' http://www.championat.com/video/football/v/87/87499.html ' ,
' md5 ' : ' fb973ecf6e4a78a67453647444222983 ' ,
' info_dict ' : {
' id ' : ' 3414141473001 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Видео. Удаление Дзагоева (ЦСКА) ' ,
' description ' : ' Онлайн-трансляция матча ЦСКА - " Волга " ' ,
' uploader ' : ' Championat ' ,
} ,
} ,
2013-12-17 12:33:55 +01:00
# Direct link to a video
{
2014-01-06 01:47:52 +01:00
' url ' : ' http://media.w3.org/2010/05/sintel/trailer.mp4 ' ,
' md5 ' : ' 67d406c2bcb6af27fa886f31aa934bbe ' ,
' info_dict ' : {
' id ' : ' trailer ' ,
2014-02-27 07:21:59 +01:00
' ext ' : ' mp4 ' ,
2014-01-06 01:47:52 +01:00
' title ' : ' trailer ' ,
' upload_date ' : ' 20100513 ' ,
2013-12-17 12:33:55 +01:00
}
2013-12-19 20:28:52 +01:00
} ,
# ooyala video
{
2014-01-06 01:47:52 +01:00
' url ' : ' http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219 ' ,
' md5 ' : ' 5644c6ca5d5782c1d0d350dad9bd840c ' ,
' info_dict ' : {
' id ' : ' BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ ' ,
' ext ' : ' mp4 ' ,
2014-01-21 01:40:34 +01:00
' title ' : ' 2cc213299525360.mov ' , # that's what we get
2013-12-19 20:28:52 +01:00
} ,
} ,
2014-02-27 07:21:59 +01:00
# google redirect
{
' url ' : ' http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http % 3A %2F %2F www.youtube.com %2F watch %3F v % 3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE ' ,
' info_dict ' : {
' id ' : ' cmQHVoWB5FY ' ,
' ext ' : ' mp4 ' ,
' upload_date ' : ' 20130224 ' ,
' uploader_id ' : ' TheVerge ' ,
' description ' : ' Chris Ziegler takes a look at the Alcatel OneTouch Fire and the ZTE Open; two of the first Firefox OS handsets to be officially announced. ' ,
' uploader ' : ' The Verge ' ,
' title ' : ' First Firefox OS phones side-by-side ' ,
} ,
' params ' : {
' skip_download ' : False ,
}
2014-03-05 14:01:53 +01:00
} ,
2014-02-24 01:15:51 +01:00
# embed.ly video
{
' url ' : ' http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/ ' ,
' info_dict ' : {
' id ' : ' 9ODmcdjQcHQ ' ,
' ext ' : ' mp4 ' ,
2014-03-05 14:05:44 +01:00
' title ' : ' Tested: Grinding Coffee at 2000 Frames Per Second ' ,
' upload_date ' : ' 20140225 ' ,
' description ' : ' md5:06a40fbf30b220468f1e0957c0f558ff ' ,
' uploader ' : ' Tested ' ,
' uploader_id ' : ' testedcom ' ,
2014-02-24 01:15:51 +01:00
} ,
# No need to test YoutubeIE here
' params ' : {
' skip_download ' : True ,
} ,
} ,
2014-03-11 16:51:36 +01:00
# funnyordie embed
{
' url ' : ' http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns ' ,
' info_dict ' : {
' id ' : ' 18e820ec3f ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Between Two Ferns with Zach Galifianakis: President Barack Obama ' ,
' description ' : ' Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet. ' ,
2014-03-16 20:00:31 +01:00
} ,
2014-03-11 16:51:36 +01:00
} ,
2014-03-16 20:00:31 +01:00
# RUTV embed
{
' url ' : ' http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html ' ,
' info_dict ' : {
' id ' : ' 776940 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Охотское море стало целиком российским ' ,
' description ' : ' md5:5ed62483b14663e2a95ebbe115eb8f43 ' ,
} ,
' params ' : {
# m3u8 download
' skip_download ' : True ,
} ,
2014-03-20 16:33:23 +01:00
} ,
# Embedded TED video
{
' url ' : ' http://en.support.wordpress.com/videos/ted-talks/ ' ,
2014-09-29 05:12:57 +02:00
' md5 ' : ' 65fdff94098e4a607385a60c5177c638 ' ,
2014-03-20 16:33:23 +01:00
' info_dict ' : {
2014-09-29 05:12:57 +02:00
' id ' : ' 1969 ' ,
2014-03-20 16:33:23 +01:00
' ext ' : ' mp4 ' ,
2014-09-29 05:12:57 +02:00
' title ' : ' Hidden miracles of the natural world ' ,
' uploader ' : ' Louie Schwartzberg ' ,
' description ' : ' md5:8145d19d320ff3e52f28401f4c4283b9 ' ,
2014-03-20 16:33:23 +01:00
}
2014-03-11 16:51:36 +01:00
} ,
2014-04-04 16:23:09 +02:00
# Embeded Ustream video
{
' url ' : ' http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm ' ,
' md5 ' : ' 27b99cdb639c9b12a79bca876a073417 ' ,
' info_dict ' : {
2014-04-04 18:56:29 +02:00
' id ' : ' 45734260 ' ,
' ext ' : ' flv ' ,
' uploader ' : ' AU SPA: The NSA and Privacy ' ,
2014-04-04 16:23:09 +02:00
' title ' : ' NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman '
}
} ,
2014-03-14 22:39:53 +01:00
# nowvideo embed hidden behind percent encoding
{
' url ' : ' http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/ ' ,
' md5 ' : ' 2baf4ddd70f697d94b1c18cf796d5107 ' ,
' info_dict ' : {
' id ' : ' 06e53103ca9aa ' ,
' ext ' : ' flv ' ,
' title ' : ' Macross Episode 001 Watch Macross Episode 001 onl ' ,
' description ' : ' No description ' ,
} ,
2014-03-21 22:14:24 +01:00
} ,
2014-03-24 22:01:47 +01:00
# arte embed
{
' url ' : ' http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html ' ,
' md5 ' : ' 7653032cbb25bf6c80d80f217055fa43 ' ,
' info_dict ' : {
' id ' : ' 048195-004_PLUS7-F ' ,
' ext ' : ' flv ' ,
' title ' : ' X:enius ' ,
' description ' : ' md5:d5fdf32ef6613cdbfd516ae658abf168 ' ,
' upload_date ' : ' 20140320 ' ,
} ,
' params ' : {
' skip_download ' : ' Requires rtmpdump '
}
} ,
2014-04-21 05:47:52 +02:00
# Condé Nast embed
{
' url ' : ' http://www.wired.com/2014/04/honda-asimo/ ' ,
' md5 ' : ' ba0dfe966fa007657bd1443ee672db0f ' ,
' info_dict ' : {
' id ' : ' 53501be369702d3275860000 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Honda’ s New Asimo Robot Is More Human Than Ever ' ,
}
2014-04-30 01:46:06 +02:00
} ,
# Dailymotion embed
{
' url ' : ' http://www.spi0n.com/zap-spi0n-com-n216/ ' ,
' md5 ' : ' 441aeeb82eb72c422c7f14ec533999cd ' ,
' info_dict ' : {
' id ' : ' k2mm4bCdJ6CQ2i7c8o2 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Le Zap de Spi0n n°216 - Zapping du Web ' ,
' uploader ' : ' Spi0n ' ,
} ,
' add_ie ' : [ ' Dailymotion ' ] ,
2014-06-09 22:06:45 +02:00
} ,
# YouTube embed
{
' url ' : ' http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html ' ,
' info_dict ' : {
' id ' : ' FXRb4ykk4S0 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' The NBL Auction 2014 ' ,
' uploader ' : ' BADMINTON England ' ,
' uploader_id ' : ' BADMINTONEvents ' ,
' upload_date ' : ' 20140603 ' ,
' description ' : ' md5:9ef128a69f1e262a700ed83edb163a73 ' ,
} ,
' add_ie ' : [ ' Youtube ' ] ,
' params ' : {
' skip_download ' : True ,
}
} ,
2014-06-22 21:38:04 +02:00
# MTVSercices embed
{
' url ' : ' http://www.gametrailers.com/news-post/76093/north-america-europe-is-getting-that-mario-kart-8-mercedes-dlc-too ' ,
' md5 ' : ' 35727f82f58c76d996fc188f9755b0d5 ' ,
' info_dict ' : {
' id ' : ' 0306a69b-8adf-4fb5-aace-75f8e8cbfca9 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Review ' ,
' description ' : ' Mario \' s life in the fast lane has never looked so good. ' ,
} ,
} ,
2014-05-21 11:55:37 +02:00
# YouTube embed via <data-embed-url="">
{
' url ' : ' https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM ' ,
' info_dict ' : {
2014-09-29 05:12:57 +02:00
' id ' : ' 4vAffPZIT44 ' ,
2014-05-21 11:55:37 +02:00
' ext ' : ' mp4 ' ,
2014-09-29 05:12:57 +02:00
' title ' : ' Asphalt 8: Airborne - Update - Welcome to Dubai! ' ,
2014-08-22 18:19:56 +02:00
' uploader ' : ' Gameloft ' ,
' uploader_id ' : ' gameloft ' ,
2014-09-29 05:12:57 +02:00
' upload_date ' : ' 20140828 ' ,
' description ' : ' md5:c80da9ed3d83ae6d1876c834de03e1c4 ' ,
2014-08-22 18:19:56 +02:00
} ,
' params ' : {
' skip_download ' : True ,
2014-05-21 11:55:37 +02:00
}
2014-08-24 02:02:17 +02:00
} ,
# Camtasia studio
{
' url ' : ' http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/ ' ,
' playlist ' : [ {
' md5 ' : ' 0c5e352edabf715d762b0ad4e6d9ee67 ' ,
' info_dict ' : {
' id ' : ' Fenn-AA_PA_Radar_Course_Lecture_1c_Final ' ,
' title ' : ' Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1 ' ,
' ext ' : ' flv ' ,
' duration ' : 2235.90 ,
}
} , {
' md5 ' : ' 10e4bb3aaca9fd630e273ff92d9f3c63 ' ,
' info_dict ' : {
' id ' : ' Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP ' ,
' title ' : ' Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip ' ,
' ext ' : ' flv ' ,
' duration ' : 2235.93 ,
}
} ] ,
' info_dict ' : {
' title ' : ' Fenn-AA_PA_Radar_Course_Lecture_1c_Final ' ,
}
2014-08-24 05:31:32 +02:00
} ,
# Flowplayer
{
' url ' : ' http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html ' ,
' md5 ' : ' 9d65602bf31c6e20014319c7d07fba27 ' ,
' info_dict ' : {
' id ' : ' 5123ea6d5e5a7 ' ,
' ext ' : ' mp4 ' ,
' age_limit ' : 18 ,
' uploader ' : ' www.handjobhub.com ' ,
2014-10-27 00:45:15 +01:00
' title ' : ' Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com ' ,
2014-08-24 05:31:32 +02:00
}
2014-08-25 18:03:01 +02:00
} ,
# RSS feed
{
' url ' : ' http://phihag.de/2014/youtube-dl/rss2.xml ' ,
' info_dict ' : {
' id ' : ' http://phihag.de/2014/youtube-dl/rss2.xml ' ,
' title ' : ' Zero Punctuation ' ,
' description ' : ' re: '
} ,
' playlist_mincount ' : 11 ,
2014-08-28 00:58:24 +02:00
} ,
# Multiple brightcove videos
# https://github.com/rg3/youtube-dl/issues/2283
{
' url ' : ' http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html ' ,
' info_dict ' : {
' id ' : ' always-never ' ,
' title ' : ' Always / Never - The New Yorker ' ,
} ,
' playlist_count ' : 3 ,
' params ' : {
' extract_flat ' : False ,
' skip_download ' : True ,
}
2014-09-02 15:19:28 +02:00
} ,
# MLB embed
{
' url ' : ' http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/ ' ,
' md5 ' : ' 96f09a37e44da40dd083e12d9a683327 ' ,
' info_dict ' : {
' id ' : ' 33322633 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Ump changes call to ball ' ,
' description ' : ' md5:71c11215384298a172a6dcb4c2e20685 ' ,
' duration ' : 48 ,
' timestamp ' : 1401537900 ,
' upload_date ' : ' 20140531 ' ,
' thumbnail ' : ' re:^https?://.* \ .jpg$ ' ,
} ,
} ,
2014-09-20 02:02:11 +02:00
# Wistia embed
{
' url ' : ' http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson ' ,
' md5 ' : ' 8788b683c777a5cf25621eaf286d0c23 ' ,
' info_dict ' : {
' id ' : ' 1cfaf6b7ea ' ,
' ext ' : ' mov ' ,
' title ' : ' md5:51364a8d3d009997ba99656004b5e20d ' ,
' duration ' : 643.0 ,
' filesize ' : 182808282 ,
' uploader ' : ' education-portal.com ' ,
} ,
} ,
2014-10-23 16:57:13 +02:00
{
' url ' : ' http://thoughtworks.wistia.com/medias/uxjb0lwrcz ' ,
' md5 ' : ' baf49c2baa8a7de5f3fc145a8506dcd4 ' ,
' info_dict ' : {
' id ' : ' uxjb0lwrcz ' ,
' ext ' : ' mp4 ' ,
2014-10-23 17:53:56 +02:00
' title ' : ' Conversation about Hexagonal Rails Part 1 - ThoughtWorks ' ,
2014-10-23 16:57:13 +02:00
' duration ' : 1715.0 ,
2014-10-23 17:53:56 +02:00
' uploader ' : ' thoughtworks.wistia.com ' ,
2014-10-26 20:49:51 +01:00
} ,
2014-10-23 16:57:13 +02:00
} ,
2014-10-26 20:49:51 +01:00
# Direct download with broken HEAD
{
' url ' : ' http://ai-radio.org:8000/radio.opus ' ,
' info_dict ' : {
' id ' : ' radio ' ,
' ext ' : ' opus ' ,
' title ' : ' radio ' ,
} ,
' params ' : {
' skip_download ' : True , # infinite live stream
} ,
' expected_warnings ' : [
r ' 501.*Not Implemented '
] ,
2014-10-29 20:27:58 +01:00
} ,
# Soundcloud embed
{
' url ' : ' http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/ ' ,
' info_dict ' : {
' id ' : ' 174391317 ' ,
' ext ' : ' mp3 ' ,
' description ' : ' md5:ff867d6b555488ad3c52572bb33d432c ' ,
' uploader ' : ' Sophos Security ' ,
' title ' : ' Chet Chat 171 - Oct 29, 2014 ' ,
' upload_date ' : ' 20141029 ' ,
}
2014-10-26 20:49:51 +01:00
}
2013-07-10 17:49:11 +02:00
]
2013-06-23 20:31:45 +02:00
def report_following_redirect ( self , new_url ) :
""" Report information extraction. """
2014-01-06 01:47:52 +01:00
self . _downloader . to_screen ( ' [redirect] Following redirect to %s ' % new_url )
2013-06-23 20:31:45 +02:00
2014-02-20 13:14:05 +01:00
def _extract_rss ( self , url , video_id , doc ) :
playlist_title = doc . find ( ' ./channel/title ' ) . text
playlist_desc_el = doc . find ( ' ./channel/description ' )
playlist_desc = None if playlist_desc_el is None else playlist_desc_el . text
entries = [ {
' _type ' : ' url ' ,
' url ' : e . find ( ' link ' ) . text ,
' title ' : e . find ( ' title ' ) . text ,
} for e in doc . findall ( ' ./channel/item ' ) ]
return {
' _type ' : ' playlist ' ,
' id ' : url ,
' title ' : playlist_title ,
' description ' : playlist_desc ,
' entries ' : entries ,
}
2014-08-24 02:02:17 +02:00
def _extract_camtasia ( self , url , video_id , webpage ) :
""" Returns None if no camtasia video can be found. """
camtasia_cfg = self . _search_regex (
r ' fo \ .addVariable \ ( \ s* " csConfigFile " , \ s* " ([^ " ]+) " \ s* \ ); ' ,
webpage , ' camtasia configuration file ' , default = None )
if camtasia_cfg is None :
return None
title = self . _html_search_meta ( ' DC.title ' , webpage , fatal = True )
camtasia_url = compat_urlparse . urljoin ( url , camtasia_cfg )
camtasia_cfg = self . _download_xml (
camtasia_url , video_id ,
note = ' Downloading camtasia configuration ' ,
errnote = ' Failed to download camtasia configuration ' )
fileset_node = camtasia_cfg . find ( ' ./playlist/array/fileset ' )
entries = [ ]
for n in fileset_node . getchildren ( ) :
url_n = n . find ( ' ./uri ' )
if url_n is None :
continue
entries . append ( {
' id ' : os . path . splitext ( url_n . text . rpartition ( ' / ' ) [ 2 ] ) [ 0 ] ,
' title ' : ' %s - %s ' % ( title , n . tag ) ,
' url ' : compat_urlparse . urljoin ( url , url_n . text ) ,
' duration ' : float_or_none ( n . find ( ' ./duration ' ) . text ) ,
} )
return {
' _type ' : ' playlist ' ,
' entries ' : entries ,
' title ' : title ,
}
2013-06-23 20:31:45 +02:00
def _real_extract ( self , url ) :
2014-04-30 01:46:06 +02:00
if url . startswith ( ' // ' ) :
return {
' _type ' : ' url ' ,
2014-05-05 03:12:41 +02:00
' url ' : self . http_scheme ( ) + url ,
2014-04-30 01:46:06 +02:00
}
2013-09-06 18:39:35 +02:00
parsed_url = compat_urlparse . urlparse ( url )
if not parsed_url . scheme :
2014-01-22 14:16:43 +01:00
default_search = self . _downloader . params . get ( ' default_search ' )
if default_search is None :
2014-07-29 17:17:43 +02:00
default_search = ' fixup_error '
2014-01-22 14:16:43 +01:00
2014-07-29 17:17:43 +02:00
if default_search in ( ' auto ' , ' auto_warning ' , ' fixup_error ' ) :
2014-01-22 14:16:43 +01:00
if ' / ' in url :
self . _downloader . report_warning ( ' The url doesn \' t specify the protocol, trying with http ' )
return self . url_result ( ' http:// ' + url )
2014-07-29 17:17:43 +02:00
elif default_search != ' fixup_error ' :
2014-03-30 15:57:31 +02:00
if default_search == ' auto_warning ' :
2014-05-19 17:10:11 +02:00
if re . match ( r ' ^(?:url|URL)$ ' , url ) :
raise ExtractorError (
' Invalid URL: %r . Call youtube-dl like this: youtube-dl -v " https://www.youtube.com/watch?v=BaW_jenozKc " ' % url ,
expected = True )
else :
self . _downloader . report_warning (
2014-07-06 11:22:44 +02:00
' Falling back to youtube search for %s . Set --default-search " auto " to suppress this warning. ' % url )
2014-01-22 14:16:43 +01:00
return self . url_result ( ' ytsearch: ' + url )
2014-07-29 17:17:43 +02:00
if default_search in ( ' error ' , ' fixup_error ' ) :
2014-07-06 11:22:44 +02:00
raise ExtractorError (
( ' %r is not a valid URL. '
2014-07-21 02:37:44 +02:00
' Set --default-search " ytsearch " (or run youtube-dl " ytsearch: %s " ) to search YouTube '
2014-07-06 11:22:44 +02:00
) % ( url , url ) , expected = True )
2014-01-22 14:16:43 +01:00
else :
2014-10-23 21:13:45 +02:00
if ' : ' not in default_search :
default_search + = ' : '
2014-01-22 14:16:43 +01:00
return self . url_result ( default_search + url )
2014-08-24 04:47:18 +02:00
url , smuggled_data = unsmuggle_url ( url )
force_videoid = None
2014-09-28 12:14:16 +02:00
is_intentional = smuggled_data and smuggled_data . get ( ' to_generic ' )
2014-08-24 04:47:18 +02:00
if smuggled_data and ' force_videoid ' in smuggled_data :
force_videoid = smuggled_data [ ' force_videoid ' ]
video_id = force_videoid
else :
video_id = os . path . splitext ( url . rstrip ( ' / ' ) . split ( ' / ' ) [ - 1 ] ) [ 0 ]
2013-09-06 18:39:35 +02:00
2014-01-06 01:47:52 +01:00
self . to_screen ( ' %s : Requesting header ' % video_id )
2013-12-27 08:38:42 +01:00
2014-08-24 06:58:11 +02:00
head_req = HEADRequest ( url )
2014-10-26 17:05:44 +01:00
head_response = self . _request_webpage (
2014-08-24 06:58:11 +02:00
head_req , video_id ,
note = False , errnote = ' Could not send HEAD request to %s ' % url ,
fatal = False )
2013-12-17 12:33:55 +01:00
2014-10-26 17:05:44 +01:00
if head_response is not False :
2013-12-17 12:33:55 +01:00
# Check for redirect
2014-10-26 17:05:44 +01:00
new_url = head_response . geturl ( )
2013-12-17 12:33:55 +01:00
if url != new_url :
self . report_following_redirect ( new_url )
2014-08-24 04:47:18 +02:00
if force_videoid :
new_url = smuggle_url (
new_url , { ' force_videoid ' : force_videoid } )
2013-12-17 12:04:33 +01:00
return self . url_result ( new_url )
2013-12-17 12:33:55 +01:00
2014-10-26 17:05:44 +01:00
full_response = None
if head_response is False :
full_response = self . _request_webpage ( url , video_id )
head_response = full_response
# Check for direct link to a video
content_type = head_response . headers . get ( ' Content-Type ' , ' ' )
m = re . match ( r ' ^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$ ' , content_type )
if m :
upload_date = unified_strdate (
head_response . headers . get ( ' Last-Modified ' ) )
return {
' id ' : video_id ,
' title ' : os . path . splitext ( url_basename ( url ) ) [ 0 ] ,
2014-11-01 12:04:15 +01:00
' direct ' : True ,
2014-10-26 17:05:44 +01:00
' formats ' : [ {
' format_id ' : m . group ( ' format_id ' ) ,
' url ' : url ,
' vcodec ' : ' none ' if m . group ( ' type ' ) == ' audio ' else None
} ] ,
' upload_date ' : upload_date ,
}
2013-12-17 12:33:55 +01:00
2014-09-28 12:14:16 +02:00
if not self . _downloader . params . get ( ' test ' , False ) and not is_intentional :
self . _downloader . report_warning ( ' Falling back on generic information extractor. ' )
2014-10-26 17:05:44 +01:00
if full_response :
2014-10-26 20:58:09 +01:00
webpage = self . _webpage_read_content ( full_response , url , video_id )
2014-10-26 17:05:44 +01:00
else :
2013-06-23 20:31:45 +02:00
webpage = self . _download_webpage ( url , video_id )
self . report_extraction ( video_id )
2013-11-18 13:28:26 +01:00
2014-02-20 13:14:05 +01:00
# Is it an RSS feed?
try :
2014-03-10 17:31:32 +01:00
doc = parse_xml ( webpage )
2014-02-20 13:14:05 +01:00
if doc . tag == ' rss ' :
return self . _extract_rss ( url , video_id , doc )
2014-02-21 16:59:10 +01:00
except compat_xml_parse_error :
2014-02-20 13:14:05 +01:00
pass
2014-08-24 02:02:17 +02:00
# Is it a Camtasia project?
camtasia_res = self . _extract_camtasia ( url , video_id , webpage )
if camtasia_res is not None :
return camtasia_res
2014-03-14 22:38:49 +01:00
# Sometimes embedded video player is hidden behind percent encoding
# (e.g. https://github.com/rg3/youtube-dl/issues/2448)
# Unescaping the whole page allows to handle those cases in a generic way
2014-02-24 17:44:31 +01:00
webpage = compat_urllib_parse . unquote ( webpage )
2013-11-18 13:28:26 +01:00
# it's tempting to parse this further, but you would
# have to take into account all the variations like
# Video Title - Site Name
# Site Name | Video Title
# Video Title - Tagline | Site Name
# and so on and so forth; it's just not practical
2013-12-06 09:15:04 +01:00
video_title = self . _html_search_regex (
2014-01-06 01:47:52 +01:00
r ' (?s)<title>(.*?)</title> ' , webpage , ' video title ' ,
default = ' video ' )
2013-12-06 09:15:04 +01:00
2014-08-24 05:31:32 +02:00
# Try to detect age limit automatically
age_limit = self . _rta_search ( webpage )
# And then there are the jokers who advertise that they use RTA,
# but actually don't.
AGE_LIMIT_MARKERS = [
r ' Proudly Labeled <a href= " http://www.rtalabel.org/ " title= " Restricted to Adults " >RTA</a> ' ,
]
if any ( re . search ( marker , webpage ) for marker in AGE_LIMIT_MARKERS ) :
age_limit = 18
2013-12-06 09:15:04 +01:00
# video uploader is domain name
video_uploader = self . _search_regex (
2014-01-06 01:47:52 +01:00
r ' ^(?:https?://)?([^/]*)/.* ' , url , ' video uploader ' )
2013-11-18 13:28:26 +01:00
2014-08-22 18:19:56 +02:00
# Helper method
def _playlist_from_matches ( matches , getter , ie = None ) :
2014-09-24 11:05:14 +02:00
urlrs = orderedSet (
self . url_result ( self . _proto_relative_url ( getter ( m ) ) , ie )
for m in matches )
2014-08-22 18:19:56 +02:00
return self . playlist_result (
urlrs , playlist_id = video_id , playlist_title = video_title )
2013-08-26 21:29:31 +02:00
# Look for BrightCove:
2014-02-03 15:19:40 +01:00
bc_urls = BrightcoveIE . _extract_brightcove_urls ( webpage )
if bc_urls :
2014-01-06 01:47:52 +01:00
self . to_screen ( ' Brightcove video detected. ' )
2014-02-03 15:19:40 +01:00
entries = [ {
' _type ' : ' url ' ,
' url ' : smuggle_url ( bc_url , { ' Referer ' : url } ) ,
' ie_key ' : ' Brightcove '
} for bc_url in bc_urls ]
return {
' _type ' : ' playlist ' ,
' title ' : video_title ,
' id ' : video_id ,
' entries ' : entries ,
}
2013-07-10 17:49:11 +02:00
2013-12-22 03:34:13 +01:00
# Look for embedded (iframe) Vimeo player
2013-10-15 12:05:13 +02:00
mobj = re . search (
2014-03-15 18:47:04 +01:00
r ' <iframe[^>]+?src=([ " \' ])(?P<url>(?:https?:)?//player \ .vimeo \ .com/video/.+?) \ 1 ' , webpage )
2013-10-15 12:05:13 +02:00
if mobj :
2014-03-15 18:47:04 +01:00
player_url = unescapeHTML ( mobj . group ( ' url ' ) )
2013-10-15 12:05:13 +02:00
surl = smuggle_url ( player_url , { ' Referer ' : url } )
2014-10-15 13:50:53 +02:00
return self . url_result ( surl )
2013-10-15 12:05:13 +02:00
2013-12-22 03:34:13 +01:00
# Look for embedded (swf embed) Vimeo player
mobj = re . search (
2014-10-15 13:50:53 +02:00
r ' <embed[^>]+?src= " ((?:https?:)?//(?:www \ .)?vimeo \ .com/moogaloop \ .swf.+?) " ' , webpage )
2013-12-22 03:34:13 +01:00
if mobj :
2014-10-15 13:50:53 +02:00
return self . url_result ( mobj . group ( 1 ) )
2013-12-22 03:34:13 +01:00
2013-10-18 11:44:57 +02:00
# Look for embedded YouTube player
2013-12-19 20:44:30 +01:00
matches = re . findall ( r ''' (?x)
2014-06-09 22:06:45 +02:00
( ? :
< iframe [ ^ > ] + ? src = |
2014-08-22 17:40:36 +02:00
data - video - url = |
2014-06-09 22:06:45 +02:00
< embed [ ^ > ] + ? src = |
2014-10-26 14:15:48 +01:00
embedSWF \( ? : \s * |
new \s + SWFObject \(
2014-06-09 22:06:45 +02:00
)
( [ " \' ])
2014-09-10 13:29:20 +02:00
( ? P < url > ( ? : https ? : ) ? / / ( ? : www \. ) ? youtube ( ? : - nocookie ) ? \. com /
2014-09-25 01:58:49 +02:00
( ? : embed | v | p ) / . + ? )
2013-12-19 20:44:30 +01:00
\1 ''' , webpage)
2013-11-18 13:28:26 +01:00
if matches :
2014-08-22 18:19:56 +02:00
return _playlist_from_matches (
2014-09-24 11:05:14 +02:00
matches , lambda m : unescapeHTML ( m [ 1 ] ) )
2013-10-18 11:44:57 +02:00
2013-12-01 01:21:33 +01:00
# Look for embedded Dailymotion player
matches = re . findall (
2013-12-06 09:15:04 +01:00
r ' <iframe[^>]+?src=([ " \' ])(?P<url>(?:https?:)?//(?:www \ .)?dailymotion \ .com/embed/video/.+?) \ 1 ' , webpage )
2013-12-01 01:21:33 +01:00
if matches :
2014-08-22 18:19:56 +02:00
return _playlist_from_matches (
matches , lambda m : unescapeHTML ( m [ 1 ] ) )
2013-12-01 01:21:33 +01:00
2014-10-02 20:42:45 +02:00
# Look for embedded Dailymotion playlist player (#3822)
m = re . search (
r ' <iframe[^>]+?src=([ " \' ])(?P<url>(?:https?:)?//(?:www \ .)?dailymotion \ .[a-z] { 2,3}/widget/jukebox \ ?.+?) \ 1 ' , webpage )
if m :
playlists = re . findall (
r ' list \ [ \ ]=/playlist/([^/]+)/ ' , unescapeHTML ( m . group ( ' url ' ) ) )
if playlists :
return _playlist_from_matches (
playlists , lambda p : ' //dailymotion.com/playlist/ %s ' % p )
2013-12-06 09:15:04 +01:00
# Look for embedded Wistia player
match = re . search (
2014-10-23 18:03:07 +02:00
r ' <(?:meta[^>]+?content|iframe[^>]+?src)=([ " \' ])(?P<url>(?:https?:)?//(?:fast \ .)?wistia \ .net/embed/iframe/.+?) \ 1 ' , webpage )
2013-12-06 09:15:04 +01:00
if match :
2014-10-18 00:52:55 +02:00
embed_url = self . _proto_relative_url (
unescapeHTML ( match . group ( ' url ' ) ) )
2013-12-06 09:15:04 +01:00
return {
' _type ' : ' url_transparent ' ,
2014-10-18 00:52:55 +02:00
' url ' : embed_url ,
2013-12-06 09:15:04 +01:00
' ie_key ' : ' Wistia ' ,
' uploader ' : video_uploader ,
' title ' : video_title ,
' id ' : video_id ,
}
2014-10-23 16:57:13 +02:00
2014-10-18 00:52:55 +02:00
match = re . search ( r ' (?:id=[ " \' ]wistia_|data-wistia-?id=[ " \' ]|Wistia \ .embed \ ([ " \' ])(?P<id>[^ " \' ]+) ' , webpage )
2014-09-20 02:02:11 +02:00
if match :
return {
' _type ' : ' url_transparent ' ,
' url ' : ' http://fast.wistia.net/embed/iframe/ {0:} ' . format ( match . group ( ' id ' ) ) ,
' ie_key ' : ' Wistia ' ,
' uploader ' : video_uploader ,
' title ' : video_title ,
' id ' : match . group ( ' id ' )
}
2013-12-06 09:15:04 +01:00
2013-12-16 20:08:23 +01:00
# Look for embedded blip.tv player
2013-12-30 06:15:02 +01:00
mobj = re . search ( r ' <meta \ s[^>]*https?://api \ .blip \ .tv/ \ w+/redirect/ \ w+/( \ d+) ' , webpage )
2013-12-16 20:08:23 +01:00
if mobj :
2013-12-30 06:15:02 +01:00
return self . url_result ( ' http://blip.tv/a/a- ' + mobj . group ( 1 ) , ' BlipTV ' )
2014-08-24 03:20:31 +02:00
mobj = re . search ( r ' <(?:iframe|embed|object) \ s[^>]*(https?://(?: \ w+ \ .)?blip \ .tv/(?:play/|api \ .swf#)[a-zA-Z0-9_]+) ' , webpage )
2013-12-16 20:08:23 +01:00
if mobj :
2013-12-30 06:15:02 +01:00
return self . url_result ( mobj . group ( 1 ) , ' BlipTV ' )
2013-12-16 20:08:23 +01:00
2014-04-21 05:47:52 +02:00
# Look for embedded condenast player
matches = re . findall (
r ' <iframe \ s+(?:[a-zA-Z-]+= " [^ " ]+ " \ s+)*?src= " (https?://player \ .cnevids \ .com/embed/[^ " ]+ " ) ' ,
webpage )
if matches :
return {
' _type ' : ' playlist ' ,
' entries ' : [ {
' _type ' : ' url ' ,
' ie_key ' : ' CondeNast ' ,
' url ' : ma ,
} for ma in matches ] ,
' title ' : video_title ,
' id ' : video_id ,
}
2013-10-27 14:40:25 +01:00
# Look for Bandcamp pages with custom domain
mobj = re . search ( r ' <meta property= " og:url " [^>]*?content= " (.*?bandcamp \ .com.*?) " ' , webpage )
if mobj is not None :
burl = unescapeHTML ( mobj . group ( 1 ) )
2013-11-22 16:05:14 +01:00
# Don't set the extractor because it can be a track url or an album
return self . url_result ( burl )
2013-10-27 14:40:25 +01:00
2013-12-16 21:45:21 +01:00
# Look for embedded Vevo player
mobj = re . search (
r ' <iframe[^>]+?src=([ " \' ])(?P<url>(?:https?:)?//(?:cache \ .)?vevo \ .com/.+?) \ 1 ' , webpage )
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) )
2013-12-19 20:28:52 +01:00
# Look for Ooyala videos
2014-03-21 21:51:33 +01:00
mobj = ( re . search ( r ' player.ooyala.com/[^ " ?]+ \ ?[^ " ]*?(?:embedCode|ec)=(?P<ec>[^ " &]+) ' , webpage ) or
re . search ( r ' OO.Player.create \ ([ \' " ].*?[ \' " ], \ s*[ \' " ](?P<ec>. {32} )[ \' " ] ' , webpage ) )
2013-12-19 20:28:52 +01:00
if mobj is not None :
2014-03-21 21:51:33 +01:00
return OoyalaIE . _build_url_result ( mobj . group ( ' ec ' ) )
2013-12-19 20:28:52 +01:00
2013-12-20 17:05:28 +01:00
# Look for Aparat videos
2014-04-21 12:37:41 +02:00
mobj = re . search ( r ' <iframe .*?src= " (http://www \ .aparat \ .com/video/[^ " ]+) " ' , webpage )
2013-12-20 17:05:28 +01:00
if mobj is not None :
return self . url_result ( mobj . group ( 1 ) , ' Aparat ' )
2014-01-07 08:07:46 +01:00
# Look for MPORA videos
2014-01-29 22:26:46 +01:00
mobj = re . search ( r ' <iframe .*?src= " (http://mpora \ .(?:com|de)/videos/[^ " ]+) " ' , webpage )
2014-01-07 08:07:46 +01:00
if mobj is not None :
return self . url_result ( mobj . group ( 1 ) , ' Mpora ' )
2014-01-08 02:11:46 +01:00
2014-04-05 12:20:05 +02:00
# Look for embedded NovaMov-based player
2014-01-08 02:07:11 +01:00
mobj = re . search (
2014-05-17 13:12:12 +02:00
r ''' (?x)<(?:pagespeed_)?iframe[^>]+?src=([ " \ ' ])
2014-04-05 12:20:05 +02:00
( ? P < url > http : / / ( ? : ( ? : embed | www ) \. ) ?
( ? : novamov \. com |
nowvideo \. ( ? : ch | sx | eu | at | ag | co ) |
videoweed \. ( ? : es | com ) |
movshare \. ( ? : net | sx | ag ) |
divxstage \. ( ? : eu | net | ch | co | at | ag ) )
/ embed \. php . + ? ) \1 ''' , webpage)
2014-01-08 02:07:11 +01:00
if mobj is not None :
2014-04-05 12:20:05 +02:00
return self . url_result ( mobj . group ( ' url ' ) )
2014-04-05 10:49:45 +02:00
2014-01-21 18:10:14 +01:00
# Look for embedded Facebook player
mobj = re . search (
2014-01-27 05:47:30 +01:00
r ' <iframe[^>]+?src=([ " \' ])(?P<url>https://www \ .facebook \ .com/video/embed.+?) \ 1 ' , webpage )
2014-01-21 18:10:14 +01:00
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) , ' Facebook ' )
2014-02-28 17:51:54 +01:00
# Look for embedded VK player
mobj = re . search ( r ' <iframe[^>]+?src=([ " \' ])(?P<url>https?://vk \ .com/video_ext \ .php.+?) \ 1 ' , webpage )
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) , ' VK ' )
2014-06-29 15:18:23 +02:00
# Look for embedded ivi player
mobj = re . search ( r ' <embed[^>]+?src=([ " \' ])(?P<url>https?://(?:www \ .)?ivi \ .ru/video/player.+?) \ 1 ' , webpage )
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) , ' Ivi ' )
2014-01-27 05:47:30 +01:00
# Look for embedded Huffington Post player
mobj = re . search (
2014-01-29 22:26:46 +01:00
r ' <iframe[^>]+?src=([ " \' ])(?P<url>https?://embed \ .live \ .huffingtonpost \ .com/.+?) \ 1 ' , webpage )
2014-01-27 05:47:30 +01:00
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) , ' HuffPost ' )
2014-02-24 01:15:51 +01:00
# Look for embed.ly
mobj = re . search ( r ' class=[ " \' ]embedly-card[ " \' ][^>]href=[ " \' ](?P<url>[^ " \' ]+) ' , webpage )
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) )
mobj = re . search ( r ' class=[ " \' ]embedly-embed[ " \' ][^>]src=[ " \' ][^ " \' ]*url=(?P<url>[^&]+) ' , webpage )
if mobj is not None :
return self . url_result ( compat_urllib_parse . unquote ( mobj . group ( ' url ' ) ) )
2014-03-11 16:51:36 +01:00
# Look for funnyordie embed
matches = re . findall ( r ' <iframe[^>]+?src= " (https?://(?:www \ .)?funnyordie \ .com/embed/[^ " ]+) " ' , webpage )
if matches :
2014-08-22 18:19:56 +02:00
return _playlist_from_matches (
matches , getter = unescapeHTML , ie = ' FunnyOrDie ' )
2014-03-11 16:51:36 +01:00
2014-03-16 20:00:31 +01:00
# Look for embedded RUTV player
rutv_url = RUTVIE . _extract_url ( webpage )
if rutv_url :
return self . url_result ( rutv_url , ' RUTV ' )
2014-03-22 10:20:44 +01:00
# Look for embedded TED player
mobj = re . search (
r ' <iframe[^>]+?src=([ " \' ])(?P<url>http://embed \ .ted \ .com/.+?) \ 1 ' , webpage )
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) , ' TED ' )
2014-04-04 16:23:09 +02:00
# Look for embedded Ustream videos
mobj = re . search (
r ' <iframe[^>]+?src=([ " \' ])(?P<url>http://www \ .ustream \ .tv/embed/.+?) \ 1 ' , webpage )
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) , ' Ustream ' )
2014-03-24 22:01:47 +01:00
# Look for embedded arte.tv player
mobj = re . search (
r ' <script [^>]*?src= " (?P<url>http://www \ .arte \ .tv/playerv2/embed[^ " ]+) " ' ,
webpage )
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) , ' ArteTVEmbed ' )
2014-03-28 13:58:49 +01:00
# Look for embedded smotri.com player
smotri_url = SmotriIE . _extract_url ( webpage )
if smotri_url :
return self . url_result ( smotri_url , ' Smotri ' )
2014-05-05 03:12:41 +02:00
# Look for embeded soundcloud player
mobj = re . search (
2014-10-29 20:27:58 +01:00
r ' <iframe \ s+(?:[a-zA-Z0-9_-]+= " [^ " ]+ " \ s+)*src= " (?P<url>https?://(?:w \ .)?soundcloud \ .com/player[^ " ]+) " ' ,
2014-05-05 03:12:41 +02:00
webpage )
if mobj is not None :
url = unescapeHTML ( mobj . group ( ' url ' ) )
return self . url_result ( url )
2014-06-09 23:06:25 +02:00
# Look for embedded vulture.com player
mobj = re . search (
r ' <iframe src= " (?P<url>https?://video \ .vulture \ .com/[^ " ]+) " ' ,
webpage )
if mobj is not None :
url = unescapeHTML ( mobj . group ( ' url ' ) )
return self . url_result ( url , ie = ' Vulture ' )
2014-06-22 21:38:04 +02:00
# Look for embedded mtvservices player
mobj = re . search (
r ' <iframe src= " (?P<url>https?://media \ .mtvnservices \ .com/embed/[^ " ]+) " ' ,
webpage )
if mobj is not None :
url = unescapeHTML ( mobj . group ( ' url ' ) )
return self . url_result ( url , ie = ' MTVServicesEmbedded ' )
2014-08-16 08:56:22 +02:00
# Look for embedded yahoo player
mobj = re . search (
r ' <iframe[^>]+?src=([ " \' ])(?P<url>https?://(?:screen|movies) \ .yahoo \ .com/.+? \ .html \ ?format=embed) \ 1 ' ,
webpage )
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) , ' Yahoo ' )
2014-08-23 15:20:49 +02:00
# Look for embedded sbs.com.au player
mobj = re . search (
r ' <iframe[^>]+?src=([ " \' ])(?P<url>https?://(?:www \ .)sbs \ .com \ .au/ondemand/video/single/.+?) \ 1 ' ,
webpage )
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) , ' SBS ' )
2014-09-02 15:19:28 +02:00
mobj = re . search (
2014-10-31 22:01:58 +01:00
r ' <iframe[^>]+?src=([ " \' ])(?P<url>https?://m(?:lb)? \ .mlb \ .com/shared/video/embed/embed \ .html \ ?.+?) \ 1 ' ,
2014-09-02 15:19:28 +02:00
webpage )
if mobj is not None :
return self . url_result ( mobj . group ( ' url ' ) , ' MLB ' )
2014-10-13 14:59:35 +02:00
mobj = re . search (
r ' <iframe[^>]+?src=([ " \' ])(?P<url> %s ) \ 1 ' % CondeNastIE . EMBED_URL ,
webpage )
if mobj is not None :
return self . url_result ( self . _proto_relative_url ( mobj . group ( ' url ' ) , scheme = ' http: ' ) , ' CondeNast ' )
2014-10-09 14:26:23 +02:00
def check_video ( vurl ) :
vpath = compat_urlparse . urlparse ( vurl ) . path
vext = determine_ext ( vpath )
return ' . ' in vpath and vext not in ( ' swf ' , ' png ' , ' jpg ' , ' srt ' , ' sbv ' , ' sub ' , ' vtt ' , ' ttml ' )
def filter_video ( urls ) :
return list ( filter ( check_video , urls ) )
2013-06-23 20:31:45 +02:00
# Start with something easy: JW Player in SWFObject
2014-10-09 14:26:23 +02:00
found = filter_video ( re . findall ( r ' flashvars: [ \' " ](?:.*&)?file=(http[^ \' " &]*) ' , webpage ) )
2014-04-30 02:23:51 +02:00
if not found :
2014-01-05 05:34:06 +01:00
# Look for gorilla-vid style embedding
2014-10-09 14:26:23 +02:00
found = filter_video ( re . findall ( r ''' (?sx)
2014-04-21 16:16:53 +02:00
( ? :
jw_plugins |
JWPlayerOptions |
jwplayer \s * \( \s * [ " ' ][^ ' " ] + [ " ' ] \ s* \ ) \ s* \ .setup
)
2014-10-09 14:26:23 +02:00
. * ? file \s * : \s * [ " \' ](.*?)[ " \' ] ' ' ' , webpage ) )
2014-04-30 02:23:51 +02:00
if not found :
2013-06-23 20:31:45 +02:00
# Broaden the search a little bit
2014-10-09 14:26:23 +02:00
found = filter_video ( re . findall ( r ' [^A-Za-z0-9]?(?:file|source)=(http[^ \' " &]*) ' , webpage ) )
2014-04-30 02:23:51 +02:00
if not found :
# Broaden the findall a little bit: JWPlayer JS loader
2014-10-09 14:26:23 +02:00
found = filter_video ( re . findall (
r ' [^A-Za-z0-9]?file[ " \' ]?: \ s*[ " \' ](http(?![^ \' " ]+ \ .[0-9]+[ \' " ])[^ \' " ]+)[ " \' ] ' , webpage ) )
2014-08-24 05:31:32 +02:00
if not found :
# Flow player
2014-10-09 14:26:23 +02:00
found = filter_video ( re . findall ( r ''' (?xs)
2014-08-24 05:31:32 +02:00
flowplayer \( " [^ " ] + " , \ s*
\{ [ ^ } ] + ? \} \s * ,
\s * { [ ^ } ] + ? [ " ' ]?clip[ " ' ]? \ s*: \ s* \ { \ s*
[ " ' ]?url[ " ' ]? \ s*: \ s*[ " ' ] ( [ ^ " ' ]+)[ " ' ]
2014-10-09 14:26:23 +02:00
''' , webpage))
2014-04-30 02:23:51 +02:00
if not found :
2013-06-23 20:31:45 +02:00
# Try to find twitter cards info
2014-10-09 14:26:23 +02:00
found = filter_video ( re . findall (
r ' <meta (?:property|name)= " twitter:player:stream " (?:content|value)= " (.+?) " ' , webpage ) )
2014-04-30 02:23:51 +02:00
if not found :
2013-06-23 20:31:45 +02:00
# We look for Open Graph info:
# We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
2014-04-30 02:23:51 +02:00
m_video_type = re . findall ( r ' <meta.*?property= " og:video:type " .*?content= " video/(.*?) " ' , webpage )
2013-06-23 20:31:45 +02:00
# We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
if m_video_type is not None :
2014-10-09 14:26:23 +02:00
found = filter_video ( re . findall ( r ' <meta.*?property= " og:video " .*?content= " (.*?) " ' , webpage ) )
2014-04-30 02:23:51 +02:00
if not found :
2013-08-21 04:32:22 +02:00
# HTML5 video
2014-10-28 10:41:24 +01:00
found = re . findall ( r ' (?s)<video[^<]*(?:>.*?<source[^>]*)? \ s+src= " ([^ " ]+) " ' , webpage )
2014-04-30 02:23:51 +02:00
if not found :
2014-05-16 15:32:53 +02:00
found = re . search (
2014-02-27 07:21:59 +01:00
r ' (?i)<meta \ s+(?=(?:[a-z-]+= " [^ " ]+ " \ s+)*http-equiv= " refresh " ) '
2014-09-16 22:53:48 +02:00
r ' (?:[a-z-]+= " [^ " ]+ " \ s+)*?content= " [0-9] { ,2};url= \' ?([^ \' " ]+) ' ,
2014-02-27 07:21:59 +01:00
webpage )
2014-04-30 02:23:51 +02:00
if found :
new_url = found . group ( 1 )
2014-02-27 07:21:59 +01:00
self . report_following_redirect ( new_url )
return {
' _type ' : ' url ' ,
' url ' : new_url ,
}
2014-04-30 02:23:51 +02:00
if not found :
2014-01-06 01:47:52 +01:00
raise ExtractorError ( ' Unsupported URL: %s ' % url )
2013-06-23 20:31:45 +02:00
2014-04-30 02:23:51 +02:00
entries = [ ]
for video_url in found :
video_url = compat_urlparse . urljoin ( url , video_url )
video_id = compat_urllib_parse . unquote ( os . path . basename ( video_url ) )
2013-06-23 20:31:45 +02:00
2014-04-30 02:23:51 +02:00
# Sometimes, jwplayer extraction will result in a YouTube URL
if YoutubeIE . suitable ( video_url ) :
entries . append ( self . url_result ( video_url , ' Youtube ' ) )
continue
2013-06-23 20:31:45 +02:00
2014-04-30 02:23:51 +02:00
# here's a fun little line of code for you:
video_id = os . path . splitext ( video_id ) [ 0 ]
2014-01-06 01:42:58 +01:00
2014-04-30 02:23:51 +02:00
entries . append ( {
' id ' : video_id ,
' url ' : video_url ,
' uploader ' : video_uploader ,
' title ' : video_title ,
2014-08-24 05:31:32 +02:00
' age_limit ' : age_limit ,
2014-04-30 02:23:51 +02:00
} )
if len ( entries ) == 1 :
2014-05-01 11:28:37 +02:00
return entries [ 0 ]
2014-04-30 02:23:51 +02:00
else :
for num , e in enumerate ( entries , start = 1 ) :
e [ ' title ' ] = ' %s ( %d ) ' % ( e [ ' title ' ] , num )
return {
' _type ' : ' playlist ' ,
' entries ' : entries ,
}
2013-06-23 20:31:45 +02:00