2013-12-09 17:08:58 +01:00
# encoding: utf-8
2013-06-23 20:57:44 +02:00
import json
import re
2013-09-14 21:41:49 +02:00
import itertools
2013-06-23 20:57:44 +02:00
from . common import InfoExtractor
from . . utils import (
compat_str ,
2013-08-21 17:06:37 +02:00
compat_urlparse ,
2013-09-14 21:41:49 +02:00
compat_urllib_parse ,
2013-06-23 20:57:44 +02:00
ExtractorError ,
unified_strdate ,
)
class SoundcloudIE ( InfoExtractor ) :
""" Information extractor for soundcloud.com
To access the media , the uid of the song and a stream token
must be extracted from the page source and the script must make
a request to media . soundcloud . com / crossdomain . xml . Then
the media can be grabbed by requesting from an url composed
of the stream token and uid
"""
2013-07-24 14:39:21 +02:00
_VALID_URL = r ''' ^(?:https?://)?
2013-12-19 16:39:01 +01:00
( ? : ( ? : ( ? : www \. | m \. ) ? soundcloud \. com /
2013-12-09 19:57:00 +01:00
( ? P < uploader > [ \w \d - ] + ) /
( ? ! sets / ) ( ? P < title > [ \w \d - ] + ) / ?
2013-12-09 17:08:58 +01:00
( ? P < token > [ ^ ? ] + ? ) ? ( ? : [ ? ] . * ) ? $ )
2013-07-24 14:39:21 +02:00
| ( ? : api \. soundcloud \. com / tracks / ( ? P < track_id > \d + ) )
2013-12-04 14:34:47 +01:00
| ( ? P < widget > w \. soundcloud \. com / player / ? . * ? url = . * )
2013-07-24 14:39:21 +02:00
)
'''
2013-06-23 20:57:44 +02:00
IE_NAME = u ' soundcloud '
2013-11-09 18:06:09 +01:00
_TESTS = [
{
u ' url ' : u ' http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy ' ,
u ' file ' : u ' 62986583.mp3 ' ,
u ' md5 ' : u ' ebef0a451b909710ed1d7787dddbf0d7 ' ,
u ' info_dict ' : {
u " upload_date " : u " 20121011 " ,
u " description " : u " No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o ' d " ,
u " uploader " : u " E.T. ExTerrestrial Music " ,
u " title " : u " Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1 "
}
} ,
# not streamable song
{
u ' url ' : u ' https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep ' ,
u ' info_dict ' : {
u ' id ' : u ' 47127627 ' ,
u ' ext ' : u ' mp3 ' ,
u ' title ' : u ' Goldrushed ' ,
u ' uploader ' : u ' The Royal Concept ' ,
u ' upload_date ' : u ' 20120521 ' ,
} ,
u ' params ' : {
# rtmp
u ' skip_download ' : True ,
} ,
} ,
2013-12-09 17:08:58 +01:00
# private link
{
u ' url ' : u ' https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp ' ,
u ' md5 ' : u ' aa0dd32bfea9b0c5ef4f02aacd080604 ' ,
u ' info_dict ' : {
u ' id ' : u ' 123998367 ' ,
u ' ext ' : u ' mp3 ' ,
u ' title ' : u ' Youtube - Dl Test Video \' \' Ä↭ ' ,
u ' uploader ' : u ' jaimeMF ' ,
u ' description ' : u ' test chars: \" \' / \\ ä↭ ' ,
u ' upload_date ' : u ' 20131209 ' ,
} ,
} ,
2013-12-10 13:04:21 +01:00
# downloadable song
{
u ' url ' : u ' https://soundcloud.com/simgretina/just-your-problem-baby-1 ' ,
u ' md5 ' : u ' 56a8b69568acaa967b4c49f9d1d52d19 ' ,
u ' info_dict ' : {
u ' id ' : u ' 105614606 ' ,
u ' ext ' : u ' wav ' ,
u ' title ' : u ' Just Your Problem Baby (Acapella) ' ,
u ' description ' : u ' Vocals ' ,
u ' uploader ' : u ' Sim Gretina ' ,
u ' upload_date ' : u ' 20130815 ' ,
} ,
} ,
2013-11-09 18:06:09 +01:00
]
2013-06-23 20:57:44 +02:00
2013-07-24 14:05:14 +02:00
_CLIENT_ID = ' b45b1aa10f1ac2941910a7f0d10f8e28 '
2013-11-21 13:16:19 +01:00
_IPHONE_CLIENT_ID = ' 376f225bf427445fc4bfb6b99b72e0bf '
2013-07-24 14:05:14 +02:00
2013-07-24 14:39:21 +02:00
@classmethod
def suitable ( cls , url ) :
return re . match ( cls . _VALID_URL , url , flags = re . VERBOSE ) is not None
2013-06-23 20:57:44 +02:00
def report_resolve ( self , video_id ) :
""" Report information extraction. """
self . to_screen ( u ' %s : Resolving id ' % video_id )
2013-07-24 14:05:14 +02:00
@classmethod
def _resolv_url ( cls , url ) :
return ' http://api.soundcloud.com/resolve.json?url= ' + url + ' &client_id= ' + cls . _CLIENT_ID
2013-12-09 17:08:58 +01:00
def _extract_info_dict ( self , info , full_title = None , quiet = False , secret_token = None ) :
2013-11-09 18:06:09 +01:00
track_id = compat_str ( info [ ' id ' ] )
name = full_title or track_id
2013-11-25 20:30:41 +01:00
if quiet :
2013-09-14 21:41:49 +02:00
self . report_extraction ( name )
2013-07-24 14:05:14 +02:00
thumbnail = info [ ' artwork_url ' ]
if thumbnail is not None :
thumbnail = thumbnail . replace ( ' -large ' , ' -t500x500 ' )
2013-12-10 11:45:13 +01:00
ext = u ' mp3 '
2013-11-09 18:06:09 +01:00
result = {
2013-11-25 20:30:41 +01:00
' id ' : track_id ,
2013-07-24 14:05:14 +02:00
' uploader ' : info [ ' user ' ] [ ' username ' ] ,
' upload_date ' : unified_strdate ( info [ ' created_at ' ] ) ,
2013-11-25 20:30:41 +01:00
' title ' : info [ ' title ' ] ,
2013-07-24 14:05:14 +02:00
' description ' : info [ ' description ' ] ,
' thumbnail ' : thumbnail ,
}
2013-11-09 18:06:09 +01:00
if info . get ( ' downloadable ' , False ) :
2013-11-21 13:16:19 +01:00
# We can build a direct link to the song
2013-11-25 20:30:41 +01:00
format_url = (
u ' https://api.soundcloud.com/tracks/ {0} /download?client_id= {1} ' . format (
track_id , self . _CLIENT_ID ) )
result [ ' formats ' ] = [ {
' format_id ' : ' download ' ,
2013-12-10 13:04:21 +01:00
' ext ' : info . get ( ' original_format ' , u ' mp3 ' ) ,
2013-11-25 20:30:41 +01:00
' url ' : format_url ,
2013-11-25 22:34:56 +01:00
' vcodec ' : ' none ' ,
2013-11-25 20:30:41 +01:00
} ]
2013-11-21 13:16:19 +01:00
else :
# We have to retrieve the url
2013-12-09 17:08:58 +01:00
streams_url = ( ' http://api.soundcloud.com/i1/tracks/ {0} /streams? '
' client_id= {1} &secret_token= {2} ' . format ( track_id , self . _IPHONE_CLIENT_ID , secret_token ) )
2013-11-09 18:06:09 +01:00
stream_json = self . _download_webpage (
2013-12-09 17:08:58 +01:00
streams_url ,
2013-11-09 18:06:09 +01:00
track_id , u ' Downloading track url ' )
2013-11-25 20:30:41 +01:00
formats = [ ]
format_dict = json . loads ( stream_json )
for key , stream_url in format_dict . items ( ) :
if key . startswith ( u ' http ' ) :
formats . append ( {
' format_id ' : key ,
' ext ' : ext ,
' url ' : stream_url ,
2013-11-25 22:34:56 +01:00
' vcodec ' : ' none ' ,
2013-11-25 20:30:41 +01:00
} )
elif key . startswith ( u ' rtmp ' ) :
# The url doesn't have an rtmp app, we have to extract the playpath
url , path = stream_url . split ( ' mp3: ' , 1 )
formats . append ( {
' format_id ' : key ,
' url ' : url ,
' play_path ' : ' mp3: ' + path ,
' ext ' : ext ,
2013-11-25 22:34:56 +01:00
' vcodec ' : ' none ' ,
2013-11-25 20:30:41 +01:00
} )
if not formats :
2013-11-21 13:16:19 +01:00
# We fallback to the stream_url in the original info, this
# cannot be always used, sometimes it can give an HTTP 404 error
2013-11-25 20:30:41 +01:00
formats . append ( {
' format_id ' : u ' fallback ' ,
' url ' : info [ ' stream_url ' ] + ' ?client_id= ' + self . _CLIENT_ID ,
' ext ' : ext ,
2013-11-25 22:34:56 +01:00
' vcodec ' : ' none ' ,
2013-11-25 20:30:41 +01:00
} )
def format_pref ( f ) :
if f [ ' format_id ' ] . startswith ( ' http ' ) :
return 2
if f [ ' format_id ' ] . startswith ( ' rtmp ' ) :
return 1
return 0
formats . sort ( key = format_pref )
result [ ' formats ' ] = formats
2013-11-21 13:16:19 +01:00
2013-11-09 18:06:09 +01:00
return result
2013-07-24 14:05:14 +02:00
2013-06-23 20:57:44 +02:00
def _real_extract ( self , url ) :
2013-07-24 14:39:21 +02:00
mobj = re . match ( self . _VALID_URL , url , flags = re . VERBOSE )
2013-06-23 20:57:44 +02:00
if mobj is None :
raise ExtractorError ( u ' Invalid URL: %s ' % url )
2013-07-24 14:39:21 +02:00
track_id = mobj . group ( ' track_id ' )
2013-12-09 17:08:58 +01:00
token = None
2013-07-24 14:39:21 +02:00
if track_id is not None :
info_json_url = ' http://api.soundcloud.com/tracks/ ' + track_id + ' .json?client_id= ' + self . _CLIENT_ID
full_title = track_id
2013-08-21 17:06:37 +02:00
elif mobj . group ( ' widget ' ) :
query = compat_urlparse . parse_qs ( compat_urlparse . urlparse ( url ) . query )
return self . url_result ( query [ ' url ' ] [ 0 ] , ie = ' Soundcloud ' )
2013-07-24 14:39:21 +02:00
else :
# extract uploader (which is in the url)
2013-12-09 17:08:58 +01:00
uploader = mobj . group ( ' uploader ' )
2013-07-24 14:39:21 +02:00
# extract simple title (uploader + slug of song title)
2013-12-09 17:08:58 +01:00
slug_title = mobj . group ( ' title ' )
token = mobj . group ( ' token ' )
full_title = resolve_title = ' %s / %s ' % ( uploader , slug_title )
if token :
resolve_title + = ' / %s ' % token
2013-07-24 14:39:21 +02:00
self . report_resolve ( full_title )
2013-12-09 17:08:58 +01:00
url = ' http://soundcloud.com/ %s ' % resolve_title
2013-07-24 14:39:21 +02:00
info_json_url = self . _resolv_url ( url )
info_json = self . _download_webpage ( info_json_url , full_title , u ' Downloading info JSON ' )
2013-06-23 20:57:44 +02:00
info = json . loads ( info_json )
2013-12-09 17:08:58 +01:00
return self . _extract_info_dict ( info , full_title , secret_token = token )
2013-06-23 20:57:44 +02:00
2013-07-24 14:05:14 +02:00
class SoundcloudSetIE ( SoundcloudIE ) :
2013-06-27 21:11:23 +02:00
_VALID_URL = r ' ^(?:https?://)?(?:www \ .)?soundcloud \ .com/([ \ w \ d-]+)/sets/([ \ w \ d-]+)(?:[?].*)?$ '
2013-06-23 20:57:44 +02:00
IE_NAME = u ' soundcloud:set '
2013-11-09 18:06:09 +01:00
# it's in tests/test_playlists.py
_TESTS = [ ]
2013-06-23 20:57:44 +02:00
def _real_extract ( self , url ) :
mobj = re . match ( self . _VALID_URL , url )
if mobj is None :
raise ExtractorError ( u ' Invalid URL: %s ' % url )
# extract uploader (which is in the url)
uploader = mobj . group ( 1 )
# extract simple title (uploader + slug of song title)
slug_title = mobj . group ( 2 )
full_title = ' %s /sets/ %s ' % ( uploader , slug_title )
self . report_resolve ( full_title )
url = ' http://soundcloud.com/ %s /sets/ %s ' % ( uploader , slug_title )
2013-07-24 14:05:14 +02:00
resolv_url = self . _resolv_url ( url )
2013-06-23 20:57:44 +02:00
info_json = self . _download_webpage ( resolv_url , full_title )
info = json . loads ( info_json )
if ' errors ' in info :
for err in info [ ' errors ' ] :
self . _downloader . report_error ( u ' unable to download video webpage: %s ' % compat_str ( err [ ' error_message ' ] ) )
return
self . report_extraction ( full_title )
2013-07-24 14:05:14 +02:00
return { ' _type ' : ' playlist ' ,
' entries ' : [ self . _extract_info_dict ( track ) for track in info [ ' tracks ' ] ] ,
' id ' : info [ ' id ' ] ,
' title ' : info [ ' title ' ] ,
}
2013-09-14 21:41:49 +02:00
class SoundcloudUserIE ( SoundcloudIE ) :
2013-12-04 14:34:47 +01:00
_VALID_URL = r ' https?://(www \ .)?soundcloud \ .com/(?P<user>[^/]+)(/?(tracks/)?)?( \ ?.*)?$ '
2013-09-14 21:41:49 +02:00
IE_NAME = u ' soundcloud:user '
# it's in tests/test_playlists.py
2013-11-09 18:06:09 +01:00
_TESTS = [ ]
2013-09-14 21:41:49 +02:00
def _real_extract ( self , url ) :
mobj = re . match ( self . _VALID_URL , url )
uploader = mobj . group ( ' user ' )
url = ' http://soundcloud.com/ %s / ' % uploader
resolv_url = self . _resolv_url ( url )
user_json = self . _download_webpage ( resolv_url , uploader ,
u ' Downloading user info ' )
user = json . loads ( user_json )
tracks = [ ]
for i in itertools . count ( ) :
data = compat_urllib_parse . urlencode ( { ' offset ' : i * 50 ,
' client_id ' : self . _CLIENT_ID ,
} )
tracks_url = ' http://api.soundcloud.com/users/ %s /tracks.json? ' % user [ ' id ' ] + data
response = self . _download_webpage ( tracks_url , uploader ,
u ' Downloading tracks page %s ' % ( i + 1 ) )
new_tracks = json . loads ( response )
tracks . extend ( self . _extract_info_dict ( track , quiet = True ) for track in new_tracks )
if len ( new_tracks ) < 50 :
break
return {
' _type ' : ' playlist ' ,
' id ' : compat_str ( user [ ' id ' ] ) ,
' title ' : user [ ' username ' ] ,
' entries ' : tracks ,
}