Madeorsk/youtube-dl
1
0
Fork 0
mirror of https://codeberg.org/polarisfm/youtube-dl synced 2024-11-29 19:47:54 +01:00
Code Issues Releases Wiki Activity

Switch codebase to use sanitized_Request instead of

Browse Source 
compat_urllib_request.Request

[downloader/dash] Use sanitized_Request

[downloader/http] Use sanitized_Request

[atresplayer] Use sanitized_Request

[bambuser] Use sanitized_Request

[bliptv] Use sanitized_Request

[brightcove] Use sanitized_Request

[cbs] Use sanitized_Request

[ceskatelevize] Use sanitized_Request

[collegerama] Use sanitized_Request

[extractor/common] Use sanitized_Request

[crunchyroll] Use sanitized_Request

[dailymotion] Use sanitized_Request

[dcn] Use sanitized_Request

[dramafever] Use sanitized_Request

[dumpert] Use sanitized_Request

[eitb] Use sanitized_Request

[escapist] Use sanitized_Request

[everyonesmixtape] Use sanitized_Request

[extremetube] Use sanitized_Request

[facebook] Use sanitized_Request

[fc2] Use sanitized_Request

[flickr] Use sanitized_Request

[4tube] Use sanitized_Request

[gdcvault] Use sanitized_Request

[extractor/generic] Use sanitized_Request

[hearthisat] Use sanitized_Request

[hotnewhiphop] Use sanitized_Request

[hypem] Use sanitized_Request

[iprima] Use sanitized_Request

[ivi] Use sanitized_Request

[keezmovies] Use sanitized_Request

[letv] Use sanitized_Request

[lynda] Use sanitized_Request

[metacafe] Use sanitized_Request

[minhateca] Use sanitized_Request

[miomio] Use sanitized_Request

[meovideo] Use sanitized_Request

[mofosex] Use sanitized_Request

[moniker] Use sanitized_Request

[mooshare] Use sanitized_Request

[movieclips] Use sanitized_Request

[mtv] Use sanitized_Request

[myvideo] Use sanitized_Request

[neteasemusic] Use sanitized_Request

[nfb] Use sanitized_Request

[niconico] Use sanitized_Request

[noco] Use sanitized_Request

[nosvideo] Use sanitized_Request

[novamov] Use sanitized_Request

[nowness] Use sanitized_Request

[nuvid] Use sanitized_Request

[played] Use sanitized_Request

[pluralsight] Use sanitized_Request

[pornhub] Use sanitized_Request

[pornotube] Use sanitized_Request

[primesharetv] Use sanitized_Request

[promptfile] Use sanitized_Request

[qqmusic] Use sanitized_Request

[rtve] Use sanitized_Request

[safari] Use sanitized_Request

[sandia] Use sanitized_Request

[shared] Use sanitized_Request

[sharesix] Use sanitized_Request

[sina] Use sanitized_Request

[smotri] Use sanitized_Request

[sohu] Use sanitized_Request

[spankwire] Use sanitized_Request

[sportdeutschland] Use sanitized_Request

[streamcloud] Use sanitized_Request

[streamcz] Use sanitized_Request

[tapely] Use sanitized_Request

[tube8] Use sanitized_Request

[tubitv] Use sanitized_Request

[twitch] Use sanitized_Request

[twitter] Use sanitized_Request

[udemy] Use sanitized_Request

[vbox7] Use sanitized_Request

[veoh] Use sanitized_Request

[vessel] Use sanitized_Request

[vevo] Use sanitized_Request

[viddler] Use sanitized_Request

[videomega] Use sanitized_Request

[viewvster] Use sanitized_Request

[viki] Use sanitized_Request

[vk] Use sanitized_Request

[vodlocker] Use sanitized_Request

[voicerepublic] Use sanitized_Request

[wistia] Use sanitized_Request

[xfileshare] Use sanitized_Request

[xtube] Use sanitized_Request

[xvideos] Use sanitized_Request

[yandexmusic] Use sanitized_Request

[youku] Use sanitized_Request

[youporn] Use sanitized_Request

[youtube] Use sanitized_Request

[patreon] Use sanitized_Request

[extractor/common] Remove unused import

[nfb] PEP 8
This commit is contained in:
Sergey M? 2015-11-21 22:18:17 +06:00 committed by Sergey M․
parent 67dda51722
commit 5c2266df4b
97 changed files with 271 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
youtube_dl/YoutubeDL.py
View File

@ -1189,7 +1189,7 @@ class YoutubeDL(object):
return res return res
def _calc_cookies(self, info_dict): def _calc_cookies(self, info_dict):
pr = compat_urllib_request.Request(info_dict['url']) pr = sanitized_Request(info_dict['url'])
self.cookiejar.add_cookie_header(pr) self.cookiejar.add_cookie_header(pr)
return pr.get_header('Cookie') return pr.get_header('Cookie')

4
youtube_dl/downloader/dash.py
View File

@ -3,7 +3,7 @@ from __future__ import unicode_literals
import re import re
from .common import FileDownloader from .common import FileDownloader
from ..compat import compat_urllib_request from ..utils import sanitized_Request
class DashSegmentsFD(FileDownloader): class DashSegmentsFD(FileDownloader):
@ -22,7 +22,7 @@ class DashSegmentsFD(FileDownloader):
def append_url_to_file(outf, target_url, target_name, remaining_bytes=None): def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name)) self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
req = compat_urllib_request.Request(target_url) req = sanitized_Request(target_url)
if remaining_bytes is not None: if remaining_bytes is not None:
req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1)) req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))

10
youtube_dl/downloader/http.py
View File

@ -7,14 +7,12 @@ import time
import re import re
from .common import FileDownloader from .common import FileDownloader
from ..compat import ( from ..compat import compat_urllib_error
compat_urllib_request,
compat_urllib_error,
)
from ..utils import ( from ..utils import (
ContentTooShortError, ContentTooShortError,
encodeFilename, encodeFilename,
sanitize_open, sanitize_open,
sanitized_Request,
) )
@ -29,8 +27,8 @@ class HttpFD(FileDownloader):
add_headers = info_dict.get('http_headers') add_headers = info_dict.get('http_headers')
if add_headers: if add_headers:
headers.update(add_headers) headers.update(add_headers)
basic_request = compat_urllib_request.Request(url, None, headers) basic_request = sanitized_Request(url, None, headers)
request = compat_urllib_request.Request(url, None, headers) request = sanitized_Request(url, None, headers)
is_test = self.params.get('test', False) is_test = self.params.get('test', False)

6
youtube_dl/extractor/atresplayer.py
View File

@ -7,11 +7,11 @@ from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_str, compat_str,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
) )
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
float_or_none, float_or_none,
sanitized_Request,
xpath_text, xpath_text,
ExtractorError, ExtractorError,
) )
@ -63,7 +63,7 @@ class AtresPlayerIE(InfoExtractor):
'j_password': password, 'j_password': password,
} }
request = compat_urllib_request.Request( request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8')) self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')
response = self._download_webpage( response = self._download_webpage(
@ -94,7 +94,7 @@ class AtresPlayerIE(InfoExtractor):
formats = [] formats = []
for fmt in ['windows', 'android_tablet']: for fmt in ['windows', 'android_tablet']:
request = compat_urllib_request.Request( request = sanitized_Request(
self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token)) self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
request.add_header('User-Agent', self._USER_AGENT) request.add_header('User-Agent', self._USER_AGENT)

6
youtube_dl/extractor/bambuser.py
View File

@ -6,13 +6,13 @@ import itertools
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
compat_str, compat_str,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
int_or_none, int_or_none,
float_or_none, float_or_none,
sanitized_Request,
) )
@ -57,7 +57,7 @@ class BambuserIE(InfoExtractor):
'pass': password, 'pass': password,
} }
request = compat_urllib_request.Request( request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8')) self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
request.add_header('Referer', self._LOGIN_URL) request.add_header('Referer', self._LOGIN_URL)
response = self._download_webpage( response = self._download_webpage(
@ -126,7 +126,7 @@ class BambuserChannelIE(InfoExtractor):
'&sort=created&access_mode=0%2C1%2C2&limit={count}' '&sort=created&access_mode=0%2C1%2C2&limit={count}'
'&method=broadcast&format=json&vid_older_than={last}' '&method=broadcast&format=json&vid_older_than={last}'
).format(user=user, count=self._STEP, last=last_id) ).format(user=user, count=self._STEP, last=last_id)
req = compat_urllib_request.Request(req_url) req = sanitized_Request(req_url)
# Without setting this header, we wouldn't get any result # Without setting this header, we wouldn't get any result
req.add_header('Referer', 'http://bambuser.com/channel/%s' % user) req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
data = self._download_json( data = self._download_json(

8
youtube_dl/extractor/bliptv.py
View File

@ -4,14 +4,12 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urlparse
compat_urllib_request,
compat_urlparse,
)
from ..utils import ( from ..utils import (
clean_html, clean_html,
int_or_none, int_or_none,
parse_iso8601, parse_iso8601,
sanitized_Request,
unescapeHTML, unescapeHTML,
xpath_text, xpath_text,
xpath_with_ns, xpath_with_ns,
@ -219,7 +217,7 @@ class BlipTVIE(InfoExtractor):
for lang, url in subtitles_urls.items(): for lang, url in subtitles_urls.items():
# For some weird reason, blip.tv serves a video instead of subtitles # For some weird reason, blip.tv serves a video instead of subtitles
# when we request with a common UA # when we request with a common UA
req = compat_urllib_request.Request(url) req = sanitized_Request(url)
req.add_header('User-Agent', 'youtube-dl') req.add_header('User-Agent', 'youtube-dl')
subtitles[lang] = [{ subtitles[lang] = [{
# The extension is 'srt' but it's actually an 'ass' file # The extension is 'srt' but it's actually an 'ass' file

6
youtube_dl/extractor/brightcove.py
View File

@ -11,7 +11,6 @@ from ..compat import (
compat_str, compat_str,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_parse_urlparse, compat_urllib_parse_urlparse,
compat_urllib_request,
compat_urlparse, compat_urlparse,
compat_xml_parse_error, compat_xml_parse_error,
) )
@ -24,6 +23,7 @@ from ..utils import (
js_to_json, js_to_json,
int_or_none, int_or_none,
parse_iso8601, parse_iso8601,
sanitized_Request,
unescapeHTML, unescapeHTML,
unsmuggle_url, unsmuggle_url,
) )
@ -250,7 +250,7 @@ class BrightcoveLegacyIE(InfoExtractor):
def _get_video_info(self, video_id, query_str, query, referer=None): def _get_video_info(self, video_id, query_str, query, referer=None):
request_url = self._FEDERATED_URL_TEMPLATE % query_str request_url = self._FEDERATED_URL_TEMPLATE % query_str
req = compat_urllib_request.Request(request_url) req = sanitized_Request(request_url)
linkBase = query.get('linkBaseURL') linkBase = query.get('linkBaseURL')
if linkBase is not None: if linkBase is not None:
referer = linkBase[0] referer = linkBase[0]
@ -443,7 +443,7 @@ class BrightcoveNewIE(InfoExtractor):
r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1', r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
webpage, 'policy key', group='pk') webpage, 'policy key', group='pk')
req = compat_urllib_request.Request( req = sanitized_Request(
'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
% (account_id, video_id), % (account_id, video_id),
headers={'Accept': 'application/json;pk=%s' % policy_key}) headers={'Accept': 'application/json;pk=%s' % policy_key})

8
youtube_dl/extractor/cbs.py
View File

@ -1,8 +1,10 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request from ..utils import (
from ..utils import smuggle_url sanitized_Request,
smuggle_url,
)
class CBSIE(InfoExtractor): class CBSIE(InfoExtractor):
@ -48,7 +50,7 @@ class CBSIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
display_id = self._match_id(url) display_id = self._match_id(url)
request = compat_urllib_request.Request(url) request = sanitized_Request(url)
# Android UA is served with higher quality (720p) streams (see # Android UA is served with higher quality (720p) streams (see
# https://github.com/rg3/youtube-dl/issues/7490) # https://github.com/rg3/youtube-dl/issues/7490)
request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)') request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)')

6
youtube_dl/extractor/ceskatelevize.py
View File

@ -5,7 +5,6 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_urllib_request,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_parse_unquote, compat_urllib_parse_unquote,
compat_urllib_parse_urlparse, compat_urllib_parse_urlparse,
@ -13,6 +12,7 @@ from ..compat import (
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
float_or_none, float_or_none,
sanitized_Request,
) )
@ -100,7 +100,7 @@ class CeskaTelevizeIE(InfoExtractor):
'requestSource': 'iVysilani', 'requestSource': 'iVysilani',
} }
req = compat_urllib_request.Request( req = sanitized_Request(
'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist', 'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
data=compat_urllib_parse.urlencode(data)) data=compat_urllib_parse.urlencode(data))
@ -115,7 +115,7 @@ class CeskaTelevizeIE(InfoExtractor):
if playlist_url == 'error_region': if playlist_url == 'error_region':
raise ExtractorError(NOT_AVAILABLE_STRING, expected=True) raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
req = compat_urllib_request.Request(compat_urllib_parse_unquote(playlist_url)) req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
req.add_header('Referer', url) req.add_header('Referer', url)
playlist_title = self._og_search_title(webpage) playlist_title = self._og_search_title(webpage)

4
youtube_dl/extractor/collegerama.py
View File

@ -3,10 +3,10 @@ from __future__ import unicode_literals
import json import json
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import ( from ..utils import (
float_or_none, float_or_none,
int_or_none, int_or_none,
sanitized_Request,
) )
@ -52,7 +52,7 @@ class CollegeRamaIE(InfoExtractor):
} }
} }
request = compat_urllib_request.Request( request = sanitized_Request(
'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions', 'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
json.dumps(player_options_request)) json.dumps(player_options_request))
request.add_header('Content-Type', 'application/json') request.add_header('Content-Type', 'application/json')

4
youtube_dl/extractor/common.py
View File

@ -19,7 +19,6 @@ from ..compat import (
compat_urllib_error, compat_urllib_error,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_parse_urlparse, compat_urllib_parse_urlparse,
compat_urllib_request,
compat_urlparse, compat_urlparse,
compat_str, compat_str,
compat_etree_fromstring, compat_etree_fromstring,
@ -37,6 +36,7 @@ from ..utils import (
int_or_none, int_or_none,
RegexNotFoundError, RegexNotFoundError,
sanitize_filename, sanitize_filename,
sanitized_Request,
unescapeHTML, unescapeHTML,
unified_strdate, unified_strdate,
url_basename, url_basename,
@ -1285,7 +1285,7 @@ class InfoExtractor(object):
def _get_cookies(self, url): def _get_cookies(self, url):
""" Return a compat_cookies.SimpleCookie with the cookies for the url """ """ Return a compat_cookies.SimpleCookie with the cookies for the url """
req = compat_urllib_request.Request(url) req = sanitized_Request(url)
self._downloader.cookiejar.add_cookie_header(req) self._downloader.cookiejar.add_cookie_header(req)
return compat_cookies.SimpleCookie(req.get_header('Cookie')) return compat_cookies.SimpleCookie(req.get_header('Cookie'))

9
youtube_dl/extractor/crunchyroll.py
View File

@ -23,6 +23,7 @@ from ..utils import (
int_or_none, int_or_none,
lowercase_escape, lowercase_escape,
remove_end, remove_end,
sanitized_Request,
unified_strdate, unified_strdate,
urlencode_postdata, urlencode_postdata,
xpath_text, xpath_text,
@ -46,7 +47,7 @@ class CrunchyrollBaseIE(InfoExtractor):
'name': username, 'name': username,
'password': password, 'password': password,
}) })
login_request = compat_urllib_request.Request(login_url, data) login_request = sanitized_Request(login_url, data)
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded') login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
self._download_webpage(login_request, None, False, 'Wrong login info') self._download_webpage(login_request, None, False, 'Wrong login info')
@ -55,7 +56,7 @@ class CrunchyrollBaseIE(InfoExtractor):
def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None): def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request) request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
else compat_urllib_request.Request(url_or_request)) else sanitized_Request(url_or_request))
# Accept-Language must be set explicitly to accept any language to avoid issues # Accept-Language must be set explicitly to accept any language to avoid issues
# similar to https://github.com/rg3/youtube-dl/issues/6797. # similar to https://github.com/rg3/youtube-dl/issues/6797.
# Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction # Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
@ -307,7 +308,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
'video_uploader', fatal=False) 'video_uploader', fatal=False)
playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url')) playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
playerdata_req = compat_urllib_request.Request(playerdata_url) playerdata_req = sanitized_Request(playerdata_url)
playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url}) playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded') playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info') playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
@ -319,7 +320,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage): for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
stream_quality, stream_format = self._FORMAT_IDS[fmt] stream_quality, stream_format = self._FORMAT_IDS[fmt]
video_format = fmt + 'p' video_format = fmt + 'p'
streamdata_req = compat_urllib_request.Request( streamdata_req = sanitized_Request(
'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s' 'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
% (stream_id, stream_format, stream_quality), % (stream_id, stream_format, stream_quality),
compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8')) compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))

8
youtube_dl/extractor/dailymotion.py
View File

@ -7,15 +7,13 @@ import itertools
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_str
compat_str,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
determine_ext, determine_ext,
int_or_none, int_or_none,
parse_iso8601, parse_iso8601,
sanitized_Request,
str_to_int, str_to_int,
unescapeHTML, unescapeHTML,
) )
@ -25,7 +23,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
@staticmethod @staticmethod
def _build_request(url): def _build_request(url):
"""Build a request with the family filter disabled""" """Build a request with the family filter disabled"""
request = compat_urllib_request.Request(url) request = sanitized_Request(url)
request.add_header('Cookie', 'family_filter=off; ff=off') request.add_header('Cookie', 'family_filter=off; ff=off')
return request return request

8
youtube_dl/extractor/dcn.py
View File

@ -2,13 +2,11 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
parse_iso8601, parse_iso8601,
sanitized_Request,
) )
@ -36,7 +34,7 @@ class DCNIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
request = compat_urllib_request.Request( request = sanitized_Request(
'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id, 'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
headers={'Origin': 'http://www.dcndigital.ae'}) headers={'Origin': 'http://www.dcndigital.ae'})

4
youtube_dl/extractor/dramafever.py
View File

@ -7,7 +7,6 @@ from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_HTTPError, compat_HTTPError,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
compat_urlparse, compat_urlparse,
) )
from ..utils import ( from ..utils import (
@ -16,6 +15,7 @@ from ..utils import (
determine_ext, determine_ext,
int_or_none, int_or_none,
parse_iso8601, parse_iso8601,
sanitized_Request,
) )
@ -51,7 +51,7 @@ class DramaFeverBaseIE(InfoExtractor):
'password': password, 'password': password,
} }
request = compat_urllib_request.Request( request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8')) self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
response = self._download_webpage( response = self._download_webpage(
request, None, 'Logging in as %s' % username) request, None, 'Logging in as %s' % username)

8
youtube_dl/extractor/dumpert.py
View File

@ -5,8 +5,10 @@ import base64
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request from ..utils import (
from ..utils import qualities qualities,
sanitized_Request,
)
class DumpertIE(InfoExtractor): class DumpertIE(InfoExtractor):
@ -32,7 +34,7 @@ class DumpertIE(InfoExtractor):
protocol = mobj.group('protocol') protocol = mobj.group('protocol')
url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id) url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id)
req = compat_urllib_request.Request(url) req = sanitized_Request(url)
req.add_header('Cookie', 'nsfw=1; cpc=10') req.add_header('Cookie', 'nsfw=1; cpc=10')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)

4
youtube_dl/extractor/eitb.py
View File

@ -2,11 +2,11 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import ( from ..utils import (
float_or_none, float_or_none,
int_or_none, int_or_none,
parse_iso8601, parse_iso8601,
sanitized_Request,
) )
@ -57,7 +57,7 @@ class EitbIE(InfoExtractor):
hls_url = media.get('HLS_SURL') hls_url = media.get('HLS_SURL')
if hls_url: if hls_url:
request = compat_urllib_request.Request( request = sanitized_Request(
'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/', 'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
headers={'Referer': url}) headers={'Referer': url})
token_data = self._download_json( token_data = self._download_json(

5
youtube_dl/extractor/escapist.py
View File

@ -3,13 +3,12 @@ from __future__ import unicode_literals
import json import json
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import ( from ..utils import (
determine_ext, determine_ext,
clean_html, clean_html,
int_or_none, int_or_none,
float_or_none, float_or_none,
sanitized_Request,
) )
@ -75,7 +74,7 @@ class EscapistIE(InfoExtractor):
video_id = ims_video['videoID'] video_id = ims_video['videoID']
key = ims_video['hash'] key = ims_video['hash']
config_req = compat_urllib_request.Request( config_req = sanitized_Request(
'http://www.escapistmagazine.com/videos/' 'http://www.escapistmagazine.com/videos/'
'vidconfig.php?videoID=%s&hash=%s' % (video_id, key)) 'vidconfig.php?videoID=%s&hash=%s' % (video_id, key))
config_req.add_header('Referer', url) config_req.add_header('Referer', url)

8
youtube_dl/extractor/everyonesmixtape.py
View File

@ -3,11 +3,9 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -42,7 +40,7 @@ class EveryonesMixtapeIE(InfoExtractor):
playlist_id = mobj.group('id') playlist_id = mobj.group('id')
pllist_url = 'http://everyonesmixtape.com/mixtape.php?a=getMixes&u=-1&linked=%s&explore=' % playlist_id pllist_url = 'http://everyonesmixtape.com/mixtape.php?a=getMixes&u=-1&linked=%s&explore=' % playlist_id
pllist_req = compat_urllib_request.Request(pllist_url) pllist_req = sanitized_Request(pllist_url)
pllist_req.add_header('X-Requested-With', 'XMLHttpRequest') pllist_req.add_header('X-Requested-With', 'XMLHttpRequest')
playlist_list = self._download_json( playlist_list = self._download_json(
@ -55,7 +53,7 @@ class EveryonesMixtapeIE(InfoExtractor):
raise ExtractorError('Playlist id not found') raise ExtractorError('Playlist id not found')
pl_url = 'http://everyonesmixtape.com/mixtape.php?a=getMix&id=%s&userId=null&code=' % playlist_no pl_url = 'http://everyonesmixtape.com/mixtape.php?a=getMix&id=%s&userId=null&code=' % playlist_no
pl_req = compat_urllib_request.Request(pl_url) pl_req = sanitized_Request(pl_url)
pl_req.add_header('X-Requested-With', 'XMLHttpRequest') pl_req.add_header('X-Requested-With', 'XMLHttpRequest')
playlist = self._download_json( playlist = self._download_json(
pl_req, playlist_id, note='Downloading playlist info') pl_req, playlist_id, note='Downloading playlist info')

4
youtube_dl/extractor/extremetube.py
View File

@ -3,9 +3,9 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
sanitized_Request,
str_to_int, str_to_int,
) )
@ -37,7 +37,7 @@ class ExtremeTubeIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
req = compat_urllib_request.Request(url) req = sanitized_Request(url)
req.add_header('Cookie', 'age_verified=1') req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)

8
youtube_dl/extractor/facebook.py
View File

@ -10,11 +10,11 @@ from ..compat import (
compat_str, compat_str,
compat_urllib_error, compat_urllib_error,
compat_urllib_parse_unquote, compat_urllib_parse_unquote,
compat_urllib_request,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
limit_length, limit_length,
sanitized_Request,
urlencode_postdata, urlencode_postdata,
get_element_by_id, get_element_by_id,
clean_html, clean_html,
@ -73,7 +73,7 @@ class FacebookIE(InfoExtractor):
if useremail is None: if useremail is None:
return return
login_page_req = compat_urllib_request.Request(self._LOGIN_URL) login_page_req = sanitized_Request(self._LOGIN_URL)
login_page_req.add_header('Cookie', 'locale=en_US') login_page_req.add_header('Cookie', 'locale=en_US')
login_page = self._download_webpage(login_page_req, None, login_page = self._download_webpage(login_page_req, None,
note='Downloading login page', note='Downloading login page',
@ -94,7 +94,7 @@ class FacebookIE(InfoExtractor):
'timezone': '-60', 'timezone': '-60',
'trynum': '1', 'trynum': '1',
} }
request = compat_urllib_request.Request(self._LOGIN_URL, urlencode_postdata(login_form)) request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')
try: try:
login_results = self._download_webpage(request, None, login_results = self._download_webpage(request, None,
@ -109,7 +109,7 @@ class FacebookIE(InfoExtractor):
r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h'), r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h'),
'name_action_selected': 'dont_save', 'name_action_selected': 'dont_save',
} }
check_req = compat_urllib_request.Request(self._CHECKPOINT_URL, urlencode_postdata(check_form)) check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
check_req.add_header('Content-Type', 'application/x-www-form-urlencoded') check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
check_response = self._download_webpage(check_req, None, check_response = self._download_webpage(check_req, None,
note='Confirming login') note='Confirming login')

5
youtube_dl/extractor/fc2.py
View File

@ -12,6 +12,7 @@ from ..compat import (
from ..utils import ( from ..utils import (
encode_dict, encode_dict,
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -57,7 +58,7 @@ class FC2IE(InfoExtractor):
} }
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8') login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
request = compat_urllib_request.Request( request = sanitized_Request(
'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data) 'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in') login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
@ -66,7 +67,7 @@ class FC2IE(InfoExtractor):
return False return False
# this is also needed # this is also needed
login_redir = compat_urllib_request.Request('http://id.fc2.com/?mode=redirect&login=done') login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
self._download_webpage( self._download_webpage(
login_redir, None, note='Login redirect', errnote='Login redirect failed') login_redir, None, note='Login redirect', errnote='Login redirect failed')

4
youtube_dl/extractor/flickr.py
View File

@ -3,10 +3,10 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
find_xpath_attr, find_xpath_attr,
sanitized_Request,
) )
@ -30,7 +30,7 @@ class FlickrIE(InfoExtractor):
video_id = mobj.group('id') video_id = mobj.group('id')
video_uploader_id = mobj.group('uploader_id') video_uploader_id = mobj.group('uploader_id')
webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
req = compat_urllib_request.Request(webpage_url) req = sanitized_Request(webpage_url)
req.add_header( req.add_header(
'User-Agent', 'User-Agent',
# it needs a more recent version # it needs a more recent version

6
youtube_dl/extractor/fourtube.py
View File

@ -3,12 +3,10 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
parse_duration, parse_duration,
parse_iso8601, parse_iso8601,
sanitized_Request,
str_to_int, str_to_int,
) )
@ -93,7 +91,7 @@ class FourTubeIE(InfoExtractor):
b'Content-Type': b'application/x-www-form-urlencoded', b'Content-Type': b'application/x-www-form-urlencoded',
b'Origin': b'http://www.4tube.com', b'Origin': b'http://www.4tube.com',
} }
token_req = compat_urllib_request.Request(token_url, b'{}', headers) token_req = sanitized_Request(token_url, b'{}', headers)
tokens = self._download_json(token_req, video_id) tokens = self._download_json(token_req, video_id)
formats = [{ formats = [{
'url': tokens[format]['token'], 'url': tokens[format]['token'],

8
youtube_dl/extractor/gdcvault.py
View File

@ -3,13 +3,11 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
remove_end, remove_end,
HEADRequest, HEADRequest,
sanitized_Request,
) )
@ -125,7 +123,7 @@ class GDCVaultIE(InfoExtractor):
'password': password, 'password': password,
} }
request = compat_urllib_request.Request(login_url, compat_urllib_parse.urlencode(login_form)) request = sanitized_Request(login_url, compat_urllib_parse.urlencode(login_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')
self._download_webpage(request, display_id, 'Logging in') self._download_webpage(request, display_id, 'Logging in')
start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page') start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')

6
youtube_dl/extractor/generic.py
View File

@ -11,7 +11,6 @@ from .youtube import YoutubeIE
from ..compat import ( from ..compat import (
compat_etree_fromstring, compat_etree_fromstring,
compat_urllib_parse_unquote, compat_urllib_parse_unquote,
compat_urllib_request,
compat_urlparse, compat_urlparse,
compat_xml_parse_error, compat_xml_parse_error,
) )
@ -22,6 +21,7 @@ from ..utils import (
HEADRequest, HEADRequest,
is_html, is_html,
orderedSet, orderedSet,
sanitized_Request,
smuggle_url, smuggle_url,
unescapeHTML, unescapeHTML,
unified_strdate, unified_strdate,
@ -1215,7 +1215,7 @@ class GenericIE(InfoExtractor):
full_response = None full_response = None
if head_response is False: if head_response is False:
request = compat_urllib_request.Request(url) request = sanitized_Request(url)
request.add_header('Accept-Encoding', '*') request.add_header('Accept-Encoding', '*')
full_response = self._request_webpage(request, video_id) full_response = self._request_webpage(request, video_id)
head_response = full_response head_response = full_response
@ -1244,7 +1244,7 @@ class GenericIE(InfoExtractor):
'%s on generic information extractor.' % ('Forcing' if force else 'Falling back')) '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
if not full_response: if not full_response:
request = compat_urllib_request.Request(url) request = sanitized_Request(url)
# Some webservers may serve compressed content of rather big size (e.g. gzipped flac) # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
# making it impossible to download only chunk of the file (yet we need only 512kB to # making it impossible to download only chunk of the file (yet we need only 512kB to
# test whether it's HTML or not). According to youtube-dl default Accept-Encoding # test whether it's HTML or not). According to youtube-dl default Accept-Encoding

8
youtube_dl/extractor/hearthisat.py
View File

@ -4,12 +4,10 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urlparse
compat_urllib_request,
compat_urlparse,
)
from ..utils import ( from ..utils import (
HEADRequest, HEADRequest,
sanitized_Request,
str_to_int, str_to_int,
urlencode_postdata, urlencode_postdata,
urlhandle_detect_ext, urlhandle_detect_ext,
@ -47,7 +45,7 @@ class HearThisAtIE(InfoExtractor):
r'intTrackId\s*=\s*(\d+)', webpage, 'track ID') r'intTrackId\s*=\s*(\d+)', webpage, 'track ID')
payload = urlencode_postdata({'tracks[]': track_id}) payload = urlencode_postdata({'tracks[]': track_id})
req = compat_urllib_request.Request(self._PLAYLIST_URL, payload) req = sanitized_Request(self._PLAYLIST_URL, payload)
req.add_header('Content-type', 'application/x-www-form-urlencoded') req.add_header('Content-type', 'application/x-www-form-urlencoded')
track = self._download_json(req, track_id, 'Downloading playlist')[0] track = self._download_json(req, track_id, 'Downloading playlist')[0]

8
youtube_dl/extractor/hotnewhiphop.py
View File

@ -3,13 +3,11 @@ from __future__ import unicode_literals
import base64 import base64
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
HEADRequest, HEADRequest,
sanitized_Request,
) )
@ -41,7 +39,7 @@ class HotNewHipHopIE(InfoExtractor):
('mediaType', 's'), ('mediaType', 's'),
('mediaId', video_id), ('mediaId', video_id),
]) ])
r = compat_urllib_request.Request( r = sanitized_Request(
'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata) 'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata)
r.add_header('Content-Type', 'application/x-www-form-urlencoded') r.add_header('Content-Type', 'application/x-www-form-urlencoded')
mkd = self._download_json( mkd = self._download_json(

10
youtube_dl/extractor/hypem.py
View File

@ -4,12 +4,10 @@ import json
import time import time
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -32,7 +30,7 @@ class HypemIE(InfoExtractor):
data = {'ax': 1, 'ts': time.time()} data = {'ax': 1, 'ts': time.time()}
data_encoded = compat_urllib_parse.urlencode(data) data_encoded = compat_urllib_parse.urlencode(data)
complete_url = url + "?" + data_encoded complete_url = url + "?" + data_encoded
request = compat_urllib_request.Request(complete_url) request = sanitized_Request(complete_url)
response, urlh = self._download_webpage_handle( response, urlh = self._download_webpage_handle(
request, track_id, 'Downloading webpage with the url') request, track_id, 'Downloading webpage with the url')
cookie = urlh.headers.get('Set-Cookie', '') cookie = urlh.headers.get('Set-Cookie', '')
@ -52,7 +50,7 @@ class HypemIE(InfoExtractor):
title = track['song'] title = track['song']
serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key) serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key)
request = compat_urllib_request.Request( request = sanitized_Request(
serve_url, '', {'Content-Type': 'application/json'}) serve_url, '', {'Content-Type': 'application/json'})
request.add_header('cookie', cookie) request.add_header('cookie', cookie)
song_data = self._download_json(request, track_id, 'Downloading metadata') song_data = self._download_json(request, track_id, 'Downloading metadata')

6
youtube_dl/extractor/iprima.py
View File

@ -6,12 +6,10 @@ from random import random
from math import floor from math import floor
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
remove_end, remove_end,
sanitized_Request,
) )
@ -61,7 +59,7 @@ class IPrimaIE(InfoExtractor):
(floor(random() * 1073741824), floor(random() * 1073741824)) (floor(random() * 1073741824), floor(random() * 1073741824))
) )
req = compat_urllib_request.Request(player_url) req = sanitized_Request(player_url)
req.add_header('Referer', url) req.add_header('Referer', url)
playerpage = self._download_webpage(req, video_id) playerpage = self._download_webpage(req, video_id)

6
youtube_dl/extractor/ivi.py
View File

@ -5,11 +5,9 @@ import re
import json import json
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -78,7 +76,7 @@ class IviIE(InfoExtractor):
] ]
} }
request = compat_urllib_request.Request(api_url, json.dumps(data)) request = sanitized_Request(api_url, json.dumps(data))
video_json_page = self._download_webpage( video_json_page = self._download_webpage(
request, video_id, 'Downloading video JSON') request, video_id, 'Downloading video JSON')

8
youtube_dl/extractor/keezmovies.py
View File

@ -4,10 +4,8 @@ import os
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse_urlparse
compat_urllib_parse_urlparse, from ..utils import sanitized_Request
compat_urllib_request,
)
class KeezMoviesIE(InfoExtractor): class KeezMoviesIE(InfoExtractor):
@ -26,7 +24,7 @@ class KeezMoviesIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
req = compat_urllib_request.Request(url) req = sanitized_Request(url)
req.add_header('Cookie', 'age_verified=1') req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)

4
youtube_dl/extractor/letv.py
View File

@ -8,13 +8,13 @@ import time
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
compat_ord, compat_ord,
) )
from ..utils import ( from ..utils import (
determine_ext, determine_ext,
ExtractorError, ExtractorError,
parse_iso8601, parse_iso8601,
sanitized_Request,
int_or_none, int_or_none,
encode_data_uri, encode_data_uri,
) )
@ -114,7 +114,7 @@ class LetvIE(InfoExtractor):
'tkey': self.calc_time_key(int(time.time())), 'tkey': self.calc_time_key(int(time.time())),
'domain': 'www.letv.com' 'domain': 'www.letv.com'
} }
play_json_req = compat_urllib_request.Request( play_json_req = sanitized_Request(
'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params) 'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
) )
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy') cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')

6
youtube_dl/extractor/lynda.py
View File

@ -7,12 +7,12 @@ from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_str, compat_str,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
clean_html, clean_html,
int_or_none, int_or_none,
sanitized_Request,
) )
@ -35,7 +35,7 @@ class LyndaBaseIE(InfoExtractor):
'remember': 'false', 'remember': 'false',
'stayPut': 'false' 'stayPut': 'false'
} }
request = compat_urllib_request.Request( request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8')) self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
login_page = self._download_webpage( login_page = self._download_webpage(
request, None, 'Logging in as %s' % username) request, None, 'Logging in as %s' % username)
@ -64,7 +64,7 @@ class LyndaBaseIE(InfoExtractor):
'remember': 'false', 'remember': 'false',
'stayPut': 'false', 'stayPut': 'false',
} }
request = compat_urllib_request.Request( request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8')) self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8'))
login_page = self._download_webpage( login_page = self._download_webpage(
request, None, request, None,

6
youtube_dl/extractor/metacafe.py
View File

@ -7,12 +7,12 @@ from ..compat import (
compat_parse_qs, compat_parse_qs,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_parse_unquote, compat_urllib_parse_unquote,
compat_urllib_request,
) )
from ..utils import ( from ..utils import (
determine_ext, determine_ext,
ExtractorError, ExtractorError,
int_or_none, int_or_none,
sanitized_Request,
) )
@ -117,7 +117,7 @@ class MetacafeIE(InfoExtractor):
'filters': '0', 'filters': '0',
'submit': "Continue - I'm over 18", 'submit': "Continue - I'm over 18",
} }
request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form)) request = sanitized_Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')
self.report_age_confirmation() self.report_age_confirmation()
self._download_webpage(request, None, False, 'Unable to confirm age') self._download_webpage(request, None, False, 'Unable to confirm age')
@ -142,7 +142,7 @@ class MetacafeIE(InfoExtractor):
return self.url_result('theplatform:%s' % ext_id, 'ThePlatform') return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')
# Retrieve video webpage to extract further information # Retrieve video webpage to extract further information
req = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id) req = sanitized_Request('http://www.metacafe.com/watch/%s/' % video_id)
# AnyClip videos require the flashversion cookie so that we get the link # AnyClip videos require the flashversion cookie so that we get the link
# to the mp4 file # to the mp4 file

8
youtube_dl/extractor/minhateca.py
View File

@ -2,14 +2,12 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
parse_duration, parse_duration,
parse_filesize, parse_filesize,
sanitized_Request,
) )
@ -39,7 +37,7 @@ class MinhatecaIE(InfoExtractor):
('fileId', video_id), ('fileId', video_id),
('__RequestVerificationToken', token), ('__RequestVerificationToken', token),
] ]
req = compat_urllib_request.Request( req = sanitized_Request(
'http://minhateca.com.br/action/License/Download', 'http://minhateca.com.br/action/License/Download',
data=compat_urllib_parse.urlencode(token_data)) data=compat_urllib_parse.urlencode(token_data))
req.add_header('Content-Type', 'application/x-www-form-urlencoded') req.add_header('Content-Type', 'application/x-www-form-urlencoded')

4
youtube_dl/extractor/miomio.py
View File

@ -4,11 +4,11 @@ from __future__ import unicode_literals
import random import random
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import ( from ..utils import (
xpath_text, xpath_text,
int_or_none, int_or_none,
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -63,7 +63,7 @@ class MioMioIE(InfoExtractor):
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)), 'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
video_id) video_id)
vid_config_request = compat_urllib_request.Request( vid_config_request = sanitized_Request(
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config), 'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
headers=http_headers) headers=http_headers)

8
youtube_dl/extractor/moevideo.py
View File

@ -5,13 +5,11 @@ import json
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
int_or_none, int_or_none,
sanitized_Request,
) )
@ -80,7 +78,7 @@ class MoeVideoIE(InfoExtractor):
] ]
r_json = json.dumps(r) r_json = json.dumps(r)
post = compat_urllib_parse.urlencode({'r': r_json}) post = compat_urllib_parse.urlencode({'r': r_json})
req = compat_urllib_request.Request(self._API_URL, post) req = sanitized_Request(self._API_URL, post)
req.add_header('Content-type', 'application/x-www-form-urlencoded') req.add_header('Content-type', 'application/x-www-form-urlencoded')
response = self._download_json(req, video_id) response = self._download_json(req, video_id)

4
youtube_dl/extractor/mofosex.py
View File

@ -7,8 +7,8 @@ from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_urllib_parse_unquote, compat_urllib_parse_unquote,
compat_urllib_parse_urlparse, compat_urllib_parse_urlparse,
compat_urllib_request,
) )
from ..utils import sanitized_Request
class MofosexIE(InfoExtractor): class MofosexIE(InfoExtractor):
@ -29,7 +29,7 @@ class MofosexIE(InfoExtractor):
video_id = mobj.group('id') video_id = mobj.group('id')
url = 'http://www.' + mobj.group('url') url = 'http://www.' + mobj.group('url')
req = compat_urllib_request.Request(url) req = sanitized_Request(url)
req.add_header('Cookie', 'age_verified=1') req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)

10
youtube_dl/extractor/moniker.py
View File

@ -5,13 +5,11 @@ import os.path
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
remove_start, remove_start,
sanitized_Request,
) )
@ -81,7 +79,7 @@ class MonikerIE(InfoExtractor):
orig_webpage, 'builtin URL', default=None, group='url') orig_webpage, 'builtin URL', default=None, group='url')
if builtin_url: if builtin_url:
req = compat_urllib_request.Request(builtin_url) req = sanitized_Request(builtin_url)
req.add_header('Referer', url) req.add_header('Referer', url)
webpage = self._download_webpage(req, video_id, 'Downloading builtin page') webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
title = self._og_search_title(orig_webpage).strip() title = self._og_search_title(orig_webpage).strip()
@ -94,7 +92,7 @@ class MonikerIE(InfoExtractor):
headers = { headers = {
b'Content-Type': b'application/x-www-form-urlencoded', b'Content-Type': b'application/x-www-form-urlencoded',
} }
req = compat_urllib_request.Request(url, post, headers) req = sanitized_Request(url, post, headers)
webpage = self._download_webpage( webpage = self._download_webpage(
req, video_id, note='Downloading video page ...') req, video_id, note='Downloading video page ...')

8
youtube_dl/extractor/mooshare.py
View File

@ -3,12 +3,10 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_request,
compat_urllib_parse,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -59,7 +57,7 @@ class MooshareIE(InfoExtractor):
'hash': hash_key, 'hash': hash_key,
} }
request = compat_urllib_request.Request( request = sanitized_Request(
'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form)) 'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')

6
youtube_dl/extractor/movieclips.py
View File

@ -2,9 +2,7 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..utils import sanitized_Request
compat_urllib_request,
)
class MovieClipsIE(InfoExtractor): class MovieClipsIE(InfoExtractor):
@ -25,7 +23,7 @@ class MovieClipsIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
display_id = self._match_id(url) display_id = self._match_id(url)
req = compat_urllib_request.Request(url) req = sanitized_Request(url)
# it doesn't work if it thinks the browser it's too old # it doesn't work if it thinks the browser it's too old
req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)') req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)')
webpage = self._download_webpage(req, display_id) webpage = self._download_webpage(req, display_id)

4
youtube_dl/extractor/mtv.py
View File

@ -5,7 +5,6 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
compat_str, compat_str,
) )
from ..utils import ( from ..utils import (
@ -13,6 +12,7 @@ from ..utils import (
find_xpath_attr, find_xpath_attr,
fix_xml_ampersands, fix_xml_ampersands,
HEADRequest, HEADRequest,
sanitized_Request,
unescapeHTML, unescapeHTML,
url_basename, url_basename,
RegexNotFoundError, RegexNotFoundError,
@ -53,7 +53,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
def _extract_mobile_video_formats(self, mtvn_id): def _extract_mobile_video_formats(self, mtvn_id):
webpage_url = self._MOBILE_TEMPLATE % mtvn_id webpage_url = self._MOBILE_TEMPLATE % mtvn_id
req = compat_urllib_request.Request(webpage_url) req = sanitized_Request(webpage_url)
# Otherwise we get a webpage that would execute some javascript # Otherwise we get a webpage that would execute some javascript
req.add_header('User-Agent', 'curl/7') req.add_header('User-Agent', 'curl/7')
webpage = self._download_webpage(req, mtvn_id, webpage = self._download_webpage(req, mtvn_id,

4
youtube_dl/extractor/myvideo.py
View File

@ -11,10 +11,10 @@ from ..compat import (
compat_ord, compat_ord,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_parse_unquote, compat_urllib_parse_unquote,
compat_urllib_request,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -83,7 +83,7 @@ class MyVideoIE(InfoExtractor):
mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage) mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
if mobj is not None: if mobj is not None:
request = compat_urllib_request.Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '') request = sanitized_Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
response = self._download_webpage(request, video_id, response = self._download_webpage(request, video_id,
'Downloading video info') 'Downloading video info')
info = json.loads(base64.b64decode(response).decode('utf-8')) info = json.loads(base64.b64decode(response).decode('utf-8'))

4
youtube_dl/extractor/neteasemusic.py
View File

@ -8,11 +8,11 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_urllib_request,
compat_urllib_parse, compat_urllib_parse,
compat_str, compat_str,
compat_itertools_count, compat_itertools_count,
) )
from ..utils import sanitized_Request
class NetEaseMusicBaseIE(InfoExtractor): class NetEaseMusicBaseIE(InfoExtractor):
@ -56,7 +56,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
return int(round(ms / 1000.0)) return int(round(ms / 1000.0))
def query_api(self, endpoint, video_id, note): def query_api(self, endpoint, video_id, note):
req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint)) req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
req.add_header('Referer', self._API_BASE) req.add_header('Referer', self._API_BASE)
return self._download_json(req, video_id, note) return self._download_json(req, video_id, note)

9
youtube_dl/extractor/nfb.py
View File

@ -1,10 +1,8 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_request, from ..utils import sanitized_Request
compat_urllib_parse,
)
class NFBIE(InfoExtractor): class NFBIE(InfoExtractor):
@ -40,7 +38,8 @@ class NFBIE(InfoExtractor):
uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>', uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>',
page, 'director name', fatal=False) page, 'director name', fatal=False)
request = compat_urllib_request.Request('https://www.nfb.ca/film/%s/player_config' % video_id, request = sanitized_Request(
'https://www.nfb.ca/film/%s/player_config' % video_id,
compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii')) compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')
request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf') request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')

6
youtube_dl/extractor/niconico.py
View File

@ -8,7 +8,6 @@ import datetime
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
compat_urlparse, compat_urlparse,
) )
from ..utils import ( from ..utils import (
@ -17,6 +16,7 @@ from ..utils import (
int_or_none, int_or_none,
parse_duration, parse_duration,
parse_iso8601, parse_iso8601,
sanitized_Request,
xpath_text, xpath_text,
determine_ext, determine_ext,
) )
@ -102,7 +102,7 @@ class NiconicoIE(InfoExtractor):
'password': password, 'password': password,
} }
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8') login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
request = compat_urllib_request.Request( request = sanitized_Request(
'https://secure.nicovideo.jp/secure/login', login_data) 'https://secure.nicovideo.jp/secure/login', login_data)
login_results = self._download_webpage( login_results = self._download_webpage(
request, None, note='Logging in', errnote='Unable to log in') request, None, note='Logging in', errnote='Unable to log in')
@ -145,7 +145,7 @@ class NiconicoIE(InfoExtractor):
'k': thumb_play_key, 'k': thumb_play_key,
'v': video_id 'v': video_id
}) })
flv_info_request = compat_urllib_request.Request( flv_info_request = sanitized_Request(
'http://ext.nicovideo.jp/thumb_watch', flv_info_data, 'http://ext.nicovideo.jp/thumb_watch', flv_info_data,
{'Content-Type': 'application/x-www-form-urlencoded'}) {'Content-Type': 'application/x-www-form-urlencoded'})
flv_info_webpage = self._download_webpage( flv_info_webpage = self._download_webpage(

4
youtube_dl/extractor/noco.py
View File

@ -9,7 +9,6 @@ from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_str, compat_str,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
) )
from ..utils import ( from ..utils import (
clean_html, clean_html,
@ -17,6 +16,7 @@ from ..utils import (
int_or_none, int_or_none,
float_or_none, float_or_none,
parse_iso8601, parse_iso8601,
sanitized_Request,
) )
@ -74,7 +74,7 @@ class NocoIE(InfoExtractor):
'username': username, 'username': username,
'password': password, 'password': password,
} }
request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form)) request = sanitized_Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8') request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
login = self._download_json(request, None, 'Logging in as %s' % username) login = self._download_json(request, None, 'Logging in as %s' % username)

6
youtube_dl/extractor/nosvideo.py
View File

@ -4,11 +4,9 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
urlencode_postdata, urlencode_postdata,
xpath_text, xpath_text,
xpath_with_ns, xpath_with_ns,
@ -41,7 +39,7 @@ class NosVideoIE(InfoExtractor):
'op': 'download1', 'op': 'download1',
'method_free': 'Continue to Video', 'method_free': 'Continue to Video',
} }
req = compat_urllib_request.Request(url, urlencode_postdata(fields)) req = sanitized_Request(url, urlencode_postdata(fields))
req.add_header('Content-type', 'application/x-www-form-urlencoded') req.add_header('Content-type', 'application/x-www-form-urlencoded')
webpage = self._download_webpage(req, video_id, webpage = self._download_webpage(req, video_id,
'Downloading download page') 'Downloading download page')

8
youtube_dl/extractor/novamov.py
View File

@ -3,14 +3,12 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urlparse
compat_urllib_request,
compat_urlparse,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
NO_DEFAULT, NO_DEFAULT,
encode_dict, encode_dict,
sanitized_Request,
urlencode_postdata, urlencode_postdata,
) )
@ -65,7 +63,7 @@ class NovaMovIE(InfoExtractor):
'post url', default=url, group='url') 'post url', default=url, group='url')
if not post_url.startswith('http'): if not post_url.startswith('http'):
post_url = compat_urlparse.urljoin(url, post_url) post_url = compat_urlparse.urljoin(url, post_url)
request = compat_urllib_request.Request( request = sanitized_Request(
post_url, urlencode_postdata(encode_dict(fields))) post_url, urlencode_postdata(encode_dict(fields)))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')
request.add_header('Referer', post_url) request.add_header('Referer', post_url)

10
youtube_dl/extractor/nowness.py
View File

@ -3,10 +3,10 @@ from __future__ import unicode_literals
from .brightcove import BrightcoveLegacyIE from .brightcove import BrightcoveLegacyIE
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ExtractorError from ..compat import compat_str
from ..compat import ( from ..utils import (
compat_str, ExtractorError,
compat_urllib_request, sanitized_Request,
) )
@ -37,7 +37,7 @@ class NownessBaseIE(InfoExtractor):
def _api_request(self, url, request_path): def _api_request(self, url, request_path):
display_id = self._match_id(url) display_id = self._match_id(url)
request = compat_urllib_request.Request( request = sanitized_Request(
'http://api.nowness.com/api/' + request_path % display_id, 'http://api.nowness.com/api/' + request_path % display_id,
headers={ headers={
'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us', 'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',

6
youtube_dl/extractor/nuvid.py
View File

@ -3,11 +3,9 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
parse_duration, parse_duration,
sanitized_Request,
unified_strdate, unified_strdate,
) )
@ -33,7 +31,7 @@ class NuvidIE(InfoExtractor):
formats = [] formats = []
for dwnld_speed, format_id in [(0, '3gp'), (5, 'mp4')]: for dwnld_speed, format_id in [(0, '3gp'), (5, 'mp4')]:
request = compat_urllib_request.Request( request = sanitized_Request(
'http://m.nuvid.com/play/%s' % video_id) 'http://m.nuvid.com/play/%s' % video_id)
request.add_header('Cookie', 'skip_download_page=1; dwnld_speed=%d; adv_show=1' % dwnld_speed) request.add_header('Cookie', 'skip_download_page=1; dwnld_speed=%d; adv_show=1' % dwnld_speed)
webpage = self._download_webpage( webpage = self._download_webpage(

6
youtube_dl/extractor/patreon.py
View File

@ -2,9 +2,7 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import js_to_json
js_to_json,
)
class PatreonIE(InfoExtractor): class PatreonIE(InfoExtractor):
@ -65,7 +63,7 @@ class PatreonIE(InfoExtractor):
'password': password, 'password': password,
} }
request = compat_urllib_request.Request( request = sanitized_Request(
'https://www.patreon.com/processLogin', 'https://www.patreon.com/processLogin',
compat_urllib_parse.urlencode(login_form).encode('utf-8') compat_urllib_parse.urlencode(login_form).encode('utf-8')
) )

8
youtube_dl/extractor/played.py
View File

@ -5,12 +5,10 @@ import re
import os.path import os.path
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -46,7 +44,7 @@ class PlayedIE(InfoExtractor):
headers = { headers = {
b'Content-Type': b'application/x-www-form-urlencoded', b'Content-Type': b'application/x-www-form-urlencoded',
} }
req = compat_urllib_request.Request(url, post, headers) req = sanitized_Request(url, post, headers)
webpage = self._download_webpage( webpage = self._download_webpage(
req, video_id, note='Downloading video page ...') req, video_id, note='Downloading video page ...')

6
youtube_dl/extractor/pluralsight.py
View File

@ -8,13 +8,13 @@ from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_str, compat_str,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
compat_urlparse, compat_urlparse,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
int_or_none, int_or_none,
parse_duration, parse_duration,
sanitized_Request,
) )
@ -73,7 +73,7 @@ class PluralsightIE(PluralsightBaseIE):
if not post_url.startswith('http'): if not post_url.startswith('http'):
post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url) post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
request = compat_urllib_request.Request( request = sanitized_Request(
post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8')) post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')
@ -181,7 +181,7 @@ class PluralsightIE(PluralsightBaseIE):
'mt': ext, 'mt': ext,
'q': '%dx%d' % (f['width'], f['height']), 'q': '%dx%d' % (f['width'], f['height']),
} }
request = compat_urllib_request.Request( request = sanitized_Request(
'%s/training/Player/ViewClip' % self._API_BASE, '%s/training/Player/ViewClip' % self._API_BASE,
json.dumps(clip_post).encode('utf-8')) json.dumps(clip_post).encode('utf-8'))
request.add_header('Content-Type', 'application/json;charset=utf-8') request.add_header('Content-Type', 'application/json;charset=utf-8')

4
youtube_dl/extractor/pornhub.py
View File

@ -8,10 +8,10 @@ from ..compat import (
compat_urllib_parse_unquote, compat_urllib_parse_unquote,
compat_urllib_parse_unquote_plus, compat_urllib_parse_unquote_plus,
compat_urllib_parse_urlparse, compat_urllib_parse_urlparse,
compat_urllib_request,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
str_to_int, str_to_int,
) )
from ..aes import ( from ..aes import (
@ -53,7 +53,7 @@ class PornHubIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
req = compat_urllib_request.Request( req = sanitized_Request(
'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id) 'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
req.add_header('Cookie', 'age_verified=1') req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)

10
youtube_dl/extractor/pornotube.py
View File

@ -3,11 +3,9 @@ from __future__ import unicode_literals
import json import json
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
sanitized_Request,
) )
@ -46,7 +44,7 @@ class PornotubeIE(InfoExtractor):
'authenticationSpaceKey': originAuthenticationSpaceKey, 'authenticationSpaceKey': originAuthenticationSpaceKey,
'credentials': 'Clip Application', 'credentials': 'Clip Application',
} }
token_req = compat_urllib_request.Request( token_req = sanitized_Request(
'https://api.aebn.net/auth/v1/token/primal', 'https://api.aebn.net/auth/v1/token/primal',
data=json.dumps(token_req_data).encode('utf-8')) data=json.dumps(token_req_data).encode('utf-8'))
token_req.add_header('Content-Type', 'application/json') token_req.add_header('Content-Type', 'application/json')
@ -56,7 +54,7 @@ class PornotubeIE(InfoExtractor):
token = token_answer['tokenKey'] token = token_answer['tokenKey']
# Get video URL # Get video URL
delivery_req = compat_urllib_request.Request( delivery_req = sanitized_Request(
'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id) 'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id)
delivery_req.add_header('Authorization', token) delivery_req.add_header('Authorization', token)
delivery_info = self._download_json( delivery_info = self._download_json(
@ -64,7 +62,7 @@ class PornotubeIE(InfoExtractor):
video_url = delivery_info['mediaUrl'] video_url = delivery_info['mediaUrl']
# Get additional info (title etc.) # Get additional info (title etc.)
info_req = compat_urllib_request.Request( info_req = sanitized_Request(
'https://api.aebn.net/content/v1/clips/%s?expand=' 'https://api.aebn.net/content/v1/clips/%s?expand='
'title,description,primaryImageNumber,startSecond,endSecond,' 'title,description,primaryImageNumber,startSecond,endSecond,'
'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,' 'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,'

10
youtube_dl/extractor/primesharetv.py
View File

@ -1,11 +1,11 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse, from ..utils import (
compat_urllib_request, ExtractorError,
sanitized_Request,
) )
from ..utils import ExtractorError
class PrimeShareTVIE(InfoExtractor): class PrimeShareTVIE(InfoExtractor):
@ -41,7 +41,7 @@ class PrimeShareTVIE(InfoExtractor):
webpage, 'wait time', default=7)) + 1 webpage, 'wait time', default=7)) + 1
self._sleep(wait_time, video_id) self._sleep(wait_time, video_id)
req = compat_urllib_request.Request( req = sanitized_Request(
url, compat_urllib_parse.urlencode(fields), headers) url, compat_urllib_parse.urlencode(fields), headers)
video_page = self._download_webpage( video_page = self._download_webpage(
req, video_id, 'Downloading video page') req, video_id, 'Downloading video page')

8
youtube_dl/extractor/promptfile.py
View File

@ -4,13 +4,11 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
determine_ext, determine_ext,
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -37,7 +35,7 @@ class PromptFileIE(InfoExtractor):
fields = self._hidden_inputs(webpage) fields = self._hidden_inputs(webpage)
post = compat_urllib_parse.urlencode(fields) post = compat_urllib_parse.urlencode(fields)
req = compat_urllib_request.Request(url, post) req = sanitized_Request(url, post)
req.add_header('Content-type', 'application/x-www-form-urlencoded') req.add_header('Content-type', 'application/x-www-form-urlencoded')
webpage = self._download_webpage( webpage = self._download_webpage(
req, video_id, 'Downloading video page') req, video_id, 'Downloading video page')

4
youtube_dl/extractor/qqmusic.py
View File

@ -7,11 +7,11 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
sanitized_Request,
strip_jsonp, strip_jsonp,
unescapeHTML, unescapeHTML,
clean_html, clean_html,
) )
from ..compat import compat_urllib_request
class QQMusicIE(InfoExtractor): class QQMusicIE(InfoExtractor):
@ -201,7 +201,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
singer_desc = None singer_desc = None
if singer_id: if singer_id:
req = compat_urllib_request.Request( req = sanitized_Request(
'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id) 'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id)
req.add_header( req.add_header(
'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html') 'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')

4
youtube_dl/extractor/rtve.py
View File

@ -6,11 +6,11 @@ import re
import time import time
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
float_or_none, float_or_none,
remove_end, remove_end,
sanitized_Request,
std_headers, std_headers,
struct_unpack, struct_unpack,
) )
@ -102,7 +102,7 @@ class RTVEALaCartaIE(InfoExtractor):
if info['state'] == 'DESPU': if info['state'] == 'DESPU':
raise ExtractorError('The video is no longer available', expected=True) raise ExtractorError('The video is no longer available', expected=True)
png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id) png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
png_request = compat_urllib_request.Request(png_url) png_request = sanitized_Request(png_url)
png_request.add_header('Referer', url) png_request.add_header('Referer', url)
png = self._download_webpage(png_request, video_id, 'Downloading url information') png = self._download_webpage(png_request, video_id, 'Downloading url information')
video_url = _decrypt_url(png) video_url = _decrypt_url(png)

8
youtube_dl/extractor/safari.py
View File

@ -6,12 +6,10 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from .brightcove import BrightcoveLegacyIE from .brightcove import BrightcoveLegacyIE
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
smuggle_url, smuggle_url,
std_headers, std_headers,
) )
@ -58,7 +56,7 @@ class SafariBaseIE(InfoExtractor):
'next': '', 'next': '',
} }
request = compat_urllib_request.Request( request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers) self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers)
login_page = self._download_webpage( login_page = self._download_webpage(
request, None, 'Logging in as %s' % username) request, None, 'Logging in as %s' % username)

8
youtube_dl/extractor/sandia.py
View File

@ -6,14 +6,12 @@ import json
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urlparse
compat_urllib_request,
compat_urlparse,
)
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
js_to_json, js_to_json,
mimetype2ext, mimetype2ext,
sanitized_Request,
unified_strdate, unified_strdate,
) )
@ -37,7 +35,7 @@ class SandiaIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
req = compat_urllib_request.Request(url) req = sanitized_Request(url)
req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4') req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)

8
youtube_dl/extractor/shared.py
View File

@ -3,13 +3,11 @@ from __future__ import unicode_literals
import base64 import base64
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
int_or_none, int_or_none,
sanitized_Request,
) )
@ -46,7 +44,7 @@ class SharedIE(InfoExtractor):
'Video %s does not exist' % video_id, expected=True) 'Video %s does not exist' % video_id, expected=True)
download_form = self._hidden_inputs(webpage) download_form = self._hidden_inputs(webpage)
request = compat_urllib_request.Request( request = sanitized_Request(
url, compat_urllib_parse.urlencode(download_form)) url, compat_urllib_parse.urlencode(download_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')

8
youtube_dl/extractor/sharesix.py
View File

@ -4,12 +4,10 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
parse_duration, parse_duration,
sanitized_Request,
) )
@ -50,7 +48,7 @@ class ShareSixIE(InfoExtractor):
'method_free': 'Free' 'method_free': 'Free'
} }
post = compat_urllib_parse.urlencode(fields) post = compat_urllib_parse.urlencode(fields)
req = compat_urllib_request.Request(url, post) req = sanitized_Request(url, post)
req.add_header('Content-type', 'application/x-www-form-urlencoded') req.add_header('Content-type', 'application/x-www-form-urlencoded')
webpage = self._download_webpage(req, video_id, webpage = self._download_webpage(req, video_id,

8
youtube_dl/extractor/sina.py
View File

@ -4,10 +4,8 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_request, from ..utils import sanitized_Request
compat_urllib_parse,
)
class SinaIE(InfoExtractor): class SinaIE(InfoExtractor):
@ -61,7 +59,7 @@ class SinaIE(InfoExtractor):
if mobj.group('token') is not None: if mobj.group('token') is not None:
# The video id is in the redirected url # The video id is in the redirected url
self.to_screen('Getting video id') self.to_screen('Getting video id')
request = compat_urllib_request.Request(url) request = sanitized_Request(url)
request.get_method = lambda: 'HEAD' request.get_method = lambda: 'HEAD'
(_, urlh) = self._download_webpage_handle(request, 'NA', False) (_, urlh) = self._download_webpage_handle(request, 'NA', False)
return self._real_extract(urlh.geturl()) return self._real_extract(urlh.geturl())

10
youtube_dl/extractor/smotri.py
View File

@ -7,13 +7,11 @@ import hashlib
import uuid import uuid
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
int_or_none, int_or_none,
sanitized_Request,
unified_strdate, unified_strdate,
) )
@ -176,7 +174,7 @@ class SmotriIE(InfoExtractor):
if video_password: if video_password:
video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest() video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
request = compat_urllib_request.Request( request = sanitized_Request(
'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form)) 'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')
@ -339,7 +337,7 @@ class SmotriBroadcastIE(InfoExtractor):
'password': password, 'password': password,
} }
request = compat_urllib_request.Request( request = sanitized_Request(
broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form)) broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')
broadcast_page = self._download_webpage( broadcast_page = self._download_webpage(

4
youtube_dl/extractor/sohu.py
View File

@ -6,11 +6,11 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_str, compat_str,
compat_urllib_request,
compat_urllib_parse, compat_urllib_parse,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -96,7 +96,7 @@ class SohuIE(InfoExtractor):
else: else:
base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid=' base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
req = compat_urllib_request.Request(base_data_url + vid_id) req = sanitized_Request(base_data_url + vid_id)
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy') cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
if cn_verification_proxy: if cn_verification_proxy:

4
youtube_dl/extractor/spankwire.py
View File

@ -6,9 +6,9 @@ from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_urllib_parse_unquote, compat_urllib_parse_unquote,
compat_urllib_parse_urlparse, compat_urllib_parse_urlparse,
compat_urllib_request,
) )
from ..utils import ( from ..utils import (
sanitized_Request,
str_to_int, str_to_int,
unified_strdate, unified_strdate,
) )
@ -51,7 +51,7 @@ class SpankwireIE(InfoExtractor):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id') video_id = mobj.group('id')
req = compat_urllib_request.Request('http://www.' + mobj.group('url')) req = sanitized_Request('http://www.' + mobj.group('url'))
req.add_header('Cookie', 'age_verified=1') req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)

6
youtube_dl/extractor/sportdeutschland.py
View File

@ -4,11 +4,9 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
parse_iso8601, parse_iso8601,
sanitized_Request,
) )
@ -54,7 +52,7 @@ class SportDeutschlandIE(InfoExtractor):
api_url = 'http://proxy.vidibusdynamic.net/sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % ( api_url = 'http://proxy.vidibusdynamic.net/sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % (
sport_id, video_id) sport_id, video_id)
req = compat_urllib_request.Request(api_url, headers={ req = sanitized_Request(api_url, headers={
'Accept': 'application/vnd.vidibus.v2.html+json', 'Accept': 'application/vnd.vidibus.v2.html+json',
'Referer': url, 'Referer': url,
}) })

8
youtube_dl/extractor/streamcloud.py
View File

@ -4,10 +4,8 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse, from ..utils import sanitized_Request
compat_urllib_request,
)
class StreamcloudIE(InfoExtractor): class StreamcloudIE(InfoExtractor):
@ -43,7 +41,7 @@ class StreamcloudIE(InfoExtractor):
headers = { headers = {
b'Content-Type': b'application/x-www-form-urlencoded', b'Content-Type': b'application/x-www-form-urlencoded',
} }
req = compat_urllib_request.Request(url, post, headers) req = sanitized_Request(url, post, headers)
webpage = self._download_webpage( webpage = self._download_webpage(
req, video_id, note='Downloading video page ...') req, video_id, note='Downloading video page ...')

6
youtube_dl/extractor/streamcz.py
View File

@ -5,11 +5,9 @@ import hashlib
import time import time
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
sanitized_Request,
) )
@ -54,7 +52,7 @@ class StreamCZIE(InfoExtractor):
video_id = self._match_id(url) video_id = self._match_id(url)
api_path = '/episode/%s' % video_id api_path = '/episode/%s' % video_id
req = compat_urllib_request.Request(self._API_URL + api_path) req = sanitized_Request(self._API_URL + api_path)
req.add_header('Api-Password', _get_api_key(api_path)) req.add_header('Api-Password', _get_api_key(api_path))
data = self._download_json(req, video_id) data = self._download_json(req, video_id)

6
youtube_dl/extractor/tapely.py
View File

@ -4,14 +4,12 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
clean_html, clean_html,
ExtractorError, ExtractorError,
float_or_none, float_or_none,
parse_iso8601, parse_iso8601,
sanitized_Request,
) )
@ -53,7 +51,7 @@ class TapelyIE(InfoExtractor):
display_id = mobj.group('id') display_id = mobj.group('id')
playlist_url = self._API_URL.format(display_id) playlist_url = self._API_URL.format(display_id)
request = compat_urllib_request.Request(playlist_url) request = sanitized_Request(playlist_url)
request.add_header('X-Requested-With', 'XMLHttpRequest') request.add_header('X-Requested-With', 'XMLHttpRequest')
request.add_header('Accept', 'application/json') request.add_header('Accept', 'application/json')
request.add_header('Referer', url) request.add_header('Referer', url)

8
youtube_dl/extractor/tube8.py
View File

@ -4,12 +4,10 @@ import json
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse_urlparse
compat_urllib_parse_urlparse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
sanitized_Request,
str_to_int, str_to_int,
) )
from ..aes import aes_decrypt_text from ..aes import aes_decrypt_text
@ -42,7 +40,7 @@ class Tube8IE(InfoExtractor):
video_id = mobj.group('id') video_id = mobj.group('id')
display_id = mobj.group('display_id') display_id = mobj.group('display_id')
req = compat_urllib_request.Request(url) req = sanitized_Request(url)
req.add_header('Cookie', 'age_verified=1') req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, display_id) webpage = self._download_webpage(req, display_id)

8
youtube_dl/extractor/tubitv.py
View File

@ -5,13 +5,11 @@ import codecs
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
int_or_none, int_or_none,
sanitized_Request,
) )
@ -44,7 +42,7 @@ class TubiTvIE(InfoExtractor):
'password': password, 'password': password,
} }
payload = compat_urllib_parse.urlencode(form_data).encode('utf-8') payload = compat_urllib_parse.urlencode(form_data).encode('utf-8')
request = compat_urllib_request.Request(self._LOGIN_URL, payload) request = sanitized_Request(self._LOGIN_URL, payload)
request.add_header('Content-Type', 'application/x-www-form-urlencoded') request.add_header('Content-Type', 'application/x-www-form-urlencoded')
login_page = self._download_webpage( login_page = self._download_webpage(
request, None, False, 'Wrong login info') request, None, False, 'Wrong login info')

6
youtube_dl/extractor/twitch.py
View File

@ -11,7 +11,6 @@ from ..compat import (
compat_str, compat_str,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_parse_urlparse, compat_urllib_parse_urlparse,
compat_urllib_request,
compat_urlparse, compat_urlparse,
) )
from ..utils import ( from ..utils import (
@ -20,6 +19,7 @@ from ..utils import (
int_or_none, int_or_none,
parse_duration, parse_duration,
parse_iso8601, parse_iso8601,
sanitized_Request,
) )
@ -48,7 +48,7 @@ class TwitchBaseIE(InfoExtractor):
for cookie in self._downloader.cookiejar: for cookie in self._downloader.cookiejar:
if cookie.name == 'api_token': if cookie.name == 'api_token':
headers['Twitch-Api-Token'] = cookie.value headers['Twitch-Api-Token'] = cookie.value
request = compat_urllib_request.Request(url, headers=headers) request = sanitized_Request(url, headers=headers)
response = super(TwitchBaseIE, self)._download_json(request, video_id, note) response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
self._handle_error(response) self._handle_error(response)
return response return response
@ -80,7 +80,7 @@ class TwitchBaseIE(InfoExtractor):
if not post_url.startswith('http'): if not post_url.startswith('http'):
post_url = compat_urlparse.urljoin(redirect_url, post_url) post_url = compat_urlparse.urljoin(redirect_url, post_url)
request = compat_urllib_request.Request( request = sanitized_Request(
post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8')) post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8'))
request.add_header('Referer', redirect_url) request.add_header('Referer', redirect_url)
response = self._download_webpage( response = self._download_webpage(

4
youtube_dl/extractor/twitter.py
View File

@ -4,13 +4,13 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import ( from ..utils import (
float_or_none, float_or_none,
xpath_text, xpath_text,
remove_end, remove_end,
int_or_none, int_or_none,
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -81,7 +81,7 @@ class TwitterCardIE(InfoExtractor):
config = None config = None
formats = [] formats = []
for user_agent in USER_AGENTS: for user_agent in USER_AGENTS:
request = compat_urllib_request.Request(url) request = sanitized_Request(url)
request.add_header('User-Agent', user_agent) request.add_header('User-Agent', user_agent)
webpage = self._download_webpage(request, video_id) webpage = self._download_webpage(request, video_id)

5
youtube_dl/extractor/udemy.py
View File

@ -9,6 +9,7 @@ from ..compat import (
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -58,7 +59,7 @@ class UdemyIE(InfoExtractor):
for header, value in headers.items(): for header, value in headers.items():
url_or_request.add_header(header, value) url_or_request.add_header(header, value)
else: else:
url_or_request = compat_urllib_request.Request(url_or_request, headers=headers) url_or_request = sanitized_Request(url_or_request, headers=headers)
response = super(UdemyIE, self)._download_json(url_or_request, video_id, note) response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
self._handle_error(response) self._handle_error(response)
@ -89,7 +90,7 @@ class UdemyIE(InfoExtractor):
'password': password.encode('utf-8'), 'password': password.encode('utf-8'),
}) })
request = compat_urllib_request.Request( request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8')) self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
request.add_header('Referer', self._ORIGIN_URL) request.add_header('Referer', self._ORIGIN_URL)
request.add_header('Origin', self._ORIGIN_URL) request.add_header('Origin', self._ORIGIN_URL)

4
youtube_dl/extractor/vbox7.py
View File

@ -4,11 +4,11 @@ from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
compat_urlparse, compat_urlparse,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -49,7 +49,7 @@ class Vbox7IE(InfoExtractor):
info_url = "http://vbox7.com/play/magare.do" info_url = "http://vbox7.com/play/magare.do"
data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id}) data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id})
info_request = compat_urllib_request.Request(info_url, data) info_request = sanitized_Request(info_url, data)
info_request.add_header('Content-Type', 'application/x-www-form-urlencoded') info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
info_response = self._download_webpage(info_request, video_id, 'Downloading info webpage') info_response = self._download_webpage(info_request, video_id, 'Downloading info webpage')
if info_response is None: if info_response is None:

6
youtube_dl/extractor/veoh.py
View File

@ -4,12 +4,10 @@ import re
import json import json
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
)
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
ExtractorError, ExtractorError,
sanitized_Request,
) )
@ -110,7 +108,7 @@ class VeohIE(InfoExtractor):
if 'class="adultwarning-container"' in webpage: if 'class="adultwarning-container"' in webpage:
self.report_age_confirmation() self.report_age_confirmation()
age_limit = 18 age_limit = 18
request = compat_urllib_request.Request(url) request = sanitized_Request(url)
request.add_header('Cookie', 'confirmedAdult=true') request.add_header('Cookie', 'confirmedAdult=true')
webpage = self._download_webpage(request, video_id) webpage = self._download_webpage(request, video_id)

4
youtube_dl/extractor/vessel.py
View File

@ -4,10 +4,10 @@ from __future__ import unicode_literals
import json import json
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
parse_iso8601, parse_iso8601,
sanitized_Request,
) )
@ -33,7 +33,7 @@ class VesselIE(InfoExtractor):
@staticmethod @staticmethod
def make_json_request(url, data): def make_json_request(url, data):
payload = json.dumps(data).encode('utf-8') payload = json.dumps(data).encode('utf-8')
req = compat_urllib_request.Request(url, payload) req = sanitized_Request(url, payload)
req.add_header('Content-Type', 'application/json; charset=utf-8') req.add_header('Content-Type', 'application/json; charset=utf-8')
return req return req

8
youtube_dl/extractor/vevo.py
View File

@ -3,13 +3,11 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_etree_fromstring
compat_etree_fromstring,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
int_or_none, int_or_none,
sanitized_Request,
) )
@ -73,7 +71,7 @@ class VevoIE(InfoExtractor):
_SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/' _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
def _real_initialize(self): def _real_initialize(self):
req = compat_urllib_request.Request( req = sanitized_Request(
'http://www.vevo.com/auth', data=b'') 'http://www.vevo.com/auth', data=b'')
webpage = self._download_webpage( webpage = self._download_webpage(
req, None, req, None,

6
youtube_dl/extractor/viddler.py
View File

@ -4,9 +4,7 @@ from .common import InfoExtractor
from ..utils import ( from ..utils import (
float_or_none, float_or_none,
int_or_none, int_or_none,
) sanitized_Request,
from ..compat import (
compat_urllib_request
) )
@ -65,7 +63,7 @@ class ViddlerIE(InfoExtractor):
'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' % 'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
video_id) video_id)
headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'} headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
request = compat_urllib_request.Request(json_url, None, headers) request = sanitized_Request(json_url, None, headers)
data = self._download_json(request, video_id)['video'] data = self._download_json(request, video_id)['video']
formats = [] formats = []

4
youtube_dl/extractor/videomega.py
View File

@ -4,7 +4,7 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request from ..utils import sanitized_Request
class VideoMegaIE(InfoExtractor): class VideoMegaIE(InfoExtractor):
@ -30,7 +30,7 @@ class VideoMegaIE(InfoExtractor):
video_id = self._match_id(url) video_id = self._match_id(url)
iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
req = compat_urllib_request.Request(iframe_url) req = sanitized_Request(iframe_url)
req.add_header('Referer', url) req.add_header('Referer', url)
req.add_header('Cookie', 'noadvtday=0') req.add_header('Cookie', 'noadvtday=0')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)

4
youtube_dl/extractor/viewster.py
View File

@ -4,7 +4,6 @@ from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_HTTPError, compat_HTTPError,
compat_urllib_request,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_parse_unquote, compat_urllib_parse_unquote,
) )
@ -13,6 +12,7 @@ from ..utils import (
ExtractorError, ExtractorError,
int_or_none, int_or_none,
parse_iso8601, parse_iso8601,
sanitized_Request,
HEADRequest, HEADRequest,
) )
@ -76,7 +76,7 @@ class ViewsterIE(InfoExtractor):
_ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01' _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True): def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
request = compat_urllib_request.Request(url) request = sanitized_Request(url)
request.add_header('Accept', self._ACCEPT_HEADER) request.add_header('Accept', self._ACCEPT_HEADER)
request.add_header('Auth-token', self._AUTH_TOKEN) request.add_header('Auth-token', self._AUTH_TOKEN)
return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal) return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal)

6
youtube_dl/extractor/viki.py
View File

@ -7,14 +7,14 @@ import hmac
import hashlib import hashlib
import itertools import itertools
from .common import InfoExtractor
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
int_or_none, int_or_none,
parse_age_limit, parse_age_limit,
parse_iso8601, parse_iso8601,
sanitized_Request,
) )
from ..compat import compat_urllib_request
from .common import InfoExtractor
class VikiBaseIE(InfoExtractor): class VikiBaseIE(InfoExtractor):
@ -43,7 +43,7 @@ class VikiBaseIE(InfoExtractor):
hashlib.sha1 hashlib.sha1
).hexdigest() ).hexdigest()
url = self._API_URL_TEMPLATE % (query, sig) url = self._API_URL_TEMPLATE % (query, sig)
return compat_urllib_request.Request( return sanitized_Request(
url, json.dumps(post_data).encode('utf-8')) if post_data else url url, json.dumps(post_data).encode('utf-8')) if post_data else url
def _call_api(self, path, video_id, note, timestamp=None, post_data=None): def _call_api(self, path, video_id, note, timestamp=None, post_data=None):

4
youtube_dl/extractor/vk.py
View File

@ -8,11 +8,11 @@ from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_str, compat_str,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
orderedSet, orderedSet,
sanitized_Request,
str_to_int, str_to_int,
unescapeHTML, unescapeHTML,
unified_strdate, unified_strdate,
@ -182,7 +182,7 @@ class VKIE(InfoExtractor):
'pass': password.encode('cp1251'), 'pass': password.encode('cp1251'),
}) })
request = compat_urllib_request.Request( request = sanitized_Request(
'https://login.vk.com/?act=login', 'https://login.vk.com/?act=login',
compat_urllib_parse.urlencode(login_form).encode('utf-8')) compat_urllib_parse.urlencode(login_form).encode('utf-8'))
login_page = self._download_webpage( login_page = self._download_webpage(

8
youtube_dl/extractor/vodlocker.py
View File

@ -2,10 +2,8 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse, from ..utils import sanitized_Request
compat_urllib_request,
)
class VodlockerIE(InfoExtractor): class VodlockerIE(InfoExtractor):
@ -31,7 +29,7 @@ class VodlockerIE(InfoExtractor):
if fields['op'] == 'download1': if fields['op'] == 'download1':
self._sleep(3, video_id) # they do detect when requests happen too fast! self._sleep(3, video_id) # they do detect when requests happen too fast!
post = compat_urllib_parse.urlencode(fields) post = compat_urllib_parse.urlencode(fields)
req = compat_urllib_request.Request(url, post) req = sanitized_Request(url, post)
req.add_header('Content-type', 'application/x-www-form-urlencoded') req.add_header('Content-type', 'application/x-www-form-urlencoded')
webpage = self._download_webpage( webpage = self._download_webpage(
req, video_id, 'Downloading video page') req, video_id, 'Downloading video page')

8
youtube_dl/extractor/voicerepublic.py
View File

@ -3,14 +3,12 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urlparse
compat_urllib_request,
compat_urlparse,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
determine_ext, determine_ext,
int_or_none, int_or_none,
sanitized_Request,
) )
@ -37,7 +35,7 @@ class VoiceRepublicIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
display_id = self._match_id(url) display_id = self._match_id(url)
req = compat_urllib_request.Request( req = sanitized_Request(
compat_urlparse.urljoin(url, '/talks/%s' % display_id)) compat_urlparse.urljoin(url, '/talks/%s' % display_id))
# Older versions of Firefox get redirected to an "upgrade browser" page # Older versions of Firefox get redirected to an "upgrade browser" page
req.add_header('User-Agent', 'youtube-dl') req.add_header('User-Agent', 'youtube-dl')

8
youtube_dl/extractor/wistia.py
View File

@ -1,8 +1,10 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request from ..utils import (
from ..utils import ExtractorError ExtractorError,
sanitized_Request,
)
class WistiaIE(InfoExtractor): class WistiaIE(InfoExtractor):
@ -23,7 +25,7 @@ class WistiaIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
request = compat_urllib_request.Request(self._API_URL.format(video_id)) request = sanitized_Request(self._API_URL.format(video_id))
request.add_header('Referer', url) # Some videos require this. request.add_header('Referer', url) # Some videos require this.
data_json = self._download_json(request, video_id) data_json = self._download_json(request, video_id)
if data_json.get('error'): if data_json.get('error'):

8
youtube_dl/extractor/xfileshare.py
View File

@ -4,14 +4,12 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse
compat_urllib_parse,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
encode_dict, encode_dict,
int_or_none, int_or_none,
sanitized_Request,
) )
@ -106,7 +104,7 @@ class XFileShareIE(InfoExtractor):
post = compat_urllib_parse.urlencode(encode_dict(fields)) post = compat_urllib_parse.urlencode(encode_dict(fields))
req = compat_urllib_request.Request(url, post) req = sanitized_Request(url, post)
req.add_header('Content-type', 'application/x-www-form-urlencoded') req.add_header('Content-type', 'application/x-www-form-urlencoded')
webpage = self._download_webpage(req, video_id, 'Downloading video page') webpage = self._download_webpage(req, video_id, 'Downloading video page')

8
youtube_dl/extractor/xtube.py
View File

@ -3,12 +3,10 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse_unquote
compat_urllib_request,
compat_urllib_parse_unquote,
)
from ..utils import ( from ..utils import (
parse_duration, parse_duration,
sanitized_Request,
str_to_int, str_to_int,
) )
@ -32,7 +30,7 @@ class XTubeIE(InfoExtractor):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
req = compat_urllib_request.Request(url) req = sanitized_Request(url)
req.add_header('Cookie', 'age_verified=1') req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id) webpage = self._download_webpage(req, video_id)

8
youtube_dl/extractor/xvideos.py
View File

@ -3,14 +3,12 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import compat_urllib_parse_unquote
compat_urllib_parse_unquote,
compat_urllib_request,
)
from ..utils import ( from ..utils import (
clean_html, clean_html,
ExtractorError, ExtractorError,
determine_ext, determine_ext,
sanitized_Request,
) )
@ -48,7 +46,7 @@ class XVideosIE(InfoExtractor):
'url': video_url, 'url': video_url,
}] }]
android_req = compat_urllib_request.Request(url) android_req = sanitized_Request(url)
android_req.add_header('User-Agent', self._ANDROID_USER_AGENT) android_req.add_header('User-Agent', self._ANDROID_USER_AGENT)
android_webpage = self._download_webpage(android_req, video_id, fatal=False) android_webpage = self._download_webpage(android_req, video_id, fatal=False)

4
youtube_dl/extractor/yandexmusic.py
View File

@ -8,11 +8,11 @@ from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_str, compat_str,
compat_urllib_parse, compat_urllib_parse,
compat_urllib_request,
) )
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
float_or_none, float_or_none,
sanitized_Request,
) )
@ -154,7 +154,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
if len(tracks) < len(track_ids): if len(tracks) < len(track_ids):
present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')]) present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids) missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
request = compat_urllib_request.Request( request = sanitized_Request(
'https://music.yandex.ru/handlers/track-entries.jsx', 'https://music.yandex.ru/handlers/track-entries.jsx',
compat_urllib_parse.urlencode({ compat_urllib_parse.urlencode({
'entries': ','.join(missing_track_ids), 'entries': ','.join(missing_track_ids),

9
youtube_dl/extractor/youku.py
View File

@ -4,12 +4,13 @@ from __future__ import unicode_literals
import base64 import base64
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ExtractorError
from ..compat import ( from ..compat import (
compat_urllib_parse, compat_urllib_parse,
compat_ord, compat_ord,
compat_urllib_request, )
from ..utils import (
ExtractorError,
sanitized_Request,
) )
@ -187,7 +188,7 @@ class YoukuIE(InfoExtractor):
video_id = self._match_id(url) video_id = self._match_id(url)
def retrieve_data(req_url, note): def retrieve_data(req_url, note):
req = compat_urllib_request.Request(req_url) req = sanitized_Request(req_url)
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy') cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
if cn_verification_proxy: if cn_verification_proxy:

4
youtube_dl/extractor/youporn.py
View File

@ -3,9 +3,9 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
sanitized_Request,
str_to_int, str_to_int,
unescapeHTML, unescapeHTML,
unified_strdate, unified_strdate,
@ -63,7 +63,7 @@ class YouPornIE(InfoExtractor):
video_id = mobj.group('id') video_id = mobj.group('id')
display_id = mobj.group('display_id') display_id = mobj.group('display_id')
request = compat_urllib_request.Request(url) request = sanitized_Request(url)
request.add_header('Cookie', 'age_verified=1') request.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(request, display_id) webpage = self._download_webpage(request, display_id)

6
youtube_dl/extractor/youtube.py
View File

@ -20,7 +20,6 @@ from ..compat import (
compat_urllib_parse_unquote, compat_urllib_parse_unquote,
compat_urllib_parse_unquote_plus, compat_urllib_parse_unquote_plus,
compat_urllib_parse_urlparse, compat_urllib_parse_urlparse,
compat_urllib_request,
compat_urlparse, compat_urlparse,
compat_str, compat_str,
) )
@ -35,6 +34,7 @@ from ..utils import (
orderedSet, orderedSet,
parse_duration, parse_duration,
remove_start, remove_start,
sanitized_Request,
smuggle_url, smuggle_url,
str_to_int, str_to_int,
unescapeHTML, unescapeHTML,
@ -114,7 +114,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii') login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
req = compat_urllib_request.Request(self._LOGIN_URL, login_data) req = sanitized_Request(self._LOGIN_URL, login_data)
login_results = self._download_webpage( login_results = self._download_webpage(
req, None, req, None,
note='Logging in', errnote='unable to log in', fatal=False) note='Logging in', errnote='unable to log in', fatal=False)
@ -147,7 +147,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii') tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data) tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
tfa_results = self._download_webpage( tfa_results = self._download_webpage(
tfa_req, None, tfa_req, None,
note='Submitting TFA code', errnote='unable to submit tfa', fatal=False) note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)