Merge branch 'master' into format-sort
This commit is contained in:
pukkandan
GitHub
commit
ad5ac0905e
|
|
@ -51,3 +51,4 @@ venv/
|
|||
|
||||
# VS Code related files
|
||||
.vscode
|
||||
*.sublime-workspace
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
pukkandan
|
||||
Zocker1999NET
|
||||
gergesh
|
||||
h-h-h-h
|
||||
gschizas
|
||||
MrDoritos
|
||||
pauldubois98
|
||||
ian
|
||||
|
|
@ -14,6 +14,7 @@ FISH_COMPLETION_FILE = 'youtube-dl.fish'
|
|||
FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in'
|
||||
|
||||
EXTRA_ARGS = {
|
||||
'remux-video': ['--arguments', 'mp4 mkv', '--exclusive'],
|
||||
'recode-video': ['--arguments', 'mp4 flv ogg webm mkv', '--exclusive'],
|
||||
|
||||
# Options that need a file parameter
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ __youtube_dl() {
|
|||
_path_files
|
||||
elif [[ ${prev} =~ ${diropts} ]]; then
|
||||
_path_files -/
|
||||
elif [[ ${prev} == "--remux-video" ]]; then
|
||||
_arguments '*: :(mp4 mkv)'
|
||||
elif [[ ${prev} == "--recode-video" ]]; then
|
||||
_arguments '*: :(mp4 flv ogg webm mkv)'
|
||||
else
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
New Single-commit Functionality
|
||||
-----------------
|
||||
Make the changes in master
|
||||
Make necessary changes in readme.md
|
||||
Change version number in version.txt
|
||||
Commit
|
||||
( 1 commit per change)
|
||||
|
||||
======================================================
|
||||
|
||||
|
||||
|
||||
New Functionality
|
||||
-----------------
|
||||
=> Upstream Update
|
||||
Rebase upstream-master to upstream/master
|
||||
Branch from upstream-master
|
||||
Do the necessary commits in the new branch
|
||||
=> Merge pull
|
||||
|
||||
======================================================
|
||||
|
||||
|
||||
|
||||
Merge pull
|
||||
-------------------------
|
||||
Create new pull request using https://github.com/pukkandan/youtube-dl/compare/master...<user>:<branch>
|
||||
Accept the request (Squash and Merge)
|
||||
Add author in AUTHORS-Fork
|
||||
Make necessary changes in readme.md
|
||||
Change version number in version.txt
|
||||
Commit
|
||||
( 2 commits per pull request)
|
||||
|
||||
|
||||
======================================================
|
||||
|
||||
|
||||
|
||||
Upstream Update
|
||||
-------------------------
|
||||
Rebase master to upstream/master
|
||||
Change version number in version.txt and top of readme.md
|
||||
Commit
|
||||
( 1 commit per update)
|
||||
|
|
@ -7,6 +7,7 @@
|
|||
"forcethumbnail": false,
|
||||
"forcetitle": false,
|
||||
"forceurl": false,
|
||||
"force_write_download_archive": false,
|
||||
"format": "best",
|
||||
"ignoreerrors": false,
|
||||
"listformats": null,
|
||||
|
|
@ -35,6 +36,11 @@
|
|||
"verbose": true,
|
||||
"writedescription": false,
|
||||
"writeinfojson": true,
|
||||
"writeannotations": false,
|
||||
"writelink": false,
|
||||
"writeurllink": false,
|
||||
"writewebloclink": false,
|
||||
"writedesktoplink": false,
|
||||
"writesubtitles": false,
|
||||
"allsubtitles": false,
|
||||
"listssubtitles": false,
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ def _make_result(formats, **kwargs):
|
|||
'title': 'testttitle',
|
||||
'extractor': 'testex',
|
||||
'extractor_key': 'TestEx',
|
||||
'webpage_url': 'http://example.com/watch?v=shenanigans',
|
||||
}
|
||||
res.update(**kwargs)
|
||||
return res
|
||||
|
|
@ -567,6 +568,7 @@ class TestYoutubeDL(unittest.TestCase):
|
|||
'subtitles': subtitles,
|
||||
'automatic_captions': auto_captions,
|
||||
'extractor': 'TEST',
|
||||
'webpage_url': 'http://example.com/watch?v=shenanigans',
|
||||
}
|
||||
|
||||
def get_info(params={}):
|
||||
|
|
@ -730,6 +732,7 @@ class TestYoutubeDL(unittest.TestCase):
|
|||
'playlist_id': '42',
|
||||
'uploader': "變態妍字幕版 太妍 тест",
|
||||
'creator': "тест ' 123 ' тест--",
|
||||
'webpage_url': 'http://example.com/watch?v=shenanigans',
|
||||
}
|
||||
second = {
|
||||
'id': '2',
|
||||
|
|
@ -741,6 +744,7 @@ class TestYoutubeDL(unittest.TestCase):
|
|||
'filesize': 5 * 1024,
|
||||
'playlist_id': '43',
|
||||
'uploader': "тест 123",
|
||||
'webpage_url': 'http://example.com/watch?v=SHENANIGANS',
|
||||
}
|
||||
videos = [first, second]
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ from youtube_dl.compat import (
|
|||
compat_shlex_split,
|
||||
compat_str,
|
||||
compat_struct_unpack,
|
||||
compat_urllib_parse_quote,
|
||||
compat_urllib_parse_quote_plus,
|
||||
compat_urllib_parse_unquote,
|
||||
compat_urllib_parse_unquote_plus,
|
||||
compat_urllib_parse_urlencode,
|
||||
|
|
@ -53,6 +55,27 @@ class TestCompat(unittest.TestCase):
|
|||
dir(youtube_dl.compat))) - set(['unicode_literals'])
|
||||
self.assertEqual(all_names, sorted(present_names))
|
||||
|
||||
def test_compat_urllib_parse_quote(self):
|
||||
self.assertEqual(compat_urllib_parse_quote('abc def'), 'abc%20def')
|
||||
self.assertEqual(compat_urllib_parse_quote('/~user/abc+def'), '/%7Euser/abc%2Bdef')
|
||||
self.assertEqual(compat_urllib_parse_quote('/~user/abc+def', safe='/~+'), '/~user/abc+def')
|
||||
self.assertEqual(compat_urllib_parse_quote(''), '')
|
||||
self.assertEqual(compat_urllib_parse_quote('%'), '%25')
|
||||
self.assertEqual(compat_urllib_parse_quote('%', safe='%'), '%')
|
||||
self.assertEqual(compat_urllib_parse_quote('津波'), '%E6%B4%A5%E6%B3%A2')
|
||||
self.assertEqual(
|
||||
compat_urllib_parse_quote('''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
|
||||
%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''', safe='<>=":%/ \r\n'),
|
||||
'''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
|
||||
%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a''')
|
||||
self.assertEqual(
|
||||
compat_urllib_parse_quote('''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%25Things%''', safe='% '),
|
||||
'''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%''')
|
||||
|
||||
def test_compat_urllib_parse_quote_plus(self):
|
||||
self.assertEqual(compat_urllib_parse_quote_plus('abc def'), 'abc+def')
|
||||
self.assertEqual(compat_urllib_parse_quote_plus('~/abc def'), '%7E%2Fabc+def')
|
||||
|
||||
def test_compat_urllib_parse_unquote(self):
|
||||
self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def')
|
||||
self.assertEqual(compat_urllib_parse_unquote('%7e/abc+def'), '~/abc+def')
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ from youtube_dl.utils import (
|
|||
cli_valueless_option,
|
||||
cli_bool_option,
|
||||
parse_codecs,
|
||||
iri_to_uri,
|
||||
)
|
||||
from youtube_dl.compat import (
|
||||
compat_chr,
|
||||
|
|
@ -1437,6 +1438,32 @@ Line 1
|
|||
self.assertEqual(get_elements_by_attribute('class', 'foo', html), [])
|
||||
self.assertEqual(get_elements_by_attribute('class', 'no-such-foo', html), [])
|
||||
|
||||
def test_iri_to_uri(self):
|
||||
self.assertEqual(
|
||||
iri_to_uri('https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b'),
|
||||
'https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b') # Same
|
||||
self.assertEqual(
|
||||
iri_to_uri('https://www.google.com/search?q=Käsesoßenrührlöffel'), # German for cheese sauce stirring spoon
|
||||
'https://www.google.com/search?q=K%C3%A4seso%C3%9Fenr%C3%BChrl%C3%B6ffel')
|
||||
self.assertEqual(
|
||||
iri_to_uri('https://www.google.com/search?q=lt<+gt>+eq%3D+amp%26+percent%25+hash%23+colon%3A+tilde~#trash=?&garbage=#'),
|
||||
'https://www.google.com/search?q=lt%3C+gt%3E+eq%3D+amp%26+percent%25+hash%23+colon%3A+tilde~#trash=?&garbage=#')
|
||||
self.assertEqual(
|
||||
iri_to_uri('http://правозащита38.рф/category/news/'),
|
||||
'http://xn--38-6kcaak9aj5chl4a3g.xn--p1ai/category/news/')
|
||||
self.assertEqual(
|
||||
iri_to_uri('http://www.правозащита38.рф/category/news/'),
|
||||
'http://www.xn--38-6kcaak9aj5chl4a3g.xn--p1ai/category/news/')
|
||||
self.assertEqual(
|
||||
iri_to_uri('https://i❤.ws/emojidomain/👍👏🤝💪'),
|
||||
'https://xn--i-7iq.ws/emojidomain/%F0%9F%91%8D%F0%9F%91%8F%F0%9F%A4%9D%F0%9F%92%AA')
|
||||
self.assertEqual(
|
||||
iri_to_uri('http://日本語.jp/'),
|
||||
'http://xn--wgv71a119e.jp/')
|
||||
self.assertEqual(
|
||||
iri_to_uri('http://导航.中国/'),
|
||||
'http://xn--fet810g.xn--fiqs8s/')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
2020.09.22.00
|
||||
2020.09.20
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"folders":
|
||||
[
|
||||
{
|
||||
"path": "./youtube_dl",
|
||||
"folder_exclude_patterns": ["__pycache__"],
|
||||
},
|
||||
{
|
||||
"path": ".",
|
||||
"name": "root-folder",
|
||||
"folder_exclude_patterns": ["youtube_dl",".github"],
|
||||
},
|
||||
]
|
||||
}
|
||||
|
|
@ -51,6 +51,9 @@ from .utils import (
|
|||
DEFAULT_OUTTMPL,
|
||||
determine_ext,
|
||||
determine_protocol,
|
||||
DOT_DESKTOP_LINK_TEMPLATE,
|
||||
DOT_URL_LINK_TEMPLATE,
|
||||
DOT_WEBLOC_LINK_TEMPLATE,
|
||||
DownloadError,
|
||||
encode_compat_str,
|
||||
encodeFilename,
|
||||
|
|
@ -61,6 +64,7 @@ from .utils import (
|
|||
formatSeconds,
|
||||
GeoRestrictedError,
|
||||
int_or_none,
|
||||
iri_to_uri,
|
||||
ISO3166Utils,
|
||||
locked_file,
|
||||
make_HTTPS_handler,
|
||||
|
|
@ -84,6 +88,7 @@ from .utils import (
|
|||
std_headers,
|
||||
str_or_none,
|
||||
subtitles_filename,
|
||||
to_high_limit_path,
|
||||
UnavailableVideoError,
|
||||
url_basename,
|
||||
version_tuple,
|
||||
|
|
@ -160,6 +165,8 @@ class YoutubeDL(object):
|
|||
forcejson: Force printing info_dict as JSON.
|
||||
dump_single_json: Force printing the info_dict of the whole playlist
|
||||
(or video) as a single JSON line.
|
||||
force_write_download_archive: Force writing download archive regardless of
|
||||
'skip_download' or 'simulate'.
|
||||
simulate: Do not download the video files.
|
||||
format: Video format code. See options.py for more information.
|
||||
outtmpl: Template for output names.
|
||||
|
|
@ -181,6 +188,11 @@ class YoutubeDL(object):
|
|||
writeannotations: Write the video annotations to a .annotations.xml file
|
||||
writethumbnail: Write the thumbnail image to a file
|
||||
write_all_thumbnails: Write all thumbnail formats to files
|
||||
writelink: Write an internet shortcut file, depending on the
|
||||
current platform (.url/.webloc/.desktop)
|
||||
writeurllink: Write a Windows internet shortcut file (.url)
|
||||
writewebloclink: Write a macOS internet shortcut file (.webloc)
|
||||
writedesktoplink: Write a Linux internet shortcut file (.desktop)
|
||||
writesubtitles: Write the video subtitles to a file
|
||||
writeautomaticsub: Write the automatically generated subtitles to a file
|
||||
allsubtitles: Downloads all the subtitles of the video
|
||||
|
|
@ -208,6 +220,8 @@ class YoutubeDL(object):
|
|||
download_archive: File name of a file where all downloads are recorded.
|
||||
Videos already present in the file are not downloaded
|
||||
again.
|
||||
break_on_existing: Stop the download process after attempting to download a file that's
|
||||
in the archive.
|
||||
cookiefile: File name where cookies should be read from and dumped to.
|
||||
nocheckcertificate:Do not verify SSL certificates
|
||||
prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
|
||||
|
|
@ -1000,8 +1014,12 @@ class YoutubeDL(object):
|
|||
|
||||
reason = self._match_entry(entry, incomplete=True)
|
||||
if reason is not None:
|
||||
self.to_screen('[download] ' + reason)
|
||||
continue
|
||||
if reason.endswith('has already been recorded in the archive') and self.params.get('break_on_existing'):
|
||||
print('[download] tried downloading a file that\'s already in the archive, stopping since --break-on-existing is set.')
|
||||
break
|
||||
else:
|
||||
self.to_screen('[download] ' + reason)
|
||||
continue
|
||||
|
||||
entry_result = self.process_ie_result(entry,
|
||||
download=download,
|
||||
|
|
@ -1753,8 +1771,11 @@ class YoutubeDL(object):
|
|||
# Forced printings
|
||||
self.__forced_printings(info_dict, filename, incomplete=False)
|
||||
|
||||
# Do nothing else if in simulate mode
|
||||
if self.params.get('simulate', False):
|
||||
if self.params.get('force_write_download_archive', False):
|
||||
self.record_download_archive(info_dict)
|
||||
|
||||
# Do nothing else if in simulate mode
|
||||
return
|
||||
|
||||
if filename is None:
|
||||
|
|
@ -1854,6 +1875,57 @@ class YoutubeDL(object):
|
|||
|
||||
self._write_thumbnails(info_dict, filename)
|
||||
|
||||
# Write internet shortcut files
|
||||
url_link = webloc_link = desktop_link = False
|
||||
if self.params.get('writelink', False):
|
||||
if sys.platform == "darwin": # macOS.
|
||||
webloc_link = True
|
||||
elif sys.platform.startswith("linux"):
|
||||
desktop_link = True
|
||||
else: # if sys.platform in ['win32', 'cygwin']:
|
||||
url_link = True
|
||||
if self.params.get('writeurllink', False):
|
||||
url_link = True
|
||||
if self.params.get('writewebloclink', False):
|
||||
webloc_link = True
|
||||
if self.params.get('writedesktoplink', False):
|
||||
desktop_link = True
|
||||
|
||||
if url_link or webloc_link or desktop_link:
|
||||
if 'webpage_url' not in info_dict:
|
||||
self.report_error('Cannot write internet shortcut file because the "webpage_url" field is missing in the media information')
|
||||
return
|
||||
ascii_url = iri_to_uri(info_dict['webpage_url'])
|
||||
|
||||
def _write_link_file(extension, template, newline, embed_filename):
|
||||
linkfn = replace_extension(filename, extension, info_dict.get('ext'))
|
||||
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(linkfn)):
|
||||
self.to_screen('[info] Internet shortcut is already present')
|
||||
else:
|
||||
try:
|
||||
self.to_screen('[info] Writing internet shortcut to: ' + linkfn)
|
||||
with io.open(encodeFilename(to_high_limit_path(linkfn)), 'w', encoding='utf-8', newline=newline) as linkfile:
|
||||
template_vars = {'url': ascii_url}
|
||||
if embed_filename:
|
||||
template_vars['filename'] = linkfn[:-(len(extension) + 1)]
|
||||
linkfile.write(template % template_vars)
|
||||
except (OSError, IOError):
|
||||
self.report_error('Cannot write internet shortcut ' + linkfn)
|
||||
return False
|
||||
return True
|
||||
|
||||
if url_link:
|
||||
if not _write_link_file('url', DOT_URL_LINK_TEMPLATE, '\r\n', embed_filename=False):
|
||||
return
|
||||
if webloc_link:
|
||||
if not _write_link_file('webloc', DOT_WEBLOC_LINK_TEMPLATE, '\n', embed_filename=False):
|
||||
return
|
||||
if desktop_link:
|
||||
if not _write_link_file('desktop', DOT_DESKTOP_LINK_TEMPLATE, '\n', embed_filename=True):
|
||||
return
|
||||
|
||||
# Download
|
||||
must_record_download_archive = False
|
||||
if not self.params.get('skip_download', False):
|
||||
try:
|
||||
def dl(name, info):
|
||||
|
|
@ -2001,7 +2073,10 @@ class YoutubeDL(object):
|
|||
except (PostProcessingError) as err:
|
||||
self.report_error('postprocessing: %s' % str(err))
|
||||
return
|
||||
self.record_download_archive(info_dict)
|
||||
must_record_download_archive = True
|
||||
|
||||
if must_record_download_archive or self.params.get('force_write_download_archive', False):
|
||||
self.record_download_archive(info_dict)
|
||||
|
||||
def download(self, url_list):
|
||||
"""Download a given list of URLs."""
|
||||
|
|
|
|||
|
|
@ -209,6 +209,9 @@ def _real_main(argv=None):
|
|||
opts.audioquality = opts.audioquality.strip('k').strip('K')
|
||||
if not opts.audioquality.isdigit():
|
||||
parser.error('invalid audio quality specified')
|
||||
if opts.remuxvideo is not None:
|
||||
if opts.remuxvideo not in ['mp4', 'mkv']:
|
||||
parser.error('invalid video container format specified')
|
||||
if opts.recodevideo is not None:
|
||||
if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']:
|
||||
parser.error('invalid video recode format specified')
|
||||
|
|
@ -261,6 +264,11 @@ def _real_main(argv=None):
|
|||
'preferredquality': opts.audioquality,
|
||||
'nopostoverwrites': opts.nopostoverwrites,
|
||||
})
|
||||
if opts.remuxvideo:
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegVideoRemuxer',
|
||||
'preferedformat': opts.remuxvideo,
|
||||
})
|
||||
if opts.recodevideo:
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegVideoConvertor',
|
||||
|
|
@ -335,6 +343,7 @@ def _real_main(argv=None):
|
|||
'forceformat': opts.getformat,
|
||||
'forcejson': opts.dumpjson or opts.print_json,
|
||||
'dump_single_json': opts.dump_single_json,
|
||||
'force_write_download_archive': opts.force_write_download_archive,
|
||||
'simulate': opts.simulate or any_getting,
|
||||
'skip_download': opts.skip_download,
|
||||
'format': opts.format,
|
||||
|
|
@ -373,6 +382,10 @@ def _real_main(argv=None):
|
|||
'writeinfojson': opts.writeinfojson,
|
||||
'writethumbnail': opts.writethumbnail,
|
||||
'write_all_thumbnails': opts.write_all_thumbnails,
|
||||
'writelink': opts.writelink,
|
||||
'writeurllink': opts.writeurllink,
|
||||
'writewebloclink': opts.writewebloclink,
|
||||
'writedesktoplink': opts.writedesktoplink,
|
||||
'writesubtitles': opts.writesubtitles,
|
||||
'writeautomaticsub': opts.writeautomaticsub,
|
||||
'allsubtitles': opts.allsubtitles,
|
||||
|
|
@ -397,6 +410,7 @@ def _real_main(argv=None):
|
|||
'youtube_print_sig_code': opts.youtube_print_sig_code,
|
||||
'age_limit': opts.age_limit,
|
||||
'download_archive': download_archive_fn,
|
||||
'break_on_existing': opts.break_on_existing,
|
||||
'cookiefile': opts.cookiefile,
|
||||
'nocheckcertificate': opts.no_check_certificate,
|
||||
'prefer_insecure': opts.prefer_insecure,
|
||||
|
|
|
|||
|
|
@ -37,15 +37,20 @@ try:
|
|||
except ImportError: # Python 2
|
||||
import urllib as compat_urllib_parse
|
||||
|
||||
try:
|
||||
import urllib.parse as compat_urlparse
|
||||
except ImportError: # Python 2
|
||||
import urlparse as compat_urlparse
|
||||
|
||||
try:
|
||||
from urllib.parse import urlparse as compat_urllib_parse_urlparse
|
||||
except ImportError: # Python 2
|
||||
from urlparse import urlparse as compat_urllib_parse_urlparse
|
||||
|
||||
try:
|
||||
import urllib.parse as compat_urlparse
|
||||
from urllib.parse import urlunparse as compat_urllib_parse_urlunparse
|
||||
except ImportError: # Python 2
|
||||
import urlparse as compat_urlparse
|
||||
from urlparse import urlunparse as compat_urllib_parse_urlunparse
|
||||
|
||||
try:
|
||||
import urllib.response as compat_urllib_response
|
||||
|
|
@ -2365,6 +2370,20 @@ try:
|
|||
except NameError:
|
||||
compat_str = str
|
||||
|
||||
try:
|
||||
from urllib.parse import quote as compat_urllib_parse_quote
|
||||
from urllib.parse import quote_plus as compat_urllib_parse_quote_plus
|
||||
except ImportError: # Python 2
|
||||
def compat_urllib_parse_quote(string, safe='/'):
|
||||
return compat_urllib_parse.quote(
|
||||
string.encode('utf-8'),
|
||||
str(safe))
|
||||
|
||||
def compat_urllib_parse_quote_plus(string, safe=''):
|
||||
return compat_urllib_parse.quote_plus(
|
||||
string.encode('utf-8'),
|
||||
str(safe))
|
||||
|
||||
try:
|
||||
from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
|
||||
from urllib.parse import unquote as compat_urllib_parse_unquote
|
||||
|
|
@ -3033,11 +3052,14 @@ __all__ = [
|
|||
'compat_tokenize_tokenize',
|
||||
'compat_urllib_error',
|
||||
'compat_urllib_parse',
|
||||
'compat_urllib_parse_quote',
|
||||
'compat_urllib_parse_quote_plus',
|
||||
'compat_urllib_parse_unquote',
|
||||
'compat_urllib_parse_unquote_plus',
|
||||
'compat_urllib_parse_unquote_to_bytes',
|
||||
'compat_urllib_parse_urlencode',
|
||||
'compat_urllib_parse_urlparse',
|
||||
'compat_urllib_parse_urlunparse',
|
||||
'compat_urllib_request',
|
||||
'compat_urllib_request_DataHandler',
|
||||
'compat_urllib_response',
|
||||
|
|
|
|||
|
|
@ -1466,6 +1466,7 @@ class InfoExtractor(object):
|
|||
else:
|
||||
audio_codec_preference -= 1
|
||||
|
||||
|
||||
prefVars = {'extractor': preference,
|
||||
'avoid_bad': avoid_bad_preference,
|
||||
'proto': proto_preference,
|
||||
|
|
@ -1479,6 +1480,7 @@ class InfoExtractor(object):
|
|||
def format_get_val(field):
|
||||
return (f.get(field + '_preference') if f.get(field) is None else f.get(field)) if prefVars.get(field) is None else prefVars.get(field)
|
||||
|
||||
|
||||
def format_get_preference(field):
|
||||
val = format_get_val(field)
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -1677,21 +1677,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
|||
def _extract_chapters_from_json(self, webpage, video_id, duration):
|
||||
if not webpage:
|
||||
return
|
||||
player = self._parse_json(
|
||||
initial_data = self._parse_json(
|
||||
self._search_regex(
|
||||
r'RELATED_PLAYER_ARGS["\']\s*:\s*({.+})\s*,?\s*\n', webpage,
|
||||
r'window\["ytInitialData"\] = (.+);\n', webpage,
|
||||
'player args', default='{}'),
|
||||
video_id, fatal=False)
|
||||
if not player or not isinstance(player, dict):
|
||||
return
|
||||
watch_next_response = player.get('watch_next_response')
|
||||
if not isinstance(watch_next_response, compat_str):
|
||||
return
|
||||
response = self._parse_json(watch_next_response, video_id, fatal=False)
|
||||
if not response or not isinstance(response, dict):
|
||||
if not initial_data or not isinstance(initial_data, dict):
|
||||
return
|
||||
chapters_list = try_get(
|
||||
response,
|
||||
initial_data,
|
||||
lambda x: x['playerOverlays']
|
||||
['playerOverlayRenderer']
|
||||
['decoratedPlayerBarRenderer']
|
||||
|
|
|
|||
|
|
@ -185,6 +185,10 @@ def parseOpts(overrideArguments=None):
|
|||
action='store_const', dest='extract_flat', const='in_playlist',
|
||||
default=False,
|
||||
help='Do not extract the videos of a playlist, only list them.')
|
||||
general.add_option(
|
||||
'--flat-videos',
|
||||
action='store_true', dest='extract_flat', default=False,
|
||||
help='Do not resolve the video urls.')
|
||||
general.add_option(
|
||||
'--mark-watched',
|
||||
action='store_true', dest='mark_watched', default=False,
|
||||
|
|
@ -344,10 +348,18 @@ def parseOpts(overrideArguments=None):
|
|||
'--download-archive', metavar='FILE',
|
||||
dest='download_archive',
|
||||
help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
|
||||
selection.add_option(
|
||||
'--break-on-existing',
|
||||
action='store_true', dest='break_on_existing', default=False,
|
||||
help="Stop the download process after attempting to download a file that's in the archive.")
|
||||
selection.add_option(
|
||||
'--include-ads',
|
||||
dest='include_ads', action='store_true',
|
||||
help='Download advertisements as well (experimental)')
|
||||
selection.add_option(
|
||||
'--no-include-ads',
|
||||
dest='include_ads', action='store_false',
|
||||
help='Do not download advertisements (default)')
|
||||
|
||||
authentication = optparse.OptionGroup(parser, 'Authentication Options')
|
||||
authentication.add_option(
|
||||
|
|
@ -422,6 +434,12 @@ def parseOpts(overrideArguments=None):
|
|||
'avoid_bad, has_video, has_audio, extractor and language. '
|
||||
'These fields normally filter out the undesirable formats. '
|
||||
'So use this option with caution. '))
|
||||
video_format.add_option(
|
||||
'--no-format-sort-force',
|
||||
action='store_false', dest='format_sort_force', metavar='FORMAT', default=False,
|
||||
help=(
|
||||
'avoid_bad, has_video, has_audio, extractor and language '
|
||||
'takes priority over any user specified sort order (default)'))
|
||||
video_format.add_option(
|
||||
'--all-formats',
|
||||
action='store_const', dest='format', const='all',
|
||||
|
|
@ -437,7 +455,7 @@ def parseOpts(overrideArguments=None):
|
|||
video_format.add_option(
|
||||
'--youtube-include-dash-manifest',
|
||||
action='store_true', dest='youtube_include_dash_manifest', default=True,
|
||||
help=optparse.SUPPRESS_HELP)
|
||||
help='Download the DASH manifests and related data on YouTube videos (default)')
|
||||
video_format.add_option(
|
||||
'--youtube-skip-dash-manifest',
|
||||
action='store_false', dest='youtube_include_dash_manifest',
|
||||
|
|
@ -455,10 +473,18 @@ def parseOpts(overrideArguments=None):
|
|||
'--write-sub', '--write-srt',
|
||||
action='store_true', dest='writesubtitles', default=False,
|
||||
help='Write subtitle file')
|
||||
subtitles.add_option(
|
||||
'--no-write-sub', '--no-write-srt',
|
||||
action='store_false', dest='writesubtitles',
|
||||
help='Do not write subtitle file (default)')
|
||||
subtitles.add_option(
|
||||
'--write-auto-sub', '--write-automatic-sub',
|
||||
action='store_true', dest='writeautomaticsub', default=False,
|
||||
help='Write automatically generated subtitle file (YouTube only)')
|
||||
subtitles.add_option(
|
||||
'--no-write-auto-sub', '--no-write-automatic-sub',
|
||||
action='store_false', dest='writeautomaticsub', default=False,
|
||||
help='Do not write automatically generated subtitle file (default)')
|
||||
subtitles.add_option(
|
||||
'--all-subs',
|
||||
action='store_true', dest='allsubtitles', default=False,
|
||||
|
|
@ -523,6 +549,10 @@ def parseOpts(overrideArguments=None):
|
|||
'--playlist-reverse',
|
||||
action='store_true',
|
||||
help='Download playlist videos in reverse order')
|
||||
downloader.add_option(
|
||||
'--no-playlist-reverse',
|
||||
action='store_false', dest='playlist_reverse',
|
||||
help='Download playlist videos in default order')
|
||||
downloader.add_option(
|
||||
'--playlist-random',
|
||||
action='store_true',
|
||||
|
|
@ -661,8 +691,11 @@ def parseOpts(overrideArguments=None):
|
|||
verbosity.add_option(
|
||||
'--print-json',
|
||||
action='store_true', dest='print_json', default=False,
|
||||
help='Be quiet and print the video information as JSON (video is still being downloaded).',
|
||||
)
|
||||
help='Be quiet and print the video information as JSON (video is still being downloaded).')
|
||||
verbosity.add_option(
|
||||
'--force-write-download-archive', '--force-write-archive',
|
||||
action='store_true', dest='force_write_download_archive', default=False,
|
||||
help='Force download archive entries to be written as far as no errors occur, even if --skip-download or any simulation switch is used.')
|
||||
verbosity.add_option(
|
||||
'--newline',
|
||||
action='store_true', dest='progress_with_newline', default=False,
|
||||
|
|
@ -729,6 +762,10 @@ def parseOpts(overrideArguments=None):
|
|||
'--restrict-filenames',
|
||||
action='store_true', dest='restrictfilenames', default=False,
|
||||
help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames')
|
||||
filesystem.add_option(
|
||||
'--no-restrict-filenames',
|
||||
action='store_false', dest='restrictfilenames', default=False,
|
||||
help='Allow Unicode characters, "&" and spaces in filenames (default)')
|
||||
filesystem.add_option(
|
||||
'-A', '--auto-number',
|
||||
action='store_true', dest='autonumber', default=False,
|
||||
|
|
@ -792,7 +829,7 @@ def parseOpts(overrideArguments=None):
|
|||
action='store_true', dest='rm_cachedir',
|
||||
help='Delete all filesystem cache files')
|
||||
|
||||
thumbnail = optparse.OptionGroup(parser, 'Thumbnail images')
|
||||
thumbnail = optparse.OptionGroup(parser, 'Thumbnail Images')
|
||||
thumbnail.add_option(
|
||||
'--write-thumbnail',
|
||||
action='store_true', dest='writethumbnail', default=False,
|
||||
|
|
@ -806,7 +843,25 @@ def parseOpts(overrideArguments=None):
|
|||
action='store_true', dest='list_thumbnails', default=False,
|
||||
help='Simulate and list all available thumbnail formats')
|
||||
|
||||
postproc = optparse.OptionGroup(parser, 'Post-processing Options')
|
||||
link = optparse.OptionGroup(parser, 'Internet Shortcut Options')
|
||||
link.add_option(
|
||||
'--write-link',
|
||||
action='store_true', dest='writelink', default=False,
|
||||
help='Write an internet shortcut file, depending on the current platform (.url/.webloc/.desktop). The URL may be cached by the OS.')
|
||||
link.add_option(
|
||||
'--write-url-link',
|
||||
action='store_true', dest='writeurllink', default=False,
|
||||
help='Write a Windows internet shortcut file (.url). Note that the OS caches the URL based on the file path.')
|
||||
link.add_option(
|
||||
'--write-webloc-link',
|
||||
action='store_true', dest='writewebloclink', default=False,
|
||||
help='Write a macOS internet shortcut file (.webloc)')
|
||||
link.add_option(
|
||||
'--write-desktop-link',
|
||||
action='store_true', dest='writedesktoplink', default=False,
|
||||
help='Write a Linux internet shortcut file (.desktop)')
|
||||
|
||||
postproc = optparse.OptionGroup(parser, 'Post-Processing Options')
|
||||
postproc.add_option(
|
||||
'-x', '--extract-audio',
|
||||
action='store_true', dest='extractaudio', default=False,
|
||||
|
|
@ -818,6 +873,10 @@ def parseOpts(overrideArguments=None):
|
|||
'--audio-quality', metavar='QUALITY',
|
||||
dest='audioquality', default='5',
|
||||
help='Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)')
|
||||
postproc.add_option(
|
||||
'--remux-video',
|
||||
metavar='FORMAT', dest='remuxvideo', default=None,
|
||||
help='Remux the video to another container format if necessary (currently supported: mp4|mkv, target container format must support video / audio encoding, remuxing may fail)')
|
||||
postproc.add_option(
|
||||
'--recode-video',
|
||||
metavar='FORMAT', dest='recodevideo', default=None,
|
||||
|
|
@ -894,6 +953,7 @@ def parseOpts(overrideArguments=None):
|
|||
parser.add_option_group(downloader)
|
||||
parser.add_option_group(filesystem)
|
||||
parser.add_option_group(thumbnail)
|
||||
parser.add_option_group(link)
|
||||
parser.add_option_group(verbosity)
|
||||
parser.add_option_group(workarounds)
|
||||
parser.add_option_group(video_format)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from .ffmpeg import (
|
|||
FFmpegMergerPP,
|
||||
FFmpegMetadataPP,
|
||||
FFmpegVideoConvertorPP,
|
||||
FFmpegVideoRemuxerPP,
|
||||
FFmpegSubtitlesConvertorPP,
|
||||
)
|
||||
from .xattrpp import XAttrMetadataPP
|
||||
|
|
@ -35,6 +36,7 @@ __all__ = [
|
|||
'FFmpegPostProcessor',
|
||||
'FFmpegSubtitlesConvertorPP',
|
||||
'FFmpegVideoConvertorPP',
|
||||
'FFmpegVideoRemuxerPP',
|
||||
'MetadataFromTitlePP',
|
||||
'XAttrMetadataPP',
|
||||
]
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from __future__ import unicode_literals
|
|||
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from .ffmpeg import FFmpegPostProcessor
|
||||
|
|
@ -55,7 +56,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
|
|||
self._downloader.to_screen(
|
||||
'[ffmpeg] Correcting extension to webp and escaping path for thumbnail "%s"' % thumbnail_filename)
|
||||
thumbnail_webp_filename = replace_extension(thumbnail_filename, 'webp')
|
||||
os.rename(encodeFilename(thumbnail_filename), encodeFilename(thumbnail_webp_filename))
|
||||
shutil.move(encodeFilename(thumbnail_filename), encodeFilename(thumbnail_webp_filename))
|
||||
thumbnail_filename = thumbnail_webp_filename
|
||||
thumbnail_ext = 'webp'
|
||||
|
||||
|
|
@ -64,20 +65,20 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
|
|||
# NB: % is supposed to be escaped with %% but this does not work
|
||||
# for input files so working around with standard substitution
|
||||
escaped_thumbnail_filename = thumbnail_filename.replace('%', '#')
|
||||
os.rename(encodeFilename(thumbnail_filename), encodeFilename(escaped_thumbnail_filename))
|
||||
shutil.move(encodeFilename(thumbnail_filename), encodeFilename(escaped_thumbnail_filename))
|
||||
escaped_thumbnail_jpg_filename = replace_extension(escaped_thumbnail_filename, 'jpg')
|
||||
self._downloader.to_screen('[ffmpeg] Converting thumbnail "%s" to JPEG' % escaped_thumbnail_filename)
|
||||
self.run_ffmpeg(escaped_thumbnail_filename, escaped_thumbnail_jpg_filename, ['-bsf:v', 'mjpeg2jpeg'])
|
||||
os.remove(encodeFilename(escaped_thumbnail_filename))
|
||||
thumbnail_jpg_filename = replace_extension(thumbnail_filename, 'jpg')
|
||||
# Rename back to unescaped for further processing
|
||||
os.rename(encodeFilename(escaped_thumbnail_jpg_filename), encodeFilename(thumbnail_jpg_filename))
|
||||
shutil.move(encodeFilename(escaped_thumbnail_jpg_filename), encodeFilename(thumbnail_jpg_filename))
|
||||
thumbnail_filename = thumbnail_jpg_filename
|
||||
os.remove(encodeFilename(escaped_thumbnail_filename))
|
||||
|
||||
if info['ext'] == 'mp3':
|
||||
options = [
|
||||
'-c', 'copy', '-map', '0', '-map', '1',
|
||||
'-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
|
||||
'-c', 'copy', '-map', '0:0', '-map', '1:0', '-id3v2_version', '3',
|
||||
'-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (front)"']
|
||||
|
||||
self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
|
||||
|
||||
|
|
@ -86,7 +87,26 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
|
|||
if not self._already_have_thumbnail:
|
||||
os.remove(encodeFilename(thumbnail_filename))
|
||||
os.remove(encodeFilename(filename))
|
||||
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
|
||||
shutil.move(encodeFilename(temp_filename), encodeFilename(filename))
|
||||
|
||||
elif info['ext'] == 'mkv':
|
||||
shutil.move(encodeFilename(thumbnail_filename), encodeFilename('cover.jpg'))
|
||||
old_thumbnail_filename = thumbnail_filename
|
||||
thumbnail_filename = 'cover.jpg'
|
||||
|
||||
options = [
|
||||
'-c', 'copy', '-attach', thumbnail_filename, '-metadata:s:t', 'mimetype=image/jpeg']
|
||||
|
||||
self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
|
||||
|
||||
self.run_ffmpeg_multiple_files([filename], temp_filename, options)
|
||||
|
||||
if not self._already_have_thumbnail:
|
||||
os.remove(encodeFilename(thumbnail_filename))
|
||||
else:
|
||||
shutil.move(encodeFilename(thumbnail_filename), encodeFilename(old_thumbnail_filename))
|
||||
os.remove(encodeFilename(filename))
|
||||
shutil.move(encodeFilename(temp_filename), encodeFilename(filename))
|
||||
|
||||
elif info['ext'] in ['m4a', 'mp4']:
|
||||
if not check_executable('AtomicParsley', ['-v']):
|
||||
|
|
@ -119,7 +139,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
|
|||
self._downloader.report_warning('The file format doesn\'t support embedding a thumbnail')
|
||||
else:
|
||||
os.remove(encodeFilename(filename))
|
||||
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
|
||||
shutil.move(encodeFilename(temp_filename), encodeFilename(filename))
|
||||
else:
|
||||
raise EmbedThumbnailPPError('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.')
|
||||
|
||||
|
|
|
|||
|
|
@ -349,6 +349,27 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
|
|||
return [path], information
|
||||
|
||||
|
||||
class FFmpegVideoRemuxerPP(FFmpegPostProcessor):
|
||||
def __init__(self, downloader=None, preferedformat=None):
|
||||
super(FFmpegVideoRemuxerPP, self).__init__(downloader)
|
||||
self._preferedformat = preferedformat
|
||||
|
||||
def run(self, information):
|
||||
path = information['filepath']
|
||||
if information['ext'] == self._preferedformat:
|
||||
self._downloader.to_screen('[ffmpeg] Not remuxing video file %s - already is in target format %s' % (path, self._preferedformat))
|
||||
return [], information
|
||||
options = ['-c', 'copy']
|
||||
prefix, sep, ext = path.rpartition('.')
|
||||
outpath = prefix + sep + self._preferedformat
|
||||
self._downloader.to_screen('[' + 'ffmpeg' + '] Remuxing video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
|
||||
self.run_ffmpeg(path, outpath, options)
|
||||
information['filepath'] = outpath
|
||||
information['format'] = self._preferedformat
|
||||
information['ext'] = self._preferedformat
|
||||
return [path], information
|
||||
|
||||
|
||||
class FFmpegVideoConvertorPP(FFmpegPostProcessor):
|
||||
def __init__(self, downloader=None, preferedformat=None):
|
||||
super(FFmpegVideoConvertorPP, self).__init__(downloader)
|
||||
|
|
|
|||
|
|
@ -60,6 +60,9 @@ from .compat import (
|
|||
compat_urllib_parse,
|
||||
compat_urllib_parse_urlencode,
|
||||
compat_urllib_parse_urlparse,
|
||||
compat_urllib_parse_urlunparse,
|
||||
compat_urllib_parse_quote,
|
||||
compat_urllib_parse_quote_plus,
|
||||
compat_urllib_parse_unquote_plus,
|
||||
compat_urllib_request,
|
||||
compat_urlparse,
|
||||
|
|
@ -5705,3 +5708,82 @@ def random_birthday(year_field, month_field, day_field):
|
|||
month_field: str(random_date.month),
|
||||
day_field: str(random_date.day),
|
||||
}
|
||||
|
||||
|
||||
# Templates for internet shortcut files, which are plain text files.
|
||||
DOT_URL_LINK_TEMPLATE = '''
|
||||
[InternetShortcut]
|
||||
URL=%(url)s
|
||||
'''.lstrip()
|
||||
|
||||
DOT_WEBLOC_LINK_TEMPLATE = '''
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
\t<key>URL</key>
|
||||
\t<string>%(url)s</string>
|
||||
</dict>
|
||||
</plist>
|
||||
'''.lstrip()
|
||||
|
||||
DOT_DESKTOP_LINK_TEMPLATE = '''
|
||||
[Desktop Entry]
|
||||
Encoding=UTF-8
|
||||
Name=%(filename)s
|
||||
Type=Link
|
||||
URL=%(url)s
|
||||
Icon=text-html
|
||||
'''.lstrip()
|
||||
|
||||
|
||||
def iri_to_uri(iri):
|
||||
"""
|
||||
Converts an IRI (Internationalized Resource Identifier, allowing Unicode characters) to a URI (Uniform Resource Identifier, ASCII-only).
|
||||
|
||||
The function doesn't add an additional layer of escaping; e.g., it doesn't escape `%3C` as `%253C`. Instead, it percent-escapes characters with an underlying UTF-8 encoding *besides* those already escaped, leaving the URI intact.
|
||||
"""
|
||||
|
||||
iri_parts = compat_urllib_parse_urlparse(iri)
|
||||
|
||||
if '[' in iri_parts.netloc:
|
||||
raise ValueError('IPv6 URIs are not, yet, supported.')
|
||||
# Querying `.netloc`, when there's only one bracket, also raises a ValueError.
|
||||
|
||||
# The `safe` argument values, that the following code uses, contain the characters that should not be percent-encoded. Everything else but letters, digits and '_.-' will be percent-encoded with an underlying UTF-8 encoding. Everything already percent-encoded will be left as is.
|
||||
|
||||
net_location = ''
|
||||
if iri_parts.username:
|
||||
net_location += compat_urllib_parse_quote(iri_parts.username, safe=r"!$%&'()*+,~")
|
||||
if iri_parts.password is not None:
|
||||
net_location += ':' + compat_urllib_parse_quote(iri_parts.password, safe=r"!$%&'()*+,~")
|
||||
net_location += '@'
|
||||
|
||||
net_location += iri_parts.hostname.encode('idna').decode('utf-8') # Punycode for Unicode hostnames.
|
||||
# The 'idna' encoding produces ASCII text.
|
||||
if iri_parts.port is not None and iri_parts.port != 80:
|
||||
net_location += ':' + str(iri_parts.port)
|
||||
|
||||
return compat_urllib_parse_urlunparse(
|
||||
(iri_parts.scheme,
|
||||
net_location,
|
||||
|
||||
compat_urllib_parse_quote_plus(iri_parts.path, safe=r"!$%&'()*+,/:;=@|~"),
|
||||
|
||||
# Unsure about the `safe` argument, since this is a legacy way of handling parameters.
|
||||
compat_urllib_parse_quote_plus(iri_parts.params, safe=r"!$%&'()*+,/:;=@|~"),
|
||||
|
||||
# Not totally sure about the `safe` argument, since the source does not explicitly mention the query URI component.
|
||||
compat_urllib_parse_quote_plus(iri_parts.query, safe=r"!$%&'()*+,/:;=?@{|}~"),
|
||||
|
||||
compat_urllib_parse_quote_plus(iri_parts.fragment, safe=r"!#$%&'()*+,/:;=?@{|}~")))
|
||||
|
||||
# Source for `safe` arguments: https://url.spec.whatwg.org/#percent-encoded-bytes.
|
||||
|
||||
|
||||
def to_high_limit_path(path):
|
||||
if sys.platform in ['win32', 'cygwin']:
|
||||
# Work around MAX_PATH limitation on Windows. The maximum allowed length for the individual path segments may still be quite limited.
|
||||
return r'\\?\ '.rstrip() + os.path.abspath(path)
|
||||
|
||||
return path
|
||||
|
|
|
|||
Loading…
Reference in New Issue