The new updates system, relies on gh-pages, secured by RSA, uses external web servers

This commit is contained in:
Filippo Valsorda 2012-12-26 23:22:49 +01:00
parent 0deac3a2d8
commit cb6ff87fbb
6 changed files with 224 additions and 58 deletions

1
.gitignore vendored
View File

@ -16,3 +16,4 @@ youtube-dl.exe
youtube-dl.tar.gz youtube-dl.tar.gz
.coverage .coverage
cover/ cover/
updates_key.pem

View File

@ -2,17 +2,48 @@
import sys, os import sys, os
import urllib2 import urllib2
import json, hashlib
def rsa_verify(message, signature, key):
from struct import pack
from hashlib import sha256
from sys import version_info
def b(x):
if version_info[0] == 2: return x
else: return x.encode('latin1')
assert(type(message) == type(b('')))
block_size = 0
n = key[0]
while n:
block_size += 1
n >>= 8
signature = pow(int(signature, 16), key[1], key[0])
raw_bytes = []
while signature:
raw_bytes.insert(0, pack("B", signature & 0xFF))
signature >>= 8
signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
if signature[0:2] != b('\x00\x01'): return False
signature = signature[2:]
if not b('\x00') in signature: return False
signature = signature[signature.index(b('\x00'))+1:]
if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
signature = signature[19:]
if signature != sha256(message).digest(): return False
return True
sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n') sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n') sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n') sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n')
raw_input() raw_input()
filename = sys.argv[0] filename = sys.argv[0]
API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe" VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
JSON_URL = UPDATE_URL + 'versions.json'
UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
if not os.access(filename, os.W_OK): if not os.access(filename, os.W_OK):
sys.exit('ERROR: no write permissions on %s' % filename) sys.exit('ERROR: no write permissions on %s' % filename)
@ -23,13 +54,35 @@ if not os.access(directory, os.W_OK):
sys.exit('ERROR: no write permissions on %s' % directory) sys.exit('ERROR: no write permissions on %s' % directory)
try: try:
urlh = urllib2.urlopen(EXE_URL) versions_info = urllib2.urlopen(JSON_URL).read().decode('utf-8')
versions_info = json.loads(versions_info)
except:
sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.')
if not 'signature' in versions_info:
sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.')
signature = versions_info['signature']
del versions_info['signature']
if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY):
sys.exit(u'ERROR: the versions file signature is invalid. Aborting.')
version = versions_info['versions'][versions_info['latest']]
try:
urlh = urllib2.urlopen(version['exe'][0])
newcontent = urlh.read() newcontent = urlh.read()
urlh.close() urlh.close()
except (IOError, OSError) as err:
sys.exit('ERROR: unable to download latest version')
newcontent_hash = hashlib.sha256(newcontent).hexdigest()
if newcontent_hash != version['exe'][1]:
sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.')
try:
with open(exe + '.new', 'wb') as outf: with open(exe + '.new', 'wb') as outf:
outf.write(newcontent) outf.write(newcontent)
except (IOError, OSError) as err: except (IOError, OSError) as err:
sys.exit('ERROR: unable to download latest version') sys.exit(u'ERROR: unable to write the new version')
try: try:
bat = os.path.join(directory, 'youtube-dl-updater.bat') bat = os.path.join(directory, 'youtube-dl-updater.bat')

View File

@ -1,15 +1,44 @@
#!/usr/bin/env python #!/usr/bin/env python
import sys, os import sys, os
import json, hashlib
try: try:
import urllib.request as compat_urllib_request import urllib.request as compat_urllib_request
except ImportError: # Python 2 except ImportError: # Python 2
import urllib2 as compat_urllib_request import urllib2 as compat_urllib_request
def rsa_verify(message, signature, key):
from struct import pack
from hashlib import sha256
from sys import version_info
def b(x):
if version_info[0] == 2: return x
else: return x.encode('latin1')
assert(type(message) == type(b('')))
block_size = 0
n = key[0]
while n:
block_size += 1
n >>= 8
signature = pow(int(signature, 16), key[1], key[0])
raw_bytes = []
while signature:
raw_bytes.insert(0, pack("B", signature & 0xFF))
signature >>= 8
signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
if signature[0:2] != b('\x00\x01'): return False
signature = signature[2:]
if not b('\x00') in signature: return False
signature = signature[signature.index(b('\x00'))+1:]
if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
signature = signature[19:]
if signature != sha256(message).digest(): return False
return True
sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n') sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n') sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n') sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n')
try: try:
raw_input() raw_input()
@ -18,19 +47,39 @@ except NameError: # Python 3
filename = sys.argv[0] filename = sys.argv[0]
API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl" VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
JSON_URL = UPDATE_URL + 'versions.json'
UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
if not os.access(filename, os.W_OK): if not os.access(filename, os.W_OK):
sys.exit('ERROR: no write permissions on %s' % filename) sys.exit('ERROR: no write permissions on %s' % filename)
try: try:
urlh = compat_urllib_request.urlopen(BIN_URL) versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
versions_info = json.loads(versions_info)
except:
sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.')
if not 'signature' in versions_info:
sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.')
signature = versions_info['signature']
del versions_info['signature']
if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY):
sys.exit(u'ERROR: the versions file signature is invalid. Aborting.')
version = versions_info['versions'][versions_info['latest']]
try:
urlh = compat_urllib_request.urlopen(version['bin'][0])
newcontent = urlh.read() newcontent = urlh.read()
urlh.close() urlh.close()
except (IOError, OSError) as err: except (IOError, OSError) as err:
sys.exit('ERROR: unable to download latest version') sys.exit('ERROR: unable to download latest version')
newcontent_hash = hashlib.sha256(newcontent).hexdigest()
if newcontent_hash != version['bin'][1]:
sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.')
try: try:
with open(filename, 'wb') as outf: with open(filename, 'wb') as outf:
outf.write(newcontent) outf.write(newcontent)

Binary file not shown.

View File

@ -41,47 +41,90 @@ from .FileDownloader import *
from .InfoExtractors import * from .InfoExtractors import *
from .PostProcessor import * from .PostProcessor import *
def updateSelf(downloader, filename): def update_self(to_screen, verbose, filename):
"""Update the program file with the latest version from the repository""" """Update the program file with the latest version from the repository"""
# TODO: at least, check https certificates
from zipimport import zipimporter from zipimport import zipimporter
import json, traceback, hashlib
API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl" VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe" JSON_URL = UPDATE_URL + 'versions.json'
UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
if hasattr(sys, "frozen"): # PY2EXE
if not os.access(filename, os.W_OK):
sys.exit('ERROR: no write permissions on %s' % filename)
downloader.to_screen(u'Updating to latest version...') if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
to_screen(u'It looks like you installed youtube-dl with pip, setup.py or a tarball. Please use that to update.')
urla = compat_urllib_request.urlopen(API_URL) return
download = filter(lambda x: x["name"] == "youtube-dl.exe", json.loads(urla.read()))
if not download: # Check if there is a new version
downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.') try:
newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
except:
if verbose: to_screen(traceback.format_exc().decode())
to_screen(u'ERROR: can\'t find the current version. Please try again later.')
return return
newversion = download[0]["description"].strip()
if newversion == __version__: if newversion == __version__:
downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
return return
urla.close()
# Download and check versions info
try:
versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
versions_info = json.loads(versions_info)
except:
if verbose: to_screen(traceback.format_exc().decode())
to_screen(u'ERROR: can\'t obtain versions info. Please try again later.')
return
if not 'signature' in versions_info:
to_screen(u'ERROR: the versions file is not signed or corrupted. Aborting.')
return
signature = versions_info['signature']
del versions_info['signature']
if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY):
to_screen(u'ERROR: the versions file signature is invalid. Aborting.')
return
to_screen(u'Updating to version ' + versions_info['latest'] + '...')
version = versions_info['versions'][versions_info['latest']]
if version.get('notes'):
to_screen(u'PLEASE NOTE:')
for note in version['notes']:
to_screen(note)
if not os.access(filename, os.W_OK):
to_screen(u'ERROR: no write permissions on %s' % filename)
return
# Py2EXE
if hasattr(sys, "frozen"):
exe = os.path.abspath(filename) exe = os.path.abspath(filename)
directory = os.path.dirname(exe) directory = os.path.dirname(exe)
if not os.access(directory, os.W_OK): if not os.access(directory, os.W_OK):
sys.exit('ERROR: no write permissions on %s' % directory) to_screen(u'ERROR: no write permissions on %s' % directory)
return
try: try:
urlh = compat_urllib_request.urlopen(EXE_URL) urlh = compat_urllib_request.urlopen(version['exe'][0])
newcontent = urlh.read() newcontent = urlh.read()
urlh.close() urlh.close()
except (IOError, OSError) as err:
if verbose: to_screen(traceback.format_exc().decode())
to_screen(u'ERROR: unable to download latest version')
return
newcontent_hash = hashlib.sha256(newcontent).hexdigest()
if newcontent_hash != version['exe'][1]:
to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
return
try:
with open(exe + '.new', 'wb') as outf: with open(exe + '.new', 'wb') as outf:
outf.write(newcontent) outf.write(newcontent)
except (IOError, OSError) as err: except (IOError, OSError) as err:
sys.exit('ERROR: unable to download latest version') if verbose: to_screen(traceback.format_exc().decode())
to_screen(u'ERROR: unable to write the new version')
return
try: try:
bat = os.path.join(directory, 'youtube-dl-updater.bat') bat = os.path.join(directory, 'youtube-dl-updater.bat')
@ -96,43 +139,35 @@ del "%s"
os.startfile(bat) os.startfile(bat)
except (IOError, OSError) as err: except (IOError, OSError) as err:
sys.exit('ERROR: unable to overwrite current version') if verbose: to_screen(traceback.format_exc().decode())
to_screen(u'ERROR: unable to overwrite current version')
elif isinstance(globals().get('__loader__'), zipimporter): # UNIX ZIP
if not os.access(filename, os.W_OK):
sys.exit('ERROR: no write permissions on %s' % filename)
downloader.to_screen(u'Updating to latest version...')
urla = compat_urllib_request.urlopen(API_URL)
download = [x for x in json.loads(urla.read().decode('utf8')) if x["name"] == "youtube-dl"]
if not download:
downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.')
return return
newversion = download[0]["description"].strip()
if newversion == __version__:
downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
return
urla.close()
# Zip unix package
elif isinstance(globals().get('__loader__'), zipimporter):
try: try:
urlh = compat_urllib_request.urlopen(BIN_URL) urlh = compat_urllib_request.urlopen(version['bin'][0])
newcontent = urlh.read() newcontent = urlh.read()
urlh.close() urlh.close()
except (IOError, OSError) as err: except (IOError, OSError) as err:
sys.exit('ERROR: unable to download latest version') if verbose: to_screen(traceback.format_exc().decode())
to_screen(u'ERROR: unable to download latest version')
return
newcontent_hash = hashlib.sha256(newcontent).hexdigest()
if newcontent_hash != version['bin'][1]:
to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
return
try: try:
with open(filename, 'wb') as outf: with open(filename, 'wb') as outf:
outf.write(newcontent) outf.write(newcontent)
except (IOError, OSError) as err: except (IOError, OSError) as err:
sys.exit('ERROR: unable to overwrite current version') if verbose: to_screen(traceback.format_exc().decode())
to_screen(u'ERROR: unable to overwrite current version')
else:
downloader.to_screen(u'It looks like you installed youtube-dl with pip or setup.py. Please use that to update.')
return return
downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
def parseOpts(): def parseOpts():
def _readOptions(filename_bytes): def _readOptions(filename_bytes):
@ -578,7 +613,7 @@ def _real_main():
# Update version # Update version
if opts.update_self: if opts.update_self:
updateSelf(fd, sys.argv[0]) update_self(fd.to_screen, opts.verbose, sys.argv[0])
# Maybe do nothing # Maybe do nothing
if len(all_urls) < 1: if len(all_urls) < 1:

View File

@ -410,6 +410,34 @@ def encodeFilename(s):
else: else:
return s.encode(sys.getfilesystemencoding(), 'ignore') return s.encode(sys.getfilesystemencoding(), 'ignore')
def rsa_verify(message, signature, key):
from struct import pack
from hashlib import sha256
from sys import version_info
def b(x):
if version_info[0] == 2: return x
else: return x.encode('latin1')
assert(type(message) == type(b('')))
block_size = 0
n = key[0]
while n:
block_size += 1
n >>= 8
signature = pow(int(signature, 16), key[1], key[0])
raw_bytes = []
while signature:
raw_bytes.insert(0, pack("B", signature & 0xFF))
signature >>= 8
signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
if signature[0:2] != b('\x00\x01'): return False
signature = signature[2:]
if not b('\x00') in signature: return False
signature = signature[signature.index(b('\x00'))+1:]
if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
signature = signature[19:]
if signature != sha256(message).digest(): return False
return True
class DownloadError(Exception): class DownloadError(Exception):
"""Download Error exception. """Download Error exception.