Madeorsk
/
youtube-dl
mirror of https://codeberg.org/polarisfm/youtube-dl
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge b734153d66 into 48c5663c5f

This commit is contained in:
yww 2020-10-22 17:59:16 +02:00 committed by GitHub
parent 48c5663c5f b734153d66
commit b699d3a833
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 313 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

517
youtube_dl/extractor/bilibili.py
View File

@ -3,38 +3,22 @@ from __future__ import unicode_literals
import hashlib
import re
from datetime import datetime
from .common import InfoExtractor
from ..compat import (
compat_parse_qs,
compat_urlparse,
)
from ..utils import (
ExtractorError,
int_or_none,
float_or_none,
parse_iso8601,
smuggle_url,
str_or_none,
strip_jsonp,
unified_timestamp,
unsmuggle_url,
urlencode_postdata,
)
class BiliBiliIE(InfoExtractor):
_VALID_URL = r'''(?x)
https?://
(?:(?:www|bangumi)\.)?
bilibili\.(?:tv|com)/
(?:
(?:
video/[aA][vV]|
anime/(?P<anime_id>\d+)/play\#
)(?P<id_bv>\d+)|
video/[bB][vV](?P<id>[^/?#&]+)
)
(?:www\.)bilibili.(?:com|tv)
/video/[aAbB][vV](?P<id>[^/?#&]+)
'''
_TESTS = [{
@ -46,26 +30,12 @@ class BiliBiliIE(InfoExtractor):
'title': '【金坷垃】金泡沫',
'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
'duration': 308.067,
'timestamp': 1398012678,
'timestamp': 1397983878,
'upload_date': '20140420',
'thumbnail': r're:^https?://.+\.jpg',
'uploader': '菊子桑',
'uploader_id': '156160',
},
}, {
# Tested in BiliBiliBangumiIE
'url': 'http://bangumi.bilibili.com/anime/1869/play#40062',
'only_matching': True,
}, {
'url': 'http://bangumi.bilibili.com/anime/5802/play#100643',
'md5': '3f721ad1e75030cc06faf73587cfec57',
'info_dict': {
'id': '100643',
'ext': 'mp4',
'title': 'CHAOS;CHILD',
'description': '如果你是神明，并且能够让妄想成为现实。那你会进行怎么样的妄想？是淫靡的世界？独裁社会？毁灭性的制裁？还是……2015年，涩谷。从6年前发生的大灾害“涩谷地震”之后复兴了的这个街区里新设立的私立高中...',
},
'skip': 'Geo-restricted to China',
}, {
# Title with double quotes
'url': 'http://www.bilibili.com/video/av8903802/',
@ -73,31 +43,29 @@ class BiliBiliIE(InfoExtractor):
'id': '8903802',
'title': '阿滴英文｜英文歌分享#6 "Closer',
'description': '滴妹今天唱Closer給你聽! 有史以来，被推最多次也是最久的歌曲，其实歌词跟我原本想像差蛮多的，不过还是好听！ 微博@阿滴英文',
'uploader': '阿滴英文',
'uploader_id': '65880958',
'upload_date': '20170301',
'timestamp': 1488353834,
},
'params': {
'skip_download': True, # Test metadata only
},
'playlist': [{
'info_dict': {
'id': '8903802_part1',
'id': '8903802_14694589_1',
'ext': 'flv',
'title': '阿滴英文｜英文歌分享#6 "Closer',
'description': 'md5:3b1b9e25b78da4ef87e9b548b88ee76a',
'uploader': '阿滴英文',
'uploader_id': '65880958',
'timestamp': 1488382634,
'upload_date': '20170301',
},
'params': {
'skip_download': True, # Test metadata only
},
}, {
'info_dict': {
'id': '8903802_part2',
'id': '8903802_14694589_2',
'ext': 'flv',
'title': '阿滴英文｜英文歌分享#6 "Closer',
'description': 'md5:3b1b9e25b78da4ef87e9b548b88ee76a',
'uploader': '阿滴英文',
'uploader_id': '65880958',
'timestamp': 1488382634,
'upload_date': '20170301',
},
'params': {
'skip_download': True, # Test metadata only
@ -107,7 +75,43 @@ class BiliBiliIE(InfoExtractor):
# new BV video id format
'url': 'https://www.bilibili.com/video/BV1JE411F741',
'only_matching': True,
}]
}, {
# multiple part video
'url': 'https://www.bilibili.com/video/BV1FJ411k7q9',
'info_dict': {
'id': '1FJ411k7q9',
'title': '【原始技术】用草木灰代替粘土（Minecraft真人版第五十一弹）',
'description': '【Primitive Technology@Youtube】\n看着上一集烧砖产生的大量草木灰，小哥有了新想法：草木灰也许可以作为粘土的又一种替代品，用来做罐子、砖块都行，不怕浸水，还不需要烧制。这是小哥第51个视频，完整合集见av2920827。更多细节见: https://youtu.be/rG6nzrksbPQ，想帮小哥制作更好的视频可以上Patreon给小哥充电：https://www.patreon.com/user?u=2945881',
'uploader': '昨梦电羊',
'uploader_id': '1388774',
'upload_date': '20191215',
'timestamp': 1576376056,
},
'params': {
'skip_download': True, # Test metadata only
},
'playlist': [{
'info_dict': {
'id': '1FJ411k7q9_135730700_1',
'ext': 'flv',
'title': '字幕版1080p',
},
'params': {
'skip_download': True, # Test metadata only
},
}, {
'info_dict': {
'id': '1FJ411k7q9_135730766_1',
'ext': 'flv',
'title': '无字幕版',
},
'params': {
'skip_download': True, # Test metadata only
},
}]
}
]
_APP_KEY = 'iVGUTjsxvpLeuDCf'
_BILIBILI_KEY = 'aHRmhWMLkdeMuILqORnYZocwMBpMEOdt'
@ -120,41 +124,44 @@ class BiliBiliIE(InfoExtractor):
else:
raise ExtractorError('Can\'t extract Bangumi episode ID')
def _aid_to_bid(self, aid):
'''
convert bilibili avid to bid
'''
api_url = 'http://api.bilibili.com/x/web-interface/view?aid=%s' % (aid, )
js = self._download_json(api_url, aid, 'convert avid to bv id', 'convert failed')
return js['data']['bvid']
def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url, {})
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id') or mobj.group('id_bv')
anime_id = mobj.group('anime_id')
video_id = self._match_id(url)
# save the origin video id
original_video_id = video_id
webpage = self._download_webpage(url, video_id)
if 'anime/' not in url:
cid = self._search_regex(
r'\bcid(?:["\']:|=)(\d+)', webpage, 'cid',
default=None
) or compat_parse_qs(self._search_regex(
[r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
r'EmbedPlayer\([^)]+,\s*\\"([^"]+)\\"\)',
r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
webpage, 'player parameters'))['cid'][0]
else:
if 'no_bangumi_tip' not in smuggled_data:
self.to_screen('Downloading episode %s. To download all videos in anime %s, re-run youtube-dl with %s' % (
video_id, anime_id, compat_urlparse.urljoin(url, '//bangumi.bilibili.com/anime/%s' % anime_id)))
headers = {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Referer': url
}
headers.update(self.geo_verification_headers())
js = self._download_json(
'http://bangumi.bilibili.com/web_api/get_source', video_id,
data=urlencode_postdata({'episode_id': video_id}),
headers=headers)
if 'result' not in js:
self._report_error(js)
cid = js['result']['cid']
title = ''
timestamp = 0
thumbnail = ''
description = ''
uploader_id = ''
uploader_name = ''
view_count = 0
part_list = []
upload_date = ''
# normal video
if re.match(r'^\d+$', video_id):
video_id = self._aid_to_bid(video_id)
self.to_screen("%s: convert to bvid %s" % (original_video_id, video_id))
list_api_url = 'https://api.bilibili.com/x/web-interface/view/detail?bvid=%s' % (video_id, )
js = self._download_json(list_api_url, original_video_id, 'downloading video list', 'downloding video list failed', fatal=False)['data']
video_list = js['View']['pages']
title = js['View']['title']
thumbnail = js.get('View', {}).get('pic')
description = js.get('View', {}).get('desc')
view_count = js.get('View', {}).get('stat', {}).get('view')
uploader_id = js.get('Card', {}).get('card', {}).get('mid')
uploader_name = js.get('Card', {}).get('card', {}).get('name')
self.to_screen("%s: video count: %d" % (original_video_id, len(video_list)))
part_list = [{'cid': x['cid'], 'title': x['part']} for x in video_list]
headers = {
'Referer': url
}
@ -163,27 +170,187 @@ class BiliBiliIE(InfoExtractor):
entries = []
RENDITIONS = ('qn=80&quality=80&type=', 'quality=2&type=mp4')
for num, rendition in enumerate(RENDITIONS, start=1):
payload = 'appkey=%s&cid=%s&otype=json&%s' % (self._APP_KEY, cid, rendition)
sign = hashlib.md5((payload + self._BILIBILI_KEY).encode('utf-8')).hexdigest()
for part_info in part_list:
# try to get video playback url, use
for num, rendition in enumerate(RENDITIONS, start=1):
payload = 'appkey=%s&cid=%s&otype=json&%s' % (self._APP_KEY, part_info['cid'], rendition)
sign = hashlib.md5((payload + self._BILIBILI_KEY).encode('utf-8')).hexdigest()
video_info = self._download_json(
'http://interface.bilibili.com/v2/playurl?%s&sign=%s' % (payload, sign),
video_id, note='Downloading video info page',
headers=headers, fatal=num == len(RENDITIONS))
video_info = self._download_json(
'http://interface.bilibili.com/v2/playurl?%s&sign=%s' % (payload, sign),
original_video_id, note='Downloading video info for cid: %s' % (part_info['cid'], ),
headers=headers, fatal=num == len(RENDITIONS))
if not video_info:
continue
if 'durl' not in video_info:
if num < len(RENDITIONS):
if not video_info:
continue
self._report_error(video_info)
for idx, durl in enumerate(video_info['durl']):
if 'durl' not in video_info:
if num < len(RENDITIONS):
continue
self._report_error(video_info)
part_title = part_info['title']
if len(part_list) == 1 or len(part_title) == 0:
# if video only got one part, use video title instead of part title
part_title = title
else:
# some video not name part title properly, so add video title in front
part_title = title + "_" + part_title
for idx, durl in enumerate(video_info['durl'], start=1):
# some video is splited to many fragments, here is this fragments
formats = [{
'url': durl['url'],
'filesize': int_or_none(durl['size']),
}]
for backup_url in durl.get('backup_url', []):
formats.append({
'url': backup_url,
# backup URLs have lower priorities
'preference': -2 if 'hd.mp4' in backup_url else -3,
})
for a_format in formats:
a_format.setdefault('http_headers', {}).update({
'Referer': url,
})
self._sort_formats(formats)
entries.append({
'id': '%s_%s_%s' % (original_video_id, part_info['cid'], idx),
'duration': float_or_none(durl.get('length'), 1000),
'formats': formats,
'title': part_title
})
break
if not title:
title = self._html_search_regex(
('<h1[^>]+\btitle=(["\'])(?P<title>(?:(?!\1).)+)\1',
'(?s)<h1[^>]*>(?P<title>.+?)</h1>'), webpage, 'title',
group='title', fatal=False)
if not timestamp:
timestamp = self._html_search_regex(
(r'"pubdate":(?P<timestamp>\d+),'), webpage, 'timestamp',
group='timestamp', fatal=False)
if not uploader_id or not uploader_name:
uploader_id = self._html_search_regex(
r'<a[^>]+href="(?:https?:)?//space\.bilibili\.com/\d+"[^>]*>(?P<name>[^<]+)',
webpage, 'id',
group='id', fatal=False)
uploader_name = self._html_search_regex(
r'<a[^>]+href="(?:https?:)?//space\.bilibili\.com/(?P<id>\d+)"',
webpage, 'name',
group='name', fatal=False)
if not thumbnail:
thumbnail = self._html_search_meta(['og:image', 'thumbnailUrl'], webpage, fatal=False)
if not description:
description = self._html_search_meta('description', webpage, fatal=False)
if timestamp:
timestamp = int_or_none(timestamp)
upload_date = datetime.fromtimestamp(timestamp).strftime('%Y%m%d')
if view_count:
view_count = int_or_none(view_count)
if len(entries) == 1:
entry = entries[0]
entry['uploader'] = uploader_name
entry['uploader_id'] = uploader_id
entry['id'] = original_video_id
entry['title'] = title
entry['description'] = description
entry['timestamp'] = timestamp
entry['thumbnail'] = thumbnail
entry['upload_date'] = upload_date
entry['view_count'] = view_count
return entry
else:
playlist_entry = self.playlist_result(entries, id, title, description)
playlist_entry['uploader'] = uploader_name
playlist_entry['uploader_id'] = uploader_id
playlist_entry['timestamp'] = timestamp
playlist_entry['thumbnail'] = thumbnail
playlist_entry['upload_date'] = upload_date
playlist_entry['view_count'] = view_count
return playlist_entry
class BiliBiliBangumiIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)bilibili.com/bangumi/media/[mD][dD](?P<id>\d+)'
IE_NAME = 'bangumi.bilibili.com'
IE_DESC = 'BiliBili番剧'
_TESTS = [{
'url': 'https://www.bilibili.com/bangumi/media/md3814',
'info_dict': {
'id': '3814',
'title': '魔动王 最后的魔法大战',
'description': 'md5:9634eb0d85d515f6930fa1c833ccee63',
},
'playlist': [{
'info_dict': {
'id': '3814_1_1',
'ext': 'flv',
'title': '最后的魔法大战 前篇'
},
}, {
'info_dict': {
'id': '3814_1_2',
'ext': 'flv',
'title': '最后的魔法大战 前篇'
},
}, {
'info_dict': {
'id': '3814_1_3',
'ext': 'flv',
'title': '最后的魔法大战 前篇'
},
}, {
'info_dict': {
'id': '3814_1_4',
'ext': 'flv',
'title': '最后的魔法大战 前篇'
},
}, {
'info_dict': {
'id': '3814_1_5',
'ext': 'flv',
'title': '最后的魔法大战 前篇'
},
}, {
'info_dict': {
'id': '3814_2_1',
'ext': 'flv',
'title': '最后的魔法大战 后篇'
},
}
]
}]
def _real_extract(self, url):
headers = {
'Referer': url
}
bangumi_id = self._match_id(url)
bangumi_info = self._download_json(
'https://api.bilibili.com/pgc/view/web/season?season_id=%s' % (bangumi_id,),
bangumi_id,
'Downloading bangumi info',
'Downloading bangumi failed')['result']
title = bangumi_info['season_title']
description = bangumi_info.get('evaluate')
episodes = bangumi_info.get('episodes')
self.to_screen('%s: episode count: %d' % (bangumi_id, len(episodes)))
entries = []
for idx, episode in enumerate(episodes, start=1):
play_back_info = self._download_json(
'http://api.bilibili.com/x/player/playurl?bvid=%s&cid=%s&qn=80' % (episode['bvid'], episode['cid']),
bangumi_id,
'downloding playback info for ep: %d' % (idx, ),
headers=headers)['data']
for fragment_idx, durl in enumerate(play_back_info['durl'], start=1):
# some video is splited to many fragments, here is this fragments
formats = [{
'url': durl['url'],
'filesize': int_or_none(durl['size']),
'filesize': int_or_none(durl.get('size')),
}]
for backup_url in durl.get('backup_url', []):
formats.append({
@ -198,130 +365,70 @@ class BiliBiliIE(InfoExtractor):
})
self._sort_formats(formats)
entries.append({
'id': '%s_part%s' % (video_id, idx),
'id': '%s_%d_%d' % (bangumi_id, idx, fragment_idx),
'duration': float_or_none(durl.get('length'), 1000),
'formats': formats,
'title': episode.get('long_title', '')
})
break
title = self._html_search_regex(
('<h1[^>]+\btitle=(["\'])(?P<title>(?:(?!\1).)+)\1',
'(?s)<h1[^>]*>(?P<title>.+?)</h1>'), webpage, 'title',
group='title')
description = self._html_search_meta('description', webpage)
timestamp = unified_timestamp(self._html_search_regex(
r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time',
default=None) or self._html_search_meta(
'uploadDate', webpage, 'timestamp', default=None))
thumbnail = self._html_search_meta(['og:image', 'thumbnailUrl'], webpage)
# TODO 'view_count' requires deobfuscating Javascript
info = {
'id': video_id,
'title': title,
'description': description,
'timestamp': timestamp,
'thumbnail': thumbnail,
'duration': float_or_none(video_info.get('timelength'), scale=1000),
}
uploader_mobj = re.search(
r'<a[^>]+href="(?:https?:)?//space\.bilibili\.com/(?P<id>\d+)"[^>]*>(?P<name>[^<]+)',
webpage)
if uploader_mobj:
info.update({
'uploader': uploader_mobj.group('name'),
'uploader_id': uploader_mobj.group('id'),
})
if not info.get('uploader'):
info['uploader'] = self._html_search_meta(
'author', webpage, 'uploader', default=None)
for entry in entries:
entry.update(info)
if len(entries) == 1:
return entries[0]
else:
for idx, entry in enumerate(entries):
entry['id'] = '%s_part%d' % (video_id, (idx + 1))
return {
'_type': 'multi_video',
'id': video_id,
'title': title,
'description': description,
'entries': entries,
}
return self.playlist_result(entries, bangumi_id, title, description)
class BiliBiliBangumiIE(InfoExtractor):
_VALID_URL = r'https?://bangumi\.bilibili\.com/anime/(?P<id>\d+)'
IE_NAME = 'bangumi.bilibili.com'
IE_DESC = 'BiliBili番剧'
class BiliBiliBangumiEpisodeIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)bilibili.com/bangumi/play/[eE][pP](?P<id>\d+)'
_TESTS = [{
'url': 'http://bangumi.bilibili.com/anime/1869',
'url': 'https://www.bilibili.com/bangumi/play/ep86635',
'info_dict': {
'id': '1869',
'title': '混沌武士',
'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
},
'playlist_count': 26,
}, {
'url': 'http://bangumi.bilibili.com/anime/1869',
'info_dict': {
'id': '1869',
'title': '混沌武士',
'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
'id': '3814',
'title': '魔动王 最后的魔法大战',
'description': 'md5:9634eb0d85d515f6930fa1c833ccee63',
},
'playlist': [{
'md5': '91da8621454dd58316851c27c68b0c13',
'info_dict': {
'id': '40062',
'ext': 'mp4',
'title': '混沌武士',
'description': '故事发生在日本的江户时代。风是一个小酒馆的打工女。一日，酒馆里来了一群恶霸，虽然他们的举动令风十分不满，但是毕竟风只是一届女流，无法对他们采取什么行动，只能在心里嘟哝。这时，酒家里又进来了个“不良份子...',
'timestamp': 1414538739,
'upload_date': '20141028',
'episode': '疾风怒涛 Tempestuous Temperaments',
'episode_number': 1,
'id': '3814_1_1',
'ext': 'flv',
'title': '最后的魔法大战 前篇'
},
}],
'params': {
'playlist_items': '1',
},
}, {
'info_dict': {
'id': '3814_1_2',
'ext': 'flv',
'title': '最后的魔法大战 前篇'
},
}, {
'info_dict': {
'id': '3814_1_3',
'ext': 'flv',
'title': '最后的魔法大战 前篇'
},
}, {
'info_dict': {
'id': '3814_1_4',
'ext': 'flv',
'title': '最后的魔法大战 前篇'
},
}, {
'info_dict': {
'id': '3814_1_5',
'ext': 'flv',
'title': '最后的魔法大战 前篇'
},
}, {
'info_dict': {
'id': '3814_2_1',
'ext': 'flv',
'title': '最后的魔法大战 后篇'
},
}
]
}]
@classmethod
def suitable(cls, url):
return False if BiliBiliIE.suitable(url) else super(BiliBiliBangumiIE, cls).suitable(url)
def _real_extract(self, url):
bangumi_id = self._match_id(url)
# Sometimes this API returns a JSONP response
season_info = self._download_json(
'http://bangumi.bilibili.com/jsonp/seasoninfo/%s.ver' % bangumi_id,
bangumi_id, transform_source=strip_jsonp)['result']
entries = [{
'_type': 'url_transparent',
'url': smuggle_url(episode['webplay_url'], {'no_bangumi_tip': 1}),
'ie_key': BiliBiliIE.ie_key(),
'timestamp': parse_iso8601(episode.get('update_time'), delimiter=' '),
'episode': episode.get('index_title'),
'episode_number': int_or_none(episode.get('index')),
} for episode in season_info['episodes']]
entries = sorted(entries, key=lambda entry: entry.get('episode_number'))
return self.playlist_result(
entries, bangumi_id,
season_info.get('bangumi_title'), season_info.get('evaluate'))
ep_id = self._match_id(url)
bangumi_id = self._download_json('https://api.bilibili.com/pgc/view/web/season?ep_id=%s' % (ep_id, ), ep_id, 'Downloading bangumi info')['result']['media_id']
return self.url_result(
'https://www.bilibili.com/bangumi/media/md%s' % bangumi_id,
ie=BiliBiliBangumiIE.ie_key(), video_id=ep_id)
class BilibiliAudioBaseIE(InfoExtractor):
@ -406,7 +513,7 @@ class BilibiliAudioAlbumIE(BilibiliAudioBaseIE):
'title': '每日新曲推荐（每日11:00更新）',
'description': '每天11:00更新，为你推送最新音乐',
},
'playlist_count': 19,
'playlist_count': 20,
}
def _real_extract(self, url):

1
youtube_dl/extractor/extractors.py
View File

@ -103,6 +103,7 @@ from .bild import BildIE
from .bilibili import (
BiliBiliIE,
BiliBiliBangumiIE,
BiliBiliBangumiEpisodeIE,
BilibiliAudioIE,
BilibiliAudioAlbumIE,
BiliBiliPlayerIE,