From 5d49d879cc59beb417d04db17b2f18bb438b52e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 9 Mar 2018 23:27:44 +0700 Subject: [PATCH 1/5] [raywenderlich] Add extractor (#15251) --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/raywenderlich.py | 103 ++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 youtube_dl/extractor/raywenderlich.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 6011fd41f..bef3b82ee 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -875,6 +875,7 @@ from .rai import ( RaiPlayPlaylistIE, RaiIE, ) +from .raywenderlich import RayWenderlichIE from .rbmaradio import RBMARadioIE from .rds import RDSIE from .redbulltv import RedBullTVIE diff --git a/youtube_dl/extractor/raywenderlich.py b/youtube_dl/extractor/raywenderlich.py new file mode 100644 index 000000000..0f061c4b2 --- /dev/null +++ b/youtube_dl/extractor/raywenderlich.py @@ -0,0 +1,103 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from .vimeo import VimeoIE +from ..utils import ( + extract_attributes, + ExtractorError, + orderedSet, + smuggle_url, + unsmuggle_url, + urljoin, +) + + +class RayWenderlichIE(InfoExtractor): + _VALID_URL = r'https?://videos\.raywenderlich\.com/courses/(?P[^/]+)/lessons/(?P\d+)' + + _TESTS = [{ + 'url': 'https://videos.raywenderlich.com/courses/105-testing-in-ios/lessons/1', + 'info_dict': { + 'id': '248377018', + 'ext': 'mp4', + 'title': 'Testing In iOS Episode 1: Introduction', + 'duration': 133, + 'uploader': 'Ray Wenderlich', + 'uploader_id': 'user3304672', + }, + 'params': { + 'noplaylist': True, + 'skip_download': True, + }, + 'add_ie': [VimeoIE.ie_key()], + 'expected_warnings': ['HTTP Error 403: Forbidden'], + }, { + 'url': 'https://videos.raywenderlich.com/courses/105-testing-in-ios/lessons/1', + 'info_dict': { + 'title': 'Testing in iOS', + 'id': '105-testing-in-ios', + }, + 'params': { + 'noplaylist': False, + }, + 'playlist_count': 29, + }] + + def _real_extract(self, url): + url, smuggled_data = unsmuggle_url(url, {}) + + mobj = re.match(self._VALID_URL, url) + course_id, lesson_id = mobj.group('course_id', 'id') + video_id = '%s/%s' % (course_id, lesson_id) + + webpage = self._download_webpage(url, video_id) + + no_playlist = self._downloader.params.get('noplaylist') + if no_playlist or smuggled_data.get('force_video', False): + if no_playlist: + self.to_screen( + 'Downloading just video %s because of --no-playlist' + % video_id) + if '>Subscribe to unlock' in webpage: + raise ExtractorError( + 'This content is only available for subscribers', + expected=True) + vimeo_id = self._search_regex( + r'data-vimeo-id=["\'](\d+)', webpage, 'video id') + return self.url_result( + VimeoIE._smuggle_referrer( + 'https://player.vimeo.com/video/%s' % vimeo_id, url), + ie=VimeoIE.ie_key(), video_id=vimeo_id) + + self.to_screen( + 'Downloading playlist %s - add --no-playlist to just download video' + % course_id) + + lesson_ids = [lesson_id] + for lesson in re.findall( + r'(]+\bclass=["\']lesson-link[^>]+>)', webpage): + attrs = extract_attributes(lesson) + if not attrs: + continue + lesson_url = attrs.get('href') + if not lesson_url: + continue + lesson_id = self._search_regex( + r'/lessons/(\d+)', lesson_url, 'lesson id', default=None) + if not lesson_id: + continue + lesson_ids.append(lesson_id) + + entries = [] + for lesson_id in orderedSet(lesson_ids): + entries.append(self.url_result( + smuggle_url(urljoin(url, lesson_id), {'force_video': True}), + ie=RayWenderlichIE.ie_key())) + + title = self._search_regex( + r'class=["\']course-title[^>]+>([^<]+)', webpage, 'course title', + default=None) + + return self.playlist_result(entries, course_id, title) From a7298f3e99c8373d8c60be42f00d3b978d9b176a Mon Sep 17 00:00:00 2001 From: Eitan Postavsky <35902204+epostavs@users.noreply.github.com> Date: Fri, 9 Mar 2018 11:57:32 -0500 Subject: [PATCH 2/5] [pornhub] Don't override session cookies (closes #15697) --- youtube_dl/extractor/pornhub.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py index 281a4f05e..a7099bbbf 100644 --- a/youtube_dl/extractor/pornhub.py +++ b/youtube_dl/extractor/pornhub.py @@ -114,13 +114,14 @@ class PornHubIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) + + self._set_cookie('pornhub.com', 'age_verified', '1') def dl_webpage(platform): + self._set_cookie('pornhub.com', 'platform', platform) return self._download_webpage( 'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id, - video_id, headers={ - 'Cookie': 'age_verified=1; platform=%s' % platform, - }) + video_id) webpage = dl_webpage('pc') From 5b1d158834f9202f29b650c4467fe144777380b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 10 Mar 2018 04:31:51 +0700 Subject: [PATCH 3/5] [raywenderlich] Extract videos in order --- youtube_dl/extractor/raywenderlich.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/raywenderlich.py b/youtube_dl/extractor/raywenderlich.py index 0f061c4b2..2a90677f4 100644 --- a/youtube_dl/extractor/raywenderlich.py +++ b/youtube_dl/extractor/raywenderlich.py @@ -75,7 +75,7 @@ class RayWenderlichIE(InfoExtractor): 'Downloading playlist %s - add --no-playlist to just download video' % course_id) - lesson_ids = [lesson_id] + lesson_ids = set((lesson_id, )) for lesson in re.findall( r'(]+\bclass=["\']lesson-link[^>]+>)', webpage): attrs = extract_attributes(lesson) @@ -88,10 +88,10 @@ class RayWenderlichIE(InfoExtractor): r'/lessons/(\d+)', lesson_url, 'lesson id', default=None) if not lesson_id: continue - lesson_ids.append(lesson_id) + lesson_ids.add(lesson_id) entries = [] - for lesson_id in orderedSet(lesson_ids): + for lesson_id in sorted(lesson_ids): entries.append(self.url_result( smuggle_url(urljoin(url, lesson_id), {'force_video': True}), ie=RayWenderlichIE.ie_key())) From 7dee4171276002b73428aa51091722665dc77101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 10 Mar 2018 04:44:46 +0700 Subject: [PATCH 4/5] [ChangeLog] Actualize [ci skip] --- ChangeLog | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ChangeLog b/ChangeLog index 56b72b31e..e05fcef9e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,27 @@ +version + +Core +* [downloader/hls] Skip uplynk ad fragments (#15748) + +Extractors +* [pornhub] Don't override session cookies (#15697) ++ [raywenderlich] Add support for videos.raywenderlich.com (#15251) +* [funk] Fix extraction and rework extractors (#15792) +* [nexx] Restore reverse engineered approach ++ [heise] Add support for kaltura embeds (#14961, #15728) ++ [tvnow] Extract series metadata (#15774) +* [ruutu] Continue formats extraction on NOT-USED URLs (#15775) +* [vrtnu] Use redirect URL for building video JSON URL (#15767, #15769) +* [vimeo] Modernize login code and improve error messaging +* [archiveorg] Fix extraction (#15770, #15772) ++ [hidive] Add support for hidive.com (#15494) +* [afreecatv] Detect deleted videos +* [afreecatv] Fix extraction (#15755) +* [vice] Fix extraction and rework extractors (#11101, #13019, #13622, #13778) ++ [vidzi] Add support for vidzi.si (#15751) +* [npo] Fix typo + + version 2018.03.03 Core From 3a0ceb32e2150e8bfaf6fc2402c995a0b138d4ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 10 Mar 2018 04:45:57 +0700 Subject: [PATCH 5/5] release 2018.03.10 --- .github/ISSUE_TEMPLATE.md | 6 +++--- ChangeLog | 2 +- docs/supportedsites.md | 6 ++++-- youtube_dl/version.py | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 520c7d163..bc0c5ef18 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -6,8 +6,8 @@ --- -### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.03.03*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected. -- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.03.03** +### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.03.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected. +- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.03.10** ### Before submitting an *issue* make sure you have: - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections @@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 -[debug] youtube-dl version 2018.03.03 +[debug] youtube-dl version 2018.03.10 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/ChangeLog b/ChangeLog index e05fcef9e..2cc256f98 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -version +version 2018.03.10 Core * [downloader/hls] Skip uplynk ad fragments (#15748) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 61cf55c2a..cb11f1b42 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -298,7 +298,8 @@ - **freespeech.org** - **FreshLive** - **Funimation** - - **Funk** + - **FunkChannel** + - **FunkMix** - **FunnyOrDie** - **Fusion** - **Fux** @@ -336,6 +337,7 @@ - **HentaiStigma** - **hetklokhuis** - **hgtv.com:show** + - **HiDive** - **HistoricFilms** - **history:topic**: History.com Topic - **hitbox** @@ -674,6 +676,7 @@ - **RaiPlay** - **RaiPlayLive** - **RaiPlayPlaylist** + - **RayWenderlich** - **RBMARadio** - **RDS**: RDS.ca - **RedBullTV** @@ -934,7 +937,6 @@ - **vice** - **vice:article** - **vice:show** - - **Viceland** - **Vidbit** - **Viddler** - **Videa** diff --git a/youtube_dl/version.py b/youtube_dl/version.py index aa5ed906e..a35d10818 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2018.03.03' +__version__ = '2018.03.10'