From 3cba49881a1aea9d222ed20e9b6d430f253b0584 Mon Sep 17 00:00:00 2001
From: zmobbie <ottoxas@hotmail.com>
Date: Thu, 11 Aug 2016 06:07:45 +0300
Subject: [PATCH 01/21] [kanal2]: Add new extractor

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/kanal2.py     | 152 +++++++++++++++++++++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 youtube_dl/extractor/kanal2.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6a5d12ab1..24c493be5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -517,6 +517,7 @@ from .jwplatform import JWPlatformIE
 from .jpopsukitv import JpopsukiIE
 from .kakao import KakaoIE
 from .kaltura import KalturaIE
+from .kanal2 import Kanal2IE
 from .kanalplay import KanalPlayIE
 from .kankan import KankanIE
 from .karaoketv import KaraoketvIE
diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
new file mode 100644
index 000000000..7a405e561
--- /dev/null
+++ b/youtube_dl/extractor/kanal2.py
@@ -0,0 +1,152 @@
+# coding: ISO-8859-15
+from __future__ import unicode_literals
+from datetime import (
+    datetime,
+    timedelta,
+)
+
+from .common import InfoExtractor
+from ..utils import (
+    update_url_query,
+    xpath_text,
+    str_to_int,
+    int_or_none,
+    HEADRequest,
+    unescapeHTML,
+)
+
+import re
+
+
+class Kanal2IE(InfoExtractor):
+    _VALID_URL = r'(?P<base>https?:\/\/.+\.postimees\.ee)[a-zA-Z0-9\/._-]+\?[a-zA-Z0-9=&._-]*id=(?P<id>[a-zA-Z0-9_-]+)[^ ]*'
+    _TESTS = [{
+        # The most ordinary case
+        'url': 'http://kanal2.postimees.ee/pluss/video/?id=40792',
+        'md5': '173e29daea5f5fab49390bddd78aaaf0',
+        'info_dict': {
+            'id': '40792',
+            'ext': 'flv',
+            'title': 'Aedniku aabits (06.08.2016 10:30)',
+            'thumbnail': 'http://kanal2.postimees.ee/imagecache/http_img.cdn.mind.ee/kanal2//14/100/00033/0053_4468c974c1010a21817c1ee37f3e7902.jpeg',
+            'description': 'Aedniku aabits" on saade, mis pakub kaasaelamist ja teadmisi nii algajatele, kui juba kogenud rohenäppudele. Kõik alates vajalikest näpunäidetest, nutikatest lahendustest, uudistoodetest kuni taimede hingeeluni ning aias kasutatava tehnikani välja.',
+            'upload_date': '20160805',
+            'timestamp': 1470434400,
+        }
+    }, {
+        # Embed player, also needs login in reality but all the streams are accessable without logging in
+        'url': 'http://kanal2.postimees.ee/video/lonelyPlayer?videoid=28848',
+        'md5': '18edb2fd235c06a60b81b3590a357ace',
+        'info_dict': {
+            'id': '28848',
+            'ext': 'flv',
+            'title': 'Viimane võmm - Rita, ära jama (24.11.2015 21:30)',
+            'thumbnail': 'http://kanal2.postimees.ee/imagecache/http_img.cdn.mind.ee/kanal2//14/100/00002/0050_4468c974c1010a21817c1ee37f3e7902.jpeg',
+            'description': 'Kinnisvaraomanik Villem Meius leitakse oma korterist tapetuna. Turvakaamera video paljastab surnukeha kõrvalt lahkumas ühe Meiuse üürniku - ei kellegi muu, kui politseinike kaitseingli Rita! Rita võetakse vahi alla ning kogu jaoskond näeb vaeva selle nimel, et teda vabastada ning tema kinniistumise ajal Rita baari käigus hoida. Uurimise käigus paljastub ulatuslik ja häbitu kinnisvarahangeldamine Kalamajas, mille niidid ulatuvad ka justiitsmaailma ladvikusse. Vastasleeri moodustavad Kalamaja põliselanikud. Organisatsiooni peakorter asub kellegi Mort Pärgi matusebüroos. Sealt hakkabki asi lõpuks hargnema.'
+        }
+    }, {
+        # Other ordinary case
+        'url': 'http://kanal2.postimees.ee/pluss/preview?id=40744',
+        'md5': '2579cdbf16013d7e7a7361a832bc818e',
+        'info_dict': {
+            'id': '40744',
+            'ext': 'flv',
+            'title': 'Kaunis Dila (10.08.2016 19:00)',
+            'thumbnail': 'http://kanal2.postimees.ee/imagecache/http_img.cdn.mind.ee/kanal2//16/300/00208/0050_4468c974c1010a21817c1ee37f3e7902.jpeg',
+        }
+    }, {
+        # Not on kanal2 subdomain like others, the site has different layout, so a lot of data can't be accessed, but the api's are same. also has rating
+        'url': 'http://kanal12.postimees.ee/vaatasaateid/Punkri-joulueri?videoid=248',
+        'md5': '4633c310980201e4d8195d22b948ad10',
+        'info_dict': {
+            'id': '248',
+            'ext': 'flv',
+            'title': 'Punkri jõulueri',
+            'thumbnail': 'http://img.cdn.mind.ee/kanal2/clips/KANAL 12/punkri joulueri.jpeg',
+            'description': 'Eestlaste lemmik-krõbesaade lõpetab aasta loodetavasti südamliku pühade-eriga! Hapukapsad ninast välja! Jeesuse sündi on tulnud tähistama Ivo Linna, pastor, saatan ja paljud teised. Saadet juhivad Marge Tava, Aleksander Ots ja Marek Reinaas.',
+            'average_rating': int,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        base = re.compile(self._VALID_URL).match(url).group('base')
+        if "pluss" not in url and "kanal2" in base:
+            url = base + '/pluss/video/?id=' + video_id
+            # This part copied from generic.py, bypasses redirects
+            head_response = self._request_webpage(HEADRequest(url), video_id)
+            if head_response is not False:
+                new_url = head_response.geturl()
+                if url != new_url:
+                    self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
+                return self.url_result(new_url)
+
+        xmlfile = self._download_xml(update_url_query(base + '/video/playerPlaylistApi', {'id': video_id}), video_id)
+        host = xmlfile.find('./playlist/video/streamItems').get('host')
+
+        formats = [{
+            'protocol': re.compile('(?P<protocol>.+):\/\/[^\0]*').match(host).group('protocol') or 'rtmp',
+            'app': re.compile(((re.compile('(?P<protocol>.+):\/\/[^\0]*').match(host).group('protocol') or 'rtmp') + ':\/\/[^\0]*\/(?P<app>.+\/)')).match(host).group('app') or 'kanal2vod',
+            'url': host + stream.get('streamName'),
+            'play_path': 'mp4:' + stream.get('streamName'),
+            'ext': 'flv',
+            'height': str_to_int(stream.get('height')),
+            'width': str_to_int(stream.get('width')),
+            'rtmp_real_time': True,
+        } for stream in xmlfile.findall('./playlist/video/streamItems/streamItem')]
+        self._sort_formats(formats)
+
+        # Remove stacked urls(e.g. http://test.comhttp://test2.com, removes everything before second http(kanal12 fix))
+        thumbnail = re.compile('[^\0]*(?P<realurl>https?:\/\/[^"]+)[^\0]*').match(base + xpath_text(xmlfile, './playlist/video/thumbUrl')).group('realurl')
+        average_rating = int_or_none(xpath_text(xmlfile, './playlist/video/rating/value'))
+
+        webpage = self._download_webpage(url, video_id)
+        if 'player-container' in webpage:
+            description = self._search_regex(r'[^\0]*<p class="full"[^>]*>([^<]*)<\/p>[^\0]*', webpage, 'description', default=None)
+            if description is not None:
+                description = description.strip()
+
+            epandseasonregex = re.compile('Osa *(?P<episode>[0-9]+) *Hooaeg *(?P<season>[0-9]+)').match(self._search_regex('[^\0]*(Osa *[0-9]+ *Hooaeg *[0-9]+)[^\0]*', webpage, 'epandseason', default=None))
+            if epandseasonregex is not None:
+                episode = int_or_none(epandseasonregex.group('episode'))
+                season = int_or_none(epandseasonregex.group('season'))
+
+            dateandtimeregex = re.compile('[^\0]*eetris[^\0]*<\/span>[^\0]*(?P<date>[0-9]{1,2}.[0-9]{1,2}.[0-9]{4,})[^0-9]*(?P<time>[0-9]{1,2}:[0-9]{1,2})[^\0]*').match(self._search_regex('[^\0]*(eetris[^\0]*<\/span>[^\0]*[0-9]{1,2}.[0-9]{1,2}.[0-9]{4,}[^0-9]*[0-9]{1,2}:[0-9]{1,2})[^\0]*', webpage, 'dateandtime', default=None))
+            if dateandtimeregex is not None:
+                date = dateandtimeregex.group('date')
+                time = dateandtimeregex.group('time')
+                timestamp = int_or_none((datetime.strptime(date + " " + time, '%d.%m.%Y %H:%M') - datetime(1970, 1, 1) + timedelta(seconds=60 * 60 * 2)).total_seconds())  # No dst support, but added the 2 default hours of estonia
+            player_url = self._search_regex('[^\0]embedSWF\("([^"]+)[^\0]', webpage, 'player_url', default=None)
+
+        else:
+            description = None
+            player_url = None
+            season = None
+            episode = None
+            timestamp = None
+
+        if description is None:
+            description = xpath_text(xmlfile, './playlist/video/description') or self._search_regex('[^\0]og:description" *content="(.*)\" *\/>', webpage, 'description', default=None)
+            if description is not None:
+                description = unescapeHTML(description).strip()
+
+        if episode is None:
+            episode = int_or_none(xpath_text(xmlfile, './playlist/video/episode'))
+
+        title = xpath_text(xmlfile, './playlist/video/name')
+        if title is None:
+            title = self._search_regex('[^\0]og:title" *content="(.*)\" *\/>', webpage, 'title', default=None) or self._search_regex('[^\0]<title>(.*)<\/title>[^\0]', webpage, 'description', default=None)
+
+        return {
+            'average_rating': average_rating,
+            'description': description,
+            'episode_number': episode,
+            'formats': formats,
+            'id': video_id,
+            'page_url': url,
+            'player_url': player_url,
+            'season_number': season,
+            'timestamp': timestamp,
+            'title': title,
+            'thumbnail': thumbnail,
+        }

From 882731cb99775f25e8daeb555633c486d9eb191e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Sun, 16 Dec 2018 17:18:03 +0200
Subject: [PATCH 02/21] [kanal2]: flake: fix W605 invalid escape sequence

---
 youtube_dl/extractor/kanal2.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 7a405e561..03b4f964b 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -19,7 +19,7 @@ import re
 
 
 class Kanal2IE(InfoExtractor):
-    _VALID_URL = r'(?P<base>https?:\/\/.+\.postimees\.ee)[a-zA-Z0-9\/._-]+\?[a-zA-Z0-9=&._-]*id=(?P<id>[a-zA-Z0-9_-]+)[^ ]*'
+    _VALID_URL = r'(?P<base>https?://.+\.postimees\.ee)[a-zA-Z0-9/._-]+\?[a-zA-Z0-9=&._-]*id=(?P<id>[a-zA-Z0-9_-]+)[^ ]*'
     _TESTS = [{
         # The most ordinary case
         'url': 'http://kanal2.postimees.ee/pluss/video/?id=40792',
@@ -85,8 +85,8 @@ class Kanal2IE(InfoExtractor):
         host = xmlfile.find('./playlist/video/streamItems').get('host')
 
         formats = [{
-            'protocol': re.compile('(?P<protocol>.+):\/\/[^\0]*').match(host).group('protocol') or 'rtmp',
-            'app': re.compile(((re.compile('(?P<protocol>.+):\/\/[^\0]*').match(host).group('protocol') or 'rtmp') + ':\/\/[^\0]*\/(?P<app>.+\/)')).match(host).group('app') or 'kanal2vod',
+            'protocol': re.compile('(?P<protocol>.+)://[^\0]*').match(host).group('protocol') or 'rtmp',
+            'app': re.compile(((re.compile('(?P<protocol>.+)://[^\0]*').match(host).group('protocol') or 'rtmp') + '://[^\0]*/(?P<app>.+/)')).match(host).group('app') or 'kanal2vod',
             'url': host + stream.get('streamName'),
             'play_path': 'mp4:' + stream.get('streamName'),
             'ext': 'flv',
@@ -97,12 +97,12 @@ class Kanal2IE(InfoExtractor):
         self._sort_formats(formats)
 
         # Remove stacked urls(e.g. http://test.comhttp://test2.com, removes everything before second http(kanal12 fix))
-        thumbnail = re.compile('[^\0]*(?P<realurl>https?:\/\/[^"]+)[^\0]*').match(base + xpath_text(xmlfile, './playlist/video/thumbUrl')).group('realurl')
+        thumbnail = re.compile('[^\0]*(?P<realurl>https?://[^"]+)[^\0]*').match(base + xpath_text(xmlfile, './playlist/video/thumbUrl')).group('realurl')
         average_rating = int_or_none(xpath_text(xmlfile, './playlist/video/rating/value'))
 
         webpage = self._download_webpage(url, video_id)
         if 'player-container' in webpage:
-            description = self._search_regex(r'[^\0]*<p class="full"[^>]*>([^<]*)<\/p>[^\0]*', webpage, 'description', default=None)
+            description = self._search_regex(r'[^\0]*<p class="full"[^>]*>([^<]*)</p>[^\0]*', webpage, 'description', default=None)
             if description is not None:
                 description = description.strip()
 
@@ -111,12 +111,12 @@ class Kanal2IE(InfoExtractor):
                 episode = int_or_none(epandseasonregex.group('episode'))
                 season = int_or_none(epandseasonregex.group('season'))
 
-            dateandtimeregex = re.compile('[^\0]*eetris[^\0]*<\/span>[^\0]*(?P<date>[0-9]{1,2}.[0-9]{1,2}.[0-9]{4,})[^0-9]*(?P<time>[0-9]{1,2}:[0-9]{1,2})[^\0]*').match(self._search_regex('[^\0]*(eetris[^\0]*<\/span>[^\0]*[0-9]{1,2}.[0-9]{1,2}.[0-9]{4,}[^0-9]*[0-9]{1,2}:[0-9]{1,2})[^\0]*', webpage, 'dateandtime', default=None))
+            dateandtimeregex = re.compile('[^\0]*eetris[^\0]*</span>[^\0]*(?P<date>[0-9]{1,2}.[0-9]{1,2}.[0-9]{4,})[^0-9]*(?P<time>[0-9]{1,2}:[0-9]{1,2})[^\0]*').match(self._search_regex('[^\0]*(eetris[^\0]*</span>[^\0]*[0-9]{1,2}.[0-9]{1,2}.[0-9]{4,}[^0-9]*[0-9]{1,2}:[0-9]{1,2})[^\0]*', webpage, 'dateandtime', default=None))
             if dateandtimeregex is not None:
                 date = dateandtimeregex.group('date')
                 time = dateandtimeregex.group('time')
                 timestamp = int_or_none((datetime.strptime(date + " " + time, '%d.%m.%Y %H:%M') - datetime(1970, 1, 1) + timedelta(seconds=60 * 60 * 2)).total_seconds())  # No dst support, but added the 2 default hours of estonia
-            player_url = self._search_regex('[^\0]embedSWF\("([^"]+)[^\0]', webpage, 'player_url', default=None)
+            player_url = self._search_regex('[^\0]embedSWF\\("([^"]+)[^\0]', webpage, 'player_url', default=None)
 
         else:
             description = None
@@ -126,7 +126,7 @@ class Kanal2IE(InfoExtractor):
             timestamp = None
 
         if description is None:
-            description = xpath_text(xmlfile, './playlist/video/description') or self._search_regex('[^\0]og:description" *content="(.*)\" *\/>', webpage, 'description', default=None)
+            description = xpath_text(xmlfile, './playlist/video/description') or self._search_regex('[^\0]og:description" *content="(.*)\" */>', webpage, 'description', default=None)
             if description is not None:
                 description = unescapeHTML(description).strip()
 
@@ -135,7 +135,7 @@ class Kanal2IE(InfoExtractor):
 
         title = xpath_text(xmlfile, './playlist/video/name')
         if title is None:
-            title = self._search_regex('[^\0]og:title" *content="(.*)\" *\/>', webpage, 'title', default=None) or self._search_regex('[^\0]<title>(.*)<\/title>[^\0]', webpage, 'description', default=None)
+            title = self._search_regex('[^\0]og:title" *content="(.*)\" */>', webpage, 'title', default=None) or self._search_regex('[^\0]<title>(.*)</title>[^\0]', webpage, 'description', default=None)
 
         return {
             'average_rating': average_rating,

From 3c110d17dba83f9047e87485b3022b80465b5e70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Sun, 16 Dec 2018 17:18:07 +0200
Subject: [PATCH 03/21] [kanal2]: fix coding hint encoding

---
 youtube_dl/extractor/kanal2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 03b4f964b..ebbfc6b83 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -1,4 +1,4 @@
-# coding: ISO-8859-15
+# coding: utf-8
 from __future__ import unicode_literals
 from datetime import (
     datetime,

From 816c8a592d36761b49d772bbddf9db414e45c7f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Mon, 17 Dec 2018 01:49:23 +0200
Subject: [PATCH 04/21] [kanal2]: extractor rewritten to use json

some scenarios dropped, test urls missing
---
 youtube_dl/extractor/kanal2.py | 243 ++++++++++++++-------------------
 1 file changed, 104 insertions(+), 139 deletions(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index ebbfc6b83..5b2d12b31 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -1,152 +1,117 @@
 # coding: utf-8
 from __future__ import unicode_literals
-from datetime import (
-    datetime,
-    timedelta,
-)
-
-from .common import InfoExtractor
-from ..utils import (
-    update_url_query,
-    xpath_text,
-    str_to_int,
-    int_or_none,
-    HEADRequest,
-    unescapeHTML,
-)
-
+from datetime import datetime
 import re
+import time
+from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class Kanal2IE(InfoExtractor):
+    SUBTITLE_DATE_RE = re.compile(r'\((\d{2}\.\d{2}\.\d{4}\s\d{2}:\d{2})\)$')
+
     _VALID_URL = r'(?P<base>https?://.+\.postimees\.ee)[a-zA-Z0-9/._-]+\?[a-zA-Z0-9=&._-]*id=(?P<id>[a-zA-Z0-9_-]+)[^ ]*'
-    _TESTS = [{
-        # The most ordinary case
-        'url': 'http://kanal2.postimees.ee/pluss/video/?id=40792',
-        'md5': '173e29daea5f5fab49390bddd78aaaf0',
-        'info_dict': {
-            'id': '40792',
-            'ext': 'flv',
-            'title': 'Aedniku aabits (06.08.2016 10:30)',
-            'thumbnail': 'http://kanal2.postimees.ee/imagecache/http_img.cdn.mind.ee/kanal2//14/100/00033/0053_4468c974c1010a21817c1ee37f3e7902.jpeg',
-            'description': 'Aedniku aabits" on saade, mis pakub kaasaelamist ja teadmisi nii algajatele, kui juba kogenud rohenäppudele. Kõik alates vajalikest näpunäidetest, nutikatest lahendustest, uudistoodetest kuni taimede hingeeluni ning aias kasutatava tehnikani välja.',
-            'upload_date': '20160805',
-            'timestamp': 1470434400,
-        }
-    }, {
-        # Embed player, also needs login in reality but all the streams are accessable without logging in
-        'url': 'http://kanal2.postimees.ee/video/lonelyPlayer?videoid=28848',
-        'md5': '18edb2fd235c06a60b81b3590a357ace',
-        'info_dict': {
-            'id': '28848',
-            'ext': 'flv',
-            'title': 'Viimane võmm - Rita, ära jama (24.11.2015 21:30)',
-            'thumbnail': 'http://kanal2.postimees.ee/imagecache/http_img.cdn.mind.ee/kanal2//14/100/00002/0050_4468c974c1010a21817c1ee37f3e7902.jpeg',
-            'description': 'Kinnisvaraomanik Villem Meius leitakse oma korterist tapetuna. Turvakaamera video paljastab surnukeha kõrvalt lahkumas ühe Meiuse üürniku - ei kellegi muu, kui politseinike kaitseingli Rita! Rita võetakse vahi alla ning kogu jaoskond näeb vaeva selle nimel, et teda vabastada ning tema kinniistumise ajal Rita baari käigus hoida. Uurimise käigus paljastub ulatuslik ja häbitu kinnisvarahangeldamine Kalamajas, mille niidid ulatuvad ka justiitsmaailma ladvikusse. Vastasleeri moodustavad Kalamaja põliselanikud. Organisatsiooni peakorter asub kellegi Mort Pärgi matusebüroos. Sealt hakkabki asi lõpuks hargnema.'
-        }
-    }, {
-        # Other ordinary case
-        'url': 'http://kanal2.postimees.ee/pluss/preview?id=40744',
-        'md5': '2579cdbf16013d7e7a7361a832bc818e',
-        'info_dict': {
-            'id': '40744',
-            'ext': 'flv',
-            'title': 'Kaunis Dila (10.08.2016 19:00)',
-            'thumbnail': 'http://kanal2.postimees.ee/imagecache/http_img.cdn.mind.ee/kanal2//16/300/00208/0050_4468c974c1010a21817c1ee37f3e7902.jpeg',
-        }
-    }, {
-        # Not on kanal2 subdomain like others, the site has different layout, so a lot of data can't be accessed, but the api's are same. also has rating
-        'url': 'http://kanal12.postimees.ee/vaatasaateid/Punkri-joulueri?videoid=248',
-        'md5': '4633c310980201e4d8195d22b948ad10',
-        'info_dict': {
-            'id': '248',
-            'ext': 'flv',
-            'title': 'Punkri jõulueri',
-            'thumbnail': 'http://img.cdn.mind.ee/kanal2/clips/KANAL 12/punkri joulueri.jpeg',
-            'description': 'Eestlaste lemmik-krõbesaade lõpetab aasta loodetavasti südamliku pühade-eriga! Hapukapsad ninast välja! Jeesuse sündi on tulnud tähistama Ivo Linna, pastor, saatan ja paljud teised. Saadet juhivad Marge Tava, Aleksander Ots ja Marek Reinaas.',
-            'average_rating': int,
-        }
-    }]
+    _TESTS = [
+        {
+            # Standard url
+            'url': 'https://kanal2.postimees.ee/pluss/video/?id=40792',
+            'md5': 'cecaf3e17706d725b1f23e886b67f8d3',
+            'info_dict': {
+                'id': '40792',
+                'ext': 'mp4',
+                'title': 'Aedniku aabits / Osa 53  (05.08.2016 20:00)',
+                'thumbnail': 'https://kanal-dl.babahhcdn.com/kanal/2016/08/05/0053_HNqKsIA/img/2.jpg',
+                'description': 'Aedniku aabits" on saade, mis pakub kaasaelamist ja teadmisi nii algajatele, kui juba kogenud rohenäppudele. Kõik alates vajalikest näpunäidetest, nutikatest lahendustest, uudistoodetest kuni taimede hingeeluni ning aias kasutatava tehnikani välja.',
+                'upload_date': '20160805',
+                'timestamp': 1470416400,
+            }
+        },
+        {
+            # Other url example
+            'url': 'http://kanal2.postimees.ee/pluss/preview?id=40744',
+            'md5': 'e1dcc6e39d17a3f04749a8158db26377',
+            'info_dict': {
+                'id': '40744',
+                'ext': 'mp4',
+                'title': 'Kaunis Dila / Osa 50  (10.08.2016 19:00)',
+                'thumbnail': 'https://kanal-dl.babahhcdn.com/kanal/2018/12/05/16_300_00208_0050-Kaunis_Dila_hamdY9I/img/2.jpg',
+                'description': u'Riza ei tea, mis oht teda ja ta pere Selcuki n\xe4ol varitseb. Azer kahtlustab, et Fatma elus on uus mees ja on valmis k\xf5igeks, et ta endale tagasi v\xf5ita. See tekitab aga Arzus suurt hirmu.',
+                'timestamp': 1470844800,
+                'upload_date': '20160810',
+            }
+        },
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        base = re.compile(self._VALID_URL).match(url).group('base')
-        if "pluss" not in url and "kanal2" in base:
-            url = base + '/pluss/video/?id=' + video_id
-            # This part copied from generic.py, bypasses redirects
-            head_response = self._request_webpage(HEADRequest(url), video_id)
-            if head_response is not False:
-                new_url = head_response.geturl()
-                if url != new_url:
-                    self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
-                return self.url_result(new_url)
+        playlist = self.get_playlist(video_id)
 
-        xmlfile = self._download_xml(update_url_query(base + '/video/playerPlaylistApi', {'id': video_id}), video_id)
-        host = xmlfile.find('./playlist/video/streamItems').get('host')
-
-        formats = [{
-            'protocol': re.compile('(?P<protocol>.+)://[^\0]*').match(host).group('protocol') or 'rtmp',
-            'app': re.compile(((re.compile('(?P<protocol>.+)://[^\0]*').match(host).group('protocol') or 'rtmp') + '://[^\0]*/(?P<app>.+/)')).match(host).group('app') or 'kanal2vod',
-            'url': host + stream.get('streamName'),
-            'play_path': 'mp4:' + stream.get('streamName'),
-            'ext': 'flv',
-            'height': str_to_int(stream.get('height')),
-            'width': str_to_int(stream.get('width')),
-            'rtmp_real_time': True,
-        } for stream in xmlfile.findall('./playlist/video/streamItems/streamItem')]
-        self._sort_formats(formats)
-
-        # Remove stacked urls(e.g. http://test.comhttp://test2.com, removes everything before second http(kanal12 fix))
-        thumbnail = re.compile('[^\0]*(?P<realurl>https?://[^"]+)[^\0]*').match(base + xpath_text(xmlfile, './playlist/video/thumbUrl')).group('realurl')
-        average_rating = int_or_none(xpath_text(xmlfile, './playlist/video/rating/value'))
-
-        webpage = self._download_webpage(url, video_id)
-        if 'player-container' in webpage:
-            description = self._search_regex(r'[^\0]*<p class="full"[^>]*>([^<]*)</p>[^\0]*', webpage, 'description', default=None)
-            if description is not None:
-                description = description.strip()
-
-            epandseasonregex = re.compile('Osa *(?P<episode>[0-9]+) *Hooaeg *(?P<season>[0-9]+)').match(self._search_regex('[^\0]*(Osa *[0-9]+ *Hooaeg *[0-9]+)[^\0]*', webpage, 'epandseason', default=None))
-            if epandseasonregex is not None:
-                episode = int_or_none(epandseasonregex.group('episode'))
-                season = int_or_none(epandseasonregex.group('season'))
-
-            dateandtimeregex = re.compile('[^\0]*eetris[^\0]*</span>[^\0]*(?P<date>[0-9]{1,2}.[0-9]{1,2}.[0-9]{4,})[^0-9]*(?P<time>[0-9]{1,2}:[0-9]{1,2})[^\0]*').match(self._search_regex('[^\0]*(eetris[^\0]*</span>[^\0]*[0-9]{1,2}.[0-9]{1,2}.[0-9]{4,}[^0-9]*[0-9]{1,2}:[0-9]{1,2})[^\0]*', webpage, 'dateandtime', default=None))
-            if dateandtimeregex is not None:
-                date = dateandtimeregex.group('date')
-                time = dateandtimeregex.group('time')
-                timestamp = int_or_none((datetime.strptime(date + " " + time, '%d.%m.%Y %H:%M') - datetime(1970, 1, 1) + timedelta(seconds=60 * 60 * 2)).total_seconds())  # No dst support, but added the 2 default hours of estonia
-            player_url = self._search_regex('[^\0]embedSWF\\("([^"]+)[^\0]', webpage, 'player_url', default=None)
-
-        else:
-            description = None
-            player_url = None
-            season = None
-            episode = None
-            timestamp = None
-
-        if description is None:
-            description = xpath_text(xmlfile, './playlist/video/description') or self._search_regex('[^\0]og:description" *content="(.*)\" */>', webpage, 'description', default=None)
-            if description is not None:
-                description = unescapeHTML(description).strip()
-
-        if episode is None:
-            episode = int_or_none(xpath_text(xmlfile, './playlist/video/episode'))
-
-        title = xpath_text(xmlfile, './playlist/video/name')
-        if title is None:
-            title = self._search_regex('[^\0]og:title" *content="(.*)\" */>', webpage, 'title', default=None) or self._search_regex('[^\0]<title>(.*)</title>[^\0]', webpage, 'description', default=None)
-
-        return {
-            'average_rating': average_rating,
-            'description': description,
-            'episode_number': episode,
-            'formats': formats,
+        # return a dict, description from here:
+        # https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303
+        info = {
             'id': video_id,
-            'page_url': url,
-            'player_url': player_url,
-            'season_number': season,
-            'timestamp': timestamp,
-            'title': title,
-            'thumbnail': thumbnail,
+            'title': self.get_title(playlist['info']),
+            'description': playlist['info'].get('description'),
+            'webpage_url': playlist['data'].get('url'),
+            'thumbnail': playlist['data'].get('image'),
+            'formats': self.get_formats(playlist, video_id),
+            'timestamp': self.get_timestamp(playlist['info']['subtitle']),
         }
+
+        return info
+
+    def get_title(self, info):
+        title = info['title']
+
+        if info['subtitle']:
+            title += ' / ' + info['subtitle']
+
+        return title
+
+    def get_timestamp(self, subtitle):
+        # Extract timestamp from:
+        #  "subtitle": "Osa 53  (05.08.2016 20:00)",
+        match = self._search_regex(self.SUBTITLE_DATE_RE, subtitle, 'dateandtime', default=None)
+        if not match:
+            return None
+
+        # https://stackoverflow.com/a/27914405/2314626
+        date = datetime.strptime(match, '%d.%m.%Y %H:%M')
+        unixtime = time.mktime(date.timetuple())
+
+        return int(unixtime)
+
+    def get_formats(self, playlist, video_id):
+        formats = []
+        session = self.get_session(playlist['data']['path'], video_id)
+        sid = session.get('session')
+        for stream in playlist['data']['streams']:
+            formats.append({
+                'protocol': 'm3u8',
+                'ext': 'mp4',
+                'url': stream.get('file') + '&s=' + sid,
+            })
+
+        return formats
+
+    def get_playlist(self, video_id):
+        url = 'https://kanal2.postimees.ee/player/playlist/%(video_id)s?type=episodes' % {'video_id': video_id}
+        headers = {
+            'X-Requested-With': 'XMLHttpRequest',
+        }
+
+        return self._download_json(url, video_id, headers=headers)
+
+    def get_session(self, path, video_id):
+        url = 'https://sts.postimees.ee/session/register'
+        headers = {
+            'X-Original-URI': path,
+            'Accept': 'application/json',
+        }
+        session = self._download_json(url, video_id, headers=headers,
+                                      note='Creating session',
+                                      errnote='Error creating session')
+        if session['reason'] != 'OK':
+            raise ExtractorError('%s: Unable to obtain session' % self.IE_NAME)
+
+        return session

From 1e27c749ef9b42a4d601f4e7787b7c138e35a899 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Mon, 17 Dec 2018 02:04:10 +0200
Subject: [PATCH 05/21] [kanal2]: _sort_formats

---
 youtube_dl/extractor/kanal2.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 5b2d12b31..6e84849d2 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -92,6 +92,8 @@ class Kanal2IE(InfoExtractor):
                 'url': stream.get('file') + '&s=' + sid,
             })
 
+        self._sort_formats(formats)
+
         return formats
 
     def get_playlist(self, video_id):

From 880efe9557274d208ae0e64201fd99e7e8e79869 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Mon, 17 Dec 2018 02:24:11 +0200
Subject: [PATCH 06/21] [kanal2]: use note key for test description

---
 youtube_dl/extractor/kanal2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 6e84849d2..efbd8871d 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -13,7 +13,7 @@ class Kanal2IE(InfoExtractor):
     _VALID_URL = r'(?P<base>https?://.+\.postimees\.ee)[a-zA-Z0-9/._-]+\?[a-zA-Z0-9=&._-]*id=(?P<id>[a-zA-Z0-9_-]+)[^ ]*'
     _TESTS = [
         {
-            # Standard url
+            'note': 'Test standard url (#18547)',
             'url': 'https://kanal2.postimees.ee/pluss/video/?id=40792',
             'md5': 'cecaf3e17706d725b1f23e886b67f8d3',
             'info_dict': {
@@ -27,7 +27,7 @@ class Kanal2IE(InfoExtractor):
             }
         },
         {
-            # Other url example
+            'note': 'Test preview url (#18547)',
             'url': 'http://kanal2.postimees.ee/pluss/preview?id=40744',
             'md5': 'e1dcc6e39d17a3f04749a8158db26377',
             'info_dict': {

From b1f933783bed9399ad09ba7cd00b9723a240a5bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Tue, 18 Dec 2018 02:22:21 +0200
Subject: [PATCH 07/21] [kanal2]: use raw utf8, unicode literals are forbidden

---
 youtube_dl/extractor/kanal2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index efbd8871d..0bc586a63 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -35,7 +35,7 @@ class Kanal2IE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Kaunis Dila / Osa 50  (10.08.2016 19:00)',
                 'thumbnail': 'https://kanal-dl.babahhcdn.com/kanal/2018/12/05/16_300_00208_0050-Kaunis_Dila_hamdY9I/img/2.jpg',
-                'description': u'Riza ei tea, mis oht teda ja ta pere Selcuki n\xe4ol varitseb. Azer kahtlustab, et Fatma elus on uus mees ja on valmis k\xf5igeks, et ta endale tagasi v\xf5ita. See tekitab aga Arzus suurt hirmu.',
+                'description': 'Riza ei tea, mis oht teda ja ta pere Selcuki näol varitseb. Azer kahtlustab, et Fatma elus on uus mees ja on valmis kõigeks, et ta endale tagasi võita. See tekitab aga Arzus suurt hirmu.',
                 'timestamp': 1470844800,
                 'upload_date': '20160810',
             }

From 9bd56961b532f9233c10ad0da0a5000f7a4c5839 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Tue, 18 Dec 2018 11:19:18 +0200
Subject: [PATCH 08/21] [kanal2]: inline internal methods

---
 youtube_dl/extractor/kanal2.py | 124 ++++++++++++++++-----------------
 1 file changed, 62 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 0bc586a63..e0bedce2d 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -43,77 +43,77 @@ class Kanal2IE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
+        def get_title(info):
+            title = info['title']
+
+            if info['subtitle']:
+                title += ' / ' + info['subtitle']
+
+            return title
+
+        def get_timestamp(subtitle):
+            # Extract timestamp from:
+            #  "subtitle": "Osa 53  (05.08.2016 20:00)",
+            match = self._search_regex(self.SUBTITLE_DATE_RE, subtitle, 'dateandtime', default=None)
+            if not match:
+                return None
+
+            # https://stackoverflow.com/a/27914405/2314626
+            date = datetime.strptime(match, '%d.%m.%Y %H:%M')
+            unixtime = time.mktime(date.timetuple())
+
+            return int(unixtime)
+
+        def get_formats(playlist, video_id):
+            formats = []
+            session = get_session(playlist['data']['path'], video_id)
+            sid = session.get('session')
+            for stream in playlist['data']['streams']:
+                formats.append({
+                    'protocol': 'm3u8',
+                    'ext': 'mp4',
+                    'url': stream.get('file') + '&s=' + sid,
+                })
+
+            self._sort_formats(formats)
+
+            return formats
+
+        def get_playlist(video_id):
+            url = 'https://kanal2.postimees.ee/player/playlist/%(video_id)s?type=episodes' % {'video_id': video_id}
+            headers = {
+                'X-Requested-With': 'XMLHttpRequest',
+            }
+
+            return self._download_json(url, video_id, headers=headers)
+
+        def get_session(path, video_id):
+            url = 'https://sts.postimees.ee/session/register'
+            headers = {
+                'X-Original-URI': path,
+                'Accept': 'application/json',
+            }
+            session = self._download_json(url, video_id, headers=headers,
+                                          note='Creating session',
+                                          errnote='Error creating session')
+            if session['reason'] != 'OK':
+                raise ExtractorError('%s: Unable to obtain session' % self.IE_NAME)
+
+            return session
+
         video_id = self._match_id(url)
-        playlist = self.get_playlist(video_id)
+        playlist = get_playlist(video_id)
 
         # return a dict, description from here:
         # https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303
         info = {
             'id': video_id,
-            'title': self.get_title(playlist['info']),
+            'title': get_title(playlist['info']),
             'description': playlist['info'].get('description'),
             'webpage_url': playlist['data'].get('url'),
             'thumbnail': playlist['data'].get('image'),
-            'formats': self.get_formats(playlist, video_id),
-            'timestamp': self.get_timestamp(playlist['info']['subtitle']),
+            'formats': get_formats(playlist, video_id),
+            'timestamp': get_timestamp(playlist['info']['subtitle']),
         }
 
         return info
-
-    def get_title(self, info):
-        title = info['title']
-
-        if info['subtitle']:
-            title += ' / ' + info['subtitle']
-
-        return title
-
-    def get_timestamp(self, subtitle):
-        # Extract timestamp from:
-        #  "subtitle": "Osa 53  (05.08.2016 20:00)",
-        match = self._search_regex(self.SUBTITLE_DATE_RE, subtitle, 'dateandtime', default=None)
-        if not match:
-            return None
-
-        # https://stackoverflow.com/a/27914405/2314626
-        date = datetime.strptime(match, '%d.%m.%Y %H:%M')
-        unixtime = time.mktime(date.timetuple())
-
-        return int(unixtime)
-
-    def get_formats(self, playlist, video_id):
-        formats = []
-        session = self.get_session(playlist['data']['path'], video_id)
-        sid = session.get('session')
-        for stream in playlist['data']['streams']:
-            formats.append({
-                'protocol': 'm3u8',
-                'ext': 'mp4',
-                'url': stream.get('file') + '&s=' + sid,
-            })
-
-        self._sort_formats(formats)
-
-        return formats
-
-    def get_playlist(self, video_id):
-        url = 'https://kanal2.postimees.ee/player/playlist/%(video_id)s?type=episodes' % {'video_id': video_id}
-        headers = {
-            'X-Requested-With': 'XMLHttpRequest',
-        }
-
-        return self._download_json(url, video_id, headers=headers)
-
-    def get_session(self, path, video_id):
-        url = 'https://sts.postimees.ee/session/register'
-        headers = {
-            'X-Original-URI': path,
-            'Accept': 'application/json',
-        }
-        session = self._download_json(url, video_id, headers=headers,
-                                      note='Creating session',
-                                      errnote='Error creating session')
-        if session['reason'] != 'OK':
-            raise ExtractorError('%s: Unable to obtain session' % self.IE_NAME)
-
-        return session

From 9ee6cd55b251bea4fb8957824550f07b2af2a1aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Tue, 18 Dec 2018 11:23:31 +0200
Subject: [PATCH 09/21] [kanal2]: avoid shadowing variables from outer scope

---
 youtube_dl/extractor/kanal2.py | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index e0bedce2d..7ddaa7b99 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -42,7 +42,7 @@ class Kanal2IE(InfoExtractor):
         },
     ]
 
-    def _real_extract(self, url):
+    def _real_extract(self, url_):
         def get_title(info):
             title = info['title']
 
@@ -101,19 +101,22 @@ class Kanal2IE(InfoExtractor):
 
             return session
 
-        video_id = self._match_id(url)
-        playlist = get_playlist(video_id)
+        def extract_info(url):
+            video_id = self._match_id(url)
+            playlist = get_playlist(video_id)
 
-        # return a dict, description from here:
-        # https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303
-        info = {
-            'id': video_id,
-            'title': get_title(playlist['info']),
-            'description': playlist['info'].get('description'),
-            'webpage_url': playlist['data'].get('url'),
-            'thumbnail': playlist['data'].get('image'),
-            'formats': get_formats(playlist, video_id),
-            'timestamp': get_timestamp(playlist['info']['subtitle']),
-        }
+            # return a dict, description from here:
+            # https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303
+            info = {
+                'id': video_id,
+                'title': get_title(playlist['info']),
+                'description': playlist['info'].get('description'),
+                'webpage_url': playlist['data'].get('url'),
+                'thumbnail': playlist['data'].get('image'),
+                'formats': get_formats(playlist, video_id),
+                'timestamp': get_timestamp(playlist['info']['subtitle']),
+            }
 
-        return info
+            return info
+
+        return extract_info(url_)

From f3bc82bcb716a24826ccff135ea9f52519680d64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 21:18:20 +0200
Subject: [PATCH 10/21] [kanal2]: Revert "[kanal2]: avoid shadowing variables
 from outer scope"

This reverts commits 4dc705f6d and 4dc705f6d
---
 youtube_dl/extractor/kanal2.py | 123 ++++++++++++++++-----------------
 1 file changed, 60 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 7ddaa7b99..0bc586a63 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -42,81 +42,78 @@ class Kanal2IE(InfoExtractor):
         },
     ]
 
-    def _real_extract(self, url_):
-        def get_title(info):
-            title = info['title']
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        playlist = self.get_playlist(video_id)
 
-            if info['subtitle']:
-                title += ' / ' + info['subtitle']
+        # return a dict, description from here:
+        # https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303
+        info = {
+            'id': video_id,
+            'title': self.get_title(playlist['info']),
+            'description': playlist['info'].get('description'),
+            'webpage_url': playlist['data'].get('url'),
+            'thumbnail': playlist['data'].get('image'),
+            'formats': self.get_formats(playlist, video_id),
+            'timestamp': self.get_timestamp(playlist['info']['subtitle']),
+        }
 
-            return title
+        return info
 
-        def get_timestamp(subtitle):
-            # Extract timestamp from:
-            #  "subtitle": "Osa 53  (05.08.2016 20:00)",
-            match = self._search_regex(self.SUBTITLE_DATE_RE, subtitle, 'dateandtime', default=None)
-            if not match:
-                return None
+    def get_title(self, info):
+        title = info['title']
 
-            # https://stackoverflow.com/a/27914405/2314626
-            date = datetime.strptime(match, '%d.%m.%Y %H:%M')
-            unixtime = time.mktime(date.timetuple())
+        if info['subtitle']:
+            title += ' / ' + info['subtitle']
 
-            return int(unixtime)
+        return title
 
-        def get_formats(playlist, video_id):
-            formats = []
-            session = get_session(playlist['data']['path'], video_id)
-            sid = session.get('session')
-            for stream in playlist['data']['streams']:
-                formats.append({
-                    'protocol': 'm3u8',
-                    'ext': 'mp4',
-                    'url': stream.get('file') + '&s=' + sid,
-                })
+    def get_timestamp(self, subtitle):
+        # Extract timestamp from:
+        #  "subtitle": "Osa 53  (05.08.2016 20:00)",
+        match = self._search_regex(self.SUBTITLE_DATE_RE, subtitle, 'dateandtime', default=None)
+        if not match:
+            return None
 
-            self._sort_formats(formats)
+        # https://stackoverflow.com/a/27914405/2314626
+        date = datetime.strptime(match, '%d.%m.%Y %H:%M')
+        unixtime = time.mktime(date.timetuple())
 
-            return formats
+        return int(unixtime)
 
-        def get_playlist(video_id):
-            url = 'https://kanal2.postimees.ee/player/playlist/%(video_id)s?type=episodes' % {'video_id': video_id}
-            headers = {
-                'X-Requested-With': 'XMLHttpRequest',
-            }
+    def get_formats(self, playlist, video_id):
+        formats = []
+        session = self.get_session(playlist['data']['path'], video_id)
+        sid = session.get('session')
+        for stream in playlist['data']['streams']:
+            formats.append({
+                'protocol': 'm3u8',
+                'ext': 'mp4',
+                'url': stream.get('file') + '&s=' + sid,
+            })
 
-            return self._download_json(url, video_id, headers=headers)
+        self._sort_formats(formats)
 
-        def get_session(path, video_id):
-            url = 'https://sts.postimees.ee/session/register'
-            headers = {
-                'X-Original-URI': path,
-                'Accept': 'application/json',
-            }
-            session = self._download_json(url, video_id, headers=headers,
-                                          note='Creating session',
-                                          errnote='Error creating session')
-            if session['reason'] != 'OK':
-                raise ExtractorError('%s: Unable to obtain session' % self.IE_NAME)
+        return formats
 
-            return session
+    def get_playlist(self, video_id):
+        url = 'https://kanal2.postimees.ee/player/playlist/%(video_id)s?type=episodes' % {'video_id': video_id}
+        headers = {
+            'X-Requested-With': 'XMLHttpRequest',
+        }
 
-        def extract_info(url):
-            video_id = self._match_id(url)
-            playlist = get_playlist(video_id)
+        return self._download_json(url, video_id, headers=headers)
 
-            # return a dict, description from here:
-            # https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303
-            info = {
-                'id': video_id,
-                'title': get_title(playlist['info']),
-                'description': playlist['info'].get('description'),
-                'webpage_url': playlist['data'].get('url'),
-                'thumbnail': playlist['data'].get('image'),
-                'formats': get_formats(playlist, video_id),
-                'timestamp': get_timestamp(playlist['info']['subtitle']),
-            }
+    def get_session(self, path, video_id):
+        url = 'https://sts.postimees.ee/session/register'
+        headers = {
+            'X-Original-URI': path,
+            'Accept': 'application/json',
+        }
+        session = self._download_json(url, video_id, headers=headers,
+                                      note='Creating session',
+                                      errnote='Error creating session')
+        if session['reason'] != 'OK':
+            raise ExtractorError('%s: Unable to obtain session' % self.IE_NAME)
 
-            return info
-
-        return extract_info(url_)
+        return session

From 8cf1fb0e185e263f4cdee00d224520b8931a3034 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 21:20:27 +0200
Subject: [PATCH 11/21] [kanal2]: cleanup "useless" comments

---
 youtube_dl/extractor/kanal2.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 0bc586a63..7415a8824 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -46,8 +46,6 @@ class Kanal2IE(InfoExtractor):
         video_id = self._match_id(url)
         playlist = self.get_playlist(video_id)
 
-        # return a dict, description from here:
-        # https://github.com/rg3/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303
         info = {
             'id': video_id,
             'title': self.get_title(playlist['info']),

From a6012b994e5c999a948f382e216430d9a1085933 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 21:32:41 +0200
Subject: [PATCH 12/21] [kanal2]: drop id=40744 test. resource is forbidden

---
 youtube_dl/extractor/kanal2.py | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 7415a8824..d34bcad39 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -26,20 +26,6 @@ class Kanal2IE(InfoExtractor):
                 'timestamp': 1470416400,
             }
         },
-        {
-            'note': 'Test preview url (#18547)',
-            'url': 'http://kanal2.postimees.ee/pluss/preview?id=40744',
-            'md5': 'e1dcc6e39d17a3f04749a8158db26377',
-            'info_dict': {
-                'id': '40744',
-                'ext': 'mp4',
-                'title': 'Kaunis Dila / Osa 50  (10.08.2016 19:00)',
-                'thumbnail': 'https://kanal-dl.babahhcdn.com/kanal/2018/12/05/16_300_00208_0050-Kaunis_Dila_hamdY9I/img/2.jpg',
-                'description': 'Riza ei tea, mis oht teda ja ta pere Selcuki näol varitseb. Azer kahtlustab, et Fatma elus on uus mees ja on valmis kõigeks, et ta endale tagasi võita. See tekitab aga Arzus suurt hirmu.',
-                'timestamp': 1470844800,
-                'upload_date': '20160810',
-            }
-        },
     ]
 
     def _real_extract(self, url):

From 330729fa5d6b0aa745e8b51f35195ee50f1e3dc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 21:33:48 +0200
Subject: [PATCH 13/21] [kanal2]: use md5:xxx in test data

---
 youtube_dl/extractor/kanal2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index d34bcad39..8fcc14dd6 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -21,7 +21,7 @@ class Kanal2IE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Aedniku aabits / Osa 53  (05.08.2016 20:00)',
                 'thumbnail': 'https://kanal-dl.babahhcdn.com/kanal/2016/08/05/0053_HNqKsIA/img/2.jpg',
-                'description': 'Aedniku aabits" on saade, mis pakub kaasaelamist ja teadmisi nii algajatele, kui juba kogenud rohenäppudele. Kõik alates vajalikest näpunäidetest, nutikatest lahendustest, uudistoodetest kuni taimede hingeeluni ning aias kasutatava tehnikani välja.',
+                'description': 'md5:53cabf3c5d73150d594747f727431248',
                 'upload_date': '20160805',
                 'timestamp': 1470416400,
             }

From b6bd9224684dc99ff216cf6ca5372b56ab443f48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 21:35:00 +0200
Subject: [PATCH 14/21] [kanal2]: use query for url query string

---
 youtube_dl/extractor/kanal2.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 8fcc14dd6..de5c30116 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -81,12 +81,15 @@ class Kanal2IE(InfoExtractor):
         return formats
 
     def get_playlist(self, video_id):
-        url = 'https://kanal2.postimees.ee/player/playlist/%(video_id)s?type=episodes' % {'video_id': video_id}
+        url = 'https://kanal2.postimees.ee/player/playlist/%(video_id)s' % {'video_id': video_id}
+        query = {
+            'type': 'episodes',
+        }
         headers = {
             'X-Requested-With': 'XMLHttpRequest',
         }
 
-        return self._download_json(url, video_id, headers=headers)
+        return self._download_json(url, video_id, headers=headers, query=query)
 
     def get_session(self, path, video_id):
         url = 'https://sts.postimees.ee/session/register'

From 7509538e645c2422edf1da354b524e297dfc05ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 21:36:20 +0200
Subject: [PATCH 15/21] [kanal2]: skip entries without file key

---
 youtube_dl/extractor/kanal2.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index de5c30116..648fd1a15 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -70,6 +70,8 @@ class Kanal2IE(InfoExtractor):
         session = self.get_session(playlist['data']['path'], video_id)
         sid = session.get('session')
         for stream in playlist['data']['streams']:
+            if not stream.get('file'):
+                continue
             formats.append({
                 'protocol': 'm3u8',
                 'ext': 'mp4',

From 379dfb9589ebb909719dc28d945e977c57265ee0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 21:39:27 +0200
Subject: [PATCH 16/21] [kanal2]: use url_or_none

---
 youtube_dl/extractor/kanal2.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 648fd1a15..468ff30da 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -4,8 +4,10 @@ from datetime import datetime
 import re
 import time
 from .common import InfoExtractor
-from ..utils import ExtractorError
-
+from ..utils import (
+    ExtractorError,
+    url_or_none,
+)
 
 class Kanal2IE(InfoExtractor):
     SUBTITLE_DATE_RE = re.compile(r'\((\d{2}\.\d{2}\.\d{4}\s\d{2}:\d{2})\)$')
@@ -75,7 +77,7 @@ class Kanal2IE(InfoExtractor):
             formats.append({
                 'protocol': 'm3u8',
                 'ext': 'mp4',
-                'url': stream.get('file') + '&s=' + sid,
+                'url': url_or_none(stream.get('file') + '&s=' + sid),
             })
 
         self._sort_formats(formats)

From c6bfc0df6b4d247a7cd5ac988a7838f1a0b93c03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 21:44:17 +0200
Subject: [PATCH 17/21] [kanal2]: handle empty data.streams key

---
 youtube_dl/extractor/kanal2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 468ff30da..0edebcbab 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -71,7 +71,7 @@ class Kanal2IE(InfoExtractor):
         formats = []
         session = self.get_session(playlist['data']['path'], video_id)
         sid = session.get('session')
-        for stream in playlist['data']['streams']:
+        for stream in playlist['data'].get('streams', []):
             if not stream.get('file'):
                 continue
             formats.append({

From 5af10e747ed0f7827bc95e1a7c01425ab3bf0f43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 21:45:22 +0200
Subject: [PATCH 18/21] [kanal2]: handle empty info.subtitle key

---
 youtube_dl/extractor/kanal2.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 0edebcbab..4a3592b9d 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -41,7 +41,7 @@ class Kanal2IE(InfoExtractor):
             'webpage_url': playlist['data'].get('url'),
             'thumbnail': playlist['data'].get('image'),
             'formats': self.get_formats(playlist, video_id),
-            'timestamp': self.get_timestamp(playlist['info']['subtitle']),
+            'timestamp': self.get_timestamp(playlist['info'].get('subtitle')),
         }
 
         return info
@@ -55,6 +55,8 @@ class Kanal2IE(InfoExtractor):
         return title
 
     def get_timestamp(self, subtitle):
+        if not subtitle:
+            return None
         # Extract timestamp from:
         #  "subtitle": "Osa 53  (05.08.2016 20:00)",
         match = self._search_regex(self.SUBTITLE_DATE_RE, subtitle, 'dateandtime', default=None)

From 565b143691a5a0f0ec343aa689d4c202db691d23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 22:06:26 +0200
Subject: [PATCH 19/21] [kanal2]: re-create _VALID_URL regex. drop unused
 groups; relaxed pattern

---
 youtube_dl/extractor/kanal2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 4a3592b9d..7058dd17c 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -12,7 +12,7 @@ from ..utils import (
 class Kanal2IE(InfoExtractor):
     SUBTITLE_DATE_RE = re.compile(r'\((\d{2}\.\d{2}\.\d{4}\s\d{2}:\d{2})\)$')
 
-    _VALID_URL = r'(?P<base>https?://.+\.postimees\.ee)[a-zA-Z0-9/._-]+\?[a-zA-Z0-9=&._-]*id=(?P<id>[a-zA-Z0-9_-]+)[^ ]*'
+    _VALID_URL = r'https?://.+\.postimees\.ee/[^?]+\?(.*?&)?id=(?P<id>\d+)'
     _TESTS = [
         {
             'note': 'Test standard url (#18547)',

From 906e4950931ff3d97803fa7d5a70d75cfc65d74f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 22:09:22 +0200
Subject: [PATCH 20/21] [kanal2]: use regexp for thumbnail test

---
 youtube_dl/extractor/kanal2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 7058dd17c..99896c067 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -22,7 +22,7 @@ class Kanal2IE(InfoExtractor):
                 'id': '40792',
                 'ext': 'mp4',
                 'title': 'Aedniku aabits / Osa 53  (05.08.2016 20:00)',
-                'thumbnail': 'https://kanal-dl.babahhcdn.com/kanal/2016/08/05/0053_HNqKsIA/img/2.jpg',
+                'thumbnail': r're:https?://.*\.jpg$',
                 'description': 'md5:53cabf3c5d73150d594747f727431248',
                 'upload_date': '20160805',
                 'timestamp': 1470416400,

From ea4023a270b83d146264c9f7cbf8dcf1cdb0fd1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Elan=20Ruusam=C3=A4e?= <glen@pld-linux.org>
Date: Wed, 2 Jan 2019 23:07:14 +0200
Subject: [PATCH 21/21] [kanal2]: adjust time with timezone local to the
 provider

---
 youtube_dl/extractor/kanal2.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kanal2.py b/youtube_dl/extractor/kanal2.py
index 99896c067..d14501b8c 100644
--- a/youtube_dl/extractor/kanal2.py
+++ b/youtube_dl/extractor/kanal2.py
@@ -9,6 +9,7 @@ from ..utils import (
     url_or_none,
 )
 
+
 class Kanal2IE(InfoExtractor):
     SUBTITLE_DATE_RE = re.compile(r'\((\d{2}\.\d{2}\.\d{4}\s\d{2}:\d{2})\)$')
 
@@ -67,7 +68,15 @@ class Kanal2IE(InfoExtractor):
         date = datetime.strptime(match, '%d.%m.%Y %H:%M')
         unixtime = time.mktime(date.timetuple())
 
-        return int(unixtime)
+        try:
+            import pytz
+            # the time is always in this timezone
+            tz = pytz.timezone('Europe/Tallinn')
+            unixtime += tz.utcoffset(date).microseconds
+        except ImportError:
+            pass
+
+        return unixtime
 
     def get_formats(self, playlist, video_id):
         formats = []