Replace PhantomJS with Node.js sandbox

PhantomJS is unmaintained and depends on the deprecated QtWebKit.

youtube_dl/YoutubeDL.py

@@ -95,7 +95,7 @@ from .utils import (
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
-from .extractor.openload import PhantomJSwrapper
+from .extractor.openload import NodejsWrapper
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
@@ -2285,7 +2285,7 @@ class YoutubeDL(object):
 
         exe_versions = FFmpegPostProcessor.get_versions(self)
         exe_versions['rtmpdump'] = rtmpdump_version()
-        exe_versions['phantomjs'] = PhantomJSwrapper._version()
+        exe_versions['nodejs'] = NodejsWrapper.version()
         exe_str = ', '.join(
             '%s %s' % (exe, v)
             for exe, v in sorted(exe_versions.items())

youtube_dl/extractor/openload.py

@@ -1,238 +1,50 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
+import base64
-import os
 import subprocess
-import tempfile
 
-from ..compat import (
-    compat_urlparse,
-    compat_kwargs,
-)
 from ..utils import (
     check_executable,
     encodeArgument,
     ExtractorError,
     get_exe_version,
-    is_outdated_version,
-    std_headers,
 )
 
 
-def cookie_to_dict(cookie):
+class NodejsWrapper(object):
-    cookie_dict = {
+    """Node.js wrapper class"""
-        'name': cookie.name,
-        'value': cookie.value,
-    }
-    if cookie.port_specified:
-        cookie_dict['port'] = cookie.port
-    if cookie.domain_specified:
-        cookie_dict['domain'] = cookie.domain
-    if cookie.path_specified:
-        cookie_dict['path'] = cookie.path
-    if cookie.expires is not None:
-        cookie_dict['expires'] = cookie.expires
-    if cookie.secure is not None:
-        cookie_dict['secure'] = cookie.secure
-    if cookie.discard is not None:
-        cookie_dict['discard'] = cookie.discard
-    try:
-        if (cookie.has_nonstandard_attr('httpOnly')
-                or cookie.has_nonstandard_attr('httponly')
-                or cookie.has_nonstandard_attr('HttpOnly')):
-            cookie_dict['httponly'] = True
-    except TypeError:
-        pass
-    return cookie_dict
-
-
-def cookie_jar_to_list(cookie_jar):
-    return [cookie_to_dict(cookie) for cookie in cookie_jar]
-
-
-class PhantomJSwrapper(object):
-    """PhantomJS wrapper class
-
-    This class is experimental.
-    """
-
-    _TEMPLATE = r'''
-        phantom.onError = function(msg, trace) {{
-          var msgStack = ['PHANTOM ERROR: ' + msg];
-          if(trace && trace.length) {{
-            msgStack.push('TRACE:');
-            trace.forEach(function(t) {{
-              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
-                + (t.function ? ' (in function ' + t.function +')' : ''));
-            }});
-          }}
-          console.error(msgStack.join('\n'));
-          phantom.exit(1);
-        }};
-        var page = require('webpage').create();
-        var fs = require('fs');
-        var read = {{ mode: 'r', charset: 'utf-8' }};
-        var write = {{ mode: 'w', charset: 'utf-8' }};
-        JSON.parse(fs.read("{cookies}", read)).forEach(function(x) {{
-          phantom.addCookie(x);
-        }});
-        page.settings.resourceTimeout = {timeout};
-        page.settings.userAgent = "{ua}";
-        page.onLoadStarted = function() {{
-          page.evaluate(function() {{
-            delete window._phantom;
-            delete window.callPhantom;
-          }});
-        }};
-        var saveAndExit = function() {{
-          fs.write("{html}", page.content, write);
-          fs.write("{cookies}", JSON.stringify(phantom.cookies), write);
-          phantom.exit();
-        }};
-        page.onLoadFinished = function(status) {{
-          if(page.url === "") {{
-            page.setContent(fs.read("{html}", read), "{url}");
-          }}
-          else {{
-            {jscode}
-          }}
-        }};
-        page.open("");
-    '''
-
-    _TMP_FILE_NAMES = ['script', 'html', 'cookies']
 
     @staticmethod
-    def _version():
+    def version():
-        return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
+        return get_exe_version('nodejs', version_re=r'([0-9.]+)')
 
-    def __init__(self, extractor, required_version=None, timeout=10000):
+    def __init__(self):
-        self._TMP_FILES = {}
+        self.exe = check_executable('nodejs', ['-v'])
-
-        self.exe = check_executable('phantomjs', ['-v'])
         if not self.exe:
-            raise ExtractorError('PhantomJS executable not found in PATH, '
+            raise ExtractorError('Node.js executable not found in PATH, '
-                                 'download it from http://phantomjs.org',
+                                 'download it from http://nodejs.org',
                                  expected=True)
 
-        self.extractor = extractor
+    def run_in_sandbox(self, jscode, timeout=10000):
-
-        if required_version:
-            version = self._version()
-            if is_outdated_version(version, required_version):
-                self.extractor._downloader.report_warning(
-                    'Your copy of PhantomJS is outdated, update it to version '
-                    '%s or newer if you encounter any errors.' % required_version)
-
-        self.options = {
-            'timeout': timeout,
-        }
-        for name in self._TMP_FILE_NAMES:
-            tmp = tempfile.NamedTemporaryFile(delete=False)
-            tmp.close()
-            self._TMP_FILES[name] = tmp
-
-    def __del__(self):
-        for name in self._TMP_FILE_NAMES:
-            try:
-                os.remove(self._TMP_FILES[name].name)
-            except (IOError, OSError, KeyError):
-                pass
-
-    def _save_cookies(self, url):
-        cookies = cookie_jar_to_list(self.extractor._downloader.cookiejar)
-        for cookie in cookies:
-            if 'path' not in cookie:
-                cookie['path'] = '/'
-            if 'domain' not in cookie:
-                cookie['domain'] = compat_urlparse.urlparse(url).netloc
-        with open(self._TMP_FILES['cookies'].name, 'wb') as f:
-            f.write(json.dumps(cookies).encode('utf-8'))
-
-    def _load_cookies(self):
-        with open(self._TMP_FILES['cookies'].name, 'rb') as f:
-            cookies = json.loads(f.read().decode('utf-8'))
-        for cookie in cookies:
-            if cookie['httponly'] is True:
-                cookie['rest'] = {'httpOnly': None}
-            if 'expiry' in cookie:
-                cookie['expire_time'] = cookie['expiry']
-            self.extractor._set_cookie(**compat_kwargs(cookie))
-
-    def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
         """
-        Downloads webpage (if needed) and executes JS
+        Executes JS in a sandbox
 
         Params:
-            url: website url
+            jscode: code to be executed
-            html: optional, html code of website
+            timeout: number of milliseconds before execution is cancelled
-            video_id: video id
-            note: optional, displayed when downloading webpage
-            note2: optional, displayed when executing JS
-            headers: custom http headers
-            jscode: code to be executed when page is loaded
 
-        Returns tuple with:
-            * downloaded website (after JS execution)
-            * anything you print with `console.log` (but not inside `page.execute`!)
-
-        In most cases you don't need to add any `jscode`.
-        It is executed in `page.onLoadFinished`.
-        `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
-        It is possible to wait for some element on the webpage, for example:
-            var check = function() {
-              var elementFound = page.evaluate(function() {
-                return document.querySelector('#b.done') !== null;
-              });
-              if(elementFound)
-                saveAndExit();
-              else
-                window.setTimeout(check, 500);
-            }
-
-            page.evaluate(function(){
-              document.querySelector('#a').click();
-            });
-            check();
         """
-        if 'saveAndExit();' not in jscode:
+        jscode_wrapper = (
-            raise ExtractorError('`saveAndExit();` not found in `jscode`')
+            'const vm = require("vm");'
-        if not html:
+            'const jscode = Buffer.from("{jscode_b64}", "base64").toString("binary");'
-            html = self.extractor._download_webpage(url, video_id, note=note, headers=headers)
+            'const options = {{timeout: {timeout:d}}};'
-        with open(self._TMP_FILES['html'].name, 'wb') as f:
+            'process.stdout.write(String(vm.runInNewContext(jscode, {{}}, options)));').format(
-            f.write(html.encode('utf-8'))
+                jscode_b64=base64.b64encode(jscode.encode('utf-8')).decode(),
-
+                timeout=timeout)
-        self._save_cookies(url)
+        args = [encodeArgument(a) for a in [self.exe, '-e', jscode_wrapper]]
-
+        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        replaces = self.options
+        stdout, stderr = p.communicate()
-        replaces['url'] = url
-        user_agent = headers.get('User-Agent') or std_headers['User-Agent']
-        replaces['ua'] = user_agent.replace('"', '\\"')
-        replaces['jscode'] = jscode
-
-        for x in self._TMP_FILE_NAMES:
-            replaces[x] = self._TMP_FILES[x].name.replace('\\', '\\\\').replace('"', '\\"')
-
-        with open(self._TMP_FILES['script'].name, 'wb') as f:
-            f.write(self._TEMPLATE.format(**replaces).encode('utf-8'))
-
-        if video_id is None:
-            self.extractor.to_screen('%s' % (note2,))
-        else:
-            self.extractor.to_screen('%s: %s' % (video_id, note2))
-
-        p = subprocess.Popen([
-            self.exe, '--ssl-protocol=any',
-            self._TMP_FILES['script'].name
-        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out, err = p.communicate()
         if p.returncode != 0:
-            raise ExtractorError(
+            raise ExtractorError(stderr.decode('utf-8', 'replace'))
-                'Executing JS failed\n:' + encodeArgument(err))
+        return stdout.decode('utf-8')
-        with open(self._TMP_FILES['html'].name, 'rb') as f:
-            html = f.read().decode('utf-8')
-
-        self._load_cookies()
-
-        return (html, encodeArgument(out))

youtube_dl/extractor/pornhub.py

@@ -11,8 +11,9 @@ from ..compat import (
     compat_HTTPError,
     compat_str,
     compat_urllib_request,
+    compat_urlparse,
 )
-from .openload import PhantomJSwrapper
+from .openload import NodejsWrapper
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -31,16 +32,26 @@ class PornHubBaseIE(InfoExtractor):
 
         webpage, urlh = dl(*args, **kwargs)
 
-        if any(re.search(p, webpage) for p in (
+        if re.search(r'<body\b[^>]+\bonload=["\']go\(\)', webpage):
-                r'<body\b[^>]+\bonload=["\']go\(\)',
-                r'document\.cookie\s*=\s*["\']RNKEY=',
-                r'document\.location\.reload\(true\)')):
             url_or_request = args[0]
             url = (url_or_request.get_full_url()
                    if isinstance(url_or_request, compat_urllib_request.Request)
                    else url_or_request)
-            phantom = PhantomJSwrapper(self, required_version='2.0')
+            domain = compat_urlparse.urlparse(url).netloc
-            phantom.get(url, html=webpage)
+            jscode = self._search_regex(
+                r'<script[^>]*>(?P<script>.*?function go\(\).*?)</script>',
+                webpage, 'jscode', flags=re.DOTALL, group='script')
+            self.to_screen('Executing JS on webpage')
+            cookie = NodejsWrapper().run_in_sandbox((
+                'let document = {{location:{{reload: function() {{}}}}}};'
+                '{jscode};//*/\n'
+                'go();'
+                'document.cookie').format(jscode=jscode))
+            cookie_name = cookie.split("=")[0].strip()
+            if cookie_name != 'RNKEY':
+                raise ExtractorError('Unexpected cookie name %r' % cookie_name)
+            cookie_value = cookie[len(cookie_name) + 1:].split(";")[0].strip()
+            self._set_cookie(domain, cookie_name, cookie_value)
             webpage, urlh = dl(*args, **kwargs)
 
         return webpage, urlh