Adding initial javascript interpreter to be able to support youtube as video backend.

MANIFEST

@@ -1,14 +1,21 @@
 .proverc
 AUTHORS
+bin/js-pruebas
 bin/peertube-dl
 bin/peertube-dl-hypnotoad
 bin/peertube-dl-web
 bin/peertube-dl-web.conf
 cpanfile
+include/javascript_builtins.h
+javascript_builtins.c
+javascript_interpreter_xs/javascript.xs
+javascript_interpreter_xs/Makefile.PL
+lib/auto/Peertube/DL/.exists
 lib/Peertube/DL.pm
 lib/Peertube/DL/Downloaders.pm
+lib/Peertube/DL/Javascript.pm
+lib/Peertube/DL/Javascript.xs
 lib/Peertube/DL/public/css/index.css
-lib/Peertube/DL/public/css/spinner.css
 lib/Peertube/DL/public/img/spinner.svg
 lib/Peertube/DL/public/index.html
 lib/Peertube/DL/public/js/peertube-dl-web.js
@@ -19,6 +26,9 @@ LICENSE
 Makefile.PL
 MANIFEST			This list of files
 README.md
+src/include/javascript_builtins.h
+src/javascript_builtins.c
+src/Makefile.PL
 t/00-use_ok.t
 t/downloaders/animeflv_example_response.html
 t/downloaders/gocdn.t

Makefile.PL

@@ -4,6 +4,16 @@ WriteMakefile(
     NAME        => 'Peertube::DL',
     VERSION     => '0.1',
     INST_SCRIPT => './bin',
+    INST_BIN    => './bin',
     test        => { TESTS => 't/*.t' },
     test        => { TESTS => 't/*/*.t' },
 );
+
+sub MY::postamble {
+    '
+src: src/Makefile
+	cd src && $(MAKE) $(PASSTHRU)
+javascript_interpreter_xs: javascript_interpreter_xs/Makefile
+	cd javascript_interpreter_xs/ && $(MAKE) $(PASSTHRU)
+';
+}

bin/js-pruebas

@@ -0,0 +1,19 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+use feature 'say';
+
+use Peertube::DL::Javascript;
+
+my $a = Peertube::DL::Javascript::_duk_create_heap_default();
+eval { Peertube::DL::Javascript::_duk_push_lstring( $a, "print(\"hola mundo\\n\");" ); };
+if ($@) {
+    warn $@;
+    $@ = "";
+}
+if ( defined $a ) {
+    printf( "0x%0x\n", $a );
+}
+Peertube::DL::Javascript::_duk_peval($a);
+Peertube::DL::Javascript::_duk_destroy_heap($a);

bin/peertube-dl

@@ -9,9 +9,12 @@ use Getopt::Long::Descriptive;
 
 use Peertube::DL::URLHandler;
 
+binmode STDOUT, ':utf8';
+
 my ( $opt, $usage ) = describe_options(
     'peertube-dl %o <url>',
-    [ 'recurse|r', 'Recursive in reproduction lists.', { default => 0 } ],
+    [ 'recurse|r',  'Recursive in reproduction lists.', { default => 0 } ],
+    [ 'format|f=s', 'Choose format by id.', ],
     [],
     [ 'help|h', 'Show this help.', { shortcircuit => 1 } ],
 );
@@ -19,10 +22,12 @@ my ( $opt, $usage ) = describe_options(
 print( $usage->text ), exit if $opt->help;
 
 my $recurse = $opt->recurse;
+my $format  = $opt->format;
 
 die "No url passed" unless @ARGV;
 
-my $download_data = Peertube::DL::URLHandler::getDownloadDataFromURL( $ARGV[0] );
+my $download_data =
+  Peertube::DL::URLHandler::getDownloadDataFromURL( $ARGV[0], { defined $format ? ( format => $format ) : () } );
 
 my $ua = Peertube::DL::URLHandler::generateUA();
 
@@ -31,13 +36,14 @@ if ( defined $download_data->{options}{list} && $download_data->{options}{list}
     if ($recurse) {
         for my $url (@$urls) {
             say "Handling nested url: $url";
-            my $url_download_data = Peertube::DL::URLHandler::getDownloadDataFromURL( $url );
+            my $url_download_data = Peertube::DL::URLHandler::getDownloadDataFromURL($url);
             downloadVideo( $ua, $url_download_data );
         }
     } else {
         say "The urls are:\n" . join "\n", @$urls;
     }
-
+} elsif ( defined $download_data->{options}{list_formats} && $download_data->{options}{list_formats} ) {
+    exit 0;
 } else {
     downloadVideo( $ua, $download_data );
 }
@@ -52,7 +58,7 @@ sub downloadVideo {
     die "Cannot retrieve video data" unless $response->is_success;
 
     my $content = $response->decoded_content;
-    
+
     say "Writing into $filename.";
     open my $fh, '>', $filename or die "Cannot open $filename";
     $fh->print( $response->decoded_content ) or die "Cannot write to $filename";

cpanfile

@@ -7,3 +7,4 @@ requires 'Test::Most';
 requires 'Test::MockObject';
 requires 'Mojo::Server::Hypnotoad';
 requires 'Getopt::Long::Descriptive';
+requires 'Perl::Tidy';

javascript_interpreter_xs/Makefile.PL

@@ -0,0 +1,11 @@
+use ExtUtils::MakeMaker;
+
+WriteMakefile(
+    NAME    => 'Peertube::DL::Javascript',
+    VERSION => '0.1',
+    LIBS    => ['-lduktape'],
+    INC     => '-Iduktape -I../src/include',
+    XS      => { 'javascript.xs' => 'javascript.o' },
+    OBJECT  => 'javascript.o ../src/javascript_builtins.o',
+    LDFLAGS => '-Wl-t',
+);

javascript_interpreter_xs/javascript.xs

@@ -0,0 +1,77 @@
+#define PERL_NO_GET_CONTEXT
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "duktape.h"
+#include "duk_config.h"
+#include "javascript_builtins.h"
+
+MODULE = Peertube::DL::Javascript  PACKAGE = Peertube::DL::Javascript
+PROTOTYPES: ENABLE
+
+SV *
+_duk_create_heap_default()
+    CODE:
+        duk_context *context = duk_create_heap_default();
+        if (context) {
+            duk_push_c_function(context, js_builtin_print, 1);
+            duk_put_global_string(context, "print");
+            RETVAL = newSVuv((size_t)context);
+        } else {
+            RETVAL = &PL_sv_undef;
+        }
+    OUTPUT:
+        RETVAL
+
+SV *
+_duk_push_lstring(SV *, SV *)
+    CODE:
+        duk_context *context = (duk_context *) SvUV(ST(0)); 
+        if(!context) {
+            croak("Javascript context undef.", 0);
+        }
+        STRLEN len;
+        char * lstring = SvPV(ST(1), len);
+        if(!len) {
+            croak("Empty string on lstring push.");
+        }
+        duk_push_lstring(context, lstring, strlen(lstring));
+//
+//      * Example subroutine call
+//      dSP;
+//      ENTER;
+//      SAVETMPS;
+//      PUSHMARK(SP);
+//      EXTEND(SP, 1);
+//      PUSHs(sv_2mortal(newSVpv("Javascript context invalid.", 0)));
+//      PUTBACK;
+//
+//      call_sv(sv_2mortal(newSVpv("::die", 0)), G_DISCARD);
+//
+//      FREETMPS;
+//      LEAVE;
+//
+//
+    OUTPUT:
+        RETVAL
+
+void
+_duk_peval(SV *)
+    CODE:
+        duk_context *context = (duk_context *) SvUV(ST(0)); 
+        if(!context) {
+            croak("Javascript context undef.", 0);
+        }
+
+        if(duk_peval(context) != 0) {
+            croak("Eval failed:\n%s\n", duk_safe_to_string(context, -1));
+        }
+
+void
+_duk_destroy_heap(SV *)
+    CODE:
+        duk_context *context = (duk_context *) SvUV(ST(0)); 
+        if (!context) {
+            croak("Cannot destroy something that is not a context.");
+        }
+        duk_destroy_heap(context);

lib/Peertube/DL/Downloaders.pm

@@ -10,6 +10,66 @@ use Peertube::DL::Utils;
 use Data::Dumper;
 use Mojo::DOM;
 
+sub youtube {
+    my $ua         = shift;
+    my $response   = shift;
+    my $options    = shift;
+    my $dom        = Mojo::DOM->new( $response->decoded_content );
+    my $script_tag = $dom->find('script')->grep(
+        sub {
+            $_[0]->text =~ /var ytInitialPlayerResponse =/;
+        }
+    )->first;
+    my ($ytInitialPlayerResponse) = $script_tag->text =~ /^var ytInitialPlayerResponse = (.*?\});var meta/;
+    $ytInitialPlayerResponse = JSON::from_json($ytInitialPlayerResponse);
+    my $microformat = $ytInitialPlayerResponse->{microformat}{playerMicroformatRenderer};
+    $ytInitialPlayerResponse = $ytInitialPlayerResponse->{streamingData};
+
+    if ( defined $options->{format} ) {
+        my $format = $options->{format};
+        ($format) = grep { $_->{itag} eq $format } (
+            scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} }
+            ? @{ $ytInitialPlayerResponse->{adaptiveFormats} }
+            : (),
+            scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : ()
+        );
+        my $url_data = $format->{signatureCipher};
+        $url_data = {
+            map {
+                my ( $a, $b ) = /(.*?)=(.*)$/;
+                ( $a => Peertube::DL::Utils::uri_decode( Peertube::DL::Utils::uri_decode($b) ) )
+            } split '&',
+            $url_data
+        };
+
+        #my ($player_url) = $response->decoded_content =~ m/"jsUrl"\s*:\s*("[^"]+")/;
+        #$player_url = JSON::from_json( $player_url, { allow_nonref => 1 } );
+        #$player_url = 'https://www.youtube.com' . $player_url
+        #  unless $player_url =~ m'^https://www.youtube.com';
+        #say $player_url;
+        say length $url_data->{s};
+    } else {
+        return {
+            options     => { list_formats => 1 },
+            title       => $microformat->{title}{simpleText},
+            description => $microformat->{description}{simpleText},
+            formats     => [
+                map {
+                    {
+                        id       => $_->{itag},
+                        mimeType => $_->{mimeType}
+                    }
+                } (
+                    scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} }
+                    ? @{ $ytInitialPlayerResponse->{adaptiveFormats} }
+                    : (),
+                    scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : ()
+                )
+            ]
+        };
+    }
+}
+
 sub kjanime {
     my $ua           = shift;
     my $response     = shift;

lib/Peertube/DL/Javascript.pm

@@ -0,0 +1,13 @@
+#!/usr/bin/env perl
+package Peertube::DL::Javascript;
+
+use strict;
+use warnings;
+
+use feature 'say';
+
+use XSLoader;
+use Data::Dumper;
+
+XSLoader::load();
+1;

lib/Peertube/DL/URLHandler.pm

@@ -11,6 +11,7 @@ use Peertube::DL::Downloaders;
 sub getDownloadDataFromURL {
     my $url_origen = shift;
     my $ua         = Peertube::DL::URLHandler::generateUA();
+    my $options    = shift;
     $ua->set_redirect_ok(1);
     my $response = $ua->get($url_origen);
     my %handlers = (
@@ -18,20 +19,23 @@ sub getDownloadDataFromURL {
         animeid => { reg => qr/animeid\.to\/streaming\.php\?/, subr => \&Peertube::DL::Downloaders::animeid },
         kjanime => {
             reg  => qr/kjanime - Anime en formato ligero y HQ - kjanime/,
-            subr => \&Peertube::DL::Downloaders::kjanime
+            subr => \&Peertube::DL::Downloaders::kjanime,
         },
         kjanime_ch => {
             reg  => qr/Link de descarga . kjanime/,
-            subr => \&Peertube::DL::Downloaders::kjanime_ch
+            subr => \&Peertube::DL::Downloaders::kjanime_ch,
+        },
+        youtube => {
+            reg  => qr/ytInitialPlayerResponse = \{/,
+            subr => \&Peertube::DL::Downloaders::youtube,
         },
-        ,
     );
     $ua->set_redirect_ok(0);
     my $handled = 0;
     my $download_data;
     for my $x ( keys %handlers ) {
         if ( $response->decoded_content =~ m/$handlers{$x}{reg}/ ) {
-            eval { $download_data = $handlers{$x}{subr}->( $ua, $response ); };
+            eval { $download_data = $handlers{$x}{subr}->( $ua, $response, $options ); };
             if ($@) {
                 warn $@;
             } else {
@@ -46,15 +50,28 @@ sub getDownloadDataFromURL {
       unless $handled;
     die "Download data not defined" unless defined $download_data;
     die "Download data not hashref" unless ref($download_data) eq 'HASH';
-    if ( defined $download_data->{options}{list}
-        && $download_data->{options}{list} )
-    {
-        say 'Reproduction list detected.';
-        die 'No url list.'
-          unless defined $download_data->{urls};
-        die 'Urls is not an array'
-          unless ref $download_data->{urls} eq 'ARRAY';
-        return $download_data;
+    if ( defined $download_data->{options} ) {
+        if ( defined $download_data->{options}{list}
+            && $download_data->{options}{list} )
+        {
+            say 'Reproduction list detected.';
+            die 'No url list.'
+              unless defined $download_data->{urls};
+            die 'Urls is not an array'
+              unless ref $download_data->{urls} eq 'ARRAY';
+            return $download_data;
+        }
+        if ( defined $download_data->{options}{list_formats} && $download_data->{options}{list_formats} ) {
+            say 'List of formats retrieved.';
+            die "No title."                   unless defined $download_data->{title};
+            die "No description."             unless defined $download_data->{description};
+            die "No formats available."       unless defined $download_data->{formats};
+            die "Formats is not an arrayref." unless ref $download_data->{formats} eq 'ARRAY';
+            say "The video title is $download_data->{title}.";
+            say "The video description is $download_data->{description}.";
+            say "The available formats are: @{[Data::Dumper::Dumper $download_data->{formats}]}.";
+            return $download_data;
+        }
     }
     die "Filename not defined"     unless exists $download_data->{filename} && $download_data->{filename};
     die "Download url not defined" unless exists $download_data->{url}      && defined $download_data->{url};

src/Makefile.PL

@@ -0,0 +1,11 @@
+use ExtUtils::MakeMaker;
+
+WriteMakefile(
+    NAME    => 'Peertube::DL::SRC',
+    VERSION => '0.1',
+    LIBS    => ['-lduktape'],
+    INC     => '-Iduktape -I./include',
+    C       => [ 'javascript_builtins.c', ],
+    OBJECT  => '${O_FILES}',
+    LDFLAGS => '-Wl-t',
+);

src/include/javascript_builtins.h

@@ -0,0 +1,5 @@
+#include <stdio.h>
+#include "duktape.h"
+#include "duk_config.h"
+
+duk_ret_t js_builtin_print(duk_context *context);

src/javascript_builtins.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+#include "duktape.h"
+#include "duk_config.h"
+
+duk_ret_t js_builtin_print(duk_context *context) {
+    const char * to_print = duk_get_string(context, 0);
+    printf("%s", to_print);
+    return 1;
+}