From 4bfca82bcd0605373d69c6d823b9bc8741c4e63a Mon Sep 17 00:00:00 2001 From: sergiotarxz Date: Wed, 6 Jan 2021 07:53:32 +0100 Subject: [PATCH 1/6] Adding initial javascript interpreter to be able to support youtube as video backend. --- MANIFEST | 12 +++- Makefile.PL | 10 ++++ bin/js-pruebas | 19 ++++++ bin/peertube-dl | 16 +++-- cpanfile | 1 + javascript_interpreter_xs/Makefile.PL | 11 ++++ javascript_interpreter_xs/javascript.xs | 77 +++++++++++++++++++++++++ lib/Peertube/DL/Downloaders.pm | 60 +++++++++++++++++++ lib/Peertube/DL/Javascript.pm | 13 +++++ lib/Peertube/DL/URLHandler.pm | 43 +++++++++----- src/Makefile.PL | 11 ++++ src/include/javascript_builtins.h | 5 ++ src/javascript_builtins.c | 9 +++ 13 files changed, 268 insertions(+), 19 deletions(-) create mode 100644 bin/js-pruebas create mode 100644 javascript_interpreter_xs/Makefile.PL create mode 100644 javascript_interpreter_xs/javascript.xs create mode 100755 lib/Peertube/DL/Javascript.pm create mode 100644 src/Makefile.PL create mode 100644 src/include/javascript_builtins.h create mode 100644 src/javascript_builtins.c diff --git a/MANIFEST b/MANIFEST index 5d38098..3a17393 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,14 +1,21 @@ .proverc AUTHORS +bin/js-pruebas bin/peertube-dl bin/peertube-dl-hypnotoad bin/peertube-dl-web bin/peertube-dl-web.conf cpanfile +include/javascript_builtins.h +javascript_builtins.c +javascript_interpreter_xs/javascript.xs +javascript_interpreter_xs/Makefile.PL +lib/auto/Peertube/DL/.exists lib/Peertube/DL.pm lib/Peertube/DL/Downloaders.pm +lib/Peertube/DL/Javascript.pm +lib/Peertube/DL/Javascript.xs lib/Peertube/DL/public/css/index.css -lib/Peertube/DL/public/css/spinner.css lib/Peertube/DL/public/img/spinner.svg lib/Peertube/DL/public/index.html lib/Peertube/DL/public/js/peertube-dl-web.js @@ -19,6 +26,9 @@ LICENSE Makefile.PL MANIFEST This list of files README.md +src/include/javascript_builtins.h +src/javascript_builtins.c +src/Makefile.PL t/00-use_ok.t t/downloaders/animeflv_example_response.html t/downloaders/gocdn.t diff --git a/Makefile.PL b/Makefile.PL index d39a7e9..830ad79 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -4,6 +4,16 @@ WriteMakefile( NAME => 'Peertube::DL', VERSION => '0.1', INST_SCRIPT => './bin', + INST_BIN => './bin', test => { TESTS => 't/*.t' }, test => { TESTS => 't/*/*.t' }, ); + +sub MY::postamble { + ' +src: src/Makefile + cd src && $(MAKE) $(PASSTHRU) +javascript_interpreter_xs: javascript_interpreter_xs/Makefile + cd javascript_interpreter_xs/ && $(MAKE) $(PASSTHRU) +'; +} diff --git a/bin/js-pruebas b/bin/js-pruebas new file mode 100644 index 0000000..1d15f59 --- /dev/null +++ b/bin/js-pruebas @@ -0,0 +1,19 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use feature 'say'; + +use Peertube::DL::Javascript; + +my $a = Peertube::DL::Javascript::_duk_create_heap_default(); +eval { Peertube::DL::Javascript::_duk_push_lstring( $a, "print(\"hola mundo\\n\");" ); }; +if ($@) { + warn $@; + $@ = ""; +} +if ( defined $a ) { + printf( "0x%0x\n", $a ); +} +Peertube::DL::Javascript::_duk_peval($a); +Peertube::DL::Javascript::_duk_destroy_heap($a); diff --git a/bin/peertube-dl b/bin/peertube-dl index 41b54cf..e85a9c7 100755 --- a/bin/peertube-dl +++ b/bin/peertube-dl @@ -9,9 +9,12 @@ use Getopt::Long::Descriptive; use Peertube::DL::URLHandler; +binmode STDOUT, ':utf8'; + my ( $opt, $usage ) = describe_options( 'peertube-dl %o ', - [ 'recurse|r', 'Recursive in reproduction lists.', { default => 0 } ], + [ 'recurse|r', 'Recursive in reproduction lists.', { default => 0 } ], + [ 'format|f=s', 'Choose format by id.', ], [], [ 'help|h', 'Show this help.', { shortcircuit => 1 } ], ); @@ -19,10 +22,12 @@ my ( $opt, $usage ) = describe_options( print( $usage->text ), exit if $opt->help; my $recurse = $opt->recurse; +my $format = $opt->format; die "No url passed" unless @ARGV; -my $download_data = Peertube::DL::URLHandler::getDownloadDataFromURL( $ARGV[0] ); +my $download_data = + Peertube::DL::URLHandler::getDownloadDataFromURL( $ARGV[0], { defined $format ? ( format => $format ) : () } ); my $ua = Peertube::DL::URLHandler::generateUA(); @@ -31,13 +36,14 @@ if ( defined $download_data->{options}{list} && $download_data->{options}{list} if ($recurse) { for my $url (@$urls) { say "Handling nested url: $url"; - my $url_download_data = Peertube::DL::URLHandler::getDownloadDataFromURL( $url ); + my $url_download_data = Peertube::DL::URLHandler::getDownloadDataFromURL($url); downloadVideo( $ua, $url_download_data ); } } else { say "The urls are:\n" . join "\n", @$urls; } - +} elsif ( defined $download_data->{options}{list_formats} && $download_data->{options}{list_formats} ) { + exit 0; } else { downloadVideo( $ua, $download_data ); } @@ -52,7 +58,7 @@ sub downloadVideo { die "Cannot retrieve video data" unless $response->is_success; my $content = $response->decoded_content; - + say "Writing into $filename."; open my $fh, '>', $filename or die "Cannot open $filename"; $fh->print( $response->decoded_content ) or die "Cannot write to $filename"; diff --git a/cpanfile b/cpanfile index 6fd4c14..ef53f94 100644 --- a/cpanfile +++ b/cpanfile @@ -7,3 +7,4 @@ requires 'Test::Most'; requires 'Test::MockObject'; requires 'Mojo::Server::Hypnotoad'; requires 'Getopt::Long::Descriptive'; +requires 'Perl::Tidy'; diff --git a/javascript_interpreter_xs/Makefile.PL b/javascript_interpreter_xs/Makefile.PL new file mode 100644 index 0000000..bec083b --- /dev/null +++ b/javascript_interpreter_xs/Makefile.PL @@ -0,0 +1,11 @@ +use ExtUtils::MakeMaker; + +WriteMakefile( + NAME => 'Peertube::DL::Javascript', + VERSION => '0.1', + LIBS => ['-lduktape'], + INC => '-Iduktape -I../src/include', + XS => { 'javascript.xs' => 'javascript.o' }, + OBJECT => 'javascript.o ../src/javascript_builtins.o', + LDFLAGS => '-Wl-t', +); diff --git a/javascript_interpreter_xs/javascript.xs b/javascript_interpreter_xs/javascript.xs new file mode 100644 index 0000000..7da8219 --- /dev/null +++ b/javascript_interpreter_xs/javascript.xs @@ -0,0 +1,77 @@ +#define PERL_NO_GET_CONTEXT +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "duktape.h" +#include "duk_config.h" +#include "javascript_builtins.h" + +MODULE = Peertube::DL::Javascript PACKAGE = Peertube::DL::Javascript +PROTOTYPES: ENABLE + +SV * +_duk_create_heap_default() + CODE: + duk_context *context = duk_create_heap_default(); + if (context) { + duk_push_c_function(context, js_builtin_print, 1); + duk_put_global_string(context, "print"); + RETVAL = newSVuv((size_t)context); + } else { + RETVAL = &PL_sv_undef; + } + OUTPUT: + RETVAL + +SV * +_duk_push_lstring(SV *, SV *) + CODE: + duk_context *context = (duk_context *) SvUV(ST(0)); + if(!context) { + croak("Javascript context undef.", 0); + } + STRLEN len; + char * lstring = SvPV(ST(1), len); + if(!len) { + croak("Empty string on lstring push."); + } + duk_push_lstring(context, lstring, strlen(lstring)); +// +// * Example subroutine call +// dSP; +// ENTER; +// SAVETMPS; +// PUSHMARK(SP); +// EXTEND(SP, 1); +// PUSHs(sv_2mortal(newSVpv("Javascript context invalid.", 0))); +// PUTBACK; +// +// call_sv(sv_2mortal(newSVpv("::die", 0)), G_DISCARD); +// +// FREETMPS; +// LEAVE; +// +// + OUTPUT: + RETVAL + +void +_duk_peval(SV *) + CODE: + duk_context *context = (duk_context *) SvUV(ST(0)); + if(!context) { + croak("Javascript context undef.", 0); + } + + if(duk_peval(context) != 0) { + croak("Eval failed:\n%s\n", duk_safe_to_string(context, -1)); + } + +void +_duk_destroy_heap(SV *) + CODE: + duk_context *context = (duk_context *) SvUV(ST(0)); + if (!context) { + croak("Cannot destroy something that is not a context."); + } + duk_destroy_heap(context); diff --git a/lib/Peertube/DL/Downloaders.pm b/lib/Peertube/DL/Downloaders.pm index af92afb..4549f4d 100644 --- a/lib/Peertube/DL/Downloaders.pm +++ b/lib/Peertube/DL/Downloaders.pm @@ -10,6 +10,66 @@ use Peertube::DL::Utils; use Data::Dumper; use Mojo::DOM; +sub youtube { + my $ua = shift; + my $response = shift; + my $options = shift; + my $dom = Mojo::DOM->new( $response->decoded_content ); + my $script_tag = $dom->find('script')->grep( + sub { + $_[0]->text =~ /var ytInitialPlayerResponse =/; + } + )->first; + my ($ytInitialPlayerResponse) = $script_tag->text =~ /^var ytInitialPlayerResponse = (.*?\});var meta/; + $ytInitialPlayerResponse = JSON::from_json($ytInitialPlayerResponse); + my $microformat = $ytInitialPlayerResponse->{microformat}{playerMicroformatRenderer}; + $ytInitialPlayerResponse = $ytInitialPlayerResponse->{streamingData}; + + if ( defined $options->{format} ) { + my $format = $options->{format}; + ($format) = grep { $_->{itag} eq $format } ( + scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} } + ? @{ $ytInitialPlayerResponse->{adaptiveFormats} } + : (), + scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : () + ); + my $url_data = $format->{signatureCipher}; + $url_data = { + map { + my ( $a, $b ) = /(.*?)=(.*)$/; + ( $a => Peertube::DL::Utils::uri_decode( Peertube::DL::Utils::uri_decode($b) ) ) + } split '&', + $url_data + }; + + #my ($player_url) = $response->decoded_content =~ m/"jsUrl"\s*:\s*("[^"]+")/; + #$player_url = JSON::from_json( $player_url, { allow_nonref => 1 } ); + #$player_url = 'https://www.youtube.com' . $player_url + # unless $player_url =~ m'^https://www.youtube.com'; + #say $player_url; + say length $url_data->{s}; + } else { + return { + options => { list_formats => 1 }, + title => $microformat->{title}{simpleText}, + description => $microformat->{description}{simpleText}, + formats => [ + map { + { + id => $_->{itag}, + mimeType => $_->{mimeType} + } + } ( + scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} } + ? @{ $ytInitialPlayerResponse->{adaptiveFormats} } + : (), + scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : () + ) + ] + }; + } +} + sub kjanime { my $ua = shift; my $response = shift; diff --git a/lib/Peertube/DL/Javascript.pm b/lib/Peertube/DL/Javascript.pm new file mode 100755 index 0000000..f4db2f4 --- /dev/null +++ b/lib/Peertube/DL/Javascript.pm @@ -0,0 +1,13 @@ +#!/usr/bin/env perl +package Peertube::DL::Javascript; + +use strict; +use warnings; + +use feature 'say'; + +use XSLoader; +use Data::Dumper; + +XSLoader::load(); +1; diff --git a/lib/Peertube/DL/URLHandler.pm b/lib/Peertube/DL/URLHandler.pm index ce63e47..115e48b 100644 --- a/lib/Peertube/DL/URLHandler.pm +++ b/lib/Peertube/DL/URLHandler.pm @@ -11,6 +11,7 @@ use Peertube::DL::Downloaders; sub getDownloadDataFromURL { my $url_origen = shift; my $ua = Peertube::DL::URLHandler::generateUA(); + my $options = shift; $ua->set_redirect_ok(1); my $response = $ua->get($url_origen); my %handlers = ( @@ -18,20 +19,23 @@ sub getDownloadDataFromURL { animeid => { reg => qr/animeid\.to\/streaming\.php\?/, subr => \&Peertube::DL::Downloaders::animeid }, kjanime => { reg => qr/kjanime - Anime en formato ligero y HQ - kjanime/, - subr => \&Peertube::DL::Downloaders::kjanime + subr => \&Peertube::DL::Downloaders::kjanime, }, kjanime_ch => { reg => qr/Link de descarga . kjanime/, - subr => \&Peertube::DL::Downloaders::kjanime_ch + subr => \&Peertube::DL::Downloaders::kjanime_ch, + }, + youtube => { + reg => qr/ytInitialPlayerResponse = \{/, + subr => \&Peertube::DL::Downloaders::youtube, }, - , ); $ua->set_redirect_ok(0); my $handled = 0; my $download_data; for my $x ( keys %handlers ) { if ( $response->decoded_content =~ m/$handlers{$x}{reg}/ ) { - eval { $download_data = $handlers{$x}{subr}->( $ua, $response ); }; + eval { $download_data = $handlers{$x}{subr}->( $ua, $response, $options ); }; if ($@) { warn $@; } else { @@ -46,15 +50,28 @@ sub getDownloadDataFromURL { unless $handled; die "Download data not defined" unless defined $download_data; die "Download data not hashref" unless ref($download_data) eq 'HASH'; - if ( defined $download_data->{options}{list} - && $download_data->{options}{list} ) - { - say 'Reproduction list detected.'; - die 'No url list.' - unless defined $download_data->{urls}; - die 'Urls is not an array' - unless ref $download_data->{urls} eq 'ARRAY'; - return $download_data; + if ( defined $download_data->{options} ) { + if ( defined $download_data->{options}{list} + && $download_data->{options}{list} ) + { + say 'Reproduction list detected.'; + die 'No url list.' + unless defined $download_data->{urls}; + die 'Urls is not an array' + unless ref $download_data->{urls} eq 'ARRAY'; + return $download_data; + } + if ( defined $download_data->{options}{list_formats} && $download_data->{options}{list_formats} ) { + say 'List of formats retrieved.'; + die "No title." unless defined $download_data->{title}; + die "No description." unless defined $download_data->{description}; + die "No formats available." unless defined $download_data->{formats}; + die "Formats is not an arrayref." unless ref $download_data->{formats} eq 'ARRAY'; + say "The video title is $download_data->{title}."; + say "The video description is $download_data->{description}."; + say "The available formats are: @{[Data::Dumper::Dumper $download_data->{formats}]}."; + return $download_data; + } } die "Filename not defined" unless exists $download_data->{filename} && $download_data->{filename}; die "Download url not defined" unless exists $download_data->{url} && defined $download_data->{url}; diff --git a/src/Makefile.PL b/src/Makefile.PL new file mode 100644 index 0000000..72482b3 --- /dev/null +++ b/src/Makefile.PL @@ -0,0 +1,11 @@ +use ExtUtils::MakeMaker; + +WriteMakefile( + NAME => 'Peertube::DL::SRC', + VERSION => '0.1', + LIBS => ['-lduktape'], + INC => '-Iduktape -I./include', + C => [ 'javascript_builtins.c', ], + OBJECT => '${O_FILES}', + LDFLAGS => '-Wl-t', +); diff --git a/src/include/javascript_builtins.h b/src/include/javascript_builtins.h new file mode 100644 index 0000000..8b279ed --- /dev/null +++ b/src/include/javascript_builtins.h @@ -0,0 +1,5 @@ +#include +#include "duktape.h" +#include "duk_config.h" + +duk_ret_t js_builtin_print(duk_context *context); diff --git a/src/javascript_builtins.c b/src/javascript_builtins.c new file mode 100644 index 0000000..db71ad7 --- /dev/null +++ b/src/javascript_builtins.c @@ -0,0 +1,9 @@ +#include +#include "duktape.h" +#include "duk_config.h" + +duk_ret_t js_builtin_print(duk_context *context) { + const char * to_print = duk_get_string(context, 0); + printf("%s", to_print); + return 1; +} -- 2.43.2 From ff2f583c2441258581db153893fa37106d1f8002 Mon Sep 17 00:00:00 2001 From: sergiotarxz Date: Wed, 6 Jan 2021 20:47:33 +0100 Subject: [PATCH 2/6] Adding support for lstring. --- MANIFEST | 6 +----- Makefile.PL | 4 +++- bin/js-pruebas | 7 ++++--- javascript_interpreter_xs/Makefile.PL | 4 ++-- javascript_interpreter_xs/javascript.xs | 21 +++++++++++++++++++++ 5 files changed, 31 insertions(+), 11 deletions(-) mode change 100644 => 100755 bin/js-pruebas diff --git a/MANIFEST b/MANIFEST index 3a17393..9a3c21c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -6,15 +6,11 @@ bin/peertube-dl-hypnotoad bin/peertube-dl-web bin/peertube-dl-web.conf cpanfile -include/javascript_builtins.h -javascript_builtins.c javascript_interpreter_xs/javascript.xs javascript_interpreter_xs/Makefile.PL -lib/auto/Peertube/DL/.exists lib/Peertube/DL.pm lib/Peertube/DL/Downloaders.pm lib/Peertube/DL/Javascript.pm -lib/Peertube/DL/Javascript.xs lib/Peertube/DL/public/css/index.css lib/Peertube/DL/public/img/spinner.svg lib/Peertube/DL/public/index.html @@ -24,7 +20,7 @@ lib/Peertube/DL/UserAgent.pm lib/Peertube/DL/Utils.pm LICENSE Makefile.PL -MANIFEST This list of files +MANIFEST README.md src/include/javascript_builtins.h src/javascript_builtins.c diff --git a/Makefile.PL b/Makefile.PL index 830ad79..8ac7fca 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -7,13 +7,15 @@ WriteMakefile( INST_BIN => './bin', test => { TESTS => 't/*.t' }, test => { TESTS => 't/*/*.t' }, + DIR => [ 'src', 'javascript_interpreter_xs/' ], ); sub MY::postamble { ' src: src/Makefile - cd src && $(MAKE) $(PASSTHRU) + src && $(MAKE) $(PASSTHRU) javascript_interpreter_xs: javascript_interpreter_xs/Makefile cd javascript_interpreter_xs/ && $(MAKE) $(PASSTHRU) '; } + diff --git a/bin/js-pruebas b/bin/js-pruebas old mode 100644 new mode 100755 index 1d15f59..10d361d --- a/bin/js-pruebas +++ b/bin/js-pruebas @@ -1,4 +1,5 @@ #!/usr/bin/env perl +# use strict; use warnings; @@ -7,13 +8,13 @@ use feature 'say'; use Peertube::DL::Javascript; my $a = Peertube::DL::Javascript::_duk_create_heap_default(); -eval { Peertube::DL::Javascript::_duk_push_lstring( $a, "print(\"hola mundo\\n\");" ); }; +eval { Peertube::DL::Javascript::_duk_push_lstring( $a, "\"hola mundo\"" ); }; if ($@) { warn $@; $@ = ""; } if ( defined $a ) { - printf( "0x%0x\n", $a ); -} Peertube::DL::Javascript::_duk_peval($a); +say Peertube::DL::Javascript::_duk_get_lstring($a, -1); Peertube::DL::Javascript::_duk_destroy_heap($a); +} diff --git a/javascript_interpreter_xs/Makefile.PL b/javascript_interpreter_xs/Makefile.PL index bec083b..88ece9a 100644 --- a/javascript_interpreter_xs/Makefile.PL +++ b/javascript_interpreter_xs/Makefile.PL @@ -3,9 +3,9 @@ use ExtUtils::MakeMaker; WriteMakefile( NAME => 'Peertube::DL::Javascript', VERSION => '0.1', - LIBS => ['-lduktape'], + LIBS => ['-lduktape -l../src'], INC => '-Iduktape -I../src/include', XS => { 'javascript.xs' => 'javascript.o' }, - OBJECT => 'javascript.o ../src/javascript_builtins.o', + OBJECT => '../src/javascript_builtins.o javascript.o', LDFLAGS => '-Wl-t', ); diff --git a/javascript_interpreter_xs/javascript.xs b/javascript_interpreter_xs/javascript.xs index 7da8219..b7c5595 100644 --- a/javascript_interpreter_xs/javascript.xs +++ b/javascript_interpreter_xs/javascript.xs @@ -75,3 +75,24 @@ _duk_destroy_heap(SV *) croak("Cannot destroy something that is not a context."); } duk_destroy_heap(context); + +SV * +_duk_get_lstring(SV *, SV *) + CODE: + duk_context *context = (duk_context *) SvUV(ST(0)); + if (!context) { + croak("Unable to get lstring from no existing context."); + } + duk_idx_t idx = SvIV(ST(1)); + if (!duk_is_string(context, idx)) { + croak("Idx is not a string."); + } + if (duk_is_symbol(context, idx)) { + croak("This is a symbol, not a string."); + } + duk_size_t *lstring_len; + const char * lstring = duk_get_lstring(context, idx, lstring_len); + RETVAL = newSVpv(lstring, (size_t) lstring_len); + OUTPUT: + RETVAL + -- 2.43.2 From 41d1dfaa6617df3b2caf88bd55ab17a6bb0aef00 Mon Sep 17 00:00:00 2001 From: sergiotarxz Date: Thu, 7 Jan 2021 16:02:34 +0100 Subject: [PATCH 3/6] Added support for calling javascript functions from perl. --- bin/js-pruebas | 20 +++-- javascript_interpreter_xs/javascript.xs | 115 +++++++++++++----------- 2 files changed, 75 insertions(+), 60 deletions(-) diff --git a/bin/js-pruebas b/bin/js-pruebas index 10d361d..ab65d63 100755 --- a/bin/js-pruebas +++ b/bin/js-pruebas @@ -8,13 +8,17 @@ use feature 'say'; use Peertube::DL::Javascript; my $a = Peertube::DL::Javascript::_duk_create_heap_default(); -eval { Peertube::DL::Javascript::_duk_push_lstring( $a, "\"hola mundo\"" ); }; -if ($@) { - warn $@; - $@ = ""; -} if ( defined $a ) { -Peertube::DL::Javascript::_duk_peval($a); -say Peertube::DL::Javascript::_duk_get_lstring($a, -1); -Peertube::DL::Javascript::_duk_destroy_heap($a); + eval { + my $return = Peertube::DL::Javascript::_duk_eval_wrapper( + $a, + "function a(nombre) { return 'Hello ' + nombre; }; a('sergio');" + ); + say $return; + }; + if ($@) { + warn $@; + } + say Peertube::DL::Javascript::_duk_call_function( $a, "a", "sergio" ); + Peertube::DL::Javascript::_duk_destroy_heap($a); } diff --git a/javascript_interpreter_xs/javascript.xs b/javascript_interpreter_xs/javascript.xs index b7c5595..9441baa 100644 --- a/javascript_interpreter_xs/javascript.xs +++ b/javascript_interpreter_xs/javascript.xs @@ -24,48 +24,80 @@ _duk_create_heap_default() RETVAL SV * -_duk_push_lstring(SV *, SV *) +_duk_eval_wrapper(SV *, SV *) CODE: + if (!ST(0)) { + croak("Javascript context undefined.", 0); + } duk_context *context = (duk_context *) SvUV(ST(0)); - if(!context) { - croak("Javascript context undef.", 0); + if (!context) { + croak("Javascript context null on function call."); } - STRLEN len; - char * lstring = SvPV(ST(1), len); - if(!len) { - croak("Empty string on lstring push."); + duk_idx_t top_index; + if ( (top_index = duk_get_top_index(context)) != DUK_INVALID_INDEX) { + duk_pop_n(context, top_index + 1); } - duk_push_lstring(context, lstring, strlen(lstring)); -// -// * Example subroutine call -// dSP; -// ENTER; -// SAVETMPS; -// PUSHMARK(SP); -// EXTEND(SP, 1); -// PUSHs(sv_2mortal(newSVpv("Javascript context invalid.", 0))); -// PUTBACK; -// -// call_sv(sv_2mortal(newSVpv("::die", 0)), G_DISCARD); -// -// FREETMPS; -// LEAVE; -// -// + duk_require_stack(context, 2); + if (!ST(1)) { + croak("Code to eval is undef."); + } + STRLEN code_len; + char *code = SvPV(ST(1), code_len); + duk_push_lstring(context, code, strlen(code)); + if ( duk_peval(context) != 0 ) { + croak("Eval failed:\n%s\n", duk_get_string(context, -1)); + } + duk_size_t *lstring_len; + const char * lstring = duk_get_lstring(context, -1, lstring_len); + RETVAL = newSVpv(lstring, (size_t) lstring_len); + top_index = duk_get_top_index(context); + duk_pop_n(context, top_index+1); OUTPUT: RETVAL -void -_duk_peval(SV *) +SV * +_duk_call_function(SV *, SV *, ...) CODE: + if (!ST(0)) { + croak("Context is undef."); + } duk_context *context = (duk_context *) SvUV(ST(0)); - if(!context) { - croak("Javascript context undef.", 0); + if (!context) { + croak("Javascript context null on function call."); } - - if(duk_peval(context) != 0) { - croak("Eval failed:\n%s\n", duk_safe_to_string(context, -1)); + STRLEN function_name_len; + if (!ST(1)) { + croak("Function name must be defined."); } + char * function_name = SvPV(ST(1), function_name_len); + if ( function_name_len == 0 ) { + croak("Function name cannot be empty."); + } + duk_idx_t top_index; + if ( (top_index = duk_get_top_index(context)) != DUK_INVALID_INDEX ) { + duk_pop_n(context, top_index+1); + } + duk_require_stack(context, items+1); + (void) duk_get_global_lstring(context, function_name, strlen(function_name)); + if (items > 2) { + for ( int i = 2; i < items; i++) { + SV *argument_SV = ST(i); + if (!argument_SV) { + croak("Argument %d undefined.", i); + } + STRLEN argument_len; + char *argument = SvPV(argument_SV, argument_len); + duk_push_lstring(context, argument, strlen(argument)); + } + } + duk_size_t *lstring_len; + duk_call(context, 1); + char * return_value = duk_get_lstring(context, -1, lstring_len); + top_index = duk_get_top_index(context); + duk_pop_n(context, top_index+1); + RETVAL = newSVpv(return_value, (size_t) lstring_len); + OUTPUT: + RETVAL void _duk_destroy_heap(SV *) @@ -75,24 +107,3 @@ _duk_destroy_heap(SV *) croak("Cannot destroy something that is not a context."); } duk_destroy_heap(context); - -SV * -_duk_get_lstring(SV *, SV *) - CODE: - duk_context *context = (duk_context *) SvUV(ST(0)); - if (!context) { - croak("Unable to get lstring from no existing context."); - } - duk_idx_t idx = SvIV(ST(1)); - if (!duk_is_string(context, idx)) { - croak("Idx is not a string."); - } - if (duk_is_symbol(context, idx)) { - croak("This is a symbol, not a string."); - } - duk_size_t *lstring_len; - const char * lstring = duk_get_lstring(context, idx, lstring_len); - RETVAL = newSVpv(lstring, (size_t) lstring_len); - OUTPUT: - RETVAL - -- 2.43.2 From 3abba1edce396d570403868083cc4355e03fafe1 Mon Sep 17 00:00:00 2001 From: sergiotarxz Date: Sat, 9 Jan 2021 03:55:59 +0100 Subject: [PATCH 4/6] Added youtube support. --- bin/js-pruebas | 20 ++-- cpanfile | 5 + javascript_interpreter_xs/javascript.xs | 7 +- lib/Peertube/DL/Downloaders.pm | 119 ++++++++++++++++++------ lib/Peertube/DL/Javascript.pm | 39 ++++++++ 5 files changed, 147 insertions(+), 43 deletions(-) diff --git a/bin/js-pruebas b/bin/js-pruebas index ab65d63..29c2824 100755 --- a/bin/js-pruebas +++ b/bin/js-pruebas @@ -7,18 +7,10 @@ use feature 'say'; use Peertube::DL::Javascript; -my $a = Peertube::DL::Javascript::_duk_create_heap_default(); -if ( defined $a ) { - eval { - my $return = Peertube::DL::Javascript::_duk_eval_wrapper( - $a, - "function a(nombre) { return 'Hello ' + nombre; }; a('sergio');" - ); - say $return; - }; - if ($@) { - warn $@; - } - say Peertube::DL::Javascript::_duk_call_function( $a, "a", "sergio" ); - Peertube::DL::Javascript::_duk_destroy_heap($a); +my $js = Peertube::DL::Javascript->new; +eval { + say $js->evalJS("if(\"undefined\"!=typeof Reflect&&Reflect.construct){if(a())return Reflect.construct;var b=Reflect.construct;return function(c,d,e){c=b(c,d);e&&Reflect.setPrototypeOf(c,e.prototype);return c}}return function(c,d,e){void 0===e&&(e=c);"); +}; +if ($@) { + warn $@; } diff --git a/cpanfile b/cpanfile index ef53f94..5a61b98 100644 --- a/cpanfile +++ b/cpanfile @@ -8,3 +8,8 @@ requires 'Test::MockObject'; requires 'Mojo::Server::Hypnotoad'; requires 'Getopt::Long::Descriptive'; requires 'Perl::Tidy'; +requires 'Params::Validate'; +requires 'Params::Util'; +requires 'Net::SSLeay'; +requires 'HTML::HeadParser'; +requires 'File::MimeInfo'; diff --git a/javascript_interpreter_xs/javascript.xs b/javascript_interpreter_xs/javascript.xs index 9441baa..b7854c5 100644 --- a/javascript_interpreter_xs/javascript.xs +++ b/javascript_interpreter_xs/javascript.xs @@ -45,7 +45,7 @@ _duk_eval_wrapper(SV *, SV *) char *code = SvPV(ST(1), code_len); duk_push_lstring(context, code, strlen(code)); if ( duk_peval(context) != 0 ) { - croak("Eval failed:\n%s\n", duk_get_string(context, -1)); + croak("Eval failed:\n%s\n%s\n", duk_safe_to_string(context, -1), duk_to_stacktrace(context, -1)); } duk_size_t *lstring_len; const char * lstring = duk_get_lstring(context, -1, lstring_len); @@ -93,8 +93,9 @@ _duk_call_function(SV *, SV *, ...) duk_size_t *lstring_len; duk_call(context, 1); char * return_value = duk_get_lstring(context, -1, lstring_len); - top_index = duk_get_top_index(context); - duk_pop_n(context, top_index+1); + if ( (top_index = duk_get_top_index(context)) != DUK_INVALID_INDEX ) { + duk_pop_n(context, top_index+1); + } RETVAL = newSVpv(return_value, (size_t) lstring_len); OUTPUT: RETVAL diff --git a/lib/Peertube/DL/Downloaders.pm b/lib/Peertube/DL/Downloaders.pm index 4549f4d..b421028 100644 --- a/lib/Peertube/DL/Downloaders.pm +++ b/lib/Peertube/DL/Downloaders.pm @@ -1,3 +1,4 @@ +## Please see file perltidy.ERR package Peertube::DL::Downloaders; use strict; @@ -5,10 +6,16 @@ use warnings; use feature 'say'; +use Symbol 'gensym'; +use IPC::Open3; +use File::Basename; use JSON; -use Peertube::DL::Utils; use Data::Dumper; use Mojo::DOM; +use File::MimeInfo; + +use Peertube::DL::Javascript; +use Peertube::DL::Utils; sub youtube { my $ua = shift; @@ -20,9 +27,11 @@ sub youtube { $_[0]->text =~ /var ytInitialPlayerResponse =/; } )->first; - my ($ytInitialPlayerResponse) = $script_tag->text =~ /^var ytInitialPlayerResponse = (.*?\});var meta/; + my ($ytInitialPlayerResponse) = + $script_tag->text =~ /^var ytInitialPlayerResponse = (.*?\});var meta/; $ytInitialPlayerResponse = JSON::from_json($ytInitialPlayerResponse); - my $microformat = $ytInitialPlayerResponse->{microformat}{playerMicroformatRenderer}; + my $microformat = + $ytInitialPlayerResponse->{microformat}{playerMicroformatRenderer}; $ytInitialPlayerResponse = $ytInitialPlayerResponse->{streamingData}; if ( defined $options->{format} ) { @@ -31,23 +40,69 @@ sub youtube { scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} } ? @{ $ytInitialPlayerResponse->{adaptiveFormats} } : (), - scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : () + scalar @{ $ytInitialPlayerResponse->{formats} } + ? @{ $ytInitialPlayerResponse->{formats} } + : () ); my $url_data = $format->{signatureCipher}; $url_data = { map { my ( $a, $b ) = /(.*?)=(.*)$/; - ( $a => Peertube::DL::Utils::uri_decode( Peertube::DL::Utils::uri_decode($b) ) ) + ( + $a => Peertube::DL::Utils::uri_decode( + Peertube::DL::Utils::uri_decode($b) + ) + ) } split '&', $url_data }; - #my ($player_url) = $response->decoded_content =~ m/"jsUrl"\s*:\s*("[^"]+")/; - #$player_url = JSON::from_json( $player_url, { allow_nonref => 1 } ); - #$player_url = 'https://www.youtube.com' . $player_url - # unless $player_url =~ m'^https://www.youtube.com'; - #say $player_url; - say length $url_data->{s}; + say Data::Dumper::Dumper $url_data; + + my ($player_url) = + $response->decoded_content =~ m/"jsUrl"\s*:\s*("[^"]+")/; + $player_url = JSON::from_json( $player_url, { allow_nonref => 1 } ); + $player_url = 'https://www.youtube.com' . $player_url + unless $player_url =~ m'^https://www.youtube.com'; + say $player_url; + my $hostname = $player_url =~ s/https?:\/\///r; + $hostname = $hostname =~ s/\/.*$//; + $hostname = JSON::encode_json($hostname); + my $js = Peertube::DL::Javascript->new; + my $player_response = $ua->get($player_url); + my $regexes_search_function_name = [ + qr/\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?(?:[a-zA-Z0-9]|\$)+)\(/, + qr/\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?(?:[a-zA-Z0-9]|\$)+)\(/, + qr/(?:\b|(?:[^a-zA-Z0-9]|\$))(?(?:[a-zA-Z0-9]|\$){2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)/, + qr/(?(?:[a-zA-Z0-9]|\$)+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)/, + ]; + my $function_name_regen_sig; + for my $regex (@$regexes_search_function_name) { + ($function_name_regen_sig) = + $player_response->decoded_content =~ /$regex/; + last if defined $function_name_regen_sig; + } + my ($extract_function_sig) = $player_response->decoded_content =~ + m/\n(${function_name_regen_sig}=function\([a-zA-Z]+\)\{.*?\};)\n/; + my ($class_decoding) = + $extract_function_sig =~ /;([a-zA-Z]+)\.[a-zA-Z]+\(/; + my ($class_decoding_code) = $player_response->decoded_content =~ + /;(var ${class_decoding}=\{.*?\}\};)/s; + $js->evalJS( join "\n", $class_decoding_code ); + $js->evalJS( join "\n", $extract_function_sig ); + my $signature = + $js->callJSFunction( $function_name_regen_sig, $url_data->{s} ); + my $url = $url_data->{url} . "&sig=${signature}"; + say Data::Dumper::Dumper $format; + my $mime_type = $format->{mimeType} =~ s/;.*$//r; + say $mime_type; + my $extension = scalar File::MimeInfo::extensions($mime_type); + $extension = 'mp4' if $mime_type eq 'video/mp4'; + say $url; + return { + url => $url, + filename => $microformat->{title}{simpleText} . '.' . $extension, + }; } else { return { options => { list_formats => 1 }, @@ -63,7 +118,9 @@ sub youtube { scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} } ? @{ $ytInitialPlayerResponse->{adaptiveFormats} } : (), - scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : () + scalar @{ $ytInitialPlayerResponse->{formats} } + ? @{ $ytInitialPlayerResponse->{formats} } + : () ) ] }; @@ -93,7 +150,8 @@ sub kjanime { 'X-Requested-With' => 'XMLHttpRequest', ); $dom = Mojo::DOM->new( $get_php_response->decoded_content ); - my $links = [ map { $_->attr('href') =~ s/^http:\/\//https:\/\//r } @{ $dom->find('a')->to_array } ]; + my $links = [ map { $_->attr('href') =~ s/^http:\/\//https:\/\//r } + @{ $dom->find('a')->to_array } ]; return { options => { list => 1 }, urls => $links }; } @@ -102,7 +160,8 @@ sub kjanime_ch { my $response = shift; my ($id) = $response->filename; say $id; - my $param = join '', map { Peertube::DL::Utils::uri_decode("%$_") } unpack( "(A2)*", $id ); + my $param = join '', + map { Peertube::DL::Utils::uri_decode("%$_") } unpack( "(A2)*", $id ); say $param; # FORMAT: KJA://70714B6E314943383135712B72582B70713768797A4B6E4D734D474775773D3D/1 @@ -118,7 +177,8 @@ sub kjanime_ch { $url_serie_php, 'X-Requested-With' => 'XMLHttpRequest', ); - my $download_data = JSON::decode_json( $response_serie_php->decoded_content ); + my $download_data = + JSON::decode_json( $response_serie_php->decoded_content ); say Data::Dumper::Dumper $download_data; $download_data->{filename} = delete $download_data->{name}; $download_data->{url} = delete $download_data->{link}; @@ -135,9 +195,11 @@ sub gocdn { my $url_gocdn = 'https://streamium.xyz/gocdn.php?v=' . $id; my $response_gocdn = $ua->get($url_gocdn); - die $response_gocdn->status_line . ' from ' . $url_gocdn unless $response_gocdn->is_success; + die $response_gocdn->status_line . ' from ' . $url_gocdn + unless $response_gocdn->is_success; - my $google_url = JSON::decode_json( $response_gocdn->decoded_content )->{file}; + my $google_url = + JSON::decode_json( $response_gocdn->decoded_content )->{file}; my ($filename) = $response->base =~ s/^.*\///gr . '.mp4'; @@ -145,12 +207,13 @@ sub gocdn { } sub animeid { - my $ua = shift; - my $response = shift; - my @order = ( 'id', 'title', 'refer' ); - my $url = $response->base; - my $url_ajax = 'https://animeid.to/ajax.php?'; - my ($get_params) = $response->decoded_content =~ /animeid\.to\/streaming\.php\?(.*?)"/; + my $ua = shift; + my $response = shift; + my @order = ( 'id', 'title', 'refer' ); + my $url = $response->base; + my $url_ajax = 'https://animeid.to/ajax.php?'; + my ($get_params) = + $response->decoded_content =~ /animeid\.to\/streaming\.php\?(.*?)"/; $get_params = { map { my ( $key, $value ) = /^(.*?)=(.*)$/; @@ -164,7 +227,10 @@ sub animeid { $get_params->{refer} = $url; my $filename = $get_params->{title}; $url_ajax .= join '&', - map { join '=', ( $_ => Peertube::DL::Utils::uri_encode( $get_params->{$_} ) ) } @order; + map { + join '=', + ( $_ => Peertube::DL::Utils::uri_encode( $get_params->{$_} ) ) + } @order; say 'Getting video location from: ' . $url_ajax; my $ajax_data = $ua->get( $url_ajax, @@ -175,8 +241,9 @@ sub animeid { say "Decoding json... $ajax_data"; $ajax_data = JSON::decode_json $ajax_data; die 'No video source found.' if ( !defined $ajax_data->{source} ); - my $download_redirect_url = $ajax_data->{source}[0]{file} // die "No url found."; - my $extension = $ajax_data->{source}[0]{type} // die "No extension found."; + my $download_redirect_url = $ajax_data->{source}[0]{file} + // die "No url found."; + my $extension = $ajax_data->{source}[0]{type} // die "No extension found."; $filename .= ".$extension"; say "Getting redirect to download url from $download_redirect_url..."; my $video_response = $ua->get( diff --git a/lib/Peertube/DL/Javascript.pm b/lib/Peertube/DL/Javascript.pm index f4db2f4..a0a1dbb 100755 --- a/lib/Peertube/DL/Javascript.pm +++ b/lib/Peertube/DL/Javascript.pm @@ -10,4 +10,43 @@ use XSLoader; use Data::Dumper; XSLoader::load(); + +sub new { + my $class = shift; + my $self = bless {}, $class; + $self->{___ContextPrivateDONOTTOUCH} = + Peertube::DL::Javascript::_duk_create_heap_default(); + return $self; +} + +sub evalJS { + my $self = shift; + my $context = $self->___ContextDONOTUSE; + my $code = shift; + return Peertube::DL::Javascript::_duk_eval_wrapper( $context, $code ); +} + +sub callJSFunction { + my $self = shift; + my $function = shift; + my @function_arguments = @_; + my $context = $self->___ContextDONOTUSE; + return Peertube::DL::Javascript::_duk_call_function( + $context, + $function, + @function_arguments + ); +} + +sub ___ContextDONOTUSE { + my $self = shift; + return $self->{___ContextPrivateDONOTTOUCH}; +} + +sub DESTROY { + my $self = shift; + my $context = $self->___ContextDONOTUSE; + Peertube::DL::Javascript::_duk_destroy_heap($context); + undef $self; +} 1; -- 2.43.2 From 702e427ef149414deba478e9b9f522fb48538056 Mon Sep 17 00:00:00 2001 From: sergiotarxz Date: Sat, 9 Jan 2021 04:06:55 +0100 Subject: [PATCH 5/6] Added youtube support at webpage. --- bin/peertube-dl-web | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/bin/peertube-dl-web b/bin/peertube-dl-web index fe8180a..1bedec1 100755 --- a/bin/peertube-dl-web +++ b/bin/peertube-dl-web @@ -16,10 +16,20 @@ get '/' => sub { post '/api' => sub { my $c = shift; - my $url = $c->req->json->{url} - // ( return $c->render( text => 'Malformed request due missing url json parameter.', status => 400 ) ); + my $url = $c->req->json->{url} // ( + return $c->render( + text => 'Malformed request due missing url json parameter.', + status => 400 + ) + ); my $render_returned; - eval { $render_returned = $c->render( json => Peertube::DL::URLHandler::getDownloadDataFromURL($url) ); }; + eval { + $render_returned = $c->render( + json => Peertube::DL::URLHandler::getDownloadDataFromURL( + $url, { format => 18 } + ) + ); + }; if ($@) { return $c->render( text => $@, status => 500 ); } @@ -28,9 +38,15 @@ post '/api' => sub { post '/proxy_to_get' => sub { my $c = shift; - my $url = $c->req->json->{url} - // ( return $c->render( text => 'Malformed request due missing url json parameter.', status => 400 ) ); - die "Not supported url" unless $url =~ s/^https:\/\///r =~ /^storage\.googleapis\.com/; + my $url = $c->req->json->{url} // ( + return $c->render( + text => 'Malformed request due missing url json parameter.', + status => 400 + ) + ); + die "Not supported url" + unless ( $url =~ s/^https?:\/\///r =~ /^storage\.googleapis\.com/ + || $url =~ s/https?:\/\///r =~ s/\/.*$//r =~ /googlevideo\.com$/ ); my $ua = Peertube::DL::URLHandler::generateUA(); return $c->render( data => $ua->get($url)->decoded_content, @@ -38,8 +54,10 @@ post '/proxy_to_get' => sub { ); }; -push @{ app->renderer->paths }, ( Peertube::DL::getDir() . '/Peertube/DL/templates' ); -push @{ app->static->paths }, ( Peertube::DL::getDir() . '/Peertube/DL/public' ); +push @{ app->renderer->paths }, + ( Peertube::DL::getDir() . '/Peertube/DL/templates' ); +push @{ app->static->paths }, + ( Peertube::DL::getDir() . '/Peertube/DL/public' ); my $config = plugin 'Config'; -- 2.43.2 From 977869a8ac4814f594617a38f97ee58dbe62928a Mon Sep 17 00:00:00 2001 From: sergiotarxz Date: Sat, 9 Jan 2021 04:09:06 +0100 Subject: [PATCH 6/6] Added support for youtube at readme. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 910b7ed..dbcb231 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ animeflv.cc kjanime.net +youtube.com (Video only in webpage, all formats on cli.) + ## Installation There is two alternative ways applying to the GNU/Linux OS: -- 2.43.2