diff --git a/MANIFEST b/MANIFEST index 5d38098..3a17393 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,14 +1,21 @@ .proverc AUTHORS +bin/js-pruebas bin/peertube-dl bin/peertube-dl-hypnotoad bin/peertube-dl-web bin/peertube-dl-web.conf cpanfile +include/javascript_builtins.h +javascript_builtins.c +javascript_interpreter_xs/javascript.xs +javascript_interpreter_xs/Makefile.PL +lib/auto/Peertube/DL/.exists lib/Peertube/DL.pm lib/Peertube/DL/Downloaders.pm +lib/Peertube/DL/Javascript.pm +lib/Peertube/DL/Javascript.xs lib/Peertube/DL/public/css/index.css -lib/Peertube/DL/public/css/spinner.css lib/Peertube/DL/public/img/spinner.svg lib/Peertube/DL/public/index.html lib/Peertube/DL/public/js/peertube-dl-web.js @@ -19,6 +26,9 @@ LICENSE Makefile.PL MANIFEST This list of files README.md +src/include/javascript_builtins.h +src/javascript_builtins.c +src/Makefile.PL t/00-use_ok.t t/downloaders/animeflv_example_response.html t/downloaders/gocdn.t diff --git a/Makefile.PL b/Makefile.PL index d39a7e9..830ad79 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -4,6 +4,16 @@ WriteMakefile( NAME => 'Peertube::DL', VERSION => '0.1', INST_SCRIPT => './bin', + INST_BIN => './bin', test => { TESTS => 't/*.t' }, test => { TESTS => 't/*/*.t' }, ); + +sub MY::postamble { + ' +src: src/Makefile + cd src && $(MAKE) $(PASSTHRU) +javascript_interpreter_xs: javascript_interpreter_xs/Makefile + cd javascript_interpreter_xs/ && $(MAKE) $(PASSTHRU) +'; +} diff --git a/bin/js-pruebas b/bin/js-pruebas new file mode 100644 index 0000000..1d15f59 --- /dev/null +++ b/bin/js-pruebas @@ -0,0 +1,19 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use feature 'say'; + +use Peertube::DL::Javascript; + +my $a = Peertube::DL::Javascript::_duk_create_heap_default(); +eval { Peertube::DL::Javascript::_duk_push_lstring( $a, "print(\"hola mundo\\n\");" ); }; +if ($@) { + warn $@; + $@ = ""; +} +if ( defined $a ) { + printf( "0x%0x\n", $a ); +} +Peertube::DL::Javascript::_duk_peval($a); +Peertube::DL::Javascript::_duk_destroy_heap($a); diff --git a/bin/peertube-dl b/bin/peertube-dl index 41b54cf..e85a9c7 100755 --- a/bin/peertube-dl +++ b/bin/peertube-dl @@ -9,9 +9,12 @@ use Getopt::Long::Descriptive; use Peertube::DL::URLHandler; +binmode STDOUT, ':utf8'; + my ( $opt, $usage ) = describe_options( 'peertube-dl %o ', - [ 'recurse|r', 'Recursive in reproduction lists.', { default => 0 } ], + [ 'recurse|r', 'Recursive in reproduction lists.', { default => 0 } ], + [ 'format|f=s', 'Choose format by id.', ], [], [ 'help|h', 'Show this help.', { shortcircuit => 1 } ], ); @@ -19,10 +22,12 @@ my ( $opt, $usage ) = describe_options( print( $usage->text ), exit if $opt->help; my $recurse = $opt->recurse; +my $format = $opt->format; die "No url passed" unless @ARGV; -my $download_data = Peertube::DL::URLHandler::getDownloadDataFromURL( $ARGV[0] ); +my $download_data = + Peertube::DL::URLHandler::getDownloadDataFromURL( $ARGV[0], { defined $format ? ( format => $format ) : () } ); my $ua = Peertube::DL::URLHandler::generateUA(); @@ -31,13 +36,14 @@ if ( defined $download_data->{options}{list} && $download_data->{options}{list} if ($recurse) { for my $url (@$urls) { say "Handling nested url: $url"; - my $url_download_data = Peertube::DL::URLHandler::getDownloadDataFromURL( $url ); + my $url_download_data = Peertube::DL::URLHandler::getDownloadDataFromURL($url); downloadVideo( $ua, $url_download_data ); } } else { say "The urls are:\n" . join "\n", @$urls; } - +} elsif ( defined $download_data->{options}{list_formats} && $download_data->{options}{list_formats} ) { + exit 0; } else { downloadVideo( $ua, $download_data ); } @@ -52,7 +58,7 @@ sub downloadVideo { die "Cannot retrieve video data" unless $response->is_success; my $content = $response->decoded_content; - + say "Writing into $filename."; open my $fh, '>', $filename or die "Cannot open $filename"; $fh->print( $response->decoded_content ) or die "Cannot write to $filename"; diff --git a/cpanfile b/cpanfile index 6fd4c14..ef53f94 100644 --- a/cpanfile +++ b/cpanfile @@ -7,3 +7,4 @@ requires 'Test::Most'; requires 'Test::MockObject'; requires 'Mojo::Server::Hypnotoad'; requires 'Getopt::Long::Descriptive'; +requires 'Perl::Tidy'; diff --git a/javascript_interpreter_xs/Makefile.PL b/javascript_interpreter_xs/Makefile.PL new file mode 100644 index 0000000..bec083b --- /dev/null +++ b/javascript_interpreter_xs/Makefile.PL @@ -0,0 +1,11 @@ +use ExtUtils::MakeMaker; + +WriteMakefile( + NAME => 'Peertube::DL::Javascript', + VERSION => '0.1', + LIBS => ['-lduktape'], + INC => '-Iduktape -I../src/include', + XS => { 'javascript.xs' => 'javascript.o' }, + OBJECT => 'javascript.o ../src/javascript_builtins.o', + LDFLAGS => '-Wl-t', +); diff --git a/javascript_interpreter_xs/javascript.xs b/javascript_interpreter_xs/javascript.xs new file mode 100644 index 0000000..7da8219 --- /dev/null +++ b/javascript_interpreter_xs/javascript.xs @@ -0,0 +1,77 @@ +#define PERL_NO_GET_CONTEXT +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "duktape.h" +#include "duk_config.h" +#include "javascript_builtins.h" + +MODULE = Peertube::DL::Javascript PACKAGE = Peertube::DL::Javascript +PROTOTYPES: ENABLE + +SV * +_duk_create_heap_default() + CODE: + duk_context *context = duk_create_heap_default(); + if (context) { + duk_push_c_function(context, js_builtin_print, 1); + duk_put_global_string(context, "print"); + RETVAL = newSVuv((size_t)context); + } else { + RETVAL = &PL_sv_undef; + } + OUTPUT: + RETVAL + +SV * +_duk_push_lstring(SV *, SV *) + CODE: + duk_context *context = (duk_context *) SvUV(ST(0)); + if(!context) { + croak("Javascript context undef.", 0); + } + STRLEN len; + char * lstring = SvPV(ST(1), len); + if(!len) { + croak("Empty string on lstring push."); + } + duk_push_lstring(context, lstring, strlen(lstring)); +// +// * Example subroutine call +// dSP; +// ENTER; +// SAVETMPS; +// PUSHMARK(SP); +// EXTEND(SP, 1); +// PUSHs(sv_2mortal(newSVpv("Javascript context invalid.", 0))); +// PUTBACK; +// +// call_sv(sv_2mortal(newSVpv("::die", 0)), G_DISCARD); +// +// FREETMPS; +// LEAVE; +// +// + OUTPUT: + RETVAL + +void +_duk_peval(SV *) + CODE: + duk_context *context = (duk_context *) SvUV(ST(0)); + if(!context) { + croak("Javascript context undef.", 0); + } + + if(duk_peval(context) != 0) { + croak("Eval failed:\n%s\n", duk_safe_to_string(context, -1)); + } + +void +_duk_destroy_heap(SV *) + CODE: + duk_context *context = (duk_context *) SvUV(ST(0)); + if (!context) { + croak("Cannot destroy something that is not a context."); + } + duk_destroy_heap(context); diff --git a/lib/Peertube/DL/Downloaders.pm b/lib/Peertube/DL/Downloaders.pm index af92afb..4549f4d 100644 --- a/lib/Peertube/DL/Downloaders.pm +++ b/lib/Peertube/DL/Downloaders.pm @@ -10,6 +10,66 @@ use Peertube::DL::Utils; use Data::Dumper; use Mojo::DOM; +sub youtube { + my $ua = shift; + my $response = shift; + my $options = shift; + my $dom = Mojo::DOM->new( $response->decoded_content ); + my $script_tag = $dom->find('script')->grep( + sub { + $_[0]->text =~ /var ytInitialPlayerResponse =/; + } + )->first; + my ($ytInitialPlayerResponse) = $script_tag->text =~ /^var ytInitialPlayerResponse = (.*?\});var meta/; + $ytInitialPlayerResponse = JSON::from_json($ytInitialPlayerResponse); + my $microformat = $ytInitialPlayerResponse->{microformat}{playerMicroformatRenderer}; + $ytInitialPlayerResponse = $ytInitialPlayerResponse->{streamingData}; + + if ( defined $options->{format} ) { + my $format = $options->{format}; + ($format) = grep { $_->{itag} eq $format } ( + scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} } + ? @{ $ytInitialPlayerResponse->{adaptiveFormats} } + : (), + scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : () + ); + my $url_data = $format->{signatureCipher}; + $url_data = { + map { + my ( $a, $b ) = /(.*?)=(.*)$/; + ( $a => Peertube::DL::Utils::uri_decode( Peertube::DL::Utils::uri_decode($b) ) ) + } split '&', + $url_data + }; + + #my ($player_url) = $response->decoded_content =~ m/"jsUrl"\s*:\s*("[^"]+")/; + #$player_url = JSON::from_json( $player_url, { allow_nonref => 1 } ); + #$player_url = 'https://www.youtube.com' . $player_url + # unless $player_url =~ m'^https://www.youtube.com'; + #say $player_url; + say length $url_data->{s}; + } else { + return { + options => { list_formats => 1 }, + title => $microformat->{title}{simpleText}, + description => $microformat->{description}{simpleText}, + formats => [ + map { + { + id => $_->{itag}, + mimeType => $_->{mimeType} + } + } ( + scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} } + ? @{ $ytInitialPlayerResponse->{adaptiveFormats} } + : (), + scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : () + ) + ] + }; + } +} + sub kjanime { my $ua = shift; my $response = shift; diff --git a/lib/Peertube/DL/Javascript.pm b/lib/Peertube/DL/Javascript.pm new file mode 100755 index 0000000..f4db2f4 --- /dev/null +++ b/lib/Peertube/DL/Javascript.pm @@ -0,0 +1,13 @@ +#!/usr/bin/env perl +package Peertube::DL::Javascript; + +use strict; +use warnings; + +use feature 'say'; + +use XSLoader; +use Data::Dumper; + +XSLoader::load(); +1; diff --git a/lib/Peertube/DL/URLHandler.pm b/lib/Peertube/DL/URLHandler.pm index ce63e47..115e48b 100644 --- a/lib/Peertube/DL/URLHandler.pm +++ b/lib/Peertube/DL/URLHandler.pm @@ -11,6 +11,7 @@ use Peertube::DL::Downloaders; sub getDownloadDataFromURL { my $url_origen = shift; my $ua = Peertube::DL::URLHandler::generateUA(); + my $options = shift; $ua->set_redirect_ok(1); my $response = $ua->get($url_origen); my %handlers = ( @@ -18,20 +19,23 @@ sub getDownloadDataFromURL { animeid => { reg => qr/animeid\.to\/streaming\.php\?/, subr => \&Peertube::DL::Downloaders::animeid }, kjanime => { reg => qr/kjanime - Anime en formato ligero y HQ - kjanime/, - subr => \&Peertube::DL::Downloaders::kjanime + subr => \&Peertube::DL::Downloaders::kjanime, }, kjanime_ch => { reg => qr/Link de descarga . kjanime/, - subr => \&Peertube::DL::Downloaders::kjanime_ch + subr => \&Peertube::DL::Downloaders::kjanime_ch, + }, + youtube => { + reg => qr/ytInitialPlayerResponse = \{/, + subr => \&Peertube::DL::Downloaders::youtube, }, - , ); $ua->set_redirect_ok(0); my $handled = 0; my $download_data; for my $x ( keys %handlers ) { if ( $response->decoded_content =~ m/$handlers{$x}{reg}/ ) { - eval { $download_data = $handlers{$x}{subr}->( $ua, $response ); }; + eval { $download_data = $handlers{$x}{subr}->( $ua, $response, $options ); }; if ($@) { warn $@; } else { @@ -46,15 +50,28 @@ sub getDownloadDataFromURL { unless $handled; die "Download data not defined" unless defined $download_data; die "Download data not hashref" unless ref($download_data) eq 'HASH'; - if ( defined $download_data->{options}{list} - && $download_data->{options}{list} ) - { - say 'Reproduction list detected.'; - die 'No url list.' - unless defined $download_data->{urls}; - die 'Urls is not an array' - unless ref $download_data->{urls} eq 'ARRAY'; - return $download_data; + if ( defined $download_data->{options} ) { + if ( defined $download_data->{options}{list} + && $download_data->{options}{list} ) + { + say 'Reproduction list detected.'; + die 'No url list.' + unless defined $download_data->{urls}; + die 'Urls is not an array' + unless ref $download_data->{urls} eq 'ARRAY'; + return $download_data; + } + if ( defined $download_data->{options}{list_formats} && $download_data->{options}{list_formats} ) { + say 'List of formats retrieved.'; + die "No title." unless defined $download_data->{title}; + die "No description." unless defined $download_data->{description}; + die "No formats available." unless defined $download_data->{formats}; + die "Formats is not an arrayref." unless ref $download_data->{formats} eq 'ARRAY'; + say "The video title is $download_data->{title}."; + say "The video description is $download_data->{description}."; + say "The available formats are: @{[Data::Dumper::Dumper $download_data->{formats}]}."; + return $download_data; + } } die "Filename not defined" unless exists $download_data->{filename} && $download_data->{filename}; die "Download url not defined" unless exists $download_data->{url} && defined $download_data->{url}; diff --git a/src/Makefile.PL b/src/Makefile.PL new file mode 100644 index 0000000..72482b3 --- /dev/null +++ b/src/Makefile.PL @@ -0,0 +1,11 @@ +use ExtUtils::MakeMaker; + +WriteMakefile( + NAME => 'Peertube::DL::SRC', + VERSION => '0.1', + LIBS => ['-lduktape'], + INC => '-Iduktape -I./include', + C => [ 'javascript_builtins.c', ], + OBJECT => '${O_FILES}', + LDFLAGS => '-Wl-t', +); diff --git a/src/include/javascript_builtins.h b/src/include/javascript_builtins.h new file mode 100644 index 0000000..8b279ed --- /dev/null +++ b/src/include/javascript_builtins.h @@ -0,0 +1,5 @@ +#include +#include "duktape.h" +#include "duk_config.h" + +duk_ret_t js_builtin_print(duk_context *context); diff --git a/src/javascript_builtins.c b/src/javascript_builtins.c new file mode 100644 index 0000000..db71ad7 --- /dev/null +++ b/src/javascript_builtins.c @@ -0,0 +1,9 @@ +#include +#include "duktape.h" +#include "duk_config.h" + +duk_ret_t js_builtin_print(duk_context *context) { + const char * to_print = duk_get_string(context, 0); + printf("%s", to_print); + return 1; +}