From 3abba1edce396d570403868083cc4355e03fafe1 Mon Sep 17 00:00:00 2001 From: sergiotarxz Date: Sat, 9 Jan 2021 03:55:59 +0100 Subject: [PATCH] Added youtube support. --- bin/js-pruebas | 20 ++-- cpanfile | 5 + javascript_interpreter_xs/javascript.xs | 7 +- lib/Peertube/DL/Downloaders.pm | 119 ++++++++++++++++++------ lib/Peertube/DL/Javascript.pm | 39 ++++++++ 5 files changed, 147 insertions(+), 43 deletions(-) diff --git a/bin/js-pruebas b/bin/js-pruebas index ab65d63..29c2824 100755 --- a/bin/js-pruebas +++ b/bin/js-pruebas @@ -7,18 +7,10 @@ use feature 'say'; use Peertube::DL::Javascript; -my $a = Peertube::DL::Javascript::_duk_create_heap_default(); -if ( defined $a ) { - eval { - my $return = Peertube::DL::Javascript::_duk_eval_wrapper( - $a, - "function a(nombre) { return 'Hello ' + nombre; }; a('sergio');" - ); - say $return; - }; - if ($@) { - warn $@; - } - say Peertube::DL::Javascript::_duk_call_function( $a, "a", "sergio" ); - Peertube::DL::Javascript::_duk_destroy_heap($a); +my $js = Peertube::DL::Javascript->new; +eval { + say $js->evalJS("if(\"undefined\"!=typeof Reflect&&Reflect.construct){if(a())return Reflect.construct;var b=Reflect.construct;return function(c,d,e){c=b(c,d);e&&Reflect.setPrototypeOf(c,e.prototype);return c}}return function(c,d,e){void 0===e&&(e=c);"); +}; +if ($@) { + warn $@; } diff --git a/cpanfile b/cpanfile index ef53f94..5a61b98 100644 --- a/cpanfile +++ b/cpanfile @@ -8,3 +8,8 @@ requires 'Test::MockObject'; requires 'Mojo::Server::Hypnotoad'; requires 'Getopt::Long::Descriptive'; requires 'Perl::Tidy'; +requires 'Params::Validate'; +requires 'Params::Util'; +requires 'Net::SSLeay'; +requires 'HTML::HeadParser'; +requires 'File::MimeInfo'; diff --git a/javascript_interpreter_xs/javascript.xs b/javascript_interpreter_xs/javascript.xs index 9441baa..b7854c5 100644 --- a/javascript_interpreter_xs/javascript.xs +++ b/javascript_interpreter_xs/javascript.xs @@ -45,7 +45,7 @@ _duk_eval_wrapper(SV *, SV *) char *code = SvPV(ST(1), code_len); duk_push_lstring(context, code, strlen(code)); if ( duk_peval(context) != 0 ) { - croak("Eval failed:\n%s\n", duk_get_string(context, -1)); + croak("Eval failed:\n%s\n%s\n", duk_safe_to_string(context, -1), duk_to_stacktrace(context, -1)); } duk_size_t *lstring_len; const char * lstring = duk_get_lstring(context, -1, lstring_len); @@ -93,8 +93,9 @@ _duk_call_function(SV *, SV *, ...) duk_size_t *lstring_len; duk_call(context, 1); char * return_value = duk_get_lstring(context, -1, lstring_len); - top_index = duk_get_top_index(context); - duk_pop_n(context, top_index+1); + if ( (top_index = duk_get_top_index(context)) != DUK_INVALID_INDEX ) { + duk_pop_n(context, top_index+1); + } RETVAL = newSVpv(return_value, (size_t) lstring_len); OUTPUT: RETVAL diff --git a/lib/Peertube/DL/Downloaders.pm b/lib/Peertube/DL/Downloaders.pm index 4549f4d..b421028 100644 --- a/lib/Peertube/DL/Downloaders.pm +++ b/lib/Peertube/DL/Downloaders.pm @@ -1,3 +1,4 @@ +## Please see file perltidy.ERR package Peertube::DL::Downloaders; use strict; @@ -5,10 +6,16 @@ use warnings; use feature 'say'; +use Symbol 'gensym'; +use IPC::Open3; +use File::Basename; use JSON; -use Peertube::DL::Utils; use Data::Dumper; use Mojo::DOM; +use File::MimeInfo; + +use Peertube::DL::Javascript; +use Peertube::DL::Utils; sub youtube { my $ua = shift; @@ -20,9 +27,11 @@ sub youtube { $_[0]->text =~ /var ytInitialPlayerResponse =/; } )->first; - my ($ytInitialPlayerResponse) = $script_tag->text =~ /^var ytInitialPlayerResponse = (.*?\});var meta/; + my ($ytInitialPlayerResponse) = + $script_tag->text =~ /^var ytInitialPlayerResponse = (.*?\});var meta/; $ytInitialPlayerResponse = JSON::from_json($ytInitialPlayerResponse); - my $microformat = $ytInitialPlayerResponse->{microformat}{playerMicroformatRenderer}; + my $microformat = + $ytInitialPlayerResponse->{microformat}{playerMicroformatRenderer}; $ytInitialPlayerResponse = $ytInitialPlayerResponse->{streamingData}; if ( defined $options->{format} ) { @@ -31,23 +40,69 @@ sub youtube { scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} } ? @{ $ytInitialPlayerResponse->{adaptiveFormats} } : (), - scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : () + scalar @{ $ytInitialPlayerResponse->{formats} } + ? @{ $ytInitialPlayerResponse->{formats} } + : () ); my $url_data = $format->{signatureCipher}; $url_data = { map { my ( $a, $b ) = /(.*?)=(.*)$/; - ( $a => Peertube::DL::Utils::uri_decode( Peertube::DL::Utils::uri_decode($b) ) ) + ( + $a => Peertube::DL::Utils::uri_decode( + Peertube::DL::Utils::uri_decode($b) + ) + ) } split '&', $url_data }; - #my ($player_url) = $response->decoded_content =~ m/"jsUrl"\s*:\s*("[^"]+")/; - #$player_url = JSON::from_json( $player_url, { allow_nonref => 1 } ); - #$player_url = 'https://www.youtube.com' . $player_url - # unless $player_url =~ m'^https://www.youtube.com'; - #say $player_url; - say length $url_data->{s}; + say Data::Dumper::Dumper $url_data; + + my ($player_url) = + $response->decoded_content =~ m/"jsUrl"\s*:\s*("[^"]+")/; + $player_url = JSON::from_json( $player_url, { allow_nonref => 1 } ); + $player_url = 'https://www.youtube.com' . $player_url + unless $player_url =~ m'^https://www.youtube.com'; + say $player_url; + my $hostname = $player_url =~ s/https?:\/\///r; + $hostname = $hostname =~ s/\/.*$//; + $hostname = JSON::encode_json($hostname); + my $js = Peertube::DL::Javascript->new; + my $player_response = $ua->get($player_url); + my $regexes_search_function_name = [ + qr/\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?(?:[a-zA-Z0-9]|\$)+)\(/, + qr/\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?(?:[a-zA-Z0-9]|\$)+)\(/, + qr/(?:\b|(?:[^a-zA-Z0-9]|\$))(?(?:[a-zA-Z0-9]|\$){2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)/, + qr/(?(?:[a-zA-Z0-9]|\$)+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)/, + ]; + my $function_name_regen_sig; + for my $regex (@$regexes_search_function_name) { + ($function_name_regen_sig) = + $player_response->decoded_content =~ /$regex/; + last if defined $function_name_regen_sig; + } + my ($extract_function_sig) = $player_response->decoded_content =~ + m/\n(${function_name_regen_sig}=function\([a-zA-Z]+\)\{.*?\};)\n/; + my ($class_decoding) = + $extract_function_sig =~ /;([a-zA-Z]+)\.[a-zA-Z]+\(/; + my ($class_decoding_code) = $player_response->decoded_content =~ + /;(var ${class_decoding}=\{.*?\}\};)/s; + $js->evalJS( join "\n", $class_decoding_code ); + $js->evalJS( join "\n", $extract_function_sig ); + my $signature = + $js->callJSFunction( $function_name_regen_sig, $url_data->{s} ); + my $url = $url_data->{url} . "&sig=${signature}"; + say Data::Dumper::Dumper $format; + my $mime_type = $format->{mimeType} =~ s/;.*$//r; + say $mime_type; + my $extension = scalar File::MimeInfo::extensions($mime_type); + $extension = 'mp4' if $mime_type eq 'video/mp4'; + say $url; + return { + url => $url, + filename => $microformat->{title}{simpleText} . '.' . $extension, + }; } else { return { options => { list_formats => 1 }, @@ -63,7 +118,9 @@ sub youtube { scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} } ? @{ $ytInitialPlayerResponse->{adaptiveFormats} } : (), - scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : () + scalar @{ $ytInitialPlayerResponse->{formats} } + ? @{ $ytInitialPlayerResponse->{formats} } + : () ) ] }; @@ -93,7 +150,8 @@ sub kjanime { 'X-Requested-With' => 'XMLHttpRequest', ); $dom = Mojo::DOM->new( $get_php_response->decoded_content ); - my $links = [ map { $_->attr('href') =~ s/^http:\/\//https:\/\//r } @{ $dom->find('a')->to_array } ]; + my $links = [ map { $_->attr('href') =~ s/^http:\/\//https:\/\//r } + @{ $dom->find('a')->to_array } ]; return { options => { list => 1 }, urls => $links }; } @@ -102,7 +160,8 @@ sub kjanime_ch { my $response = shift; my ($id) = $response->filename; say $id; - my $param = join '', map { Peertube::DL::Utils::uri_decode("%$_") } unpack( "(A2)*", $id ); + my $param = join '', + map { Peertube::DL::Utils::uri_decode("%$_") } unpack( "(A2)*", $id ); say $param; # FORMAT: KJA://70714B6E314943383135712B72582B70713768797A4B6E4D734D474775773D3D/1 @@ -118,7 +177,8 @@ sub kjanime_ch { $url_serie_php, 'X-Requested-With' => 'XMLHttpRequest', ); - my $download_data = JSON::decode_json( $response_serie_php->decoded_content ); + my $download_data = + JSON::decode_json( $response_serie_php->decoded_content ); say Data::Dumper::Dumper $download_data; $download_data->{filename} = delete $download_data->{name}; $download_data->{url} = delete $download_data->{link}; @@ -135,9 +195,11 @@ sub gocdn { my $url_gocdn = 'https://streamium.xyz/gocdn.php?v=' . $id; my $response_gocdn = $ua->get($url_gocdn); - die $response_gocdn->status_line . ' from ' . $url_gocdn unless $response_gocdn->is_success; + die $response_gocdn->status_line . ' from ' . $url_gocdn + unless $response_gocdn->is_success; - my $google_url = JSON::decode_json( $response_gocdn->decoded_content )->{file}; + my $google_url = + JSON::decode_json( $response_gocdn->decoded_content )->{file}; my ($filename) = $response->base =~ s/^.*\///gr . '.mp4'; @@ -145,12 +207,13 @@ sub gocdn { } sub animeid { - my $ua = shift; - my $response = shift; - my @order = ( 'id', 'title', 'refer' ); - my $url = $response->base; - my $url_ajax = 'https://animeid.to/ajax.php?'; - my ($get_params) = $response->decoded_content =~ /animeid\.to\/streaming\.php\?(.*?)"/; + my $ua = shift; + my $response = shift; + my @order = ( 'id', 'title', 'refer' ); + my $url = $response->base; + my $url_ajax = 'https://animeid.to/ajax.php?'; + my ($get_params) = + $response->decoded_content =~ /animeid\.to\/streaming\.php\?(.*?)"/; $get_params = { map { my ( $key, $value ) = /^(.*?)=(.*)$/; @@ -164,7 +227,10 @@ sub animeid { $get_params->{refer} = $url; my $filename = $get_params->{title}; $url_ajax .= join '&', - map { join '=', ( $_ => Peertube::DL::Utils::uri_encode( $get_params->{$_} ) ) } @order; + map { + join '=', + ( $_ => Peertube::DL::Utils::uri_encode( $get_params->{$_} ) ) + } @order; say 'Getting video location from: ' . $url_ajax; my $ajax_data = $ua->get( $url_ajax, @@ -175,8 +241,9 @@ sub animeid { say "Decoding json... $ajax_data"; $ajax_data = JSON::decode_json $ajax_data; die 'No video source found.' if ( !defined $ajax_data->{source} ); - my $download_redirect_url = $ajax_data->{source}[0]{file} // die "No url found."; - my $extension = $ajax_data->{source}[0]{type} // die "No extension found."; + my $download_redirect_url = $ajax_data->{source}[0]{file} + // die "No url found."; + my $extension = $ajax_data->{source}[0]{type} // die "No extension found."; $filename .= ".$extension"; say "Getting redirect to download url from $download_redirect_url..."; my $video_response = $ua->get( diff --git a/lib/Peertube/DL/Javascript.pm b/lib/Peertube/DL/Javascript.pm index f4db2f4..a0a1dbb 100755 --- a/lib/Peertube/DL/Javascript.pm +++ b/lib/Peertube/DL/Javascript.pm @@ -10,4 +10,43 @@ use XSLoader; use Data::Dumper; XSLoader::load(); + +sub new { + my $class = shift; + my $self = bless {}, $class; + $self->{___ContextPrivateDONOTTOUCH} = + Peertube::DL::Javascript::_duk_create_heap_default(); + return $self; +} + +sub evalJS { + my $self = shift; + my $context = $self->___ContextDONOTUSE; + my $code = shift; + return Peertube::DL::Javascript::_duk_eval_wrapper( $context, $code ); +} + +sub callJSFunction { + my $self = shift; + my $function = shift; + my @function_arguments = @_; + my $context = $self->___ContextDONOTUSE; + return Peertube::DL::Javascript::_duk_call_function( + $context, + $function, + @function_arguments + ); +} + +sub ___ContextDONOTUSE { + my $self = shift; + return $self->{___ContextPrivateDONOTTOUCH}; +} + +sub DESTROY { + my $self = shift; + my $context = $self->___ContextDONOTUSE; + Peertube::DL::Javascript::_duk_destroy_heap($context); + undef $self; +} 1;