Adding initial javascript interpreter to be able to support youtube as video backend.

This commit is contained in:
sergiotarxz 2021-01-06 07:53:32 +01:00
parent 22f2aef630
commit 4bfca82bcd
Signed by untrusted user who does not match committer: sergiotarxz
GPG Key ID: E5903508B6510AC2
13 changed files with 268 additions and 19 deletions

View File

@ -1,14 +1,21 @@
.proverc .proverc
AUTHORS AUTHORS
bin/js-pruebas
bin/peertube-dl bin/peertube-dl
bin/peertube-dl-hypnotoad bin/peertube-dl-hypnotoad
bin/peertube-dl-web bin/peertube-dl-web
bin/peertube-dl-web.conf bin/peertube-dl-web.conf
cpanfile cpanfile
include/javascript_builtins.h
javascript_builtins.c
javascript_interpreter_xs/javascript.xs
javascript_interpreter_xs/Makefile.PL
lib/auto/Peertube/DL/.exists
lib/Peertube/DL.pm lib/Peertube/DL.pm
lib/Peertube/DL/Downloaders.pm lib/Peertube/DL/Downloaders.pm
lib/Peertube/DL/Javascript.pm
lib/Peertube/DL/Javascript.xs
lib/Peertube/DL/public/css/index.css lib/Peertube/DL/public/css/index.css
lib/Peertube/DL/public/css/spinner.css
lib/Peertube/DL/public/img/spinner.svg lib/Peertube/DL/public/img/spinner.svg
lib/Peertube/DL/public/index.html lib/Peertube/DL/public/index.html
lib/Peertube/DL/public/js/peertube-dl-web.js lib/Peertube/DL/public/js/peertube-dl-web.js
@ -19,6 +26,9 @@ LICENSE
Makefile.PL Makefile.PL
MANIFEST This list of files MANIFEST This list of files
README.md README.md
src/include/javascript_builtins.h
src/javascript_builtins.c
src/Makefile.PL
t/00-use_ok.t t/00-use_ok.t
t/downloaders/animeflv_example_response.html t/downloaders/animeflv_example_response.html
t/downloaders/gocdn.t t/downloaders/gocdn.t

View File

@ -4,6 +4,16 @@ WriteMakefile(
NAME => 'Peertube::DL', NAME => 'Peertube::DL',
VERSION => '0.1', VERSION => '0.1',
INST_SCRIPT => './bin', INST_SCRIPT => './bin',
INST_BIN => './bin',
test => { TESTS => 't/*.t' }, test => { TESTS => 't/*.t' },
test => { TESTS => 't/*/*.t' }, test => { TESTS => 't/*/*.t' },
); );
sub MY::postamble {
'
src: src/Makefile
cd src && $(MAKE) $(PASSTHRU)
javascript_interpreter_xs: javascript_interpreter_xs/Makefile
cd javascript_interpreter_xs/ && $(MAKE) $(PASSTHRU)
';
}

19
bin/js-pruebas Normal file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env perl
use strict;
use warnings;
use feature 'say';
use Peertube::DL::Javascript;
my $a = Peertube::DL::Javascript::_duk_create_heap_default();
eval { Peertube::DL::Javascript::_duk_push_lstring( $a, "print(\"hola mundo\\n\");" ); };
if ($@) {
warn $@;
$@ = "";
}
if ( defined $a ) {
printf( "0x%0x\n", $a );
}
Peertube::DL::Javascript::_duk_peval($a);
Peertube::DL::Javascript::_duk_destroy_heap($a);

View File

@ -9,9 +9,12 @@ use Getopt::Long::Descriptive;
use Peertube::DL::URLHandler; use Peertube::DL::URLHandler;
binmode STDOUT, ':utf8';
my ( $opt, $usage ) = describe_options( my ( $opt, $usage ) = describe_options(
'peertube-dl %o <url>', 'peertube-dl %o <url>',
[ 'recurse|r', 'Recursive in reproduction lists.', { default => 0 } ], [ 'recurse|r', 'Recursive in reproduction lists.', { default => 0 } ],
[ 'format|f=s', 'Choose format by id.', ],
[], [],
[ 'help|h', 'Show this help.', { shortcircuit => 1 } ], [ 'help|h', 'Show this help.', { shortcircuit => 1 } ],
); );
@ -19,10 +22,12 @@ my ( $opt, $usage ) = describe_options(
print( $usage->text ), exit if $opt->help; print( $usage->text ), exit if $opt->help;
my $recurse = $opt->recurse; my $recurse = $opt->recurse;
my $format = $opt->format;
die "No url passed" unless @ARGV; die "No url passed" unless @ARGV;
my $download_data = Peertube::DL::URLHandler::getDownloadDataFromURL( $ARGV[0] ); my $download_data =
Peertube::DL::URLHandler::getDownloadDataFromURL( $ARGV[0], { defined $format ? ( format => $format ) : () } );
my $ua = Peertube::DL::URLHandler::generateUA(); my $ua = Peertube::DL::URLHandler::generateUA();
@ -31,13 +36,14 @@ if ( defined $download_data->{options}{list} && $download_data->{options}{list}
if ($recurse) { if ($recurse) {
for my $url (@$urls) { for my $url (@$urls) {
say "Handling nested url: $url"; say "Handling nested url: $url";
my $url_download_data = Peertube::DL::URLHandler::getDownloadDataFromURL( $url ); my $url_download_data = Peertube::DL::URLHandler::getDownloadDataFromURL($url);
downloadVideo( $ua, $url_download_data ); downloadVideo( $ua, $url_download_data );
} }
} else { } else {
say "The urls are:\n" . join "\n", @$urls; say "The urls are:\n" . join "\n", @$urls;
} }
} elsif ( defined $download_data->{options}{list_formats} && $download_data->{options}{list_formats} ) {
exit 0;
} else { } else {
downloadVideo( $ua, $download_data ); downloadVideo( $ua, $download_data );
} }

View File

@ -7,3 +7,4 @@ requires 'Test::Most';
requires 'Test::MockObject'; requires 'Test::MockObject';
requires 'Mojo::Server::Hypnotoad'; requires 'Mojo::Server::Hypnotoad';
requires 'Getopt::Long::Descriptive'; requires 'Getopt::Long::Descriptive';
requires 'Perl::Tidy';

View File

@ -0,0 +1,11 @@
use ExtUtils::MakeMaker;
WriteMakefile(
NAME => 'Peertube::DL::Javascript',
VERSION => '0.1',
LIBS => ['-lduktape'],
INC => '-Iduktape -I../src/include',
XS => { 'javascript.xs' => 'javascript.o' },
OBJECT => 'javascript.o ../src/javascript_builtins.o',
LDFLAGS => '-Wl-t',
);

View File

@ -0,0 +1,77 @@
#define PERL_NO_GET_CONTEXT
#include "EXTERN.h"
#include "perl.h"
#include "XSUB.h"
#include "duktape.h"
#include "duk_config.h"
#include "javascript_builtins.h"
MODULE = Peertube::DL::Javascript PACKAGE = Peertube::DL::Javascript
PROTOTYPES: ENABLE
SV *
_duk_create_heap_default()
CODE:
duk_context *context = duk_create_heap_default();
if (context) {
duk_push_c_function(context, js_builtin_print, 1);
duk_put_global_string(context, "print");
RETVAL = newSVuv((size_t)context);
} else {
RETVAL = &PL_sv_undef;
}
OUTPUT:
RETVAL
SV *
_duk_push_lstring(SV *, SV *)
CODE:
duk_context *context = (duk_context *) SvUV(ST(0));
if(!context) {
croak("Javascript context undef.", 0);
}
STRLEN len;
char * lstring = SvPV(ST(1), len);
if(!len) {
croak("Empty string on lstring push.");
}
duk_push_lstring(context, lstring, strlen(lstring));
//
// * Example subroutine call
// dSP;
// ENTER;
// SAVETMPS;
// PUSHMARK(SP);
// EXTEND(SP, 1);
// PUSHs(sv_2mortal(newSVpv("Javascript context invalid.", 0)));
// PUTBACK;
//
// call_sv(sv_2mortal(newSVpv("::die", 0)), G_DISCARD);
//
// FREETMPS;
// LEAVE;
//
//
OUTPUT:
RETVAL
void
_duk_peval(SV *)
CODE:
duk_context *context = (duk_context *) SvUV(ST(0));
if(!context) {
croak("Javascript context undef.", 0);
}
if(duk_peval(context) != 0) {
croak("Eval failed:\n%s\n", duk_safe_to_string(context, -1));
}
void
_duk_destroy_heap(SV *)
CODE:
duk_context *context = (duk_context *) SvUV(ST(0));
if (!context) {
croak("Cannot destroy something that is not a context.");
}
duk_destroy_heap(context);

View File

@ -10,6 +10,66 @@ use Peertube::DL::Utils;
use Data::Dumper; use Data::Dumper;
use Mojo::DOM; use Mojo::DOM;
sub youtube {
my $ua = shift;
my $response = shift;
my $options = shift;
my $dom = Mojo::DOM->new( $response->decoded_content );
my $script_tag = $dom->find('script')->grep(
sub {
$_[0]->text =~ /var ytInitialPlayerResponse =/;
}
)->first;
my ($ytInitialPlayerResponse) = $script_tag->text =~ /^var ytInitialPlayerResponse = (.*?\});var meta/;
$ytInitialPlayerResponse = JSON::from_json($ytInitialPlayerResponse);
my $microformat = $ytInitialPlayerResponse->{microformat}{playerMicroformatRenderer};
$ytInitialPlayerResponse = $ytInitialPlayerResponse->{streamingData};
if ( defined $options->{format} ) {
my $format = $options->{format};
($format) = grep { $_->{itag} eq $format } (
scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} }
? @{ $ytInitialPlayerResponse->{adaptiveFormats} }
: (),
scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : ()
);
my $url_data = $format->{signatureCipher};
$url_data = {
map {
my ( $a, $b ) = /(.*?)=(.*)$/;
( $a => Peertube::DL::Utils::uri_decode( Peertube::DL::Utils::uri_decode($b) ) )
} split '&',
$url_data
};
#my ($player_url) = $response->decoded_content =~ m/"jsUrl"\s*:\s*("[^"]+")/;
#$player_url = JSON::from_json( $player_url, { allow_nonref => 1 } );
#$player_url = 'https://www.youtube.com' . $player_url
# unless $player_url =~ m'^https://www.youtube.com';
#say $player_url;
say length $url_data->{s};
} else {
return {
options => { list_formats => 1 },
title => $microformat->{title}{simpleText},
description => $microformat->{description}{simpleText},
formats => [
map {
{
id => $_->{itag},
mimeType => $_->{mimeType}
}
} (
scalar @{ $ytInitialPlayerResponse->{adaptiveFormats} }
? @{ $ytInitialPlayerResponse->{adaptiveFormats} }
: (),
scalar @{ $ytInitialPlayerResponse->{formats} } ? @{ $ytInitialPlayerResponse->{formats} } : ()
)
]
};
}
}
sub kjanime { sub kjanime {
my $ua = shift; my $ua = shift;
my $response = shift; my $response = shift;

13
lib/Peertube/DL/Javascript.pm Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env perl
package Peertube::DL::Javascript;
use strict;
use warnings;
use feature 'say';
use XSLoader;
use Data::Dumper;
XSLoader::load();
1;

View File

@ -11,6 +11,7 @@ use Peertube::DL::Downloaders;
sub getDownloadDataFromURL { sub getDownloadDataFromURL {
my $url_origen = shift; my $url_origen = shift;
my $ua = Peertube::DL::URLHandler::generateUA(); my $ua = Peertube::DL::URLHandler::generateUA();
my $options = shift;
$ua->set_redirect_ok(1); $ua->set_redirect_ok(1);
my $response = $ua->get($url_origen); my $response = $ua->get($url_origen);
my %handlers = ( my %handlers = (
@ -18,20 +19,23 @@ sub getDownloadDataFromURL {
animeid => { reg => qr/animeid\.to\/streaming\.php\?/, subr => \&Peertube::DL::Downloaders::animeid }, animeid => { reg => qr/animeid\.to\/streaming\.php\?/, subr => \&Peertube::DL::Downloaders::animeid },
kjanime => { kjanime => {
reg => qr/kjanime - Anime en formato ligero y HQ - kjanime/, reg => qr/kjanime - Anime en formato ligero y HQ - kjanime/,
subr => \&Peertube::DL::Downloaders::kjanime subr => \&Peertube::DL::Downloaders::kjanime,
}, },
kjanime_ch => { kjanime_ch => {
reg => qr/Link de descarga . kjanime/, reg => qr/Link de descarga . kjanime/,
subr => \&Peertube::DL::Downloaders::kjanime_ch subr => \&Peertube::DL::Downloaders::kjanime_ch,
},
youtube => {
reg => qr/ytInitialPlayerResponse = \{/,
subr => \&Peertube::DL::Downloaders::youtube,
}, },
,
); );
$ua->set_redirect_ok(0); $ua->set_redirect_ok(0);
my $handled = 0; my $handled = 0;
my $download_data; my $download_data;
for my $x ( keys %handlers ) { for my $x ( keys %handlers ) {
if ( $response->decoded_content =~ m/$handlers{$x}{reg}/ ) { if ( $response->decoded_content =~ m/$handlers{$x}{reg}/ ) {
eval { $download_data = $handlers{$x}{subr}->( $ua, $response ); }; eval { $download_data = $handlers{$x}{subr}->( $ua, $response, $options ); };
if ($@) { if ($@) {
warn $@; warn $@;
} else { } else {
@ -46,6 +50,7 @@ sub getDownloadDataFromURL {
unless $handled; unless $handled;
die "Download data not defined" unless defined $download_data; die "Download data not defined" unless defined $download_data;
die "Download data not hashref" unless ref($download_data) eq 'HASH'; die "Download data not hashref" unless ref($download_data) eq 'HASH';
if ( defined $download_data->{options} ) {
if ( defined $download_data->{options}{list} if ( defined $download_data->{options}{list}
&& $download_data->{options}{list} ) && $download_data->{options}{list} )
{ {
@ -56,6 +61,18 @@ sub getDownloadDataFromURL {
unless ref $download_data->{urls} eq 'ARRAY'; unless ref $download_data->{urls} eq 'ARRAY';
return $download_data; return $download_data;
} }
if ( defined $download_data->{options}{list_formats} && $download_data->{options}{list_formats} ) {
say 'List of formats retrieved.';
die "No title." unless defined $download_data->{title};
die "No description." unless defined $download_data->{description};
die "No formats available." unless defined $download_data->{formats};
die "Formats is not an arrayref." unless ref $download_data->{formats} eq 'ARRAY';
say "The video title is $download_data->{title}.";
say "The video description is $download_data->{description}.";
say "The available formats are: @{[Data::Dumper::Dumper $download_data->{formats}]}.";
return $download_data;
}
}
die "Filename not defined" unless exists $download_data->{filename} && $download_data->{filename}; die "Filename not defined" unless exists $download_data->{filename} && $download_data->{filename};
die "Download url not defined" unless exists $download_data->{url} && defined $download_data->{url}; die "Download url not defined" unless exists $download_data->{url} && defined $download_data->{url};
return $download_data; return $download_data;

11
src/Makefile.PL Normal file
View File

@ -0,0 +1,11 @@
use ExtUtils::MakeMaker;
WriteMakefile(
NAME => 'Peertube::DL::SRC',
VERSION => '0.1',
LIBS => ['-lduktape'],
INC => '-Iduktape -I./include',
C => [ 'javascript_builtins.c', ],
OBJECT => '${O_FILES}',
LDFLAGS => '-Wl-t',
);

View File

@ -0,0 +1,5 @@
#include <stdio.h>
#include "duktape.h"
#include "duk_config.h"
duk_ret_t js_builtin_print(duk_context *context);

View File

@ -0,0 +1,9 @@
#include <stdio.h>
#include "duktape.h"
#include "duk_config.h"
duk_ret_t js_builtin_print(duk_context *context) {
const char * to_print = duk_get_string(context, 0);
printf("%s", to_print);
return 1;
}