367 lines
11 KiB
Perl
367 lines
11 KiB
Perl
|
package YTYT;
|
||
|
|
|
||
|
|
use strict;
|
||
|
|
use warnings;
|
||
|
|
|
||
|
|
use Data::Dump;
|
||
|
|
use utf8;
|
||
|
|
use DBI;
|
||
|
|
use JSON::Any;
|
||
|
|
use WWW::Mechanize;
|
||
|
|
use HTTP::Cookies;
|
||
|
|
|
||
|
|
sub new {
|
||
|
|
my %params = @_;
|
||
|
|
|
||
|
|
if (!defined $params{host}) {
|
||
|
|
$params{host} = 'www.youtube.com';
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!defined $params{db_path}) {
|
||
|
|
$params{db_path} = "/var/lib/youtube/db/youtube.sqlite";
|
||
|
|
}
|
||
|
|
if (! -e $params{db_path} ) {
|
||
|
|
create_db(\%params);
|
||
|
|
}
|
||
|
|
|
||
|
|
my $cookie_jar = HTTP::Cookies->new('/var/www/yt/yt_cookie.txt');
|
||
|
|
if (!defined $params{mechanize}) {
|
||
|
|
if ($params{host} eq 'www.youtube.com') {
|
||
|
|
#$params{mechanize} = WWW::Mechanize->new(autocheck => 1, cookie_jar => $cookie_jar, agent => 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; MDDRJS; rv:11.0) like Gecko');
|
||
|
|
$params{mechanize} = LWP::UserAgent->new();
|
||
|
|
} elsif ($params{host} eq 'm.youtube.com') {
|
||
|
|
$params{mechanize} = WWW::Mechanize->new(autocheck => 1, cookie_jar => $cookie_jar, agent => 'Mozilla/5.0 (Android 7.1.2; Mobile; rv:64.0) Gecko/64.0 Firefox/64.0');
|
||
|
|
} else {
|
||
|
|
die "Invalid host: $params{host}\nAcceptable options: 'www.youtube.com' or 'm.youtube.com'";
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
my $self = { db_path => $params{db_path}, mechanize => $params{mechanize}, location => "https://$params{host}" };
|
||
|
|
bless $self;
|
||
|
|
}
|
||
|
|
|
||
|
|
sub get_videos_page {
|
||
|
|
|
||
|
|
my $self = shift;
|
||
|
|
my %params = @_;
|
||
|
|
my $get;
|
||
|
|
my $target;
|
||
|
|
|
||
|
|
if (defined $params{channelId}) {
|
||
|
|
$target = $params{channelId};
|
||
|
|
$get = $self->{location} . '/' .
|
||
|
|
'channel/' . $params{channelId} .
|
||
|
|
'/videos';
|
||
|
|
} elsif (defined $params{channelName}) {
|
||
|
|
$target = $params{channelName};
|
||
|
|
$get = $self->{location} . '/' .
|
||
|
|
'c/' . $params{channelName} .
|
||
|
|
'/videos';
|
||
|
|
} elsif (defined $params{userName}) {
|
||
|
|
$target = $params{userName};
|
||
|
|
$get = $self->{location} . '/' .
|
||
|
|
'user/' . $params{userName} .
|
||
|
|
'/videos';
|
||
|
|
} else {
|
||
|
|
die "Failed to fetch video page.\n" .
|
||
|
|
"get_video_pages requires either channelId or channelName as an argument.\n";
|
||
|
|
}
|
||
|
|
|
||
|
|
my $response = $self->{mechanize}->get($get)->decoded_content;
|
||
|
|
|
||
|
|
my @lines = split '\n', $response;
|
||
|
|
my $initial_data = (grep {/var ytInitialData/} @lines)[0];
|
||
|
|
if (!defined $initial_data) {
|
||
|
|
die "Error: Channel $target may not exist\n";
|
||
|
|
}
|
||
|
|
|
||
|
|
$initial_data =~ s/.*var ytInitialData = (.*?);\s*<\/script>.*/$1/;
|
||
|
|
|
||
|
|
my $json = JSON::Any->new( utf8 => 1 );
|
||
|
|
return $json->decode($initial_data);
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
sub latest_videos {
|
||
|
|
|
||
|
|
my $self = shift;
|
||
|
|
my %params = @_;
|
||
|
|
|
||
|
|
my $content = get_videos_page($self, %params);
|
||
|
|
|
||
|
|
#my $list_ref = $content->{contents}->{twoColumnBrowseResultsRenderer}->{tabs}->[1]->{tabRenderer}->{content}->{sectionListRenderer}->{contents}->[0]->{itemSectionRenderer}->{contents}->[0]->{gridRenderer}->{items};
|
||
|
|
my $list_ref = $content->{contents}->{twoColumnBrowseResultsRenderer}->{tabs}->[1]->{tabRenderer}->{content}->{richGridRenderer}->{contents};
|
||
|
|
my ($offset, $last) = (0, 0);
|
||
|
|
my @videos;
|
||
|
|
|
||
|
|
foreach my $item (@$list_ref) {
|
||
|
|
|
||
|
|
use Data::Dump 'dump';
|
||
|
|
#die (dump($item->{richItemRenderer}->{content}->{videoRenderer}->{videoId}));
|
||
|
|
# Premium don't provide age and/or view count. Skip.
|
||
|
|
my $badges = $item->{richItemRenderer}->{content}->{videoRenderer}->{badges}[0];
|
||
|
|
my $premium;
|
||
|
|
if (defined $badges) {
|
||
|
|
foreach ($badges) {
|
||
|
|
if ($_->{metadataBadgeRenderer}->{label} eq "Premium") {
|
||
|
|
$premium = 1;
|
||
|
|
last;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (defined $premium) {
|
||
|
|
next;
|
||
|
|
}
|
||
|
|
|
||
|
|
my %video = (
|
||
|
|
channelId => $params{channelId},
|
||
|
|
);
|
||
|
|
$video{videoId} = $item->{richItemRenderer}->{content}->{videoRenderer}->{videoId} || next;
|
||
|
|
$video{videoThumbnail} = $item->{richItemRenderer}->{content}->{videoRenderer}->{thumbnail}->{thumbnails}[0]->{url};
|
||
|
|
foreach my $overlay (@{$item->{richItemRenderer}->{content}->{videoRenderer}->{thumbnailOverlays}}) {
|
||
|
|
if (defined ($overlay->{thumbnailOverlayTimeStatusRenderer})) {
|
||
|
|
$video{lengthText} = $overlay->{thumbnailOverlayTimeStatusRenderer}->{text}->{simpleText};
|
||
|
|
last;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
foreach ( qw| publishedTimeText shortViewCountText | ) {
|
||
|
|
$video{$_} = $item->{richItemRenderer}->{content}->{videoRenderer}->{$_}->{simpleText};
|
||
|
|
}
|
||
|
|
|
||
|
|
foreach ( qw| title | ) {
|
||
|
|
$video{$_} = $item->{richItemRenderer}->{content}->{videoRenderer}->{$_}->{runs}[0]->{text};
|
||
|
|
}
|
||
|
|
|
||
|
|
if (defined $video{publishedTimeText}) {
|
||
|
|
$video{age} = $video{publishedTimeText};
|
||
|
|
if ($video{publishedTimeText} =~ m/Streamed /) {
|
||
|
|
$video{age} =~ s/^Streamed //;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
# YouTube Premium doesn't show a publishTimeText, but I can't watch them anyways. Just skip
|
||
|
|
#This shows up for currently live streams
|
||
|
|
#print $video{videoId} . " looks like premium\n";
|
||
|
|
next;
|
||
|
|
}
|
||
|
|
if ($video{age} eq $last) {
|
||
|
|
$offset++;
|
||
|
|
} else {
|
||
|
|
$offset = 0;
|
||
|
|
}
|
||
|
|
$last = $video{age};
|
||
|
|
|
||
|
|
my $now = time();
|
||
|
|
|
||
|
|
if ($video{age} =~ m/\d+ seconds? ago/) {
|
||
|
|
$video{age} =~ s/(\d+) seconds? ago/$1/eeg;
|
||
|
|
if ($offset) {
|
||
|
|
$video{age} += $offset;
|
||
|
|
}
|
||
|
|
} elsif ($video{age} =~ m/\d+ minutes? ago/) {
|
||
|
|
$video{age} =~ s/(\d+) minutes? ago/$1/eeg;
|
||
|
|
$video{age} *= 60;
|
||
|
|
if ($offset) {
|
||
|
|
$video{age} += $offset;
|
||
|
|
}
|
||
|
|
} elsif ($video{age} =~ m/\d+ hours? ago/) {
|
||
|
|
$video{age} =~ s/(\d+) hours? ago/$1/eeg;
|
||
|
|
$video{age} *= 60*60;
|
||
|
|
if ($offset) {
|
||
|
|
$video{age} += $offset*60;
|
||
|
|
}
|
||
|
|
} elsif ($video{age} =~ m/\d+ days? ago/) {
|
||
|
|
$video{age} =~ s/(\d+) days? ago/$1/eeg;
|
||
|
|
$video{age} *= 60*60*24;
|
||
|
|
if ($offset) {
|
||
|
|
$video{age} += $offset*60*60;
|
||
|
|
}
|
||
|
|
} elsif ($video{age} =~ m/\d+ weeks? ago/) {
|
||
|
|
$video{age} =~ s/(\d+) weeks? ago/$1/eeg;
|
||
|
|
$video{age} *= 60*60*24*7;
|
||
|
|
if ($offset) {
|
||
|
|
$video{age} += $offset*60*60*24;
|
||
|
|
}
|
||
|
|
} elsif ($video{age} =~ m/\d+ months? ago/) {
|
||
|
|
$video{age} =~ s/(\d+) months? ago/$1/eeg;
|
||
|
|
$video{age} *= 60*60*24*30;
|
||
|
|
if ($offset) {
|
||
|
|
$video{age} += $offset*60*60*24;
|
||
|
|
}
|
||
|
|
} elsif ($video{age} =~ m/\d+ years? ago/) {
|
||
|
|
$video{age} =~ s/(\d+) years? ago/$1/eeg;
|
||
|
|
$video{age} *= 60*60*24*365;
|
||
|
|
if ($offset) {
|
||
|
|
$video{age} += $offset*60*60*24;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
die "Invalid age $video{age}. Cannot convert to seconds.";
|
||
|
|
}
|
||
|
|
|
||
|
|
$video{age} = $now - $video{age};
|
||
|
|
|
||
|
|
my $video_ref = \%video;
|
||
|
|
push @videos, $video_ref;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
return @videos;
|
||
|
|
}
|
||
|
|
|
||
|
|
sub db_connect {
|
||
|
|
my $self = shift;
|
||
|
|
my $db = 'dbi:SQLite:dbname=' . $self->{db_path};
|
||
|
|
$self->{dbh} = DBI->connect($db, '', '', {AutoCommit=>1, PrintError=>0})
|
||
|
|
or die "Unable to connect: $!\n";
|
||
|
|
return 1;
|
||
|
|
}
|
||
|
|
|
||
|
|
sub db_disconnect {
|
||
|
|
my $self = shift;
|
||
|
|
$self->{dbh}->disconnect();
|
||
|
|
return 1;
|
||
|
|
}
|
||
|
|
|
||
|
|
sub create_db {
|
||
|
|
|
||
|
|
my $self = shift;
|
||
|
|
|
||
|
|
my @path = split '/', $self->{db_path};
|
||
|
|
my $filename = pop @path;
|
||
|
|
shift @path;
|
||
|
|
|
||
|
|
# Ensure that entire path exists
|
||
|
|
my $path;
|
||
|
|
foreach (@path) {
|
||
|
|
$path .= "/" . $_ ;
|
||
|
|
if ($_ eq 'db') {
|
||
|
|
$) = getgrnam('www-data');
|
||
|
|
$> = getpwnam('www-data');
|
||
|
|
}
|
||
|
|
if (! -e $path) {
|
||
|
|
mkdir $path;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (-e $self->{db_path}) {
|
||
|
|
die "Database already exists: $self->{db_path}\n";
|
||
|
|
} else {
|
||
|
|
db_connect($self);
|
||
|
|
$self->{dbh}->do("CREATE TABLE videos(channelId, videoId, title, videoThumbnail, publishedTimeText, lengthText, shortViewCountText, age INTERGER, seen BOOL);");
|
||
|
|
$self->{dbh}->do("CREATE TABLE channels(channelId, channelName, channelThumbnail, regex, category);");
|
||
|
|
$self->{dbh}->do("CREATE TABLE settings(enable4 BOOL, read4, write4, enable6 BOOL, read6, write6, refresh INTEGER, player, embed_type, theme);");
|
||
|
|
$self->{dbh}->do("INSERT INTO settings(enable4, read4, write4, enable6, read6, write6, refresh, player, embed_type, theme) values(1, '0.0.0.0/0', '0.0.0.0/0', 1, '::0/0', '::0/0', 15, 'web', 'proxy', 'default');");
|
||
|
|
db_disconnect($self);
|
||
|
|
}
|
||
|
|
$) = $(;
|
||
|
|
$> = $<;
|
||
|
|
}
|
||
|
|
|
||
|
|
sub db_update {
|
||
|
|
|
||
|
|
my $self = shift;
|
||
|
|
my %params = @_;
|
||
|
|
|
||
|
|
if (!defined $self->{dbh}) {
|
||
|
|
die "Missing database handle\n";
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!defined $params{table}) {
|
||
|
|
die "Missing 'table' parameter\n";
|
||
|
|
}
|
||
|
|
|
||
|
|
my @cols;
|
||
|
|
my $end;
|
||
|
|
if ($params{table} eq 'videos') {
|
||
|
|
@cols = ( qw | channelId videoId title publishedTimeText lengthText age seen | );
|
||
|
|
$end = "videoId = '$params{videoId}'";
|
||
|
|
} elsif ($params{table} eq 'channels') {
|
||
|
|
@cols = ( qw | channelId channelName channelThumbnail | );
|
||
|
|
$end = "channelId = '$params{channelId}'";
|
||
|
|
} else {
|
||
|
|
die "Invalid table: $params{table}\n";
|
||
|
|
}
|
||
|
|
|
||
|
|
foreach (@cols) {
|
||
|
|
if (!defined $params{$_}) {
|
||
|
|
print STDERR "Missing necessary column $_\n";
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
my $query = "UPDATE $params{table} SET";
|
||
|
|
|
||
|
|
foreach (@cols) {
|
||
|
|
if ($_ ne 'channelId' && $_ ne 'videoId' && $_ ne 'seen') {
|
||
|
|
$query .= " $_ = '$params{$_}',";
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if ($params{table} eq 'channels' && defined $params{regex}) {
|
||
|
|
$query .= " regex = '$params{regex}'";
|
||
|
|
}
|
||
|
|
|
||
|
|
$query =~ s/,$/ /;
|
||
|
|
$query .= "WHERE $end";
|
||
|
|
|
||
|
|
$self->{dbh}->do($query);
|
||
|
|
}
|
||
|
|
|
||
|
|
sub db_insert {
|
||
|
|
|
||
|
|
my $self = shift;
|
||
|
|
my %params = @_;
|
||
|
|
|
||
|
|
if (!defined $self->{dbh}) {
|
||
|
|
die "Missing database handle\n";
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!defined $params{table}) {
|
||
|
|
die "Missing 'table' parameter\n";
|
||
|
|
}
|
||
|
|
|
||
|
|
my @cols;
|
||
|
|
if ($params{table} eq 'videos') {
|
||
|
|
@cols = ( qw | channelId videoId title publishedTimeText lengthText age seen | );
|
||
|
|
} elsif ($params{table} eq 'channels') {
|
||
|
|
@cols = ( qw | channelId channelName channelThumbnail | );
|
||
|
|
} else {
|
||
|
|
die "Invalid table: $params{table}\n";
|
||
|
|
}
|
||
|
|
|
||
|
|
foreach (@cols) {
|
||
|
|
if (!defined $params{$_}) {
|
||
|
|
print STDERR "Missing necessary column $_\n";
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
my $query = "INSERT INTO $params{table}(";
|
||
|
|
my $values = "VALUES(";
|
||
|
|
|
||
|
|
foreach (@cols) {
|
||
|
|
$query .= "$_,";
|
||
|
|
if ($_ ne 'age' && $_ ne 'seen') {
|
||
|
|
my $param = $params{$_};
|
||
|
|
$param =~ s/'/''/g;
|
||
|
|
$values .= "'$param',";
|
||
|
|
} else {
|
||
|
|
$values .= "$params{$_},";
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if ($params{table} eq 'channels' && defined $params{regex}) {
|
||
|
|
$query .= "regex,";
|
||
|
|
$values .= "$params{regex}";
|
||
|
|
}
|
||
|
|
|
||
|
|
$query =~ s/,$/\) /;
|
||
|
|
$values =~ s/,$/\);/;
|
||
|
|
$query .= $values;
|
||
|
|
|
||
|
|
my $response = $self->{dbh}->do($query);
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
1;
|