ytyt-legacy/lib/YTYT.pm

367 lines
11 KiB
Perl
Raw Normal View History

package YTYT;
use strict;
use warnings;
use Data::Dump;
use utf8;
use DBI;
use JSON::Any;
use WWW::Mechanize;
use HTTP::Cookies;
sub new {
my %params = @_;
if (!defined $params{host}) {
$params{host} = 'www.youtube.com';
}
if (!defined $params{db_path}) {
$params{db_path} = "/var/lib/youtube/db/youtube.sqlite";
}
if (! -e $params{db_path} ) {
create_db(\%params);
}
my $cookie_jar = HTTP::Cookies->new('/var/www/yt/yt_cookie.txt');
if (!defined $params{mechanize}) {
if ($params{host} eq 'www.youtube.com') {
#$params{mechanize} = WWW::Mechanize->new(autocheck => 1, cookie_jar => $cookie_jar, agent => 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; MDDRJS; rv:11.0) like Gecko');
$params{mechanize} = LWP::UserAgent->new();
} elsif ($params{host} eq 'm.youtube.com') {
$params{mechanize} = WWW::Mechanize->new(autocheck => 1, cookie_jar => $cookie_jar, agent => 'Mozilla/5.0 (Android 7.1.2; Mobile; rv:64.0) Gecko/64.0 Firefox/64.0');
} else {
die "Invalid host: $params{host}\nAcceptable options: 'www.youtube.com' or 'm.youtube.com'";
}
}
my $self = { db_path => $params{db_path}, mechanize => $params{mechanize}, location => "https://$params{host}" };
bless $self;
}
sub get_videos_page {
my $self = shift;
my %params = @_;
my $get;
my $target;
if (defined $params{channelId}) {
$target = $params{channelId};
$get = $self->{location} . '/' .
'channel/' . $params{channelId} .
'/videos';
} elsif (defined $params{channelName}) {
$target = $params{channelName};
$get = $self->{location} . '/' .
'c/' . $params{channelName} .
'/videos';
} elsif (defined $params{userName}) {
$target = $params{userName};
$get = $self->{location} . '/' .
'user/' . $params{userName} .
'/videos';
} else {
die "Failed to fetch video page.\n" .
"get_video_pages requires either channelId or channelName as an argument.\n";
}
my $response = $self->{mechanize}->get($get)->decoded_content;
my @lines = split '\n', $response;
my $initial_data = (grep {/var ytInitialData/} @lines)[0];
if (!defined $initial_data) {
die "Error: Channel $target may not exist\n";
}
$initial_data =~ s/.*var ytInitialData = (.*?);\s*<\/script>.*/$1/;
my $json = JSON::Any->new( utf8 => 1 );
return $json->decode($initial_data);
}
sub latest_videos {
my $self = shift;
my %params = @_;
my $content = get_videos_page($self, %params);
#my $list_ref = $content->{contents}->{twoColumnBrowseResultsRenderer}->{tabs}->[1]->{tabRenderer}->{content}->{sectionListRenderer}->{contents}->[0]->{itemSectionRenderer}->{contents}->[0]->{gridRenderer}->{items};
my $list_ref = $content->{contents}->{twoColumnBrowseResultsRenderer}->{tabs}->[1]->{tabRenderer}->{content}->{richGridRenderer}->{contents};
my ($offset, $last) = (0, 0);
my @videos;
foreach my $item (@$list_ref) {
use Data::Dump 'dump';
#die (dump($item->{richItemRenderer}->{content}->{videoRenderer}->{videoId}));
# Premium don't provide age and/or view count. Skip.
my $badges = $item->{richItemRenderer}->{content}->{videoRenderer}->{badges}[0];
my $premium;
if (defined $badges) {
foreach ($badges) {
if ($_->{metadataBadgeRenderer}->{label} eq "Premium") {
$premium = 1;
last;
}
}
}
if (defined $premium) {
next;
}
my %video = (
channelId => $params{channelId},
);
$video{videoId} = $item->{richItemRenderer}->{content}->{videoRenderer}->{videoId} || next;
$video{videoThumbnail} = $item->{richItemRenderer}->{content}->{videoRenderer}->{thumbnail}->{thumbnails}[0]->{url};
foreach my $overlay (@{$item->{richItemRenderer}->{content}->{videoRenderer}->{thumbnailOverlays}}) {
if (defined ($overlay->{thumbnailOverlayTimeStatusRenderer})) {
$video{lengthText} = $overlay->{thumbnailOverlayTimeStatusRenderer}->{text}->{simpleText};
last;
}
}
foreach ( qw| publishedTimeText shortViewCountText | ) {
$video{$_} = $item->{richItemRenderer}->{content}->{videoRenderer}->{$_}->{simpleText};
}
foreach ( qw| title | ) {
$video{$_} = $item->{richItemRenderer}->{content}->{videoRenderer}->{$_}->{runs}[0]->{text};
}
if (defined $video{publishedTimeText}) {
$video{age} = $video{publishedTimeText};
if ($video{publishedTimeText} =~ m/Streamed /) {
$video{age} =~ s/^Streamed //;
}
} else {
# YouTube Premium doesn't show a publishTimeText, but I can't watch them anyways. Just skip
#This shows up for currently live streams
#print $video{videoId} . " looks like premium\n";
next;
}
if ($video{age} eq $last) {
$offset++;
} else {
$offset = 0;
}
$last = $video{age};
my $now = time();
if ($video{age} =~ m/\d+ seconds? ago/) {
$video{age} =~ s/(\d+) seconds? ago/$1/eeg;
if ($offset) {
$video{age} += $offset;
}
} elsif ($video{age} =~ m/\d+ minutes? ago/) {
$video{age} =~ s/(\d+) minutes? ago/$1/eeg;
$video{age} *= 60;
if ($offset) {
$video{age} += $offset;
}
} elsif ($video{age} =~ m/\d+ hours? ago/) {
$video{age} =~ s/(\d+) hours? ago/$1/eeg;
$video{age} *= 60*60;
if ($offset) {
$video{age} += $offset*60;
}
} elsif ($video{age} =~ m/\d+ days? ago/) {
$video{age} =~ s/(\d+) days? ago/$1/eeg;
$video{age} *= 60*60*24;
if ($offset) {
$video{age} += $offset*60*60;
}
} elsif ($video{age} =~ m/\d+ weeks? ago/) {
$video{age} =~ s/(\d+) weeks? ago/$1/eeg;
$video{age} *= 60*60*24*7;
if ($offset) {
$video{age} += $offset*60*60*24;
}
} elsif ($video{age} =~ m/\d+ months? ago/) {
$video{age} =~ s/(\d+) months? ago/$1/eeg;
$video{age} *= 60*60*24*30;
if ($offset) {
$video{age} += $offset*60*60*24;
}
} elsif ($video{age} =~ m/\d+ years? ago/) {
$video{age} =~ s/(\d+) years? ago/$1/eeg;
$video{age} *= 60*60*24*365;
if ($offset) {
$video{age} += $offset*60*60*24;
}
} else {
die "Invalid age $video{age}. Cannot convert to seconds.";
}
$video{age} = $now - $video{age};
my $video_ref = \%video;
push @videos, $video_ref;
}
return @videos;
}
sub db_connect {
my $self = shift;
my $db = 'dbi:SQLite:dbname=' . $self->{db_path};
$self->{dbh} = DBI->connect($db, '', '', {AutoCommit=>1, PrintError=>0})
or die "Unable to connect: $!\n";
return 1;
}
sub db_disconnect {
my $self = shift;
$self->{dbh}->disconnect();
return 1;
}
sub create_db {
my $self = shift;
my @path = split '/', $self->{db_path};
my $filename = pop @path;
shift @path;
# Ensure that entire path exists
my $path;
foreach (@path) {
$path .= "/" . $_ ;
if ($_ eq 'db') {
$) = getgrnam('www-data');
$> = getpwnam('www-data');
}
if (! -e $path) {
mkdir $path;
}
}
if (-e $self->{db_path}) {
die "Database already exists: $self->{db_path}\n";
} else {
db_connect($self);
$self->{dbh}->do("CREATE TABLE videos(channelId, videoId, title, videoThumbnail, publishedTimeText, lengthText, shortViewCountText, age INTERGER, seen BOOL);");
$self->{dbh}->do("CREATE TABLE channels(channelId, channelName, channelThumbnail, regex, category);");
$self->{dbh}->do("CREATE TABLE settings(enable4 BOOL, read4, write4, enable6 BOOL, read6, write6, refresh INTEGER, player, embed_type, theme);");
$self->{dbh}->do("INSERT INTO settings(enable4, read4, write4, enable6, read6, write6, refresh, player, embed_type, theme) values(1, '0.0.0.0/0', '0.0.0.0/0', 1, '::0/0', '::0/0', 15, 'web', 'proxy', 'default');");
db_disconnect($self);
}
$) = $(;
$> = $<;
}
sub db_update {
my $self = shift;
my %params = @_;
if (!defined $self->{dbh}) {
die "Missing database handle\n";
}
if (!defined $params{table}) {
die "Missing 'table' parameter\n";
}
my @cols;
my $end;
if ($params{table} eq 'videos') {
@cols = ( qw | channelId videoId title publishedTimeText lengthText age seen | );
$end = "videoId = '$params{videoId}'";
} elsif ($params{table} eq 'channels') {
@cols = ( qw | channelId channelName channelThumbnail | );
$end = "channelId = '$params{channelId}'";
} else {
die "Invalid table: $params{table}\n";
}
foreach (@cols) {
if (!defined $params{$_}) {
print STDERR "Missing necessary column $_\n";
return;
}
}
my $query = "UPDATE $params{table} SET";
foreach (@cols) {
if ($_ ne 'channelId' && $_ ne 'videoId' && $_ ne 'seen') {
$query .= " $_ = '$params{$_}',";
}
}
if ($params{table} eq 'channels' && defined $params{regex}) {
$query .= " regex = '$params{regex}'";
}
$query =~ s/,$/ /;
$query .= "WHERE $end";
$self->{dbh}->do($query);
}
sub db_insert {
my $self = shift;
my %params = @_;
if (!defined $self->{dbh}) {
die "Missing database handle\n";
}
if (!defined $params{table}) {
die "Missing 'table' parameter\n";
}
my @cols;
if ($params{table} eq 'videos') {
@cols = ( qw | channelId videoId title publishedTimeText lengthText age seen | );
} elsif ($params{table} eq 'channels') {
@cols = ( qw | channelId channelName channelThumbnail | );
} else {
die "Invalid table: $params{table}\n";
}
foreach (@cols) {
if (!defined $params{$_}) {
print STDERR "Missing necessary column $_\n";
return;
}
}
my $query = "INSERT INTO $params{table}(";
my $values = "VALUES(";
foreach (@cols) {
$query .= "$_,";
if ($_ ne 'age' && $_ ne 'seen') {
my $param = $params{$_};
$param =~ s/'/''/g;
$values .= "'$param',";
} else {
$values .= "$params{$_},";
}
}
if ($params{table} eq 'channels' && defined $params{regex}) {
$query .= "regex,";
$values .= "$params{regex}";
}
$query =~ s/,$/\) /;
$values =~ s/,$/\);/;
$query .= $values;
my $response = $self->{dbh}->do($query);
}
1;