parent
b2dfce8db2
commit
c54d6897e5
GPG Key ID:
E9C5EA2D867501AB
|
|
@ -0,0 +1,70 @@
|
|||
#!/usr/bin/perl
|
||||
use warnings;
|
||||
use strict;
|
||||
use lib '../lib';
|
||||
|
||||
use Data::Dump qw% dump %;
|
||||
use YTYT;
|
||||
|
||||
my %params;
|
||||
|
||||
# Setup YouTube object
|
||||
my $youtube = YTYT::new( host => 'www.youtube.com' );
|
||||
|
||||
my $content;
|
||||
|
||||
unless (defined $ARGV[0]) {
|
||||
die "Requires at least one argument with a User ID, Channel ID or URL string\n" .
|
||||
"with either of these values included. RegEx argument optional" . (scalar @ARGV) . "\n";
|
||||
}
|
||||
|
||||
if (defined $ARGV[1]) {
|
||||
$params{regex} = $ARGV[1];
|
||||
}
|
||||
|
||||
if ($ARGV[0] =~ m%^UC[a-zA-Z0-9\-\_]{22}$%) {
|
||||
$params{channelId} = $ARGV[0];
|
||||
#print "Found raw channelId " . $params{channelId} . "\n";
|
||||
} elsif ($ARGV[0] =~ m%^(?:(?:https://)?(?:www\.|m\.)?youtube\.com/)?\@([^\/]+)%) {
|
||||
$params{channelHandle} = $1;
|
||||
die("Handles are not yet supported. Found \@$params{channelHandle}\n");
|
||||
} elsif ($ARGV[0] =~ m%^(https://)?(www\.|m\.)?youtube\.com/channel/UC[a-zA-Z0-9\-\_]{22}%) {
|
||||
$params{channelId} = $ARGV[0];
|
||||
$params{channelId} =~ s%(https://)?(www\.|m\.)?youtube\.com/channel/(UC[^/]{22})(/.*)?%$3%;
|
||||
#print "Found URL encoded channelId " . $params{channelId} . "\n";
|
||||
} elsif ($ARGV[0] =~ m%^(https://)?(www\.|m\.)?youtube\.com/c/%) {
|
||||
$params{channelName} = $ARGV[0];
|
||||
$params{channelName} =~ s%(https://)?(www\.|m\.)?youtube\.com/c/([^/]*)/?.*%$4%;
|
||||
#print "Found URL encoded channelName " . $params{channelName} . "\n";
|
||||
} elsif ($ARGV[0] =~ m%^(https://)?(www\.|m\.)?youtube\.com/user/%) {
|
||||
$params{userName} = $ARGV[0];
|
||||
$params{userName} =~ s%(https://)?(www\.|m\.)?youtube\.com/user/([^/]*)/?.*%$4%;
|
||||
#print "Found URL encoded channelName " . $params{userName} . "\n";
|
||||
} else {
|
||||
$params{channelName} = $ARGV[0];
|
||||
#print "Found raw channelName " . $params{channelName} . "\n";
|
||||
}
|
||||
|
||||
$content = $youtube->get_videos_page( %params );
|
||||
|
||||
unless (defined $params{channelName}) {
|
||||
$params{channelName} = $content->{header}->{c4TabbedHeaderRenderer}->{title};
|
||||
}
|
||||
unless (defined $params{channelId}) {
|
||||
$params{channelId} = $content->{header}->{c4TabbedHeaderRenderer}->{channelId};
|
||||
}
|
||||
|
||||
$params{channelThumbnail} = $content->{header}->{c4TabbedHeaderRenderer}->{avatar}->{thumbnails}[0]->{url};
|
||||
|
||||
$youtube->db_connect();
|
||||
|
||||
my @check = $youtube->{dbh}->selectrow_array("SELECT channelId FROM channels WHERE channelId = '$params{channelId}';");
|
||||
|
||||
if (scalar @check) {
|
||||
$youtube->db_disconnect();
|
||||
die "Channel already exists in database.\n";
|
||||
}
|
||||
|
||||
$youtube->db_insert( table => 'channels', %params ) || die "Failed to insert: $!\n";
|
||||
$youtube->db_disconnect();
|
||||
print "Channel added successfully\n";
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
#!/usr/bin/perl
|
||||
use warnings;
|
||||
use strict;
|
||||
use lib '../lib';
|
||||
|
||||
use Data::Dump qw% dump %;
|
||||
use YTYT;
|
||||
|
||||
my %params;
|
||||
|
||||
# Simply initializing the object will create the DB. This is only needed for the PHP application
|
||||
if (my $youtube = YTYT::new( host => 'www.youtube.com' )) {
|
||||
print "Success\n";
|
||||
} else {
|
||||
print "Failed\n";
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
#!/usr/bin/perl
|
||||
use warnings;
|
||||
use strict;
|
||||
|
||||
use Data::Dump qw| dump |;
|
||||
|
||||
my $lib = $0;
|
||||
$lib =~ s#^(.*)/[^/]*$#$1/../lib#;
|
||||
|
||||
use File::Spec;
|
||||
require File::Spec->catfile($lib, "YTYT.pm");;
|
||||
|
||||
# Setup YouTube object
|
||||
my $youtube = YTYT::new();
|
||||
$youtube->db_connect();
|
||||
|
||||
# Get Channel list and Setup Hash Ref for Videos
|
||||
my $channels = $youtube->{dbh}->selectall_arrayref("SELECT channelId FROM channels;");
|
||||
my @channels;
|
||||
|
||||
foreach my $channel_ref (@$channels) {
|
||||
push @channels, @$channel_ref[0];
|
||||
}
|
||||
|
||||
# Get list of Videos from all subscriptions
|
||||
foreach my $channelId (@channels) {
|
||||
my @videos = $youtube->latest_videos( channelId => $channelId );
|
||||
unless (scalar @videos) {
|
||||
print(" no videos found for $channelId\n");
|
||||
next;
|
||||
}
|
||||
foreach my $video_ref (@videos) {
|
||||
my %video = %$video_ref;
|
||||
$video{channelId} = $channelId;
|
||||
$video{seen} = 0;
|
||||
|
||||
if (scalar $youtube->{dbh}->selectrow_array("SELECT videoId FROM videos WHERE videoId = '$video{videoId}';")) {
|
||||
$youtube->db_update( table => 'videos', %video );
|
||||
} else {
|
||||
$youtube->db_insert( table => 'videos', %video );
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
$youtube->db_disconnect();
|
||||
|
||||
print "Completed without error\n";
|
||||
|
|
@ -0,0 +1,366 @@
|
|||
package YTYT;
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use Data::Dump;
|
||||
use utf8;
|
||||
use DBI;
|
||||
use JSON::Any;
|
||||
use WWW::Mechanize;
|
||||
use HTTP::Cookies;
|
||||
|
||||
sub new {
|
||||
my %params = @_;
|
||||
|
||||
if (!defined $params{host}) {
|
||||
$params{host} = 'www.youtube.com';
|
||||
}
|
||||
|
||||
if (!defined $params{db_path}) {
|
||||
$params{db_path} = "/var/lib/youtube/db/youtube.sqlite";
|
||||
}
|
||||
if (! -e $params{db_path} ) {
|
||||
create_db(\%params);
|
||||
}
|
||||
|
||||
my $cookie_jar = HTTP::Cookies->new('/var/www/yt/yt_cookie.txt');
|
||||
if (!defined $params{mechanize}) {
|
||||
if ($params{host} eq 'www.youtube.com') {
|
||||
#$params{mechanize} = WWW::Mechanize->new(autocheck => 1, cookie_jar => $cookie_jar, agent => 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; MDDRJS; rv:11.0) like Gecko');
|
||||
$params{mechanize} = LWP::UserAgent->new();
|
||||
} elsif ($params{host} eq 'm.youtube.com') {
|
||||
$params{mechanize} = WWW::Mechanize->new(autocheck => 1, cookie_jar => $cookie_jar, agent => 'Mozilla/5.0 (Android 7.1.2; Mobile; rv:64.0) Gecko/64.0 Firefox/64.0');
|
||||
} else {
|
||||
die "Invalid host: $params{host}\nAcceptable options: 'www.youtube.com' or 'm.youtube.com'";
|
||||
}
|
||||
}
|
||||
|
||||
my $self = { db_path => $params{db_path}, mechanize => $params{mechanize}, location => "https://$params{host}" };
|
||||
bless $self;
|
||||
}
|
||||
|
||||
sub get_videos_page {
|
||||
|
||||
my $self = shift;
|
||||
my %params = @_;
|
||||
my $get;
|
||||
my $target;
|
||||
|
||||
if (defined $params{channelId}) {
|
||||
$target = $params{channelId};
|
||||
$get = $self->{location} . '/' .
|
||||
'channel/' . $params{channelId} .
|
||||
'/videos';
|
||||
} elsif (defined $params{channelName}) {
|
||||
$target = $params{channelName};
|
||||
$get = $self->{location} . '/' .
|
||||
'c/' . $params{channelName} .
|
||||
'/videos';
|
||||
} elsif (defined $params{userName}) {
|
||||
$target = $params{userName};
|
||||
$get = $self->{location} . '/' .
|
||||
'user/' . $params{userName} .
|
||||
'/videos';
|
||||
} else {
|
||||
die "Failed to fetch video page.\n" .
|
||||
"get_video_pages requires either channelId or channelName as an argument.\n";
|
||||
}
|
||||
|
||||
my $response = $self->{mechanize}->get($get)->decoded_content;
|
||||
|
||||
my @lines = split '\n', $response;
|
||||
my $initial_data = (grep {/var ytInitialData/} @lines)[0];
|
||||
if (!defined $initial_data) {
|
||||
die "Error: Channel $target may not exist\n";
|
||||
}
|
||||
|
||||
$initial_data =~ s/.*var ytInitialData = (.*?);\s*<\/script>.*/$1/;
|
||||
|
||||
my $json = JSON::Any->new( utf8 => 1 );
|
||||
return $json->decode($initial_data);
|
||||
|
||||
}
|
||||
|
||||
sub latest_videos {
|
||||
|
||||
my $self = shift;
|
||||
my %params = @_;
|
||||
|
||||
my $content = get_videos_page($self, %params);
|
||||
|
||||
#my $list_ref = $content->{contents}->{twoColumnBrowseResultsRenderer}->{tabs}->[1]->{tabRenderer}->{content}->{sectionListRenderer}->{contents}->[0]->{itemSectionRenderer}->{contents}->[0]->{gridRenderer}->{items};
|
||||
my $list_ref = $content->{contents}->{twoColumnBrowseResultsRenderer}->{tabs}->[1]->{tabRenderer}->{content}->{richGridRenderer}->{contents};
|
||||
my ($offset, $last) = (0, 0);
|
||||
my @videos;
|
||||
|
||||
foreach my $item (@$list_ref) {
|
||||
|
||||
use Data::Dump 'dump';
|
||||
#die (dump($item->{richItemRenderer}->{content}->{videoRenderer}->{videoId}));
|
||||
# Premium don't provide age and/or view count. Skip.
|
||||
my $badges = $item->{richItemRenderer}->{content}->{videoRenderer}->{badges}[0];
|
||||
my $premium;
|
||||
if (defined $badges) {
|
||||
foreach ($badges) {
|
||||
if ($_->{metadataBadgeRenderer}->{label} eq "Premium") {
|
||||
$premium = 1;
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (defined $premium) {
|
||||
next;
|
||||
}
|
||||
|
||||
my %video = (
|
||||
channelId => $params{channelId},
|
||||
);
|
||||
$video{videoId} = $item->{richItemRenderer}->{content}->{videoRenderer}->{videoId} || next;
|
||||
$video{videoThumbnail} = $item->{richItemRenderer}->{content}->{videoRenderer}->{thumbnail}->{thumbnails}[0]->{url};
|
||||
foreach my $overlay (@{$item->{richItemRenderer}->{content}->{videoRenderer}->{thumbnailOverlays}}) {
|
||||
if (defined ($overlay->{thumbnailOverlayTimeStatusRenderer})) {
|
||||
$video{lengthText} = $overlay->{thumbnailOverlayTimeStatusRenderer}->{text}->{simpleText};
|
||||
last;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ( qw| publishedTimeText shortViewCountText | ) {
|
||||
$video{$_} = $item->{richItemRenderer}->{content}->{videoRenderer}->{$_}->{simpleText};
|
||||
}
|
||||
|
||||
foreach ( qw| title | ) {
|
||||
$video{$_} = $item->{richItemRenderer}->{content}->{videoRenderer}->{$_}->{runs}[0]->{text};
|
||||
}
|
||||
|
||||
if (defined $video{publishedTimeText}) {
|
||||
$video{age} = $video{publishedTimeText};
|
||||
if ($video{publishedTimeText} =~ m/Streamed /) {
|
||||
$video{age} =~ s/^Streamed //;
|
||||
}
|
||||
} else {
|
||||
# YouTube Premium doesn't show a publishTimeText, but I can't watch them anyways. Just skip
|
||||
#This shows up for currently live streams
|
||||
#print $video{videoId} . " looks like premium\n";
|
||||
next;
|
||||
}
|
||||
if ($video{age} eq $last) {
|
||||
$offset++;
|
||||
} else {
|
||||
$offset = 0;
|
||||
}
|
||||
$last = $video{age};
|
||||
|
||||
my $now = time();
|
||||
|
||||
if ($video{age} =~ m/\d+ seconds? ago/) {
|
||||
$video{age} =~ s/(\d+) seconds? ago/$1/eeg;
|
||||
if ($offset) {
|
||||
$video{age} += $offset;
|
||||
}
|
||||
} elsif ($video{age} =~ m/\d+ minutes? ago/) {
|
||||
$video{age} =~ s/(\d+) minutes? ago/$1/eeg;
|
||||
$video{age} *= 60;
|
||||
if ($offset) {
|
||||
$video{age} += $offset;
|
||||
}
|
||||
} elsif ($video{age} =~ m/\d+ hours? ago/) {
|
||||
$video{age} =~ s/(\d+) hours? ago/$1/eeg;
|
||||
$video{age} *= 60*60;
|
||||
if ($offset) {
|
||||
$video{age} += $offset*60;
|
||||
}
|
||||
} elsif ($video{age} =~ m/\d+ days? ago/) {
|
||||
$video{age} =~ s/(\d+) days? ago/$1/eeg;
|
||||
$video{age} *= 60*60*24;
|
||||
if ($offset) {
|
||||
$video{age} += $offset*60*60;
|
||||
}
|
||||
} elsif ($video{age} =~ m/\d+ weeks? ago/) {
|
||||
$video{age} =~ s/(\d+) weeks? ago/$1/eeg;
|
||||
$video{age} *= 60*60*24*7;
|
||||
if ($offset) {
|
||||
$video{age} += $offset*60*60*24;
|
||||
}
|
||||
} elsif ($video{age} =~ m/\d+ months? ago/) {
|
||||
$video{age} =~ s/(\d+) months? ago/$1/eeg;
|
||||
$video{age} *= 60*60*24*30;
|
||||
if ($offset) {
|
||||
$video{age} += $offset*60*60*24;
|
||||
}
|
||||
} elsif ($video{age} =~ m/\d+ years? ago/) {
|
||||
$video{age} =~ s/(\d+) years? ago/$1/eeg;
|
||||
$video{age} *= 60*60*24*365;
|
||||
if ($offset) {
|
||||
$video{age} += $offset*60*60*24;
|
||||
}
|
||||
} else {
|
||||
die "Invalid age $video{age}. Cannot convert to seconds.";
|
||||
}
|
||||
|
||||
$video{age} = $now - $video{age};
|
||||
|
||||
my $video_ref = \%video;
|
||||
push @videos, $video_ref;
|
||||
|
||||
}
|
||||
|
||||
return @videos;
|
||||
}
|
||||
|
||||
sub db_connect {
|
||||
my $self = shift;
|
||||
my $db = 'dbi:SQLite:dbname=' . $self->{db_path};
|
||||
$self->{dbh} = DBI->connect($db, '', '', {AutoCommit=>1, PrintError=>0})
|
||||
or die "Unable to connect: $!\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
sub db_disconnect {
|
||||
my $self = shift;
|
||||
$self->{dbh}->disconnect();
|
||||
return 1;
|
||||
}
|
||||
|
||||
sub create_db {
|
||||
|
||||
my $self = shift;
|
||||
|
||||
my @path = split '/', $self->{db_path};
|
||||
my $filename = pop @path;
|
||||
shift @path;
|
||||
|
||||
# Ensure that entire path exists
|
||||
my $path;
|
||||
foreach (@path) {
|
||||
$path .= "/" . $_ ;
|
||||
if ($_ eq 'db') {
|
||||
$) = getgrnam('www-data');
|
||||
$> = getpwnam('www-data');
|
||||
}
|
||||
if (! -e $path) {
|
||||
mkdir $path;
|
||||
}
|
||||
}
|
||||
|
||||
if (-e $self->{db_path}) {
|
||||
die "Database already exists: $self->{db_path}\n";
|
||||
} else {
|
||||
db_connect($self);
|
||||
$self->{dbh}->do("CREATE TABLE videos(channelId, videoId, title, videoThumbnail, publishedTimeText, lengthText, shortViewCountText, age INTERGER, seen BOOL);");
|
||||
$self->{dbh}->do("CREATE TABLE channels(channelId, channelName, channelThumbnail, regex, category);");
|
||||
$self->{dbh}->do("CREATE TABLE settings(enable4 BOOL, read4, write4, enable6 BOOL, read6, write6, refresh INTEGER, player, embed_type, theme);");
|
||||
$self->{dbh}->do("INSERT INTO settings(enable4, read4, write4, enable6, read6, write6, refresh, player, embed_type, theme) values(1, '0.0.0.0/0', '0.0.0.0/0', 1, '::0/0', '::0/0', 15, 'web', 'proxy', 'default');");
|
||||
db_disconnect($self);
|
||||
}
|
||||
$) = $(;
|
||||
$> = $<;
|
||||
}
|
||||
|
||||
sub db_update {
|
||||
|
||||
my $self = shift;
|
||||
my %params = @_;
|
||||
|
||||
if (!defined $self->{dbh}) {
|
||||
die "Missing database handle\n";
|
||||
}
|
||||
|
||||
if (!defined $params{table}) {
|
||||
die "Missing 'table' parameter\n";
|
||||
}
|
||||
|
||||
my @cols;
|
||||
my $end;
|
||||
if ($params{table} eq 'videos') {
|
||||
@cols = ( qw | channelId videoId title publishedTimeText lengthText age seen | );
|
||||
$end = "videoId = '$params{videoId}'";
|
||||
} elsif ($params{table} eq 'channels') {
|
||||
@cols = ( qw | channelId channelName channelThumbnail | );
|
||||
$end = "channelId = '$params{channelId}'";
|
||||
} else {
|
||||
die "Invalid table: $params{table}\n";
|
||||
}
|
||||
|
||||
foreach (@cols) {
|
||||
if (!defined $params{$_}) {
|
||||
print STDERR "Missing necessary column $_\n";
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
my $query = "UPDATE $params{table} SET";
|
||||
|
||||
foreach (@cols) {
|
||||
if ($_ ne 'channelId' && $_ ne 'videoId' && $_ ne 'seen') {
|
||||
$query .= " $_ = '$params{$_}',";
|
||||
}
|
||||
}
|
||||
|
||||
if ($params{table} eq 'channels' && defined $params{regex}) {
|
||||
$query .= " regex = '$params{regex}'";
|
||||
}
|
||||
|
||||
$query =~ s/,$/ /;
|
||||
$query .= "WHERE $end";
|
||||
|
||||
$self->{dbh}->do($query);
|
||||
}
|
||||
|
||||
sub db_insert {
|
||||
|
||||
my $self = shift;
|
||||
my %params = @_;
|
||||
|
||||
if (!defined $self->{dbh}) {
|
||||
die "Missing database handle\n";
|
||||
}
|
||||
|
||||
if (!defined $params{table}) {
|
||||
die "Missing 'table' parameter\n";
|
||||
}
|
||||
|
||||
my @cols;
|
||||
if ($params{table} eq 'videos') {
|
||||
@cols = ( qw | channelId videoId title publishedTimeText lengthText age seen | );
|
||||
} elsif ($params{table} eq 'channels') {
|
||||
@cols = ( qw | channelId channelName channelThumbnail | );
|
||||
} else {
|
||||
die "Invalid table: $params{table}\n";
|
||||
}
|
||||
|
||||
foreach (@cols) {
|
||||
if (!defined $params{$_}) {
|
||||
print STDERR "Missing necessary column $_\n";
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
my $query = "INSERT INTO $params{table}(";
|
||||
my $values = "VALUES(";
|
||||
|
||||
foreach (@cols) {
|
||||
$query .= "$_,";
|
||||
if ($_ ne 'age' && $_ ne 'seen') {
|
||||
my $param = $params{$_};
|
||||
$param =~ s/'/''/g;
|
||||
$values .= "'$param',";
|
||||
} else {
|
||||
$values .= "$params{$_},";
|
||||
}
|
||||
}
|
||||
|
||||
if ($params{table} eq 'channels' && defined $params{regex}) {
|
||||
$query .= "regex,";
|
||||
$values .= "$params{regex}";
|
||||
}
|
||||
|
||||
$query =~ s/,$/\) /;
|
||||
$values =~ s/,$/\);/;
|
||||
$query .= $values;
|
||||
|
||||
my $response = $self->{dbh}->do($query);
|
||||
|
||||
}
|
||||
|
||||
1;
|
||||
Loading…
Reference in New Issue