Linux ip-172-26-7-228 5.4.0-1103-aws #111~18.04.1-Ubuntu SMP Tue May 23 20:04:10 UTC 2023 x86_64
Your IP : 18.118.151.112
#!/usr/bin/perl
# Copyright (C) 2010 Modestas Vainius <modax@debian.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>
use strict;
use warnings;
use Debian::PkgKde;
use Getopt::Long;
use File::Copy qw();
use File::Temp qw();
use File::Spec;
use IO::Uncompress::Inflate qw(inflate);
use IO::Uncompress::Gunzip qw(gunzip);
# Load extra modules (from libwww-perl and its dependencies)
INIT {
eval "use HTTP::Request";
if ($@) {
error "in order to use this utility, you have to install libwww-perl package";
}
eval "use URI; use URI::QueryParam; use URI::Escape;";
eval "use HTTP::Response; use LWP::UserAgent; use HTML::LinkExtor; use HTML::Parser;";
}
sub usage {
usageerr "[ -d destdir ] [ -v version ] [ -a arch ] [ -o ] [ package ] [ distribution ]";
}
sub construct_url {
my ($path, %params) = @_;
my $url = URI->new($path);
foreach my $param (keys %params) {
my @value = (ref $params{$param} eq "ARRAY") ?
@{$params{$param}} : ( $params{$param} );
$url->query_param_append($param, @value);
}
return $url->as_string();
}
sub as_array {
my $scalar = shift;
my @ret;
if (defined $scalar) {
if (ref($scalar) eq 'ARRAY') {
@ret = @$scalar;
} else {
push @ret, $scalar;
}
}
return @ret;
}
sub in_array {
my $value = shift;
foreach my $member (@_) {
return 1 if $member eq $value;
}
return 0;
}
sub get_command_output {
my @lines;
open(my $cmd, "-|", @_) or syserr("unable to execute command %s", $_[0]);
while (<$cmd>) {
chop;
push @lines, $_;
}
close $cmd;
return @lines;
}
sub get_rfc822_field_value {
my ($field, $input) = @_;
foreach my $line (@$input) {
if ($line =~ /^\Q$field\E:\s*(.*)$/) {
return "$1"
}
}
}
sub html2text {
my ($in, $out) = @_;
my $body;
my $parser = HTML::Parser->new( api_version => 3,
start_h => [ sub { if (shift() eq "body") { $body = 1 } }, "tagname" ],
end_h => [ sub { if (shift() eq "body") { $body = 0 } }, "tagname" ],
text_h => [ sub { if ($body) { print $out shift(); } }, "dtext" ]
);
$parser->ignore_elements("head", "a", "img");
return defined($parser->parse_file($in)) && defined($body);
}
sub fetch_log {
my ($browser, $uri, %opts) = @_;
my $ok = 0;
my @status;
my ($filename, $file);
# Construct log filename
$filename = sprintf("%s_%s_%s_%s.build",
$uri->query_param("pkg"),
$uri->query_param("ver"),
$uri->query_param("arch"),
$uri->query_param("stamp"));
$file = File::Spec->catfile($opts{destdir}, $filename)
if defined $opts{destdir};
# Check overwrite option
if (!$opts{overwrite} && -e $file) {
info "Not overwriting existing build log at $file ...";
} else {
# Create a temporary file
my %tmpfileopts = ( TEMPLATE => $filename . ".XXXXXX" );
$tmpfileopts{DIR} = $opts{destdir} if exists $opts{destdir};
my $tmpfile1 = File::Temp->new(%tmpfileopts);
my $tmpfile2 = File::Temp->new(%tmpfileopts);
info "Fetching build log to $filename ...";
my $request = HTTP::Request->new(GET => $uri);
$request->header("Accept-Encoding" => "gzip, deflate, identity");
$browser->show_progress(1);
$tmpfile1->close();
my $response = $browser->request($request, $tmpfile1->filename);
if ($response->is_success()) {
# Inflate/gunzip contents if needed
my $content_encoding = $response->header("Content-Encoding");
if (defined $content_encoding && $content_encoding ne "identity") {
my $uncompress_func;
if ($content_encoding eq "deflate") {
push @status, "decoded:deflate";
$uncompress_func = \&inflate;
} elsif ($content_encoding eq "gzip") {
push @status, "decoded:gzip";
$uncompress_func = \&gunzip;
} else {
push @status, "unsupported content encoding: $content_encoding";
}
if (defined $uncompress_func) {
if (&$uncompress_func($tmpfile1->filename => $tmpfile2, BinModeOut => 1)) {
$tmpfile2->close();
($tmpfile1, $tmpfile2) = ($tmpfile2, $tmpfile1);
open($tmpfile2, ">:utf8", $tmpfile2->filename) or
syserr "unable to reopen temporary file";
$ok = 1;
}
}
} else {
# Identity encoding (no compression)
$ok = 1;
binmode($tmpfile2, ":utf8");
}
if ($ok) {
open($tmpfile1, "<:utf8", $tmpfile1->filename);
if ($ok = html2text($tmpfile1 => $tmpfile2)) {
$tmpfile1->close();
$tmpfile1 = $tmpfile2;
} else {
push @status, "html unstripped";
}
$tmpfile1->close();
$tmpfile2->close();
File::Copy::move($tmpfile1->filename, $file) or
error "unable to rename '%s' to '%s'", $tmpfile1->filename, $file;
}
}
}
return ($ok, [ $filename, @status ]);
}
sub download_logs {
my ($pkg, %opts) = @_;
my $distro = $opts{distro};
my @arches = as_array($opts{arch});
my @vers = as_array($opts{ver});
my $url;
# Construct index URL
if (defined $distro) {
$url = construct_url('https://buildd.debian.org/status/package.php',
p => $pkg, suite => $distro);
} elsif (defined $opts{ver}) {
if (scalar(@arches) == 1 && scalar(@vers) == 1) {
$url = construct_url('https://buildd.debian.org/build.php',
pkg => $pkg, ver => \@vers, arch => \@arches);
} else {
# Only single ver + single arch query is supported by build.php.
# For anything else fallback to url filtering (see below)
$url = construct_url('https://buildd.debian.org/build.php', pkg => $pkg);
}
} else {
error "neither version(s) nor distribution was specified";
}
# Download index document and extract links
info "Downloading build log index from $url ...";
my $browser = LWP::UserAgent->new(
agent => get_program_name(),
timeout => 10,
keep_alive => 1,
env_proxy => 1,
);
my $request = HTTP::Request->new(GET => $url);
if (my $response = $browser->request($request)) {
error "unable to access log index at URL $url: ".$response->status_line
unless $response->is_success();
my $urlbase = $url;
$urlbase =~ s,[^/]*$,,;
my $linkextor = HTML::LinkExtor->new(undef, $urlbase);
$linkextor->parse($response->content());
if (my @links = grep { $_->[0] eq "a" } $linkextor->links()) {
@links = map { shift @{$_}; +{ @{$_} }->{href} } @links;
my @ok;
my @failed;
my %latest_logs;
# Collect links we need to fetch
foreach my $link (@links) {
# Check if it is the link we need
if ($link =~ m,/fetch\.(cgi|php)(\?[^/]+)$,) {
# We might need to filter out links based on the arch and
# version we need
my $loguri = URI->new($link);
my $ver = $loguri->query_param("ver");
my $arch = $loguri->query_param("arch");
my $stamp = $loguri->query_param("stamp");
next if @vers && !in_array($ver, @vers);
next if @arches && !in_array($arch, @arches);
# Ensure that we are fetching the latest log for ver/arch
my $k = $ver . "_" . $arch;
if (!exists $latest_logs{$k} || $latest_logs{$k}{stamp} < $stamp) {
$latest_logs{$k} = { stamp => $stamp, uri => $loguri };
}
}
}
# Fetch logs
foreach my $k (sort keys %latest_logs) {
my $loguri = $latest_logs{$k}{uri};
my ($ok, $status) = fetch_log($browser, $loguri, %opts);
if ($ok) {
push @ok, $status;
} else {
push @failed, $status;
}
}
return (@ok || @failed) ? (\@ok, \@failed) : ();
}
return ();
} else {
error "unable to access log index URL $url";
}
}
sub print_summary {
my $logs = shift;
my $is_warning = shift;
my $msg = shift;
if (@$logs) {
info $msg, @_ unless $is_warning;
warning $msg, @_ if $is_warning;
foreach my $log_info (@$logs) {
my ($filename, @info) = @$log_info;
if (@info) {
printmsg " - %s [%s]", $filename, join(", ", @info);
} else {
printmsg " - %s", $filename;
}
}
}
}
my $opt_destdir;
my @opt_versions;
my @opt_archs;
my $opt_force;
# Get and verify options
unless (GetOptions(
"destdir|d=s" => \$opt_destdir,
"version|v=s" => \@opt_versions,
"arch|a=s" => \@opt_archs,
"force|f!" => \$opt_force))
{
usage();
}
my ($opt_package, $opt_distro) = @ARGV;
my @dpkg_parsechangelog;
if (!$opt_package && -f "debian/changelog") {
@dpkg_parsechangelog = get_command_output("dpkg-parsechangelog");
$opt_package = get_rfc822_field_value("Source", \@dpkg_parsechangelog);
}
if (!$opt_package) {
errormsg "source package was not specified and could not be autoguessed";
usage();
}
if ($opt_distro && @opt_versions) {
errormsg "version and distribution options are mutually exclusive";
usage();
}
if (!@opt_versions) {
if (!$opt_distro && -f "debian/changelog") {
@dpkg_parsechangelog = get_command_output("dpkg-parsechangelog") unless @dpkg_parsechangelog;
$opt_distro = get_rfc822_field_value("Distribution", \@dpkg_parsechangelog);
if ($opt_distro eq "UNRELEASED") {
# Get distro from the next to current entry
$opt_distro = get_rfc822_field_value("Distribution",
[ get_command_output("dpkg-parsechangelog", "-c1", "-o1") ]);
}
}
if (!$opt_distro) {
errormsg "neither distribution nor version(s) was specified and could not be autoguessed";
usage();
}
}
# Determine destination directory to store logs
unless ($opt_destdir) {
$opt_destdir = sprintf("%s_%s_logs", $opt_package,
($opt_distro) ? $opt_distro : $opt_versions[0]);
}
info("Selected output directory for logs: %s/", $opt_destdir);
unless (-d $opt_destdir) {
mkdir $opt_destdir;
}
my ($ok_logs, $failed_logs) =
download_logs($opt_package, overwrite => $opt_force, destdir => $opt_destdir,
distro => $opt_distro, ver => \@opt_versions, arch => \@opt_archs);
if (defined $ok_logs) {
print_summary $ok_logs, 0, "Successfully downloaded build logs (stored to %s):", $opt_destdir;
print_summary $failed_logs, 1, "Failed to fetch/ignored the following build logs:";
} else {
error "no build logs referenced in the build log index";
}
END {
rmdir $opt_destdir if $opt_destdir && $opt_destdir ne ".";
}
exit 0
# vim: noexpandtab tabstop=8 shiftwidth=4
|