Hi,
On Fri, Nov 29, 2013 at 12:09:19PM +0100, Rafael Laboissiere wrote:
> Package: devscripts
> Version: 2.13.4
> Severity: wishlist
> Tags: patch
>
> Please, find enclosed below a patch for allowing uscan to use a
> different compression scheme when repacking upstream tarballs. The
> patch should apply cleanly to the master branch of the Git
> repository, as today.
Feel free to find a patch which applies to the current state of Git
commit 418128c981768829145034df14469df03e6e7fd0
to simplify your work to approve the patch.
Thanks for maintaining uscan in devscripts
Andreas.
--
http://fam-tille.de
>From 432b3cecb27663293ea6132b0233de6ebd941f95 Mon Sep 17 00:00:00 2001
From: Andreas Tille <[email protected]>
Date: Fri, 10 Jan 2014 17:00:21 +0100
Subject: [PATCH] [uscan] Allow a different compression scheme when repacking
upstream tarballs
The changes introduced by this commit were originally done by Andreas
Tille <[email protected]> and were discussed in the context of
Bug#685787. I just merged the relevant changes from Andreas' private
Git repository into the master branch of the Git repository of
devscripts.
The --repack-compression option of uscan accepts now an argument that
indicates the type of the compression that will be used when repacking
the upstream tarball, either because there are files listed in the
Files-Excluded field of debian/copyright or because the --repack
option was given and there is need for repacking. The current
alternatives are gz, bz, xz, and lzma.
The documentation in scripts/uscan.1 has been changed accordingly and
the test in test/test_uscan has been extended to exercise the option
--repack-compression=xz, which will repack the gz upstream tarball
using xz. For that to work, a build-dependency on xz-utils has also
been added.
---
debian/control | 3 +-
scripts/uscan.1 | 6 +++
scripts/uscan.pl | 112 +++++++++++++++++++++++++++++++++++++------------------
test/test_uscan | 8 ++--
4 files changed, 88 insertions(+), 41 deletions(-)
diff --git a/debian/control b/debian/control
index 5502ce7..d5e153d 100644
--- a/debian/control
+++ b/debian/control
@@ -28,7 +28,8 @@ Build-Depends: debhelper (>= 9),
python3-setuptools,
shunit2 (>= 2.1.6),
wdiff,
- xsltproc
+ xsltproc,
+ xz-utils
XS-Testsuite: autopkgtest
Vcs-Browser: http://anonscm.debian.org/gitweb/?p=collab-maint/devscripts.git
Vcs-Git: git://anonscm.debian.org/collab-maint/devscripts.git
diff --git a/scripts/uscan.1 b/scripts/uscan.1
index 17e7941..9adee41 100644
--- a/scripts/uscan.1
+++ b/scripts/uscan.1
@@ -401,6 +401,12 @@ a zip archive (i.e. it doesn't match a .tlz, .tlzm, .tlzma, .tar.lzma, .txz,
installed in order to repack .zip archives, the xz-utils package must be
installed to repack lzma or xz tar archives.
.TP
+\fB\-\-repack-compression\fR \fICOMP\fR
+In the case where the upstream sources are repacked (either because
+\fB\-\-repack\fR option is given or \fIdebian/copyright\fR contains the
+field \fBFiles-Excluded\fR), it is possible to control the compression
+method via the \fICOMP\fR parameter (defaults to \fBgz\fR).
+.TP
.B \-\-no\-symlink
Don't make these symlinks and don't rename the files.
.TP
diff --git a/scripts/uscan.pl b/scripts/uscan.pl
index 433f5e2..3ecc9aa 100755
--- a/scripts/uscan.pl
+++ b/scripts/uscan.pl
@@ -75,6 +75,7 @@ sub dehs_output ();
sub quoted_regex_replace ($);
sub safe_replace ($$);
sub get_main_source_dir($$$$);
+sub compress_archive($$$);
sub usage {
print <<"EOF";
@@ -103,6 +104,9 @@ Options:
--repack Repack downloaded archives from orig.tar.bz2, orig.tar.lzma,
orig.tar.xz or orig.zip to orig.tar.gz
(does nothing if downloaded archive orig.tar.gz)
+ --repack-compression COMP
+ When the upstream sources are repacked, use compression COMP
+ for the resulting tarball
--no-symlink Don\'t make symlink or rename
--verbose Give verbose output
--no-verbose Don\'t give verbose output (default)
@@ -174,6 +178,8 @@ my $download_version;
my $force_download = 0;
my $report = 0; # report even on up-to-date packages?
my $repack = 0; # repack .tar.bz2, .tar.lzma, .tar.xz or .zip to .tar.gz
+my $default_compression = 'gz' ;
+my $repack_compression = $default_compression;
my $symlink = 'symlink';
my $verbose = 0;
my $check_dirname_level = 1;
@@ -273,7 +279,8 @@ if (@ARGV and $ARGV[0] =~ /^--no-?conf$/) {
# Now read the command line arguments
my $debug = 0;
my ($opt_h, $opt_v, $opt_destdir, $opt_download, $opt_force_download,
- $opt_report, $opt_passive, $opt_symlink, $opt_repack, $opt_exclusion);
+ $opt_report, $opt_passive, $opt_symlink, $opt_repack,
+ $opt_repack_compression, $opt_exclusion);
my ($opt_verbose, $opt_level, $opt_regex, $opt_noconf);
my ($opt_package, $opt_uversion, $opt_watchfile, $opt_dehs, $opt_timeout);
my $opt_download_version;
@@ -293,6 +300,7 @@ GetOptions("help" => \$opt_h,
"symlink!" => sub { $opt_symlink = $_[1] ? 'symlink' : 'no'; },
"rename" => sub { $opt_symlink = 'rename'; },
"repack" => sub { $opt_repack = 1; },
+ "repack-compression=s" => \$opt_repack_compression,
"package=s" => \$opt_package,
"upstream-version=s" => \$opt_uversion,
"watchfile=s" => \$opt_watchfile,
@@ -328,6 +336,16 @@ $timeout = $opt_timeout if defined $opt_timeout;
$timeout = 20 unless defined $timeout and $timeout > 0;
$symlink = $opt_symlink if defined $opt_symlink;
$verbose = $opt_verbose if defined $opt_verbose;
+if ( defined $opt_repack_compression ) {
+ if ( $opt_repack_compression =~ /^gz$/ or
+ $opt_repack_compression =~ /^bz2$/ or
+ $opt_repack_compression =~ /^xz$/ or
+ $opt_repack_compression =~ /^lzma$/ ) {
+ $repack_compression = $opt_repack_compression;
+ } else {
+ print "-- Invalid compression $opt_repack_compression given. Use default $default_compression instead.\n" if $verbose ;
+ }
+}
$dehs = $opt_dehs if defined $opt_dehs;
$exclusion = $opt_exclusion if defined $opt_exclusion;
$user_agent_string = $opt_user_agent if defined $opt_user_agent;
@@ -1414,78 +1432,70 @@ EOF
or uscan_die("$progname warning: OpenPGP signature did not verify.\n");
}
- if ($repack and $newfile_base =~ /^(.*)\.(tar\.bz2|tbz2?)$/) {
+ if ($repack and $newfile_base =~ /^(.*)\.(tar\.bz2|tbz2?)$/ and
+ $repack_compression !~ /^bz2$/ ) {
print "-- Repacking from bzip2 to gzip\n" if $verbose;
- my $newfile_base_gz = "$1.tar.gz";
+ my $newfile_base_compression = "$1.tar.".$repack_compression;
my (undef, $fname) = tempfile(UNLINK => 1);
spawn(exec => ['bunzip2', '-c', "$destdir/$newfile_base"],
to_file => $fname,
wait_child => 1);
spawn(exec => ['gzip', '-n', '-9'],
from_file => $fname,
- to_file => "$destdir/$newfile_base_gz",
wait_child => 1);
unlink "$destdir/$newfile_base";
- $newfile_base = $newfile_base_gz;
+ $newfile_base = $newfile_base_compression;
}
- if ($repack and $newfile_base =~ /^(.*)\.(tar\.lzma|tlz(?:ma?)?)$/) {
- print "-- Repacking from lzma to gzip\n" if $verbose;
- my $newfile_base_gz = "$1.tar.gz";
+ if ($repack and $newfile_base =~ /^(.*)\.(tar\.lzma|tlz(?:ma?)?)$/ and
+ $repack_compression !~ /^lzma$/ ) {
+ print "-- Repacking from lzma to $repack_compression\n" if $verbose;
+ my $newfile_base_compression = "$1.tar.".$repack_compression;
my (undef, $fname) = tempfile(UNLINK => 1);
spawn(exec => ['xz', '-F', 'lzma', '-cd', "$destdir/$newfile_base"],
to_file => $fname,
wait_child => 1);
- spawn(exec => ['gzip', '-n', '-9'],
- from_file => $fname,
- to_file => "$destdir/$newfile_base_gz",
- wait_child => 1);
- unlink "$destdir/$newfile_base";
- $newfile_base = $newfile_base_gz;
+ compress_archive("$fname", "$destdir/$newfile_base_compression", $repack_compression);
+ $newfile_base = $newfile_base_compression;
}
- if ($repack and $newfile_base =~ /^(.*)\.(tar\.xz|txz)$/) {
- print "-- Repacking from xz to gzip\n" if $verbose;
- my $newfile_base_gz = "$1.tar.gz";
+ if ($repack and $newfile_base =~ /^(.*)\.(tar\.xz|txz)$/ and
+ $repack_compression !~ /^xz$/ ) {
+ print "-- Repacking from xz to $repack_compression\n" if $verbose;
+ my $newfile_base_compression = "$1.tar.".$repack_compression;
my (undef, $fname) = tempfile(UNLINK => 1);
spawn(exec => ['xz', '-cd', "$destdir/$newfile_base"],
to_file => $fname,
wait_child => 1);
- spawn(exec => ['gzip', '-n', '-9'],
- from_file => $fname,
- to_file => "$destdir/$newfile_base_gz",
- wait_child => 1);
- unlink "$destdir/$newfile_base";
- $newfile_base = $newfile_base_gz;
+ compress_archive("$fname", "$destdir/$newfile_base_compression", $repack_compression);
+ $newfile_base = $newfile_base_compression;
}
if ($repack and $newfile_base =~ /^(.*)\.(zip|jar)$/) {
- print "-- Repacking from zip to .tar.gz\n" if $verbose;
+ print "-- Repacking from zip to .tar.$repack_compression\n" if $verbose;
system('command -v unzip >/dev/null 2>&1') >> 8 == 0
or uscan_die("unzip binary not found. You need to install the package unzip to be able to repack .zip upstream archives.\n");
- my $newfile_base_gz = "$1.tar.gz";
+ my $compress_file_base = "$1.tar" ;
+ my $newfile_base_compression = "$compress_file_base.".$repack_compression;
my $tempdir = tempdir ("uscanXXXX", TMPDIR => 1, CLEANUP => 1);
# Parent of the target directory should be under our control
$tempdir .= '/repack';
mkdir $tempdir or uscan_die("Unable to mkdir($tempdir): $!\n");
my $absdestdir = abs_path($destdir);
system('unzip', '-q', '-a', '-d', $tempdir, "$destdir/$newfile_base") == 0
- or uscan_die("Repacking from zip or jar to tar.gz failed (could not unzip)\n");
+ or uscan_die("Repacking from zip or jar to tar.$repack_compression failed (could not unzip)\n");
my $cwd = cwd();
chdir($tempdir) or uscan_die("Unable to chdir($tempdir): $!\n");
eval {
- spawn(exec => ['tar', '--owner=root', '--group=root', '--mode=a+rX', '-czf', "$absdestdir/$newfile_base_gz", glob('* .[!.]*')],
- env => { GZIP => '-n -9' },
- wait_child => 1);
+ compress_archive("$absdestdir/$compress_file_base", "$absdestdir/$newfile_base_compression", $repack_compression);
};
if ($@) {
- uscan_die("Repacking from zip or jar to tar.gz failed (could not create tarball)\n");
+ uscan_die("Repacking from zip or jar to tar.$repack_compression failed (could not create tarball)\n");
}
chdir($cwd);
- unlink "$destdir/$newfile_base";
- $newfile_base = $newfile_base_gz;
+ $newfile_base = $newfile_base_compression;
}
if ($newfile_base =~ /\.(tar\.gz|tgz
@@ -1556,14 +1566,13 @@ EOF
if ( $nfiles_before == $nfiles_after ) {
print "-- Source tree remains identical - no need for repacking.\n" if $verbose;
} else {
- my $suffix = 'gz' ;
- my $newfile_base_dfsg = "${pkg}_${newversion}${excludesuffix}.orig.tar.$suffix" ;
+ my $newfile_base_dfsg = "${pkg}_${newversion}${excludesuffix}.orig.tar" ;
my $cwd = cwd();
chdir($tempdir) or uscan_die("Unable to chdir($tempdir): $!\n");
eval {
- spawn(exec => ['tar', '--owner=root', '--group=root', '--mode=a+rX', '-czf', "$absdestdir/$newfile_base_dfsg", glob('* .[!.]*')],
- env => { GZIP => '-n -9' },
- wait_child => 1);
+ spawn(exec => ['tar', '--owner=root', '--group=root', '--mode=a+rX', '-cf', "$absdestdir/$newfile_base_dfsg", glob('* .[!.]*')],
+ wait_child => 1);
+ compress_archive("$absdestdir/$newfile_base_dfsg", "$absdestdir/$newfile_base_dfsg.$repack_compression", $repack_compression);
};
if ($@) {
uscan_die("Excluding files failed (could not create tarball)\n");
@@ -2215,3 +2222,34 @@ sub get_main_source_dir($$$$) {
}
return $main_source_dir;
}
+
+sub compress_archive($$$) {
+ my ($from_file, $to_file, $compression) = @_;
+ if ( $compression =~ /^gz$/ ) {
+ spawn(exec => ['gzip', '-n', '-9'],
+ from_file => $from_file,
+ to_file => $to_file,
+ wait_child => 1);
+ } elsif ( $compression =~ /^bz2$/ ) {
+ # The actual options should be discussed - supporting small memory seems reasonable
+ spawn(exec => ['bzip2', '--small'],
+ from_file => $from_file,
+ to_file => $to_file,
+ wait_child => 1);
+ } elsif ( $compression =~ /^xz$/ ) {
+ # The actual options should be discussed - supporting small memory seems reasonable
+ spawn(exec => ['xz', '--memlimit=150MiB'],
+ from_file => $from_file,
+ to_file => $to_file,
+ wait_child => 1);
+ } elsif ( $compression =~ /^lzma$/ ) {
+ # The actual options should be discussed - no idea what might be reasonable here
+ spawn(exec => ['lzma'],
+ from_file => $from_file,
+ to_file => $to_file,
+ wait_child => 1);
+ } else {
+ die("Unknown compression method $compression.");
+ }
+ unlink "$from_file";
+}
diff --git a/test/test_uscan b/test/test_uscan
index 2f27bef..c0abf7c 100755
--- a/test/test_uscan
+++ b/test/test_uscan
@@ -20,7 +20,7 @@ if test "$1" = --installed; then
shift
else
top_srcdir=$(readlink -f "${0%/*}/..")
- COMMAND="perl -I $top_srcdir $top_srcdir/scripts/uscan.pl --no-conf"
+ COMMAND="perl -I $top_srcdir $top_srcdir/scripts/uscan.pl --no-conf --repack-compression=xz"
fi
cleanup(){
@@ -84,10 +84,12 @@ END
OUTPUT=$( (cd $TMPDIR/$PKG ; $COMMAND) | grep baz)
- TARBALL=${PKG}_1+dfsg.orig.tar.gz
+ TARBALL=${PKG}_1+dfsg.orig.tar.xz
assertTrue 'pristine tarball is not created' "[ -f $TMPDIR/$TARBALL ]"
+ assertNotNull 'pristine tarball is not xz-compressed' \
+ "$( file $TMPDIR/$TARBALL | grep 'XZ compressed data' )"
assertNull 'file that must be excluded is present in the tarball' \
- "$( tar tzf $TMPDIR/$TARBALL | grep exclude-this )"
+ "$( tar atf $TMPDIR/$TARBALL | grep exclude-this )"
assertNull "dir that must be excluded is present in the tarball" \
"$(tar atf $TMPDIR/$TARBALL | grep exclude-dir)"
# 731849
--
1.8.5.1
_______________________________________________
devscripts-devel mailing list
[email protected]
http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/devscripts-devel
