Hello community, here is the log from the commit of package storeBackup for openSUSE:13.1 checked in at 2013-09-29 17:53:37 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:13.1/storeBackup (Old) and /work/SRC/openSUSE:13.1/.storeBackup.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "storeBackup" Changes: -------- --- /work/SRC/openSUSE:13.1/storeBackup/storeBackup.changes 2013-09-23 11:09:40.000000000 +0200 +++ /work/SRC/openSUSE:13.1/.storeBackup.new/storeBackup.changes 2013-09-29 17:53:38.000000000 +0200 @@ -1,0 +2,11 @@ +Fri Sep 27 14:36:26 UTC 2013 - [email protected] + +- Update to 3.4.2: + * Fixed reading output of external programs + * Fixed option --debug in storeBackupUpdateBackup.pl + * Added time frame support to --progressReport option + * Added --createSparseFiles option to storeBackupRecover.pl and + linkToDirs.pl to create sparse files when restoring blocked + files and copying files, respectively + +------------------------------------------------------------------- Old: ---- storeBackup-3.4.1.config.default storeBackup-3.4.1.diff storeBackup-3.4.1.tar.bz2 New: ---- storeBackup-3.4.2.config.default storeBackup-3.4.2.diff storeBackup-3.4.2.tar.bz2 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ storeBackup.spec ++++++ --- /var/tmp/diff_new_pack.JiDEFd/_old 2013-09-29 17:53:38.000000000 +0200 +++ /var/tmp/diff_new_pack.JiDEFd/_new 2013-09-29 17:53:38.000000000 +0200 @@ -17,7 +17,7 @@ Name: storeBackup -Version: 3.4.1 +Version: 3.4.2 Release: 0 Summary: A disk-to-disk backup tool for Linux License: GPL-3.0+ ++++++ storeBackup-3.4.1.config.default -> storeBackup-3.4.2.config.default ++++++ --- /work/SRC/openSUSE:13.1/storeBackup/storeBackup-3.4.1.config.default 2013-09-23 11:09:40.000000000 +0200 +++ /work/SRC/openSUSE:13.1/.storeBackup.new/storeBackup-3.4.2.config.default 2013-09-29 17:53:38.000000000 +0200 @@ -1,5 +1,5 @@ # configuration file for storeBackup.pl -# Generated by storeBackup.pl, 3.4.1 +# Generated by storeBackup.pl, 3.4.2 ### explanations ### @@ -218,7 +218,7 @@ # default = no: no late compression ;lateCompress= -# repair simple inconsistencies (from lateLinks) automaticly +# repair simple inconsistencies (from lateLinks) automatically # without requesting the action # default = no, no automatic repair ;autorepair= @@ -541,6 +541,9 @@ # print progress report after each 'number' files # Default is 0, which means no reports. +# additional you may add a time frame after which a message is printed +# if you want to print a report each 1000 files and after +# one minute and 10 seconds, use: -P 1000,1m10s ;progressReport= # print depth of actual readed directory during backup ++++++ storeBackup-3.4.1.diff -> storeBackup-3.4.2.diff ++++++ ++++++ storeBackup-3.4.1.tar.bz2 -> storeBackup-3.4.2.tar.bz2 ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/bin/linkToDirs.pl new/storeBackup/bin/linkToDirs.pl --- old/storeBackup/bin/linkToDirs.pl 2013-08-09 18:56:23.000000000 +0200 +++ new/storeBackup/bin/linkToDirs.pl 2013-09-22 17:12:25.000000000 +0200 @@ -118,6 +118,7 @@ --targetDir targetForSourceDir [--progressReport number] [--dontLinkSymlinks] [--ignoreErrors] [--saveRAM] [-T tmpdir] + [--createSparseFiles [--blockSize]] sourceDir ... =head1 DESCRIPTION @@ -172,6 +173,20 @@ =item B<--tmpdir>, B<-T> directory for temporary files, default is </tmp> + + +=item B<--createSparseFiles>, B<-s> + + if a file is indicated as a sparse file and that file has to be + copied, then external program 'cp' is called to copy that file + (gnucp / linux does some inspection about sparse files, if your + OS related version of cp does not support this functionality, + then this option will not work for you) + +=item B<--blockSize> + + block size used to check if a file is / may be a sparse file + default is 512 bytes (which should be fine for most file systems) =item B<sourceDir> @@ -218,6 +233,14 @@ '-cl_alias' => '--ignoreErrors'), Option->new('-name' => 'saveRAM', '-cl_option' => '--saveRAM'), + Option->new('-name' => 'createSparseFiles', + '-cl_option' => '--createSparseFiles', + '-cl_alias' => '-s'), + Option->new('-name' => 'blockSize', + '-cl_option' => '--blockSize', + '-default' => 512, + '-only_if' => '[createSparseFiles]', + '-pattern' => '\A\d+\Z'), Option->new('-name' => 'tmpdir', '-cl_option' => '-T', '-cl_alias' => '--tmpdir', @@ -237,9 +260,10 @@ my $ignoreErrors = $CheckPar->getOptWithoutPar('ignoreErrors'); my (@sourceDir) = $CheckPar->getListPar(); my $saveRAM = $CheckPar->getOptWithoutPar('saveRAM'); +my $createSparseFiles = $CheckPar->getOptWithoutPar('createSparseFiles'); +my $blockSize = $CheckPar->getOptWithPar('blockSize'); $tmpdir = $CheckPar->getOptWithPar('tmpdir'); - my $prLog; my ($prLogKind) = ['A:BEGIN', 'Z:END', @@ -528,11 +552,11 @@ elsif ($t eq 'f' or $t eq 'l') # normal file or symlink { my ($dev, $inode, $mode, $uid, $gid, $size, $atime, - $mtime, $ctime); + $mtime, $ctime, $blocks); if ($t eq 'f') { ($dev, $inode, $mode, $uid, $gid, $size, $atime, - $mtime, $ctime) = (stat($f))[0,1,2,4,5,7,8,9,10]; + $mtime, $ctime, $blocks) = (stat($f))[0,1,2,4,5,7,8,9,10,12]; } elsif ($t eq 'l') { @@ -559,7 +583,11 @@ if ($t eq 'f') { - unless (&::copyFile($f, $fnew, $prLog)) +# unless (&::copyFile($f, $fnew, $prLog)) + unless (&::checkSparseAndCopyFiles( + $f, $fnew, $prLog, $size, + $blocks, $createSparseFiles, + $blockSize, $tmpdir)) { $prLog->print('-kind' => 'E', '-str' => @@ -617,7 +645,11 @@ if ($t eq 'f') { - unless (&::copyFile($f, $fnew, $prLog)) +# unless (&::copyFile($f, $fnew, $prLog)) + unless (&::checkSparseAndCopyFiles( + $f, $fnew, $prLog, $size, + $blocks, $createSparseFiles, + $blockSize, $tmpdir)) { $prLog->print('-kind' => 'E', '-str' => @@ -652,7 +684,11 @@ #print "\tcopy $f =>$fnew\n"; if ($t eq 'f') { - unless (&::copyFile($f, $fnew, $prLog)) +# unless (&::copyFile($f, $fnew, $prLog)) + unless (&::checkSparseAndCopyFiles( + $f, $fnew, $prLog, $size, + $blocks, $createSparseFiles, + $blockSize, $tmpdir)) { $prLog->print('-kind' => 'E', '-str' => @@ -744,6 +780,59 @@ ################################################## +# retunrs 1 on success, 0 if copy fails +sub checkSparseAndCopyFiles +{ + my $source = shift; + my $target = shift; + my $prLog = shift; + my $size = shift; + my $blocks = shift; + my $createSparseFiles = shift; + my $blockSize = shift; + my $tmpdir = shift; + + if ($createSparseFiles and $blocks * $blockSize < $size) # maybe sparse file + { + my $ret = 1; # success + my $cp = forkProc->new('-exec' => 'cp', + '-param' => + ["$source", "$target"], + '-outRandom' => "$tmpdir/linkTo-cp-", + '-prLog' => $prLog); + $cp->wait(); + my $out = $cp->getSTDOUT(); + if (@$out) + { + $prLog->print('-kind' => 'W', + '-str' => + ["copying of <$source> to <$target> reported:", + @$out]); + } + $out = $cp->getSTDERR(); + if (@$out) + { + $prLog->print('-kind' => 'E', + '-str' => + ["copying (with 'cp -r') of <$source> to <$target> reported:", + @$out]); + $ret = 0; + } + $prLog->print('-kind' => 'I', + '-str' => ["file <$source> copied as sparse file"]) + if $ret; + + return $ret; + } + else + { + return &::copyFile($source, $target, $prLog); + } +} + + + +################################################## package Statistic; ######################################## diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/bin/storeBackup.pl new/storeBackup/bin/storeBackup.pl --- old/storeBackup/bin/storeBackup.pl 2013-09-01 08:49:08.000000000 +0200 +++ new/storeBackup/bin/storeBackup.pl 2013-09-25 11:00:22.000000000 +0200 @@ -239,7 +239,7 @@ [--exceptSuffix suffixes] [--addExceptSuffix suffixes] [--compressSuffix] [--minCompressSize size] [--comprRule] [--doNotCompressMD5File] [--chmodMD5File] [-v] - [-d level][--progressReport number] [--printDepth] + [-d level][--progressReport number,[timeframe]] [--printDepth] [--ignoreReadError] [--suppressWarning key] [--linkToRecent name] [--doNotDelete] [--deleteNotFinishedDirs] @@ -309,6 +309,10 @@ lock file, if exists, new instances will finish if an old is already running, default is $lockFile + this type of lock files does not work across multiple servers + and is not designed to separate storeBackup.pl and + storeBackupUpdateBackup.pl or any other storeBackup + process in a separate PID space =item B<--unlockBeforeDel> @@ -439,7 +443,7 @@ =item B<--autorepair>, B<-a> - repair simple inconsistencies (from lateLinks) automaticly + repair simple inconsistencies (from lateLinks) automatically without requesting the action =item B<--checkBlocksSuffix> @@ -830,6 +834,10 @@ =item B<--progressReport>, B<-P> print progress report after each 'number' files + additional you may add a time frame after which a message is + printed + if you want to print a report each 1000 files and after + one minute and 10 seconds, use: -P 1000,1m10s =item B<--printDepth>, B<-D> @@ -1294,7 +1302,7 @@ # default = no: no late compression ;lateCompress= -# repair simple inconsistencies (from lateLinks) automaticly +# repair simple inconsistencies (from lateLinks) automatically # without requesting the action # default = no, no automatic repair ;autorepair= @@ -1523,6 +1531,9 @@ # print progress report after each 'number' files # Default is 0, which means no reports. +# additional you may add a time frame after which a message is printed +# if you want to print a report each 1000 files and after +# one minute and 10 seconds, use: -P 1000,1m10s ;progressReport= # print depth of actual readed directory during backup @@ -2084,8 +2095,7 @@ '-cl_option' => '--progressReport', '-cl_alias' => '-P', '-cf_key' => 'progressReport', - '-default' => 0, - '-pattern' => '\A\d+\Z'), + '-default' => 0), Option->new('-name' => 'printDepth', '-cl_option' => '--printDepth', '-cl_alias' => '-D', @@ -2425,7 +2435,7 @@ $prLog->print('-kind' => 'E', '-str' => ["cannot create directory for series <$targetDir>"], '-exit' => 1) - unless mkdir $targetDir, 0700; + unless &::makeDirPath($targetDir, $prLog); $prLog->print('-kind' => 'W', '-str' => ["created directory <$targetDir>"]) unless exists $suppressWarning{'crSeries'}; @@ -2454,6 +2464,33 @@ '-exit' => 1); } +# check progressReport settings +my $progressDeltaTime = 0; +if ($progressReport) +{ + my ($count, $t); + if ($progressReport =~ /,/) + { + ($count, $t) = $progressReport =~ /\A(.*?),(.*)\Z/; + $prLog->print('-kind' => 'E', + '-str' => ["wrong format for option progressReport " . + "time period <$t>"], + '-exit' => 1) + unless &dateTools::checkStr('-str' => $t); + $progressDeltaTime = &dateTools::strToSec('-str' => $t); + } + else + { + $count = $progressReport; + } + $prLog->print('-kind' => 'E', + '-str' => ["counter <$count> for progress report " . + "must be a positive integer"], + '-exit' => 1) + unless $count =~ /\A\d+\Z/; + $progressReport = $count; +} +print "progressReport=<$progressReport>, progressDeltaTime=<$progressDeltaTime>\n"; $main::IOCompressDirect = 0; @@ -2926,6 +2963,7 @@ '-exceptSuffix' => $exceptSuffix, '-prLog' => $prLog, '-progressReport' => $progressReport, + '-progressDeltaTime' => $progressDeltaTime, '-withUserGroupStat' => $withUserGroupStat, '-userGroupStatFile' => $userGroupStatFile, '-compress' => $compress); @@ -3815,7 +3853,7 @@ if ($old) { my ($digest, $file, $postfix) = @{$old->get('-what' => 'info')}; - $statSizeNew += (stat("$targetDir/$file$postfix"))[7]; + $statSizeNew += (stat("$targetDir/$file$postfix"))[7] || 0; #print "-1- md5 $file <$postfix>\n"; } @@ -3894,7 +3932,7 @@ #print "-10.5-\n"; ::copyOneBlock($buffer, "$targetDir/$relPath/$blockName", $prLog); - $statSizeNew += (stat("$targetDir/$relPath/$blockName"))[7]; + $statSizeNew += (stat("$targetDir/$relPath/$blockName"))[7] || 0; } #print "-10.6-\n"; @@ -3943,7 +3981,7 @@ { ::copyOneBlock($buffer, "$targetDir/$relPath/$blockName", $prLog); - $statSizeNew += (stat("$targetDir/$relPath/$blockName"))[7]; + $statSizeNew += (stat("$targetDir/$relPath/$blockName"))[7] || 0; } else { @@ -3983,7 +4021,7 @@ $bz->eof(); $statSizeNew += - (stat("$targetDir/$blockFile$postfix"))[7]; + (stat("$targetDir/$blockFile$postfix"))[7] || 0; } else { @@ -4011,7 +4049,7 @@ { ::copyOneBlock($buffer, "$targetDir/$blockFile", $prLog); - $statSizeNew += (stat("$targetDir/$blockFile"))[7]; + $statSizeNew += (stat("$targetDir/$blockFile"))[7] || 0; } else { @@ -7015,6 +7053,7 @@ } + $main::stat->checkPrintTimeProgressReport(); # Wait in case we did nothing in this loop run $main::tinyWaitScheduler->wait(); } @@ -7168,6 +7207,7 @@ '-exceptSuffix' => undef, # Filename (if set) '-prLog' => undef, '-progressReport' => undef, + '-progressDeltaTime' => 0, '-withUserGroupStat' => undef, '-userGroupStatFile' => undef, '-compress' => undef @@ -7284,6 +7324,9 @@ $self->{'md5CheckSum'} = 0; $self->{'sumDBMFiles'} = 0; + $self->{'timeProgrReport'} = + ($self->{'progressDeltaTime'} > 0) ? time : 0; + bless $self, $class; } @@ -7519,18 +7562,43 @@ if ($self->{'progressReport'} > 0) { - my $s = $self->{'sumNewCompr'} + $self->{'sumNewCopy'}; - $self->{'prLog'}->print('-kind' => 'P', - '-str' => - [$self->{'noFiles'} . ' files processed (' . - (&::humanReadable($self->{'sumOrigFiles'}))[0] . - ', ' . - (&::humanReadable($s))[0] . ') (' . - $self->{'sumOrigFiles'} . ', ' . $s . ')']); + $self->__printProgressReport(); } } +######################################## +sub checkPrintTimeProgressReport +{ + my $self = shift; + + if ($self->{'timeProgrReport'} > 0 and + time >= $self->{'timeProgrReport'} + $self->{'progressDeltaTime'}) + { + $self->__printProgressReport(); + } +} + + +######################################## +sub __printProgressReport +{ + my $self = shift; + + my $s = $self->{'sumNewCompr'} + $self->{'sumNewCopy'}; + $self->{'prLog'}->print('-kind' => 'P', + '-str' => + [$self->{'noFiles'} . ' files processed (' . + (&::humanReadable($self->{'sumOrigFiles'}))[0] . + ', ' . + (&::humanReadable($s))[0] . ') (' . + $self->{'sumOrigFiles'} . ', ' . $s . ')']); + + $self->{'timeProgrReport'} = time + if $self->{'timeProgrReport'} > 0; +} + + ######################################## sub addSumMD5Sum # in byte { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/bin/storeBackupCheckBackup.pl new/storeBackup/bin/storeBackupCheckBackup.pl --- old/storeBackup/bin/storeBackupCheckBackup.pl 2013-08-03 10:42:46.000000000 +0200 +++ new/storeBackup/bin/storeBackupCheckBackup.pl 2013-09-21 05:36:59.000000000 +0200 @@ -991,8 +991,7 @@ '-str' => ["strange line in <$f/.md5BlockCheckSums.bz2> " . "in line " . $fileIn->get('-what' => 'lineNr') . - ":", "\t<$l>"], - '-exit' => 1) + ":", "\t<$l>"]); } if (-e "$dirToCheck/$l_f") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/bin/storeBackupDel.pl new/storeBackup/bin/storeBackupDel.pl --- old/storeBackup/bin/storeBackupDel.pl 2013-07-03 21:03:45.000000000 +0200 +++ new/storeBackup/bin/storeBackupDel.pl 2013-09-22 13:33:05.000000000 +0200 @@ -152,6 +152,10 @@ lock file, if exists, new instances will finish if an old is already running, default is $lockFile + this type of lock files does not work across multiple servers + and is not designed to separate storeBackup.pl and + storeBackupUpdateBackup.pl or any other storeBackup + process in a separate PID space =item B<--doNotDelete> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/bin/storeBackupRecover.pl new/storeBackup/bin/storeBackupRecover.pl --- old/storeBackup/bin/storeBackupRecover.pl 2013-08-16 21:01:08.000000000 +0200 +++ new/storeBackup/bin/storeBackupRecover.pl 2013-09-21 08:57:24.000000000 +0200 @@ -24,6 +24,7 @@ use strict; use DB_File; # Berkeley DB version 1 +use Digest::MD5 qw(md5_hex); sub libPath @@ -113,7 +114,7 @@ storeBackupRecover.pl -r restore [-b root] -t targetDir [--flat] [-o] [--tmpdir] [--noHardLinks] [-p number] [-v] [-n] - [--cpIsGnu] [--noGnuCp] + [--cpIsGnu] [--noGnuCp] [-s] =head1 OPTIONS @@ -169,6 +170,11 @@ installed (only relevant for sockets, block and character devices) +=item B<--createSparseFiles>, B<-s> + + creates sparse file from blocked files if full blockes + are filled with zeros + =back =head1 COPYRIGHT @@ -223,7 +229,10 @@ '-cl_option' => '-n', '-cl_alias' => '--noRestored'), Option->new('-name' => 'noGnuCp', - '-cl_option' => '--noGnuCp') + '-cl_option' => '--noGnuCp'), + Option->new('-name' => 'createSparseFiles', + '-cl_option' => '--createSparseFiles', + '-cl_alias' => '-s') ] ); @@ -243,6 +252,7 @@ my $verbose = $CheckPar->getOptWithoutPar('verbose'); my $noRestored = $CheckPar->getOptWithoutPar('noRestored'); my $noGnuCp = $CheckPar->getOptWithoutPar('noGnuCp'); +my $createSparseFiles = $CheckPar->getOptWithoutPar('createSparseFiles'); my $prLog = printLog->new('-kind' => ['I:INFO', 'W:WARNING', 'E:ERROR', @@ -693,8 +703,9 @@ ($old, $new) = $fork->add_block('-function' => \&uncompressCatBlock, '-funcPar' => ["$backupRoot/$filename", + $backupRoot, $createSparseFiles, $targetFile, '\A\d.*', $uncompr, \@uncomprPar, - $postfix, $prLog], + $postfix, $size, $prLog], '-info' => [$targetFile, $uid, $gid, $mode, $atime, $mtime, $devInode, $comm]); $prLog->print('-kind' => 'I', @@ -873,30 +884,53 @@ sub uncompressCatBlock { my $fromDir = shift; + my $backupRoot = shift; + my $createSparseFiles = shift; my $toFile = shift; my $mask = shift; my $umcompr = shift; my $uncomprPar = shift; my $postfix = shift; + my $size = shift; my $prLog = shift; - local *DIR; - my $ret = 0; - unless (opendir(DIR, $fromDir)) + my $nBlocks = 0; + my (@entries, @md5, @compr); + + my $fileIn = + pipeFromFork->new('-exec' => 'bzip2', + '-param' => ['-d'], + '-stdin' => "$fromDir/.md5BlockCheckSums.bz2", + '-outRandom' => '/tmp/stbuPipeFrom11-', + '-prLog' => $prLog); + my $l; + while ($l = $fileIn->read()) { - $prLog->print('-kind' => 'E', - '-str' => ["cannot open <$fromDir>"]); - return 1; + my ($l_md5, $l_compr, $l_f, $n); + chomp $l; + $n = ($l_md5, $l_compr, $l_f) = split(/\s/, $l, 3); + if ($n != 3) + { + $prLog->print('-kind' => 'E', + '-str' => + ["strange line in <$fromDir/.md5BlockCheckSums.bz2> " . + "in line " . $fileIn->get('-what' => 'lineNr') . + ":", "\t<$l>"]); + return 1; + } + ++$nBlocks; + push @md5, $l_md5; + push @compr, $l_compr; + push @entries, $l_f; } - my ($entry, @entries); - while ($entry = readdir DIR) - { - next unless $entry =~ /$mask/; + $fileIn->close(); + $fileIn = undef; - push @entries, $entry; - } - close(DIR); + &::createSparseFile($toFile, $size, $prLog) or exit 1; + + my $md5null = ''; + my $ret = 0; local *TO; unless (sysopen(TO, $toFile, O_CREAT | O_WRONLY)) { @@ -905,61 +939,78 @@ $ret = 1; } - foreach $entry (sort @entries) + my $blockSize = 0; + for (my $iBlock = 0 ; $iBlock < @entries ; ++$iBlock) { + my $entry = $entries[$iBlock]; + my $md5 = $md5[$iBlock]; + my $compr = $compr[$iBlock]; + my $actSeekPos = $blockSize * $iBlock; + + sysseek(TO, $actSeekPos, 0); + + next if ($md5 eq $md5null); + my $buffer; local *FROM; my $fileIn = undef; - if ($entry =~ /$postfix\Z/) # compressed block + if ($compr eq 'c') # compressed block { if ($main::IOCompressDirect) { - my $input = "$fromDir/$entry"; + my $input = "$backupRoot/$entry"; my $uc = new IO::Uncompress::Bunzip2 $input; while ($uc->read($buffer, 10*1024**2)) { - unless (syswrite(TO, $buffer)) + my $n; + unless ($n = syswrite(TO, $buffer)) { $prLog->print('-kind' => 'E', '-str' => ["writing to <$toFile> failed"]); $ret = 1; } + $blockSize += $n if $iBlock == 0; } - next; } - - $fileIn = - pipeFromFork->new('-exec' => $uncompr, - '-param' => \@uncomprPar, - '-stdin' => "$fromDir/$entry", - '-outRandom' => '/tmp/stbuPipeFrom11-', - '-prLog' => $prLog); - while ($fileIn->sysread(\$buffer, 10*1024**2)) + else { - unless (syswrite(TO, $buffer)) + $fileIn = + pipeFromFork->new('-exec' => $uncompr, + '-param' => \@uncomprPar, + '-stdin' => "$backupRoot/$entry", + '-outRandom' => '/tmp/stbuPipeFrom11-', + '-prLog' => $prLog); + while ($fileIn->sysread(\$buffer, 10*1024**2)) { - $prLog->print('-kind' => 'E', - '-str' => ["writing to <$toFile> failed"]); - $ret = 1; + my $n; + unless ($n = syswrite(TO, $buffer)) + { + $prLog->print('-kind' => 'E', + '-str' => ["writing to <$toFile> failed"]); + $ret = 1; + } + $blockSize += $n if $iBlock == 0; } } } else # block not compressed { - unless (sysopen(FROM, "$fromDir/$entry", O_RDONLY)) + unless (sysopen(FROM, "$backupRoot/$entry", O_RDONLY)) { $prLog->print('-kind' => 'E', - '-str' => ["cannot read <$fromDir/$entry>"]); + '-str' => ["cannot read <$backupRoot/$entry>"]); return 1; } while (sysread(FROM, $buffer, 10*1024**2)) { - unless (syswrite(TO, $buffer)) + my $n; + unless ($n = syswrite(TO, $buffer)) { $prLog->print('-kind' => 'E', '-str' => ["writing to <$toFile> failed"]); $ret = 1; } + $blockSize += $n if $iBlock == 0; } } @@ -972,7 +1023,27 @@ { close(FROM); } + + if ($iBlock == 0) + { + $md5null = &::calcMD5null($blockSize) + if $createSparseFiles; + } } close(TO); return $ret; } + + +############################################################ +# calc md5 sum of block with $size time zero +sub calcMD5null +{ + my $size = shift; + + my $null = pack('C', 0) x $size; + + my $md5 = Digest::MD5->new(); + $md5->add($null); + return $md5->hexdigest(); +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/bin/storeBackupUpdateBackup.pl new/storeBackup/bin/storeBackupUpdateBackup.pl --- old/storeBackup/bin/storeBackupUpdateBackup.pl 2013-08-12 12:36:31.000000000 +0200 +++ new/storeBackup/bin/storeBackupUpdateBackup.pl 2013-09-22 13:33:34.000000000 +0200 @@ -173,6 +173,10 @@ If set to the same file as in storeBackup it will prevent $prog from running in parallel to storeBackup, default is $lockFile + this type of lock files does not work across multiple servers + and is not designed to separate storeBackup.pl and + storeBackupUpdateBackup.pl or any other storeBackup + process in a separate PID space =item B<--noCompress> @@ -329,7 +333,7 @@ '-cl_alias' => '--verbose'), Option->new('-name' => 'debug', '-cl_option' => '--debug', - '-cl_option' => '-d', + '-cl_alias' => '-d', '-only_if' => 'not [interactive]'), Option->new('-name' => 'logFile', '-cl_option' => '-l', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/doc/ChangeLog new/storeBackup/doc/ChangeLog --- old/storeBackup/doc/ChangeLog 2013-09-01 10:31:29.000000000 +0200 +++ new/storeBackup/doc/ChangeLog 2013-09-22 17:14:55.000000000 +0200 @@ -1095,3 +1095,19 @@ documentation - new chapter "internals" + +---------------------------- +version 3.4.2 2013.09 + storeBackup.pl + - fixed bug when reading output files of external programs + (heuristical bug) + - option --progressReport now accepts additionaly a time frame + + storeBackupUpdateBackup.pl + - option --debug now works like -d (typo) + + storeBackupRecover.pl + - new option --createSparseFiles + + linkToDirs.pl + - new options --createSparseFiles and --blockSize Files old/storeBackup/doc/storeBackupDE.pdf and new/storeBackup/doc/storeBackupDE.pdf differ Files old/storeBackup/doc/storeBackupEN.pdf and new/storeBackup/doc/storeBackupEN.pdf differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/lib/fileDir.pl new/storeBackup/lib/fileDir.pl --- old/storeBackup/lib/fileDir.pl 2013-08-31 10:28:38.000000000 +0200 +++ new/storeBackup/lib/fileDir.pl 2013-09-21 05:07:11.000000000 +0200 @@ -749,6 +749,45 @@ ############################################################ +# creates sparse file with specified size +# deletes existing file +# returns 1 if success / returns 0 if no success +sub createSparseFile +{ + my $file = shift; + my $size = shift; + my $prLog = shift; + + if (-e $file) + { + return 0 if unlink $file != 1; + } + + unless (sysopen(TO, $file, O_CREAT | O_WRONLY)) + { + $prLog->print('-kind' => 'E', + '-str' => ["cannot create sparse file <$file>"]); + return 0; + } + + sysseek(TO, $size - 1, 0); + + my $buffer = pack('C', 0); + + unless (syswrite(TO, $buffer)) + { + $prLog->print('-kind' => 'E', + '-str' => + ["cannot syswrite when creating sparse file <$file>"]); + return 0; + } + + close(TO); + return 1; +} + + +############################################################ # Objekt kann zum (wiederholten) Abfragen von Informationen # �ber eine Datei verwendet werden. # Liefert: alles von stat, md5sum diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/lib/forkProc.pl new/storeBackup/lib/forkProc.pl --- old/storeBackup/lib/forkProc.pl 2013-07-16 15:42:57.000000000 +0200 +++ new/storeBackup/lib/forkProc.pl 2013-09-16 20:25:14.000000000 +0200 @@ -66,7 +66,7 @@ .05, .05, .05, .05, .05, .05, .05, .05, .05, .05, .05, .05, .05, .05, .05, .05, 0); # 1 second - if ((not -e $existingFile) and (stat($existingFile))[7] == 0) + if ((not -e $existingFile) or (stat($existingFile))[7] == 0) { my $i; foreach $i (@intervall) @@ -83,7 +83,7 @@ if ($i == 0) { system "/bin/sync"; - return (not -e $existingFile) and (stat($existingFile))[7] == 0; + return (not -e $existingFile) or (stat($existingFile))[7] == 0; } } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/lib/version.pl new/storeBackup/lib/version.pl --- old/storeBackup/lib/version.pl 2013-08-29 12:32:32.000000000 +0200 +++ new/storeBackup/lib/version.pl 2013-09-21 07:26:15.000000000 +0200 @@ -19,7 +19,7 @@ # -$main::STOREBACKUPVERSION = "3.4.1"; +$main::STOREBACKUPVERSION = "3.4.2"; sub printVersion diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/man/man1/linkToDirs.pl.1 new/storeBackup/man/man1/linkToDirs.pl.1 --- old/storeBackup/man/man1/linkToDirs.pl.1 2013-09-07 13:36:31.000000000 +0200 +++ new/storeBackup/man/man1/linkToDirs.pl.1 2013-09-25 17:00:59.000000000 +0200 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "LINKTODIRS 1" -.TH LINKTODIRS 1 "2013-08-09" "perl v5.14.2" "User Contributed Perl Documentation" +.TH LINKTODIRS 1 "2013-09-22" "perl v5.14.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -133,11 +133,12 @@ linkToDirs.pl \- hard links files in directories with others .SH "SYNOPSIS" .IX Header "SYNOPSIS" -.Vb 5 +.Vb 6 \& linkToDirs.pl [\-\-linkWith copyBackupDir] [\-\-linkWith ...] \& \-\-targetDir targetForSourceDir \& [\-\-progressReport number] [\-\-dontLinkSymlinks] \& [\-\-ignoreErrors] [\-\-saveRAM] [\-T tmpdir] +\& [\-\-createSparseFiles [\-\-blockSize]] \& sourceDir ... .Ve .SH "DESCRIPTION" @@ -197,6 +198,21 @@ .Vb 1 \& directory for temporary files, default is </tmp> .Ve +.IP "\fB\-\-createSparseFiles\fR, \fB\-s\fR" 8 +.IX Item "--createSparseFiles, -s" +.Vb 5 +\& if a file is indicated as a sparse file and that file has to be +\& copied, then external program \*(Aqcp\*(Aq is called to copy that file +\& (gnucp / linux does some inspection about sparse files, if your +\& OS related version of cp does not support this functionality, +\& then this option will not work for you) +.Ve +.IP "\fB\-\-blockSize\fR" 8 +.IX Item "--blockSize" +.Vb 2 +\& block size used to check if a file is / may be a sparse file +\& default is 512 bytes (which should be fine for most file systems) +.Ve .IP "\fBsourceDir\fR" 8 .IX Item "sourceDir" .Vb 2 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/man/man1/storeBackup.pl.1 new/storeBackup/man/man1/storeBackup.pl.1 --- old/storeBackup/man/man1/storeBackup.pl.1 2013-09-07 13:36:30.000000000 +0200 +++ new/storeBackup/man/man1/storeBackup.pl.1 2013-09-25 17:00:58.000000000 +0200 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "STOREBACKUP 1" -.TH STOREBACKUP 1 "2013-09-01" "perl v5.14.2" "User Contributed Perl Documentation" +.TH STOREBACKUP 1 "2013-09-25" "perl v5.14.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -203,7 +203,7 @@ \& [\-\-exceptSuffix suffixes] [\-\-addExceptSuffix suffixes] \& [\-\-compressSuffix] [\-\-minCompressSize size] [\-\-comprRule] \& [\-\-doNotCompressMD5File] [\-\-chmodMD5File] [\-v] -\& [\-d level][\-\-progressReport number] [\-\-printDepth] +\& [\-d level][\-\-progressReport number,[timeframe]] [\-\-printDepth] \& [\-\-ignoreReadError] \& [\-\-suppressWarning key] [\-\-linkToRecent name] \& [\-\-doNotDelete] [\-\-deleteNotFinishedDirs] @@ -277,9 +277,13 @@ .Ve .IP "\fB\-\-lockFile\fR, \fB\-L\fR" 8 .IX Item "--lockFile, -L" -.Vb 2 +.Vb 6 \& lock file, if exists, new instances will finish if an old \& is already running, default is $lockFile +\& this type of lock files does not work across multiple servers +\& and is not designed to separate storeBackup.pl and +\& storeBackupUpdateBackup.pl or any other storeBackup +\& process in a separate PID space .Ve .IP "\fB\-\-unlockBeforeDel\fR" 8 .IX Item "--unlockBeforeDel" @@ -430,7 +434,7 @@ .IP "\fB\-\-autorepair\fR, \fB\-a\fR" 8 .IX Item "--autorepair, -a" .Vb 2 -\& repair simple inconsistencies (from lateLinks) automaticly +\& repair simple inconsistencies (from lateLinks) automatically \& without requesting the action .Ve .IP "\fB\-\-checkBlocksSuffix\fR" 8 @@ -875,8 +879,12 @@ .Ve .IP "\fB\-\-progressReport\fR, \fB\-P\fR" 8 .IX Item "--progressReport, -P" -.Vb 1 +.Vb 5 \& print progress report after each \*(Aqnumber\*(Aq files +\& additional you may add a time frame after which a message is +\& printed +\& if you want to print a report each 1000 files and after +\& one minute and 10 seconds, use: \-P 1000,1m10s .Ve .IP "\fB\-\-printDepth\fR, \fB\-D\fR" 8 .IX Item "--printDepth, -D" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/man/man1/storeBackupCheckBackup.pl.1 new/storeBackup/man/man1/storeBackupCheckBackup.pl.1 --- old/storeBackup/man/man1/storeBackupCheckBackup.pl.1 2013-09-07 13:36:30.000000000 +0200 +++ new/storeBackup/man/man1/storeBackupCheckBackup.pl.1 2013-09-25 17:00:58.000000000 +0200 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "STOREBACKUPCHECKBACKUP 1" -.TH STOREBACKUPCHECKBACKUP 1 "2013-08-03" "perl v5.14.2" "User Contributed Perl Documentation" +.TH STOREBACKUPCHECKBACKUP 1 "2013-09-21" "perl v5.14.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/man/man1/storeBackupDel.pl.1 new/storeBackup/man/man1/storeBackupDel.pl.1 --- old/storeBackup/man/man1/storeBackupDel.pl.1 2013-09-07 13:36:30.000000000 +0200 +++ new/storeBackup/man/man1/storeBackupDel.pl.1 2013-09-25 17:00:58.000000000 +0200 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "STOREBACKUPDEL 1" -.TH STOREBACKUPDEL 1 "2013-07-03" "perl v5.14.2" "User Contributed Perl Documentation" +.TH STOREBACKUPDEL 1 "2013-09-22" "perl v5.14.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -177,9 +177,13 @@ .Ve .IP "\fB\-\-lockFile\fR, \fB\-L\fR" 8 .IX Item "--lockFile, -L" -.Vb 2 +.Vb 6 \& lock file, if exists, new instances will finish if \& an old is already running, default is $lockFile +\& this type of lock files does not work across multiple servers +\& and is not designed to separate storeBackup.pl and +\& storeBackupUpdateBackup.pl or any other storeBackup +\& process in a separate PID space .Ve .IP "\fB\-\-doNotDelete\fR" 8 .IX Item "--doNotDelete" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/man/man1/storeBackupRecover.pl.1 new/storeBackup/man/man1/storeBackupRecover.pl.1 --- old/storeBackup/man/man1/storeBackupRecover.pl.1 2013-09-07 13:36:30.000000000 +0200 +++ new/storeBackup/man/man1/storeBackupRecover.pl.1 2013-09-25 17:00:59.000000000 +0200 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "STOREBACKUPRECOVER 1" -.TH STOREBACKUPRECOVER 1 "2013-08-16" "perl v5.14.2" "User Contributed Perl Documentation" +.TH STOREBACKUPRECOVER 1 "2013-09-21" "perl v5.14.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -136,7 +136,7 @@ .Vb 3 \& storeBackupRecover.pl \-r restore [\-b root] \-t targetDir [\-\-flat] \& [\-o] [\-\-tmpdir] [\-\-noHardLinks] [\-p number] [\-v] [\-n] -\& [\-\-cpIsGnu] [\-\-noGnuCp] +\& [\-\-cpIsGnu] [\-\-noGnuCp] [\-s] .Ve .SH "OPTIONS" .IX Header "OPTIONS" @@ -201,6 +201,12 @@ \& installed \& (only relevant for sockets, block and character devices) .Ve +.IP "\fB\-\-createSparseFiles\fR, \fB\-s\fR" 8 +.IX Item "--createSparseFiles, -s" +.Vb 2 +\& creates sparse file from blocked files if full blockes +\& are filled with zeros +.Ve .SH "COPYRIGHT" .IX Header "COPYRIGHT" Copyright (c) 2002\-2013 by Heinz-Josef Claes (see \s-1README\s0). diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/storeBackup/man/man1/storeBackupUpdateBackup.pl.1 new/storeBackup/man/man1/storeBackupUpdateBackup.pl.1 --- old/storeBackup/man/man1/storeBackupUpdateBackup.pl.1 2013-09-07 13:36:30.000000000 +0200 +++ new/storeBackup/man/man1/storeBackupUpdateBackup.pl.1 2013-09-25 17:00:59.000000000 +0200 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "STOREBACKUPUPDATEBACKUP 1" -.TH STOREBACKUPUPDATEBACKUP 1 "2013-08-12" "perl v5.14.2" "User Contributed Perl Documentation" +.TH STOREBACKUPUPDATEBACKUP 1 "2013-09-22" "perl v5.14.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -193,12 +193,16 @@ .Ve .IP "\fB\-\-lockFile\fR, \fB\-L\fR" 8 .IX Item "--lockFile, -L" -.Vb 5 +.Vb 9 \& lock file, if exist, new instances will finish if \& an old is already running \& If set to the same file as in storeBackup it will \& prevent $prog from running in parallel \& to storeBackup, default is $lockFile +\& this type of lock files does not work across multiple servers +\& and is not designed to separate storeBackup.pl and +\& storeBackupUpdateBackup.pl or any other storeBackup +\& process in a separate PID space .Ve .IP "\fB\-\-noCompress\fR" 8 .IX Item "--noCompress" -- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
