Hello community, here is the log from the commit of package dd_rescue for openSUSE:Factory checked in at 2014-06-01 18:58:09 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/dd_rescue (Old) and /work/SRC/openSUSE:Factory/.dd_rescue.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "dd_rescue" Changes: -------- --- /work/SRC/openSUSE:Factory/dd_rescue/dd_rescue.changes 2014-05-26 10:28:12.000000000 +0200 +++ /work/SRC/openSUSE:Factory/.dd_rescue.new/dd_rescue.changes 2014-06-01 18:58:09.000000000 +0200 @@ -1,0 +2,10 @@ +Tue May 27 13:45:42 CEST 2014 - [email protected] + +- Update to dd_rescue-1.45: + * ddr_hash: Fix potential sha512/sha384 buffer overflow. + * ddr_hash: Support sha1 + * ddr_hash: Support checking and storing hash in xattrs and + md5sum/sha256sum/... style files. + * New ddr_null plugin. + +------------------------------------------------------------------- Old: ---- dd_rescue-1.44.tar.gz New: ---- dd_rescue-1.45.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ dd_rescue.spec ++++++ --- /var/tmp/diff_new_pack.TBSnEk/_old 2014-06-01 18:58:10.000000000 +0200 +++ /var/tmp/diff_new_pack.TBSnEk/_new 2014-06-01 18:58:10.000000000 +0200 @@ -17,7 +17,7 @@ Name: dd_rescue -Version: 1.44 +Version: 1.45 Release: 0 Summary: Data Copying in the Presence of I/O Errors License: GPL-2.0 or GPL-3.0 @@ -114,6 +114,7 @@ #EndUsrMerge %{_libdir}/libddr_hash.so %{_libdir}/libddr_MD5.so +%{_libdir}/libddr_null.so %doc %{_mandir}/man1/dd_rescue.1%{ext_man} %files lzo ++++++ dd_rescue-1.44.tar.gz -> dd_rescue-1.45.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/Makefile new/dd_rescue/Makefile --- old/dd_rescue/Makefile 2014-05-23 12:27:19.000000000 +0200 +++ new/dd_rescue/Makefile 2014-05-27 13:31:21.000000000 +0200 @@ -1,8 +1,8 @@ # Makefile for dd_rescue # (c) [email protected], 99/10/09, GNU GPL -# $Id: Makefile,v 1.165 2014/05/23 10:27:19 garloff Exp $ +# $Id: Makefile,v 1.172 2014/05/27 11:31:21 garloff Exp $ -VERSION = 1.44 +VERSION = 1.45 DESTDIR = @@ -15,13 +15,13 @@ prefix = $(DESTDIR)/usr INSTALLDIR = $(prefix)/bin #INSTALLDIR = $(DESTDIR)/bin -#INSTALLLIBDIR = $(prefix)/$(LIB) -INSTALLLIBDIR = $(DESTDIR)/$(LIBDIR) +INSTALLLIBDIR = $(prefix)/$(LIB) +#INSTALLLIBDIR = $(DESTDIR)/$(LIBDIR) MANDIR = $(prefix)/share/man #MYDIR = dd_rescue-$(VERSION) MYDIR = dd_rescue BINTARGETS = dd_rescue -LIBTARGETS = libddr_hash.so libddr_MD5.so +LIBTARGETS = libddr_hash.so libddr_MD5.so libddr_null.so #TARGETS = libfalloc-dl OTHTARGETS = find_nonzero fiemap file_zblock fmt_no md5 sha256 sha512 sha224 sha384 sha1 OBJECTS = frandom.o fmt_no.o find_nonzero.o @@ -152,6 +152,9 @@ libddr_lzo.so: libddr_lzo.po $(CC) -shared -o $@ $^ -llzo2 +libddr_null.so: libddr_null.po + $(CC) -shared -o $@ $^ + find_nonzero.o: find_nonzero.c $(FNZ_HEADERS) config.h $(CC) $(CFLAGS_OPT) -c $< $(SSE) @@ -211,7 +214,7 @@ static: dd_rescue.c $(HEADERS) $(OBJECTS) $(CC) $(CFLAGS) -DNO_LIBDL -DNO_LIBFALLOCATE -static $(DEFINES) $< $(OUT) $(OBJECTS) $(OBJECTS2) -strip: $(TARGETS) +strip: $(TARGETS) $(LIBTARGETS) strip -S $^ strip-all: $(OTHTARGETS) @@ -299,7 +302,7 @@ #MD5=$$(./dd_rescue -c0 -a -b16k -L ./libddr_MD5.so TEST TEST2 2>&1 | grep 'MD5(0)': | tail -n1 | sed 's/^dd_rescue: (info): MD5(0):[^:]*: //'); MD5S=$$(md5sum TEST | sed 's/ .*$$//'); echo $$MD5 $$MD5S; if test "$$MD5" != "$$MD5S"; then false; fi ./sha1 /dev/null ./sha1 /dev/null | sha1sum -c - ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=output:alg=sha1 TEST TEST2 >HASH.TEST + ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=outnm=HASH.TEST:alg=sha1 TEST TEST2 sha1sum -c HASH.TEST if test $(HAVE_SHA256SUM) = 1; then $(MAKE) check_sha2; fi ./sha256 /dev/null @@ -309,20 +312,36 @@ if test $(HAVE_LZO) = 1; then $(MAKE) check_lzo_algos; fi #if test $(HAVE_LZO) = 1; then $(MAKE) check_lzo_test; fi if test $(HAVE_LZO) = 1; then $(MAKE) check_lzo_fuzz; fi + # Tests for libddr_null + ./dd_rescue -L ./libddr_null.so=debug dd_rescue /dev/null + # Tests with hash set_xattr and chk_xattr (with fallback as not all filesystems support xattrs ...) + ./dd_rescue -tL ./libddr_hash.so=sha256:set_xattr:fallback dd_rescue /tmp/dd_rescue + ./dd_rescue -L ./libddr_hash.so=sha256:chk_xattr:fallback /tmp/dd_rescue /dev/null + rm -f /tmp/dd_rescue CHECKSUMS.sha256 + # Tests with prepend and append + ./dd_rescue -tL ./libddr_hash.so=sha512:set_xattr:fallback:prepend=abc:append=xyz dd_rescue /tmp/dd_rescue + ./dd_rescue -L ./libddr_hash.so=sha512:chk_xattr:fallback /tmp/dd_rescue /dev/null && false || true + ./dd_rescue -L ./libddr_hash.so=sha512:chk_xattr:fallback:prepend=abc:append=xyz /tmp/dd_rescue /dev/null + rm -f /tmp/dd_rescue CHECKSUMS.sha512 check_sha2: $(TARGETS) sha224 sha384 ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=output:alg=sha224 TEST TEST2 >HASH.TEST sha224sum -c HASH.TEST - ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=output:alg=sha256 TEST TEST2 >HASH.TEST - sha256sum -c HASH.TEST + ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=outnm=:alg=sha256 TEST TEST2 >HASH.TEST + sha256sum -c CHECKSUMS.sha256 ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=output:alg=sha384 TEST TEST2 >HASH.TEST sha384sum -c HASH.TEST - ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=output:alg=sha512 TEST TEST2 >HASH.TEST - sha512sum -c HASH.TEST + ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=outnm=:alg=sha512 TEST TEST2 + ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=outnm=:alg=sha512,./libddr_null.so=change dd_rescue /dev/null + sha512sum -c CHECKSUMS.sha512 + ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=sha512:chknm=CHECKSUMS.sha512 TEST2 /dev/null + ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=alg=sha512:chknm= dd_rescue /dev/null + ./dd_rescue -c0 -a -b16k -t -L ./libddr_hash.so=sha512:check dd_rescue /dev/null <CHECKSUMS.sha512 ./sha224 /dev/null | sha224sum -c ./sha256 /dev/null | sha256sum -c ./sha384 /dev/null | sha384sum -c ./sha512 /dev/null | sha512sum -c + rm -f HASH.TEST CHECKSUMS.sha256 CHECKSUMS.sha512 TEST2 check_lzo: $(TARGETS) @echo "***** dd_rescue lzo (and MD5) plugin tests *****" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/Makefile.android new/dd_rescue/Makefile.android --- old/dd_rescue/Makefile.android 2014-03-01 16:16:26.000000000 +0100 +++ new/dd_rescue/Makefile.android 2014-05-27 13:41:59.000000000 +0200 @@ -43,7 +43,7 @@ $(MAKE) -f Makefile PATH=$(CBIN):$(PATH) CC="$(CC) $(CFLG)" LD="$(CC) -L $(LIBD)" MACH=arm $@ strip: dd_rescue - strip -S dd_rescue + strip -S dd_rescue libddr_*.so find_nonzero: $(DEPS) $(MAKE) -f Makefile PATH=$(CBIN):$(PATH) CC="$(CC) $(CFLG)" LD="$(CC) -L $(LIBD)" MACH=arm $@ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/README.dd_rescue new/dd_rescue/README.dd_rescue --- old/dd_rescue/README.dd_rescue 2014-05-23 10:18:39.000000000 +0200 +++ new/dd_rescue/README.dd_rescue 2014-05-27 13:33:48.000000000 +0200 @@ -190,6 +190,9 @@ to test error recovery), and appending to existing .lzo files. * Version 1.44 renamed the MD5 plugin to the hash plugin, as we support the SHA-2 family of cryptographic hashes now. +* Version 1.45 added sha1 to ddr_hash as well as the chk_xattr:chknm=:check + and outnm:set_xattr parameters that allow to conveniently store and validate + hashes on files. Copyright @@ -235,4 +238,4 @@ Kurt Garloff <[email protected]>, 2000-08-30 -$Id: README.dd_rescue,v 1.35 2014/05/23 08:18:39 garloff Exp $ +$Id: README.dd_rescue,v 1.36 2014/05/27 11:33:48 garloff Exp $ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/configure.in new/dd_rescue/configure.in --- old/dd_rescue/configure.in 2014-05-23 12:39:05.000000000 +0200 +++ new/dd_rescue/configure.in 2014-05-27 12:36:22.000000000 +0200 @@ -7,7 +7,7 @@ #AC_PROG_INSTALL #CFLAGS="$CFLAGS -DHAVE_CONFIG_H" AC_CHECK_HEADERS([fallocate.h dlfcn.h unistd.h attr/xattr.h sys/acl.h sys/ioctl.h endian.h linux/fs.h linux/fiemap.h stdint.h lzo/lzo1x.h]) -AC_CHECK_FUNCS([ffs ffsl basename fallocate64 splice getopt_long open64 pread pread64 lseek64 stat64 posix_fadvise posix_fadvise64 __builtin_prefetch htobe64]) +AC_CHECK_FUNCS([ffs ffsl basename fallocate64 splice getopt_long open64 pread pread64 lseek64 stat64 posix_fadvise posix_fadvise64 __builtin_prefetch htobe64 feof_unlocked getline]) AC_CHECK_LIB(dl,dlsym) AC_CHECK_LIB(fallocate,linux_fallocate64) AC_CHECK_LIB(lzo2,lzo1x_1_compress) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/dd_rescue.1 new/dd_rescue/dd_rescue.1 --- old/dd_rescue/dd_rescue.1 2014-05-23 11:06:36.000000000 +0200 +++ new/dd_rescue/dd_rescue.1 2014-05-27 13:34:49.000000000 +0200 @@ -1,4 +1,4 @@ -.\" $Id: dd_rescue.1,v 1.39 2014/05/23 09:06:36 garloff Exp $ +.\" $Id: dd_rescue.1,v 1.46 2014/05/27 11:34:49 garloff Exp $ . .TH dd_rescue 1 "2014-04-30" "Kurt Garloff" "Data recovery and protection tool" . @@ -370,10 +370,13 @@ ownership, access rights, xattrs) to be copied, similar to the option with the same name in the cp program. .br -Note that this option is only available if +Note that ACLs and xattrs will only be copied if .B dd_rescue -has been compiled with libxattr support and the library can by dynamically -loaded on the system. +has been compiled with libxattr support and the library can be dynamically +loaded on the system. Also note that failing to copy the attributes with +.IR -p +is not considered a failure and thus won't negatively affect the exit code +of dd_rescue. .TP 8 .BR \-t ", " \-\-truncate tells @@ -539,6 +542,22 @@ . .SH PLUGINS +.SS null +The null plugin (ddr_null) does nothing, except if you specify the +.B [no]lnchange +or the +.B [no]change +options in which case the plugin indicates to others that it transforms the +length of the output or the data of the stream. (With the no prefix, it's +reset to the default no-change indication again.) +This may be helpful for testing or to influence which file the hash plugin +considers for reading/writing extended attributes from/to +and for plugins to change their behavior with respect to hole detection. +.br +ddr_null_ddr also allows you to specify +.B debug +in which case it just reports the blocks that it passes on. +. .SS hash When the hash plugin (subsequently referred to as ddr_hash) is loaded, it will calculate a cryptographic hash over the @@ -549,11 +568,28 @@ alg=help to get a list.) To abbreviate the syntax, the alg= piece can be omitted. .br +For backwards compatibility, the hash plugin can also be referred to with the +old MD5 name; it then defaults to the md5 algorithm. +.br The computed value should be identical to calling md5sum/sha256sum/... on the target file (unless you only write part of the file), but saves time by not accessing the (possibly large) file a second time. The hash plugin handles sparse writes and arbitrary offsets fine. -.br +.PP +ddr_hash also supports the parameter +.B append=STRING +which appends the given STRING to the output before computing the cryptographic +hash. Treating the STRING as a shared secret, this can actually be used to protect +against someone not knowing the secret altering the contents (and recomputing the +hash) without anyone noticing. It's thus a cheap way of a cryptographic signature +(but with preshared secrets as opposed to public key cryptography). +.br +ddr_hash also supports +.B prepend=STRING +which is likely harder to attack with brute force than an appended string. +Note that ddr_hash always prepends multiples of the hash algorithm's block +size and pads the STRING with 0 to match. +.PP ddr_hash accepts the parameter .B output , which will cause ddr_hash to output @@ -565,9 +601,79 @@ always processes data in binary mode and correctly indicates this with a star (*) in the output generated with output/outfd=. .br -For backwards compatibility, the hash plugin can also be referred to with the -old MD5 name; it then defaults to the md5 algorithm. -.br +The checksum can also be written to a file by giving the +.B outnm=CHKNAME +parameter. Then a file with CHKNAME will be created and a md5sum/sha256sum/... +compatible line will be printed to the file. If the file exists and contains +an entry for the file, it will be updated. If the file exists and does not +contain an entry for the file, one will be appended. If NAME is omitted, the +filename CHECKSUMS.alg will be used (alg is replaced by the chosen algorithm). +If the checksum can't be written, a warning will be printed and the exit code +of dd_rescue will become non-zero. +.PP +The checksum can be validated using +.B checknm=CHKNAME . +The file will be read and ddr_hash will look for an md5sum/sha256sum/... +compatible line with a matching file name to take the checksum from and +compare it to the one computed. If NAME is omitted, the same default +as described above (in outnm=...) will be used. You can also read the +checksum from stdin if you prefer by specifying the +.B check +option. +.br +Note that in any case, the check is only performed after the copy operation +is completed -- a faulty checksum will thus NOT result in the copy not +taking place. However, the exit code of dd_rescue will indicate the +error. (If you want to avoid copying data with a broken checksum into +the final target, use a temporary target that you delete upon error and +only move to the final location if dd_rescue's exit value is 0; you can +of course also copy to /dev/null for testing beforehand, but it might +be too costly reading the input file twice.) +.PP +You can store the cryptographic hash into the files by using the +.B set_xattr +option. The hash will be stored into the extended attribute user.checksum.ALG +by default, but you can override the name of the attribute by specifying +.B set_xattr=XATTR\.NAME +instead. If the xattr can't be written, an error will be reported, unless +you also specify the +.B fallb[ack][=CHKNAME] +option. In that case, ddr_hash tries to write the checksum to the CHKNAME +checksums file. (For the default for CHKNAME, see outnm= option above.) +.br +.B chk_xattr +will validate that the computed hash matches the one read from the extended +attribute. The same default attribute name applies and you can likewise override +it with +.B chk_xattr=XATTR\.NAME . +A missing attribute is considered an error (although the same fallback is +tried if you specify the fallback option). A broken checksum is of course +considered an error as well, but just like with checknm=CHKNAME won't +prevent the copy. See the discussion there. +.PP +Note that for output,outfd,outnm=,set_xattr ddr_hash will use the +output file name to attach the checksum to (be it by setting xattr or the +file name used in the checksum file), unless a plugin +in the chain after ddr_hash indicates that it changes the data. +In that case, it will warn and associate the checksum with the input file +name, unless there's another plugin before ddr_hash in the chain which +indicates data transformation as well. In that case, there is no file that +the checksum could be associated with and ddr_hash will report an error. +.br +Likewise for chknm=,check,chk_xattr ddr_hash will use the input file +name to get the checksum (be it by reading the xattr or by looking for +the input file name in a checksums file) unless there's a plugin in the +chain before ddr_hash that indicates that it changes the data. The output +file name will then be used, unless there's another plugin after ddr_hash +indicating data change as well, in which case there's no file we could +get the checksum for and thus an error is reported. +.PP +If your system supports extended attributes, those have the advantage +of travelling with the files; thus a rename or copy (with dd_rescue -p) +will maintain the checksum. Checksum files on the other hand can be +handled everywhere (including the transfer via ftp or http) and can +be cryptographically signed with PGP/GnuPG. +.PP Please note that the md5 algorithm is NOT recommended any more for good protetction against malicious attempts to hide data modification; it's not considered strong enough any more to prevent hash collisions. @@ -638,6 +744,17 @@ it from the filenames. This example shows that you can specify multiple plugins with multiple parameters; the plugins are forming a filter chain. You can specify the same plugin multiple times. +.TP +.BI dd_rescue\ \-L\ hash=sha512:set_xattr:fallb,null=change\ infile\ /dev/null +reads the file +.IR infile +and computes its sha512 hash. It stores it in the input file's user.checksum.sha512 +attribute (and falls back to writing it to CHECKSUMS.sha512 if xattrs can't be +written). Note the use of the null plugin with faking data change with +the change parameter; this causes the hash plugin to write to the input +file which it would not normally have done. Of course this +will fail if you don't have the appropriate privileges to write xattrs to +infile nor to write the checksum to CHECKSUMS.sha512. .PP See also README.dd_rescue and ddr_lzo(1) to learn about the possibilities. . @@ -760,7 +877,8 @@ .br Plugins exist since 1.42, the MD5 plugin came with 1.42, the lzo plugin with 1.43. 1.44 renamed the MD5 plugin to hash and -added support for the SHA-2 family of hashes. +added support for the SHA-2 family of hashes. 1.45 added SHA-1 +and the ability to store and validate checksums. .PP Some additional information can be found on .br diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/dd_rescue.c new/dd_rescue/dd_rescue.c --- old/dd_rescue/dd_rescue.c 2014-05-23 11:56:30.000000000 +0200 +++ new/dd_rescue/dd_rescue.c 2014-05-27 10:27:21.000000000 +0200 @@ -49,7 +49,7 @@ # define __COMPILER__ "(unknown compiler)" #endif -#define ID "$Id: dd_rescue.c,v 1.319 2014/05/22 12:29:07 garloff Exp $" +#define ID "$Id: dd_rescue.c,v 1.323 2014/05/27 08:27:21 garloff Exp $" #ifndef BUF_SOFTBLOCKSIZE # define BUF_SOFTBLOCKSIZE 131072 @@ -349,8 +349,10 @@ int plug_max_neg_slack_pre = 0; unsigned int plug_max_slack_post = 0; int plug_max_neg_slack_post = 0; -int plug_first_lenchg = -1; +int plug_first_lenchg = 9999; int plug_last_lenchg = -1; +int plug_first_chg = 9999; +int plug_last_chg = -1; unsigned int plug_max_req_align = 0; char plug_not_sparse = 0; char plug_output_chg = 0; @@ -378,16 +380,18 @@ */ int err = LISTDATA(plug).open_callback(op, (plugins_opened > plug_first_lenchg? 1: 0), (plugins_opened < plug_last_lenchg ? 1: 0), + (plugins_opened > plug_first_chg? 1: 0), + (plugins_opened < plug_last_chg ? 1: 0), plug_max_slack_pre-slk_pre, plug_max_slack_post-slk_post, &LISTDATA(plug).state); if (err < 0) { - fplog(stderr, WARN, "Error initializing plugin %s: %s!\n", - LISTDATA(plug).name, strerror(err)); + fplog(stderr, WARN, "Error initializing plugin %s(%i): %s!\n", + LISTDATA(plug).name, plugins_opened, strerror(-err)); exit(13); } else if (err>0) { fst->ipos += err; - fplog(stderr, WARN, "Plugin %s skipping %i bytes might break other plugins!\n", - LISTDATA(plug).name, err); + fplog(stderr, WARN, "Plugin %s(%i) skipping %i bytes might break other plugins!\n", + LISTDATA(plug).name, plugins_opened, err); } } ++plugins_opened; @@ -396,20 +400,26 @@ assert(slk_post == plug_max_slack_post); } -void call_plugins_close(opt_t *op, fstate_t *fst) +int call_plugins_close(opt_t *op, fstate_t *fst) { + int errs = 0; + int seq = 0; if (!plugins_opened) - return; + return 0; LISTTYPE(ddr_plugin_t) *plug; LISTFOREACH(ddr_plugins, plug) { if (LISTDATA(plug).close_callback) { int err = LISTDATA(plug).close_callback(fst->opos, &LISTDATA(plug).state); - if (err) - fplog(stderr, WARN, "Error closing plugin %s: %s!\n", - LISTDATA(plug).name, strerror(err)); + if (err) { + fplog(stderr, WARN, "Plugin %s(%i) reported error on close: %s!\n", + LISTDATA(plug).name, seq, strerror(-err)); + ++errs; + } } + ++seq; --plugins_opened; } + return errs; } /** Call the plugin block processing chain ... @@ -459,7 +469,19 @@ if (!plug->name) plug->name = nm; plug->fplog = fplog; - + /* Call init after dd_rescue-filled fields have been set; this allows the + * init_callback to adjust fiels like slack, align_needs, output_chg + * depending on parameters and options ... */ + if (param && !plug->init_callback) { + fplog(stderr, FATAL, "Plugin %s has no init callback to consume passed param %s\n", + nm, param); + exit(13); + } + if (plug->init_callback) { + int ret = plug->init_callback(&plug->state, param, plugins_loaded, op); + if (ret) + exit(-ret); + } if (plug->slack_pre > 0) plug_max_slack_pre += plug->slack_pre; else if (plug->slack_pre < 0) @@ -475,16 +497,6 @@ plug_not_sparse = 1; if (plug->changes_output) plug_output_chg = 1; - if (param && !plug->init_callback) { - fplog(stderr, FATAL, "Plugin %s has no init callback to consume passed param %s\n", - nm, param); - exit(13); - } - if (plug->init_callback) { - int ret = plug->init_callback(&plug->state, param, plugins_loaded, op); - if (ret) - exit(ret); - } plugins_loaded++; LISTAPPEND(ddr_plugins, *plug, ddr_plugin_t); if (param && !memcmp(param, "help", 4)) @@ -536,8 +548,12 @@ } if (plug->changes_output_len) plug_last_lenchg = plugno; - if (plug_first_lenchg == -1 && plug->changes_output_len) + if (plug_first_lenchg == 9999 && plug->changes_output_len) plug_first_lenchg = plugno; + if (plug->changes_output) + plug_last_chg = plugno; + if (plug_first_chg == 9999 && plug->changes_output) + plug_first_chg = plugno; ++plugno; } plugs = next; @@ -1186,7 +1202,7 @@ else errs++; /* And finalize */ - call_plugins_close(op, fst); + errs += call_plugins_close(op, fst); } errs += sync_close(fst->odes, op->oname, fst->o_chr, op, fst); if (fst->ides != -1) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/ddr_lzo.1 new/dd_rescue/ddr_lzo.1 --- old/dd_rescue/ddr_lzo.1 2014-05-23 10:18:40.000000000 +0200 +++ new/dd_rescue/ddr_lzo.1 2014-05-25 00:01:52.000000000 +0200 @@ -1,4 +1,4 @@ -.\" $Id: ddr_lzo.1,v 1.10 2014/05/23 08:18:40 garloff Exp $ +.\" $Id: ddr_lzo.1,v 1.11 2014/05/24 22:01:52 garloff Exp $ . .TH ddr_lzo 1 "2014-05-12" "Kurt Garloff" "LZO de/compression plugin for dd_rescue" . @@ -244,7 +244,7 @@ it will be emptied before (if the file happens to exist). The output file won't have encoded holes; errors in the infile will result in zeros. .TP -.BI dd_rescue\ \-aL\ MD5,lzo=compress:bench,MD5,lzo=decompress,MD5\ infile\ infile2 +.BI dd_rescue\ \-aL\ MD5,lzo=compr:bench,MD5,lzo=decompress,MD5\ infile\ infile2 will copy .IR infile to diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/ddr_plugin.h new/dd_rescue/ddr_plugin.h --- old/dd_rescue/ddr_plugin.h 2014-05-22 14:10:14.000000000 +0200 +++ new/dd_rescue/ddr_plugin.h 2014-05-26 10:05:13.000000000 +0200 @@ -22,18 +22,20 @@ #endif /** init callback parameters: - * opaque handle, parameters from commandline, sequece in filter chain + * opaque handle, parameters from commandline, sequence in filter chain, + * pointer to options. + * Return value: 0 = OK, -x = ERROR */ typedef int (_init_callback)(void **stat, char* param, int seq, const opt_t *opt); -/** open_callback parameters: input file descriptor, input file name, - * initial offset input file, same 3 params for output file, - * soft (large) block size, hard (fallback) block size, - * estimated xfer size, flag that olen will change after writing, - * total slack size for all plugins, - * ptr to buffer ptr, opaque handle. +/** open_callback parameters: pointer to options, four flags telling the + * plugin whether length, and/or contents of the stream are changed + * by other plugins before (i) or after (o) this one, + * required extra buffer memory before and after the main buffer + * and the opaque handle * Return value: 0 = OK, -x = ERROR, +x = Bytes consumed from input file. */ typedef int (_open_callback)(const opt_t *opt, int ilnchange, int olnchange, + int ichange, int ochange, unsigned int totslack_pre, unsigned int totslack_post, void **stat); /** block_callback parameters: file state (contains file descriptors, positions, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/hash.h new/dd_rescue/hash.h --- old/dd_rescue/hash.h 2014-05-23 10:30:30.000000000 +0200 +++ new/dd_rescue/hash.h 2014-05-27 12:29:40.000000000 +0200 @@ -21,6 +21,7 @@ uint32_t sha1_h[5]; uint32_t sha256_h[8]; uint64_t sha512_h[8]; + //uint64_t sha3_h[25]; }; } hash_t ALIGNED(32); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/libddr_MD5.c new/dd_rescue/libddr_MD5.c --- old/dd_rescue/libddr_MD5.c 2014-05-23 11:05:18.000000000 +0200 +++ new/dd_rescue/libddr_MD5.c 2014-05-27 14:03:18.000000000 +0200 @@ -7,6 +7,13 @@ * License: GNU GPLv2 or v3 */ +#ifndef _GNU_SOURCE +# define _GNU_SOURCE 1 +#endif + +#define _LARGEFILE64_SOURCE 1 +#define _FILE_OFFSET_BITS 64 + #include "ddr_plugin.h" #include "ddr_ctrl.h" #include "hash.h" @@ -16,9 +23,15 @@ #include "sha1.h" #include <stdlib.h> -#include <unistd.h> #include <string.h> +#include <libgen.h> +#include <ctype.h> #include <assert.h> +#include <errno.h> +#include <unistd.h> +#ifdef HAVE_ATTR_XATTR_H +# include <attr/xattr.h> +#endif // TODO: pass at runtime rather than compile time #define HASH_DEBUG(x) if (state->debug) x @@ -49,26 +62,40 @@ { "sha224", sha224_init, sha256_64 , sha256_calc, sha224_out, 64 }, { "sha512", sha512_init, sha512_128, sha512_calc, sha512_out, 128 }, { "sha384", sha384_init, sha512_128, sha512_calc, sha384_out, 128 } + // SHA3 ... }; typedef struct _hash_state { hash_t hash; loff_t hash_pos; - const char* fname; + const char *fname; + const char *append, *prepend; hashalg_t *alg; - uint8_t buf[128]; + uint8_t buf[288]; // enough for SHA-3 with max blksz of 144Bytes int seq; int outfd; unsigned char buflen; - unsigned char ilnchg, olnchg, debug; + unsigned char ilnchg, olnchg, ichg, ochg, debug, outf, chkf, chkfalloc; + const char* chkfnm; const opt_t *opts; +#ifdef HAVE_ATTR_XATTR_H + char chk_xattr, set_xattr, xnmalloc, xfallback; + char* xattr_name; +#endif } hash_state; +/* TO DO: Add updating and reading MD5SUMS style files, optionally + * as fallback for unavailable xattrs + */ const char *hash_help = "The HASH plugin for dd_rescue calculates a cryptographic checksum on the fly.\n" - " It supports unaligned blocks (arbitrary offsets) and sparse writing.\n" - " Parameters: output:outfd=FNO:debug:alg[o[rithm]=ALG\n" + " It supports unaligned blocks (arbitrary offsets) and holes(sparse writing).\n" + " Parameters: output:outfd=FNO:outnm=FILE:check:chknm=FILE:debug:[alg[o[rithm]=]ALG\n" + "\t:append=STR:prepend=STR\n" +#ifdef HAVE_ATTR_XATTR_H + "\t:chk_xattr[=xattr_name]:set_xattr[=xattr_name]:fallb[ack][=FILE]\n" +#endif " Use algorithm=help to get a list of supported hash algorithms\n"; @@ -78,7 +105,6 @@ const char help = !strcmp(nm, "help"); if (help) FPLOG(INFO, "Supported algorithms:"); - // TODO: Handle alg=help for (i = 0; i < sizeof(hashes)/sizeof(hashalg_t); ++i) { if (help) fprintf(stderr, " %s", hashes[i].name); @@ -113,6 +139,34 @@ state->outfd = 1; else if (!memcmp(param, "outfd=", 6)) state->outfd = atoi(param+6); + else if (!memcmp(param, "append=", 7)) + state->append = param+7; + else if (!memcmp(param, "prepend=", 8)) + state->prepend = param+8; +#ifdef HAVE_ATTR_XATTR_H + else if (!memcmp(param, "chk_xattr=", 10)) { + state->chk_xattr = 1; state->xattr_name = param+10; } + else if (!strcmp(param, "chk_xattr")) + state->chk_xattr = 1; + else if (!memcmp(param, "set_xattr=", 10)) { + state->set_xattr = 1; state->xattr_name = param+10; } + else if (!strcmp(param, "set_xattr")) + state->set_xattr = 1; + else if (!strcmp(param, "fallb")) + state->xfallback = 1; + else if (!strcmp(param, "fallback")) + state->xfallback = 1; + else if (!memcmp(param, "fallback=", 9)) { + state->xfallback = 1; state->chkfnm = param+9; } + else if (!memcmp(param, "fallb=", 6)) { + state->xfallback = 1; state->chkfnm = param+6; } +#endif + else if (!memcmp(param, "outnm=", 6)) { + state->outf = 1; state->chkfnm=param+6; } + else if (!memcmp(param, "chknm=", 6)) { + state->chkf = 1; state->chkfnm=param+6; } + else if (!strcmp(param, "check")) { + state->chkf = 1; state->chkfnm="-"; } else if (!memcmp(param, "algo=", 5)) state->alg = get_hashalg(state, param+5); else if (!memcmp(param, "alg=", 4)) @@ -128,38 +182,93 @@ else { FPLOG(FATAL, "plugin doesn't understand param %s\n", param); - ++err; + --err; } } param = next; } if (!state->alg) { FPLOG(FATAL, "No hash algorithm specified\n"); - ++err; + --err; + } +#ifdef HAVE_ATTR_XATTR_H + if ((state->chk_xattr || state->set_xattr) && !state->xattr_name) { + state->xattr_name = (char*)malloc(24); + state->xnmalloc = 1; + snprintf(state->xattr_name, 24, "user.checksum.%s", state->alg->name); + } +#endif + if ((!state->chkfnm || !*state->chkfnm) && (state->chkf || state->outf +#ifdef HAVE_ATTR_XATTR_H + || state->xfallback +#endif + )) { + char cfnm[32]; + // if (!strcmp(state->alg->name, "md5")) strcpy(cfnm, "MD5SUMS"); else + snprintf(cfnm, 32, "CHECKSUMS.%s", state->alg->name); + state->chkfalloc = 1; + state->chkfnm = strdup(cfnm); } if (state->debug) - FPLOG(DEBUG, "Initialized plugin %s\n", ddr_plug.name); + FPLOG(DEBUG, "Initialized plugin %s (%s)\n", ddr_plug.name, state->alg->name); return err; } -int hash_open(const opt_t *opt, int ilnchg, int olnchg, +#define MIN(a,b) ((a)<(b)? (a): (b)) + +int hash_open(const opt_t *opt, int ilnchg, int olnchg, int ichg, int ochg, unsigned int totslack_pre, unsigned int totslack_post, void **stat) { + int err = 0; hash_state *state = (hash_state*)*stat; state->opts = opt; state->alg->hash_init(&state->hash); state->hash_pos = 0; - state->fname = (state->seq == 0? opt->iname: opt->oname); - memset(state->buf, 0, 128); + if (!ochg && state->seq != 0) + state->fname = opt->oname; + else if (!ichg) + state->fname = opt->iname; + else { + char* nnm = (char*)malloc(strlen(opt->iname)+strlen(opt->oname)+3); + strcpy(nnm, opt->iname); + strcat(nnm, "->"); + strcat(nnm, opt->oname); + state->fname = nnm; +#ifdef HAVE_ATTR_XATTR_H + if (state->chk_xattr || state->set_xattr) { + --err; + FPLOG(WARN, "Can't access xattr in the middle of a plugin chain!"); + } +#endif + } + if (state->prepend) { + const int blksz = state->alg->blksz; + int done = 0; int remain = strlen(state->prepend); + while (remain >= blksz) { + state->alg->hash_block((uint8_t*)(state->prepend)+done, &state->hash); + remain -= blksz; + done += blksz; + } + HASH_DEBUG(FPLOG(DEBUG, "Prepending %i+%i bytes (padded with %i bytes)\n", + done, remain, blksz-remain)); + if (remain) { + memcpy(state->buf, state->prepend+done, remain); + memset(state->buf+remain, 0, blksz-remain); + state->alg->hash_block(state->buf, &state->hash); + } + } + memset(state->buf, 0, sizeof(state->buf)); state->buflen = 0; state->ilnchg = ilnchg; state->olnchg = olnchg; - if (ilnchg && olnchg && (state->opts->sparse || !state->opts->nosparse)) { + state->ichg = ichg; + state->ochg = ochg; + if (ichg && ochg && (state->opts->sparse || !state->opts->nosparse)) { FPLOG(WARN, "Size of potential holes may not be correct due to other plugins;\n"); FPLOG(WARN, " MD5 hash may be miscomputed! Avoid holes (remove -a, use -A).\n"); } - return 0; + return err; } #if __WORDSIZE == 64 @@ -170,6 +279,13 @@ #error __WORDSIZE unknown #endif +inline int round_down(int val, const int gran) +{ + return val-val%gran; +} + +#define round_up(v, g) round_down(v+g-1, g) + void hash_last(hash_state *state, loff_t pos) { //hash_block(0, 0, ooff, stat); @@ -180,7 +296,15 @@ state->fname, len, state->hash_pos); */ HASH_DEBUG(FPLOG(DEBUG, "Last block with %i bytes\n", state->buflen)); - state->alg->hash_calc(state->buf, state->buflen, state->hash_pos+state->buflen, &state->hash); + if (state->append) { + memcpy(state->buf+state->buflen, state->append, strlen(state->append)); + state->buflen += strlen(state->append); + HASH_DEBUG(FPLOG(DEBUG, "Append string with %i bytes for hash\n", strlen(state->append))); + } + int preln = state->prepend? round_up(strlen(state->prepend), state->alg->blksz): 0; + if (preln) + HASH_DEBUG(FPLOG(DEBUG, "Account for %i extra prepended bytes\n", preln)); + state->alg->hash_calc(state->buf, state->buflen, state->hash_pos+state->buflen+preln, &state->hash); state->hash_pos += state->buflen; } @@ -193,8 +317,6 @@ memset(state->buf, 0, clear); } -#define MIN(a,b) ((a)<(b)? (a): (b)) - void hash_hole(fstate_t *fst, hash_state *state, loff_t holelen) { if (state->buflen) { @@ -287,31 +409,284 @@ return bf; } +/* + * XXXSUM file parsing and updating routines + */ + +#ifndef HAVE_FEOF_UNLOCKED +#define feof_unlocked(x) feof(x) +#endif + +#ifndef HAVE_GETLINE +ssize_t getline(char **bf, size_t *sz, FILE *f) +{ + if (*sz == 0) { + *bf = (char*)malloc(1024); + *sz = 1024; + } + char* bret = fgets(*bf, *sz, f); + if (!bret) + return -1; + int ln = strlen(bret); + //if (bret[ln-1] != '\n') increase_buffersize(); + return ln; +} +#endif + +/* file offset in the chksum file which has the chksum for nm, -1 = not found */ +off_t find_chks(hash_state* st, FILE* f, const char* nm, char* res) +{ + char *lnbf = NULL; + size_t lln = 0; + char* bnm = basename((char*)nm); + while (!feof_unlocked(f)) { + char *fnm, *fwh; + off_t pos = ftello(f); + ssize_t n = getline(&lnbf, &lln, f); + if (n <= 0) + break; + fwh = strchr(lnbf, ' '); + if (!fwh) + continue; + fnm = fwh; + ++fnm; + if (*fnm == '*' || *fnm == ' ') + fnm++; + int last = strlen(fnm)-1; + // Remove trailing \n\r + while (last > 0 && (fnm[last] == '\n' || fnm[last] == '\r')) + fnm[last--] = 0; + if (!strcmp(fnm, nm) || !strcmp(fnm, bnm)) { + if (res && fwh-lnbf <= 2*sizeof(hash_t)) { + memcpy(res, lnbf, fwh-lnbf); + res[fwh-lnbf] = 0; + } + free(lnbf); + return pos; + } + } + if (lnbf) + free(lnbf); + return -1; +} +FILE* fopen_chks(hash_state *state, const char* mode) +{ + const char* fnm = state->chkfnm; + assert(fnm); + if (!strcmp("-", fnm)) + return stdin; + else + return fopen(fnm, mode); +} + +static char _chks[129]; +/* get chksum */ +char* get_chks(hash_state* state, const char* nm) +{ + FILE *f = fopen_chks(state, "r"); + if (!f) + return NULL; + *_chks = 0; + find_chks(state, f, nm, _chks); + if (f != stdin) + fclose(f); + return *_chks? _chks: NULL; +} + +/* update chksum */ +int upd_chks(hash_state* state, const char *nm, const char *chks) +{ + FILE *f = fopen_chks(state, "r+"); + int err = 0; + if (!f) { + errno = 0; + f = fopen_chks(state, "w"); + if (!f) + return -errno; + fprintf(f, "%s *%s\n", chks, nm); + err = -errno; + } else { + off_t pos = find_chks(state, f, nm, _chks); + if (pos == -1 || strlen(chks) != strlen(_chks)) { + fclose(f); + f = fopen_chks(state, "a"); + fprintf(f, "%s *%s\n", chks, nm); + err = -errno; + } else { + if (strcmp(chks, _chks)) { + if (pwrite(fileno(f), chks, strlen(chks), pos) <= 0) + err = -errno; + //pwrite(fileno(f), "*", 1, pos+strlen(chks)+1); + } + } + } + fclose(f); + return err; +} + + +#ifdef HAVE_ATTR_XATTR_H +int check_xattr(hash_state* state, const char* res) +{ + char xatstr[128]; + strcpy(xatstr, "xattr"); + const char* name = state->opts->iname; + if (state->ichg && !state->ochg) { + name = state->opts->oname; + if (!state->opts->quiet) + FPLOG(INFO, "Read xattr from output file %s\n", name); + } else if (state->ichg) { + FPLOG(WARN, "Can't read xattrs in the middle of plugin chain (%s)\n", state->fname); + return -ENOENT; + } + /* Longest is 128byte hex for SHA512 (8x64byte numbers -> 8x16 digits) */ + char chksum[129]; + ssize_t itln = getxattr(name, state->xattr_name, chksum, 129); + const int rln = strlen(res); + if (itln <= 0) { + if (state->xfallback) { + char* cks = get_chks(state, name); + snprintf(xatstr, 128, "chksum file %s", state->chkfnm); + if (!cks) { + FPLOG(WARN, "no hash found in xattr nor %s for %s\n", xatstr, name); + return -ENOENT; + } else if (strcmp(cks, res)) { + FPLOG(WARN, "Hash from %s for %s does not match\n", xatstr, name); + return -EBADF; + } + } else { + FPLOG(WARN, "Hash could not be read from xattr of %s\n", name); + return -ENOENT; + } + } else if (itln < rln || memcmp(res, chksum, rln)) { + FPLOG(WARN, "Hash from xattr of %s does not match\n", name); + return -EBADF; + } + if (!state->opts->quiet || state->debug) + FPLOG(INFO, "Successfully validated hash from %s for %s\n", xatstr, name); + return 0; +} + +int write_xattr(hash_state* state, const char* res) +{ + const char* name = state->opts->oname; + char xatstr[128]; + snprintf(xatstr, 128, "xattr %s", state->xattr_name); + if (state->ochg && !state->ichg) { + name = state->opts->iname; + if (!state->opts->quiet) + FPLOG(INFO, "Write xattr to input file %s\n", name); + } else if (state->ochg) { + FPLOG(WARN, "Can't write xattr in the middle of plugin chain (%s)\n", + state->fname); + return -ENOENT; + } + if (setxattr(name, state->xattr_name, res, strlen(res), 0)) { + if (state->xfallback) { + int err = upd_chks(state, name, res); + snprintf(xatstr, 128, "chksum file %s", state->chkfnm); + if (err) { + FPLOG(WARN, "Failed writing to %s for %s: %s\n", + xatstr, name, strerror(-err)); + return err; + } + } else { + FPLOG(WARN, "Failed writing hash to xattr of %s\n", name); + return -errno; + } + } + if (state->debug) + FPLOG(DEBUG, "Set %s for %s to %s\n", + xatstr, name, res); + return 0; +} +#endif + +int check_chkf(hash_state *state, const char* res) +{ + const char* name = state->opts->iname; + if (state->ichg && !state->ochg) { + name = state->opts->oname; + if (!state->opts->quiet) + FPLOG(INFO, "Read checksum from %s for output file %s\n", state->chkfnm, name); + } else if (state->ichg) { + FPLOG(WARN, "Can't read checksum in the middle of plugin chain (%s)\n", state->fname); + return -ENOENT; + } + char* cks = get_chks(state, name); + if (!cks) { + FPLOG(WARN, "Can't find checksum in %s for %s\n", state->chkfnm, name); + return -ENOENT; + } + if (strcmp(cks, res)) { + FPLOG(WARN, "Hash from chksum file %s for %s does not match\n", state->chkfnm, name); + return -EBADF; + } + return 0; +} + +int write_chkf(hash_state *state, const char *res) +{ + const char* name = state->opts->oname; + if (state->ochg && !state->ichg) { + name = state->opts->iname; + if (!state->opts->quiet) + FPLOG(INFO, "Write checksum to %s for input file %s\n", state->chkfnm, name); + } else if (state->ochg) { + FPLOG(WARN, "Can't write checksum in the middle of plugin chain (%s)\n", + state->fname); + return -ENOENT; + } + int err = upd_chks(state, name, res); + if (err) + FPLOG(WARN, "Hash writing to %s for %s failed\n", state->chkfnm, name); + return err; +} int hash_close(loff_t ooff, void **stat) { + int err = 0; hash_state *state = (hash_state*)*stat; char res[129]; loff_t firstpos = (state->seq == 0? state->opts->init_ipos: state->opts->init_opos); - FPLOG(INFO, "%s %s (%" LL "i-%" LL "i): %s\n", - state->alg->name, state->fname, firstpos, firstpos+state->hash_pos, - state->alg->hash_out(res, &state->hash)); + state->alg->hash_out(res, &state->hash); + if (!state->opts->quiet) + FPLOG(INFO, "%s %s (%" LL "i-%" LL "i): %s\n", + state->alg->name, state->fname, firstpos, firstpos+state->hash_pos, res); if (state->outfd) { char outbuf[512]; - snprintf(outbuf, 511, "%s *%s\n", state->alg->hash_out(res, &state->hash), state->fname); - if (write(state->outfd, outbuf, strlen(outbuf)) <= 0) + snprintf(outbuf, 511, "%s *%s\n", res, state->fname); + if (write(state->outfd, outbuf, strlen(outbuf)) <= 0) { FPLOG(WARN, "Could not write HASH result to fd %i\n", state->outfd); + --err; + } } + if (state->chkf) + err += check_chkf(state, res); + if (state->outf) + err += write_chkf(state, res); +#ifdef HAVE_ATTR_XATTR_H + if (state->chk_xattr) + err += check_xattr(state, res); + if (state->set_xattr) + err += write_xattr(state, res); + if (state->xnmalloc) + free((void*)state->xattr_name); +#endif + if (state->chkfalloc) + free((void*)state->chkfnm); + if (strcmp(state->fname, state->opts->iname) && strcmp(state->fname, state->opts->oname)) + free((void*)state->fname); free(*stat); - return 0; + return err; } ddr_plugin_t ddr_plug = { //.name = "MD5", - .slack_pre = 128, // not yet used - .slack_post = 128, // not yet used + .slack_pre = 144, // not yet used + .slack_post = 288, // not yet used .needs_align = 0, .handles_sparse = 1, .init_callback = hash_plug_init, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/libddr_lzo.c new/dd_rescue/libddr_lzo.c --- old/dd_rescue/libddr_lzo.c 2014-05-23 11:57:26.000000000 +0200 +++ new/dd_rescue/libddr_lzo.c 2014-05-26 10:19:09.000000000 +0200 @@ -438,7 +438,7 @@ } else { FPLOG(FATAL, "plugin doesn't understand param %s\n", param); - ++err; + --err; } param = next; } @@ -495,7 +495,7 @@ /* TO DO: We could as well adjust to real max (2*softbs) */ #define MAXBLOCKSZ 16UL*1024UL*1024UL -int lzo_open(const opt_t *opt, int ilnchg, int olnchg, +int lzo_open(const opt_t *opt, int ilnchg, int olnchg, int ichg, int ochg, unsigned int totslack_pre, unsigned int totslack_post, void **stat) { @@ -650,10 +650,12 @@ hlen = sizeof(blockhdr_t)-4+((state->flags&(F_ADLER32_C|F_CRC32_C))? 4: 0); /* Overwrite EOF */ if (state->flags & F_MULTIPART) { - FPLOG(INFO, "extending by writing next part (MULTIPART)\n"); + if (!state->opts->quiet) + FPLOG(INFO, "extending by writing next part (MULTIPART)\n"); state->hdr_seen = 0; } else { - FPLOG(INFO, "extending by overwriting EOF\n"); + if (!state->opts->quiet) + FPLOG(INFO, "extending by overwriting EOF\n"); fst->opos -= 4; } } @@ -782,7 +784,8 @@ && fst->buf[off+8] == lzop_hdr[8]) { loff_t hole; int hlen = lzo_parse_hdr(fst->buf+off+sizeof(lzop_hdr), &hole, state); - FPLOG(INFO, "lzop header at %i (sz %i/hole %li)\n", fst->ipos+off, + if (!state->opts->quiet) + FPLOG(INFO, "lzop header at %i (sz %i/hole %li)\n", fst->ipos+off, hlen+sizeof(lzop_hdr), hole); fst->opos += hole; off += hlen+sizeof(lzop_hdr); @@ -803,14 +806,14 @@ !check_blklen_and_next(state, fst, *towr, off-state->hdroff, 12, unc_len, cmp_len)) { if (state->debug) - FPLOG(INFO, "Blk Cand @ %i failed chain tests ...\n", + FPLOG(DEBUG, "Blk Cand @ %i failed chain tests ...\n", fst->ipos+off); continue; } /* Candidate found but we can't decode it with our buffer sizes ... */ if (cmp_len > 2*state->opts->softbs) { if (state->debug) - FPLOG(INFO, "Blk Cand @ %i with large size %i, increase softblocksize\n", + FPLOG(DEBUG, "Blk Cand @ %i with large size %i, increase softblocksize\n", fst->ipos+off, cmp_len); continue; } @@ -835,7 +838,7 @@ int err = state->algo->decompr(fst->buf+off+12, cmp_len, state->dbuf, &dst_len, NULL); if (err != LZO_E_OK || dst_len != unc_len) { if (state->debug) - FPLOG(INFO, "Blk Cand @ %i failed decompression\n", + FPLOG(DEBUG, "Blk Cand @ %i failed decompression\n", fst->ipos + off); continue; } @@ -849,15 +852,16 @@ state->flags = F_OS_UNIX | F_CRC32_D | F_MULTIPART; else { //if (state->debug) - FPLOG(INFO, "Blk Cand @ %i fails decomp chksum test\n", + FPLOG(DEBUG, "Blk Cand @ %i fails decomp chksum test\n", fst->ipos+off); continue; } } } } - FPLOG(INFO, "Found block @ %i (flags %08x)\n", - fst->ipos+off, state->flags); + if (!state->opts->quiet) + FPLOG(INFO, "Found block @ %i (flags %08x)\n", + fst->ipos+off, state->flags); //*towr -= off; state->hdroff = off; state->do_search = 0; @@ -868,7 +872,8 @@ const size_t totbufln = state->opts->softbs - ddr_plug.slack_post*((state->opts->softbs+15)/16); const size_t left = totbufln - (*towr-off); if (left < state->opts->softbs) { - FPLOG(INFO, "Buffer exhausted Blk Cand @ %i\n", fst->ipos+off); + if (!state->opts->quiet) + FPLOG(INFO, "Buffer exhausted Blk Cand @ %i\n", fst->ipos+off); off += fst->buf-state->obuf; fst->buf = state->obuf; assert(off >= 0); @@ -944,7 +949,7 @@ #define QUIT { raise(SIGQUIT); ++do_break; break; } #define BREAK if (!state->nodiscard) ++do_break; break #define DRAIN(x) do { ++do_break; *recall=1; \ - LZO_DEBUG(FPLOG(INFO, "Drain %i bytes before %s error handling\n", d_off, x)); \ + LZO_DEBUG(FPLOG(DEBUG, "Drain %i bytes before %s error handling\n", d_off, x)); \ eof = 0; \ break; } while(0); \ if (do_break) break @@ -1004,7 +1009,7 @@ char is_err = 0; effbf = bf+c_off+state->hdroff; lzo_uint dst_len; - LZO_DEBUG(FPLOG(INFO, "dec blk @ %p (offs %i, stoffs %i, bln %zi, tbw %i)\n", + LZO_DEBUG(FPLOG(DEBUG, "dec blk @ %p (offs %i, stoffs %i, bln %zi, tbw %i)\n", effbf, effbf-state->obuf, state->hdroff, totbufln, inlen)); blockhdr_t *hdr = (blockhdr_t*)effbf; have_len = inlen-state->hdroff-c_off; @@ -1024,7 +1029,7 @@ } if (!unc_len && state->flags & F_MULTIPART && have_len > 32) { /* EOF with new LZOP sig */ - LZO_DEBUG(FPLOG(INFO, "Next part ...\n")); + LZO_DEBUG(FPLOG(DEBUG, "Next part ...\n")); if (memcmp(effbf+4, lzop_hdr, sizeof(lzop_hdr))) { FPLOG(FATAL, "EOF with MULTIPART, but no new hdr\n"); raise(SIGQUIT); @@ -1034,7 +1039,8 @@ int hln = lzo_parse_hdr(effbf+4+sizeof(lzop_hdr), &hsz, state); bhsz = hln+sizeof(lzop_hdr)+4; if (!hsz) { - FPLOG(INFO, "MULTIPART, just append ...\n"); + if (!state->opts->quiet) + FPLOG(INFO, "MULTIPART, just append ...\n"); c_off += bhsz; state->cmp_hdr += bhsz; continue; @@ -1063,7 +1069,7 @@ if (cmp_len == unc_len) cmp_cksum = unc_cksum; - LZO_DEBUG(FPLOG(INFO, "dec blk @ %p (hdroff %i, cln %i, uln %i, have %i)\n", + LZO_DEBUG(FPLOG(DEBUG, "dec blk @ %p (hdroff %i, cln %i, uln %i, have %i)\n", effbf, c_off+state->hdroff, unc_len, cmp_len, have_len)); /* Block incomplete? Then we're done for this round ... */ if (have_len < bhsz+cmp_len) @@ -1133,7 +1139,7 @@ FPLOG(WARN, "compressed %i > uncompressed %i breaks lzop\n", cmp_len, unc_len); err = state->algo->decompr(effbf+bhsz, cmp_len, (unsigned char*)state->dbuf+d_off, &dst_len, NULL); - LZO_DEBUG(FPLOG(INFO, "decompressed %i@%p -> %i\n", + LZO_DEBUG(FPLOG(DEBUG, "decompressed %i@%p -> %i\n", cmp_len, effbf+bhsz, dst_len)); if (dst_len != unc_len) { fst->nrerr++; @@ -1271,7 +1277,7 @@ /* We have enough space to just append */ state->hdroff -= inlen-c_off; fst->buf += inlen; - LZO_DEBUG(FPLOG(INFO, "append @ %p\n", fst->buf)); + LZO_DEBUG(FPLOG(DEBUG, "append @ %p\n", fst->buf)); /* OK, now for the bad cases: * (a) We can't append, but everything fits if we memmove to start @@ -1281,7 +1287,7 @@ } else if (bhsz+cmp_len < totbufln && have_len+nextrd < totbufln) { /* We need to move block to beg of buffer */ - LZO_DEBUG(FPLOG(INFO, "move %i bytes to buffer head\n", have_len)); + LZO_DEBUG(FPLOG(DEBUG, "move %i bytes to buffer head\n", have_len)); if (effbf != state->obuf) { memmove(state->obuf, effbf, have_len); ++state->nr_memmove; @@ -1336,30 +1342,32 @@ slackfree(state->dbuf, state); if (state->workspace) free(state->workspace); - if (state->mode == COMPRESS) - FPLOG(INFO, "%s_compress %.1fkiB (%1.f%%) + %i <- %.1fkiB\n", - state->algo->name, - state->cmp_ln/1024.0, - 100.0*((double)state->cmp_ln/state->unc_ln), - state->cmp_hdr, - state->unc_ln/1024.0); - else { - FPLOG(INFO, "%s_decompr %.1fkiB (%.1f%%) + %i -> %.1fkiB\n", - state->algo->name, - state->cmp_ln/1024.0, - 100.0*((double)state->cmp_ln/state->unc_ln), - state->cmp_hdr, - state->unc_ln/1024.0); - if (state->do_bench) - FPLOG(INFO, "%i reallocs (%ikiB), %i(+%i) moves\n", - state->nr_realloc, state->dbuflen/1024, - state->nr_memmove, state->nr_cheapmemmove); - } - /* Only output if it took us more than 0.05s, otherwise it's completely meaningless */ - if (state->do_bench && state->cpu/(CLOCKS_PER_SEC/20) > 0) - FPLOG(INFO, "%.2fs CPU time, %.1fMiB/s\n", + if (state->do_bench || !state->opts->quiet) { + if (state->mode == COMPRESS) + FPLOG(INFO, "%s_compress %.1fkiB (%1.f%%) + %i <- %.1fkiB\n", + state->algo->name, + state->cmp_ln/1024.0, + 100.0*((double)state->cmp_ln/state->unc_ln), + state->cmp_hdr, + state->unc_ln/1024.0); + else { + FPLOG(INFO, "%s_decompr %.1fkiB (%.1f%%) + %i -> %.1fkiB\n", + state->algo->name, + state->cmp_ln/1024.0, + 100.0*((double)state->cmp_ln/state->unc_ln), + state->cmp_hdr, + state->unc_ln/1024.0); + if (state->do_bench) + FPLOG(INFO, "%i reallocs (%ikiB), %i(+%i) moves\n", + state->nr_realloc, state->dbuflen/1024, + state->nr_memmove, state->nr_cheapmemmove); + } + /* Only output if it took us more than 0.05s, otherwise it's completely meaningless */ + if (state->do_bench && state->cpu/(CLOCKS_PER_SEC/20) > 0) + FPLOG(INFO, "%.2fs CPU time, %.1fMiB/s\n", (double)state->cpu/CLOCKS_PER_SEC, state->unc_ln/1024 / (state->cpu/(CLOCKS_PER_SEC/1024.0))); + } free(*stat); return 0; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/libddr_null.c new/dd_rescue/libddr_null.c --- old/dd_rescue/libddr_null.c 1970-01-01 01:00:00.000000000 +0100 +++ new/dd_rescue/libddr_null.c 2014-05-26 10:28:12.000000000 +0200 @@ -0,0 +1,121 @@ +/* libddr_null.c + * + * plugin for dd_rescue, doing nothing (except optionally setting changes_length) + * + * (c) Kurt Garloff <[email protected]>, 2014 + * License: GNU GPLv2 or v3 + */ + +#include "ddr_plugin.h" +#include "ddr_ctrl.h" +#include <string.h> +#include <stdlib.h> + +/* fwd decl */ +extern ddr_plugin_t ddr_plug; + +typedef struct _null_state { + int seq; + char debug; +} null_state; + +#define FPLOG(lvl, fmt, args...) \ + ddr_plug.fplog(stderr, lvl, "%s(%i): " fmt, ddr_plug.name, state->seq, ##args) + +const char* null_help = "The null plugin does nothing ...\n" + "Options: debug:[no]lnchange:[no]change. [no]lnchange indicates that the length\n" + " may [not] be changed by ddr_null; [no]change indicates that the contents may\n" + " [not] be changed by ddr_null. (Both is not true, but influences the behavior\n" + " of other plugins)\n"; + +int null_plug_init(void **stat, char* param, int seq, const opt_t *opt) +{ + null_state *state = (null_state*)malloc(sizeof(null_state)); + *stat = (void*)state; + memset(state, 0, sizeof(null_state)); + state->seq = seq; + while (param) { + char* next = strchr(param, ':'); + if (next) + *next++ = 0; + if (!strcmp(param, "help")) + FPLOG(INFO, "%s", null_help); + else if (!strcmp(param, "lnchange")) + ddr_plug.changes_output_len = 1; + else if (!strcmp(param, "lnchg")) + ddr_plug.changes_output_len = 1; + /* Do we need this if loaded multiple times? */ + else if (!strcmp(param, "nolnchange")) + ddr_plug.changes_output_len = 0; + else if (!strcmp(param, "nolnchg")) + ddr_plug.changes_output_len = 0; + else if (!strcmp(param, "change")) + ddr_plug.changes_output = 1; + else if (!strcmp(param, "chg")) + ddr_plug.changes_output = 1; + /* Do we need this if loaded multiple times? */ + else if (!strcmp(param, "nochange")) + ddr_plug.changes_output = 0; + else if (!strcmp(param, "nochg")) + ddr_plug.changes_output = 0; + else if (!strcmp(param, "debug")) + state->debug = 1; + else { + FPLOG(FATAL, "plugin doesn't understand param %s\n", + param); + return 1; + + } + param = next; + } + /* If the length changes, so does the contents ... */ + if (ddr_plug.changes_output_len && !ddr_plug.changes_output) + FPLOG(WARN, "Change indication for length without contents change?\n"); + return 0; +} + +int null_open(const opt_t *opt, int ilnchg, int olnchg, int ichg, int ochg, + unsigned int totslack_pre, unsigned int totslack_post, + void **stat) +{ + return 0; +} + +#if __WORDSIZE == 64 +#define LL "l" +#elif __WORDSIZE == 32 +#define LL "ll" +#else +#error __WORDSIZE unknown +#endif + + +unsigned char* null_blk_cb(fstate_t *fst, unsigned char* bf, + int *towr, int eof, int *recall, void **stat) +{ + /* TODO: Could actually add debugging output here if wanted ... */ + null_state *state = (null_state*)*stat; + if (state->debug) + FPLOG(DEBUG, "Block ipos %" LL "i opos %" LL "i with %i bytes %s\n", + fst->ipos, fst->opos, *towr, (eof? "EOF": "")); + return bf; +} + +int null_close(loff_t ooff, void **stat) +{ + //null_state *state = (null_state*)*stat; + free(*stat); + return 0; +} + +ddr_plugin_t ddr_plug = { + .name = "null", + .needs_align = 0, + .handles_sparse = 1, + .init_callback = null_plug_init, + .open_callback = null_open, + .block_callback = null_blk_cb, + .close_callback = null_close, +}; + + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/dd_rescue/md5.c new/dd_rescue/md5.c --- old/dd_rescue/md5.c 2014-05-23 12:12:48.000000000 +0200 +++ new/dd_rescue/md5.c 2014-05-27 13:37:33.000000000 +0200 @@ -51,10 +51,12 @@ { *(uint32_t *)bytes = val; } +#if 0 // Unused static inline uint32_t to_int32(const uint8_t *bytes) { return *(const uint32_t *)bytes; } +#endif #else /* Store val into bytes in little endian fmt */ static inline void to_bytes(uint32_t val, uint8_t *bytes) -- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
