Package: gzip
Version: 1.6-2
Followup-For: Bug #708423
Tags: patch
I compared squeeze and wheezy patches side-by-side and noticed
that block flushing method differs.
In squeeze version blocks are aligned on byte boundary after flushes.
In wheezy version internal structures are reinitialized before flush
instead.
I do not dig very deep into this, but I think this could led to
misaligned data.
So, I recreate patch by restoring flush method.
This patch allows rsyncable to work again, and if window is set to 4k,
it produces byte to byte identical result with previous version.
Also patch contains minor fix to texinfo doc (paragraph about rsyncable
option was included three times).
--- a/deflate.c
+++ b/deflate.c
@@ -131,6 +131,14 @@
#endif
/* Matches of length 3 are discarded if their distance exceeds TOO_FAR */
+#ifndef RSYNC_WIN
+# define RSYNC_WIN 8192
+#endif
+/* Size of rsync window, must be < MAX_DIST */
+
+#define RSYNC_SUM_MATCH(sum) (((sum) & (RSYNC_WIN - 1)) == 0)
+/* Whether window sum matches magic value */
+
/* ===========================================================================
* Local data used by the "longest match" routines.
*/
@@ -212,6 +220,8 @@
unsigned good_match;
/* Use a faster search when the previous match is longer than this */
+local ulg rsync_sum; /* rolling sum of rsync window */
+local ulg rsync_chunk_end; /* next rsync sequence point */
/* Values for max_lazy_match, good_match and max_chain_length, depending on
* the desired pack level (0..9). The values given below have been tuned to
@@ -314,6 +324,10 @@
#endif
/* prev will be initialized on the fly */
+ /* rsync params */
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ rsync_sum = 0;
+
/* Set the default configuration parameters:
*/
max_lazy_match = configuration_table[pack_level].max_lazy;
@@ -550,6 +564,8 @@
memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE);
match_start -= WSIZE;
strstart -= WSIZE; /* we now have strstart >= MAX_DIST: */
+ if (rsync_chunk_end != 0xFFFFFFFFUL)
+ rsync_chunk_end -= WSIZE;
block_start -= (long) WSIZE;
@@ -579,13 +595,44 @@
}
}
+local void rsync_roll(unsigned start, unsigned num)
+{
+ unsigned i;
+
+ if (start < RSYNC_WIN) {
+ /* before window fills. */
+ for (i = start; i < RSYNC_WIN; i++) {
+ if (i == start + num) return;
+ rsync_sum += (ulg)window[i];
+ }
+ num -= (RSYNC_WIN - start);
+ start = RSYNC_WIN;
+ }
+
+ /* buffer after window full */
+ for (i = start; i < start+num; i++) {
+ /* New character in */
+ rsync_sum += (ulg)window[i];
+ /* Old character out */
+ rsync_sum -= (ulg)window[i - RSYNC_WIN];
+ if (rsync_chunk_end == 0xFFFFFFFFUL && RSYNC_SUM_MATCH(rsync_sum))
+ rsync_chunk_end = i;
+ }
+}
+
+/* ===========================================================================
+ * Set rsync_chunk_end if window sum matches magic value.
+ */
+#define RSYNC_ROLL(s, n) \
+ do { if (rsync) rsync_roll((s), (n)); } while(0)
+
/* ===========================================================================
* Flush the current block, with given end-of-file flag.
* IN assertion: strstart is set to the end of the current match.
*/
#define FLUSH_BLOCK(eof) \
flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \
- (char*)NULL, (long)strstart - block_start, (eof))
+ (char*)NULL, (long)strstart - block_start, flush-1, (eof))
/* ===========================================================================
* Processes a new input file and return its compressed length. This
@@ -596,7 +643,7 @@
local off_t deflate_fast()
{
IPos hash_head; /* head of the hash chain */
- int flush; /* set if current block must be flushed */
+ int flush = 0; /* set if current block must be flushed, 2=>and padded */
unsigned match_length = 0; /* length of best match */
prev_length = MIN_MATCH-1;
@@ -626,7 +673,8 @@
lookahead -= match_length;
- /* Insert new strings in the hash table only if the match length
+ RSYNC_ROLL(strstart, match_length);
+ /* Insert new strings in the hash table only if the match length
* is not too large. This saves time but degrades compression.
*/
if (match_length <= max_insert_length) {
@@ -654,9 +702,14 @@
/* No match, output a literal byte */
Tracevv((stderr,"%c",window[strstart]));
flush = ct_tally (0, window[strstart]);
+ RSYNC_ROLL(strstart, 1);
lookahead--;
strstart++;
}
+ if (rsync && strstart > rsync_chunk_end) {
+ flush = 2;
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ }
if (flush) FLUSH_BLOCK(0), block_start = strstart;
/* Make sure that we always have enough lookahead, except
@@ -679,7 +732,7 @@
{
IPos hash_head; /* head of hash chain */
IPos prev_match; /* previous match */
- int flush; /* set if current block must be flushed */
+ int flush = 0; /* set if current block must be flushed */
int match_available = 0; /* set if previous match exists */
register unsigned match_length = MIN_MATCH-1; /* length of best match */
@@ -730,6 +783,7 @@
*/
lookahead -= prev_length-1;
prev_length -= 2;
+ RSYNC_ROLL(strstart, prev_length+1);
do {
strstart++;
INSERT_STRING(strstart, hash_head);
@@ -742,24 +796,38 @@
match_available = 0;
match_length = MIN_MATCH-1;
strstart++;
- if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
+ if (flush) FLUSH_BLOCK(0), block_start = strstart;
} else if (match_available) {
/* If there was no match at the previous position, output a
* single literal. If there was a match but the current match
* is longer, truncate the previous match to a single literal.
*/
Tracevv((stderr,"%c",window[strstart-1]));
- if (ct_tally (0, window[strstart-1])) {
- FLUSH_BLOCK(0), block_start = strstart;
- }
+ flush = ct_tally (0, window[strstart-1]);
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
+ if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ RSYNC_ROLL(strstart, 1);
strstart++;
lookahead--;
} else {
/* There is no previous match to compare with, wait for
* the next step to decide.
*/
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ FLUSH_BLOCK(0), block_start = strstart;
+ }
match_available = 1;
+ RSYNC_ROLL(strstart, 1);
strstart++;
lookahead--;
}
--- a/doc/gzip.texi
+++ b/doc/gzip.texi
@@ -208,6 +208,7 @@
-V, --version display version number
-1, --fast compress faster
-9, --best compress better
+ --rsyncable Make rsync-friendly archive
With no FILE, or when FILE is -, read standard input.
@@ -358,6 +359,14 @@
into the directory and compress all the files it finds there (or
decompress them in the case of @command{gunzip}).
+@item --rsyncable
+While compressing, synchronize the output occasionally based on
+the input. This increases size by less than 1 percent most
+cases, but means that the @command{rsync} program can much more efficiently
+synchronize files compressed with this flag. @command{gunzip}
+cannot tell the difference between a compressed file created
+with this option, and one created without it.
+
@item --suffix @var{suf}
@itemx -S @var{suf}
Use suffix @var{suf} instead of @samp{.gz}. Any suffix can be
--- a/gzip.c
+++ b/gzip.c
@@ -215,6 +215,7 @@
unsigned insize; /* valid bytes in inbuf */
unsigned inptr; /* index of next byte to be processed in inbuf */
unsigned outcnt; /* bytes in output buffer */
+int rsync = 0; /* make ryncable chunks */
static int handled_sig[] =
{
@@ -275,7 +276,7 @@
{"best", 0, 0, '9'}, /* compress better */
{"lzw", 0, 0, 'Z'}, /* make output compatible with old compress */
{"bits", 1, 0, 'b'}, /* max number of bits per code (implies -Z) */
-
+ {"rsyncable", 0, 0, 'R'}, /* make rsync-friendly archive */
{ 0, 0, 0, 0 }
};
@@ -359,6 +360,7 @@
" -Z, --lzw produce output compatible with old compress",
" -b, --bits=BITS max number of bits per code (implies -Z)",
#endif
+ " --rsyncable Make rsync-friendly archive",
"",
"With no FILE, or when FILE is -, read standard input.",
"",
@@ -489,8 +491,11 @@
#else
recursive = 1;
#endif
- break;
- case 'S':
+ break;
+ case 'R':
+ rsync = 1; break;
+
+ case 'S':
#ifdef NO_MULTIPLE_DOTS
if (*optarg == '.') optarg++;
#endif
--- a/gzip.h
+++ b/gzip.h
@@ -140,6 +140,7 @@
extern unsigned insize; /* valid bytes in inbuf */
extern unsigned inptr; /* index of next byte to be processed in inbuf */
extern unsigned outcnt; /* bytes in output buffer */
+extern int rsync; /* deflate into rsyncable chunks */
extern off_t bytes_in; /* number of input bytes */
extern off_t bytes_out; /* number of output bytes */
@@ -287,7 +288,7 @@
/* in trees.c */
extern void ct_init (ush *attr, int *method);
extern int ct_tally (int dist, int lc);
-extern off_t flush_block (char *buf, ulg stored_len, int eof);
+extern off_t flush_block (char *buf, ulg stored_len, int pad, int eof);
/* in bits.c */
extern void bi_init (file_t zipfile);
--- a/gzip.1
+++ b/gzip.1
@@ -5,6 +5,7 @@
.ll +8
.B gzip
.RB [ " \-acdfhklLnNrtvV19 " ]
+.RB [ --rsyncable ]
.RB [ \-S\ suffix ]
[
.I "name \&..."
@@ -287,6 +288,16 @@
.I gunzip
).
.TP
+.B --rsyncable
+While compressing, synchronize the output occasionally based on the input.
+This increases size by less than 1 percent most cases, but means that the
+.BR rsync (1)
+program can take advantage of similarities in the uncompressed input
+when syncronizing two files compressed with this flag.
+.I gunzip
+cannot tell the difference between a compressed file created with this option,
+and one created without it.
+.TP
.B \-S .suf --suffix .suf
When compressing, use suffix .suf instead of .gz.
Any non-empty suffix can be given, but suffixes
--- a/trees.c
+++ b/trees.c
@@ -856,9 +856,10 @@
* trees or store, and output the encoded block to the zip file. This function
* returns the total compressed length for the file so far.
*/
-off_t flush_block(buf, stored_len, eof)
+off_t flush_block(buf, stored_len, pad, eof)
char *buf; /* input block, or NULL if too old */
ulg stored_len; /* length of input block */
+ int pad; /* pad output to byte boundary */
int eof; /* true if this is the last block for a file */
{
ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
@@ -951,6 +952,10 @@
Assert (input_len == bytes_in, "bad input size");
bi_windup();
compressed_len += 7; /* align on byte boundary */
+ } else if (pad && (compressed_len % 8) != 0) {
+ send_bits((STORED_BLOCK<<1)+eof, 3); /* send block type */
+ compressed_len = (compressed_len + 3 + 7) & ~7L;
+ copy_block(buf, 0, 1); /* with header */
}
return compressed_len >> 3;
