The branch, master has been updated
via 64d5ea39 More compress changes
from 4af8403a Fix negotiation of none & improve NSTR debug msgs.
https://git.samba.org/?p=rsync.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 64d5ea39c0bf4e0e15aeb1f4dae37fc5125fc0c7
Author: Wayne Davison <wa...@opencoder.net>
Date: Sun May 24 15:45:59 2020 -0700
More compress changes
- Add the zlibx (external-code compatible) compression name.
- Re-enable zlib support with the external library so it can be
tried as a fallback if zlibx isn't available.
- Add --compress-choice=STR (aka -zz=STR) option.
- Make --cc=STR an alias for --checksum-choice=STR.
- Hook up the new compression negotiation logic.
-----------------------------------------------------------------------
Summary of changes:
compat.c | 38 ++++++++++++++++++++++++++++++++++++++
flist.c | 1 +
options.c | 62 ++++++++++++++++++++++++++++++++++++++++++--------------------
rsync.yo | 57 ++++++++++++++++++++++++++++++++++++++++++---------------
token.c | 16 ++++------------
5 files changed, 127 insertions(+), 47 deletions(-)
Changeset truncated at 500 lines:
diff --git a/compat.c b/compat.c
index 410e6f69..3a5ad46d 100644
--- a/compat.c
+++ b/compat.c
@@ -89,10 +89,17 @@ int filesfrom_convert = 0;
#define CPRES_NONE 0
#define CPRES_ZLIB 1
+#define CPRES_ZLIBX 2
struct name_num_obj valid_compressions = {
"compress", NULL, NULL, 0, 0, {
+#ifndef EXTERNAL_ZLIB
{ CPRES_ZLIB, "zlib", NULL },
+#endif
+ { CPRES_ZLIBX, "zlibx", NULL },
+#ifdef EXTERNAL_ZLIB
+ { CPRES_ZLIB, "zlib", NULL },
+#endif
{ CPRES_NONE, "none", NULL },
{ 0, NULL, NULL }
}
@@ -160,6 +167,37 @@ void set_allow_inc_recurse(void)
allow_inc_recurse = 0;
}
+void parse_compress_choice(int final_call)
+{
+ int num;
+
+ if (valid_compressions.negotiated_name)
+ num = valid_compressions.negotiated_num;
+ else if (compress_choice) {
+ struct name_num_item *nni =
get_nni_by_name(&valid_compressions, compress_choice, -1);
+ if (!nni) {
+ rprintf(FERROR, "unknown compress name: %s\n",
compress_choice);
+ exit_cleanup(RERR_UNSUPPORTED);
+ }
+ num = nni->num;
+ } else
+ num = CPRES_NONE;
+
+ if (num == CPRES_NONE) {
+ do_compression = 0;
+ compress_choice = NULL;
+ } else if (num > 0)
+ do_compression = num != CPRES_ZLIB ? 2 : 1;
+
+ if (final_call && DEBUG_GTE(NSTR, am_server ? 2 : 1)) {
+ const char *c_s = am_server ? "Server" : "Client";
+ if (valid_compressions.negotiated_name)
+ rprintf(FINFO, "%s negotiated compress: %s\n", c_s,
valid_compressions.negotiated_name);
+ else
+ rprintf(FINFO, "%s compress: %s\n", c_s, do_compression
? compress_choice : "none");
+ }
+}
+
struct name_num_item *get_nni_by_name(struct name_num_obj *nno, const char
*name, int len)
{
struct name_num_item *nni;
diff --git a/flist.c b/flist.c
index 940071fd..a1b3e4bb 100644
--- a/flist.c
+++ b/flist.c
@@ -143,6 +143,7 @@ void init_flist(void)
(int)FILE_STRUCT_LEN, (int)EXTRA_LEN);
}
parse_checksum_choice(1); /* Sets checksum_type && xfersum_type */
+ parse_compress_choice(1); /* Sets do_compression */
flist_csum_len = csum_len_for_type(checksum_type, 1);
show_filelist_progress = INFO_GTE(FLIST, 1) && xfer_dirs && !am_server
&& !inc_recurse;
diff --git a/options.c b/options.c
index 62fe0fb5..469b9fb6 100644
--- a/options.c
+++ b/options.c
@@ -840,6 +840,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER,
OPT_EXCLUDE, OPT_EXCLUDE_FROM,
OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG,
OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT,
+ OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS,
OPT_SERVER, OPT_REFUSED_BASE = 9000};
static struct poptOption long_options[] = {
@@ -981,11 +982,12 @@ static struct poptOption long_options[] = {
{"cvs-exclude", 'C', POPT_ARG_NONE, &cvs_exclude, 0, 0, 0 },
{"whole-file", 'W', POPT_ARG_VAL, &whole_file, 1, 0, 0 },
{"no-whole-file", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 },
- {"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
{"no-W", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 },
{"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
{"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
{"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
+ {"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
+ {"cc", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
{"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 },
{"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
{"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
@@ -994,10 +996,12 @@ static struct poptOption long_options[] = {
{"no-fuzzy", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 },
{"no-y", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 },
{"compress", 'z', POPT_ARG_NONE, 0, 'z', 0, 0 },
- {"old-compress", 0, POPT_ARG_VAL, &do_compression, 1, 0, 0 },
- {"new-compress", 0, POPT_ARG_VAL, &do_compression, 2, 0, 0 },
- {"no-compress", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 },
- {"no-z", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 },
+ {"old-compress", 0, POPT_ARG_NONE, 0, OPT_OLD_COMPRESS, 0, 0 },
+ {"new-compress", 0, POPT_ARG_NONE, 0, OPT_NEW_COMPRESS, 0, 0 },
+ {"no-compress", 0, POPT_ARG_NONE, 0, OPT_NO_COMPRESS, 0, 0 },
+ {"no-z", 0, POPT_ARG_NONE, 0, OPT_NO_COMPRESS, 0, 0 },
+ {"compress-choice", 0, POPT_ARG_STRING, &compress_choice, 0, 0, 0 },
+ {"zz", 0, POPT_ARG_STRING, &compress_choice, 0, 0, 0 },
{"skip-compress", 0, POPT_ARG_STRING, &skip_compress, 0, 0, 0 },
{"compress-level", 0, POPT_ARG_INT, &def_compress_level, 0, 0, 0 },
{0, 'P', POPT_ARG_NONE, 0, 'P', 0, 0 },
@@ -1668,6 +1672,19 @@ int parse_arguments(int *argc_p, const char ***argv_p)
do_compression++;
break;
+ case OPT_OLD_COMPRESS:
+ compress_choice = "zlib";
+ break;
+
+ case OPT_NEW_COMPRESS:
+ compress_choice = "zlibx";
+ break;
+
+ case OPT_NO_COMPRESS:
+ do_compression = 0;
+ compress_choice = NULL;
+ break;
+
case 'M':
arg = poptGetOptArg(pc);
if (*arg != '-') {
@@ -1948,6 +1965,13 @@ int parse_arguments(int *argc_p, const char ***argv_p)
exit_cleanup(0);
}
+ if (!compress_choice && do_compression > 1)
+ compress_choice = "zlibx";
+ if (compress_choice && strcasecmp(compress_choice, "auto") != 0)
+ parse_compress_choice(0); /* Can twiddle do_compression and
possibly NULL-out compress_choice */
+ else
+ compress_choice = NULL;
+
if (do_compression || def_compress_level != NOT_SPECIFIED) {
if (def_compress_level == NOT_SPECIFIED)
def_compress_level = Z_DEFAULT_COMPRESSION;
@@ -1955,24 +1979,15 @@ int parse_arguments(int *argc_p, const char ***argv_p)
snprintf(err_buf, sizeof err_buf, "--compress-level
value is invalid: %d\n",
def_compress_level);
return 0;
- } else if (def_compress_level == Z_NO_COMPRESSION)
+ } else if (def_compress_level == Z_NO_COMPRESSION) {
do_compression = 0;
- else if (!do_compression)
+ compress_choice = NULL;
+ } else if (!do_compression)
do_compression = 1;
if (do_compression && refused_compress) {
create_refuse_error(refused_compress);
return 0;
}
-#ifdef EXTERNAL_ZLIB
- if (do_compression == 1) {
- snprintf(err_buf, sizeof err_buf,
- "This rsync lacks old-style --compress due to
its external zlib. Try -zz.\n");
- if (am_server)
- return 0;
- fprintf(stderr, "%s" "Continuing without
compression.\n\n", err_buf);
- do_compression = 0;
- }
-#endif
}
#ifdef HAVE_SETVBUF
@@ -2750,6 +2765,16 @@ void server_options(char **args, int *argc_p)
args[ac++] = arg;
}
+ if ((!compress_choice && do_compression > 1) || (compress_choice &&
strcasecmp(compress_choice, "zlibx") == 0))
+ args[ac++] = "--new-compress";
+ else if (compress_choice && strcasecmp(compress_choice, "zlib") == 0)
+ args[ac++] = "--old-compress";
+ else if (compress_choice) {
+ if (asprintf(&arg, "--compress-choice=%s", compress_choice) < 0)
+ goto oom;
+ args[ac++] = arg;
+ }
+
if (am_sender) {
if (max_delete > 0) {
if (asprintf(&arg, "--max-delete=%d", max_delete) < 0)
@@ -2930,9 +2955,6 @@ void server_options(char **args, int *argc_p)
exit_cleanup(RERR_MALLOC);
}
- if (do_compression > 1)
- args[ac++] = "--new-compress";
-
if (remote_option_cnt) {
int j;
if (ac + remote_option_cnt > MAX_SERVER_ARGS) {
diff --git a/rsync.yo b/rsync.yo
index ff8bca04..0b6c8d65 100644
--- a/rsync.yo
+++ b/rsync.yo
@@ -1370,7 +1370,7 @@ destination machines is higher than the bandwidth to disk
(especially when the
the source and destination are specified as local paths, but only if no
batch-writing option is in effect.
-dit(bf(--checksum-choice=STR)) This option overrides the checksum algorithms.
+dit(bf(--checksum-choice=STR, --cc=STR)) This option overrides the checksum
algorithms.
If one algorithm name is specified, it is used for both the transfer checksums
and (assuming bf(--checksum) is specified) the pre-transfer checksums. If two
comma-separated names are supplied, the first name affects the transfer
@@ -1402,7 +1402,7 @@ error. This method does not allow you to specify the
transfer checksum
separately from the pre-transfer checksum, and it ignores "auto" and all
unknown checksum names. If the remote rsync is not new enough to handle a
checksum negotiation list, the list is silently ignored unless it contains the
-string "FAIL" in it.
+string "FAIL".
The use of the bf(--checksum-choice) option overrides this environment list.
@@ -2056,27 +2056,54 @@ dit(bf(-z, --compress)) With this option, rsync
compresses the file data
as it is sent to the destination machine, which reduces the amount of data
being transmitted -- something that is useful over a slow connection.
-Note that this option typically achieves better compression ratios than can
-be achieved by using a compressing remote shell or a compressing transport
+The "zlib" compression method typically achieves better compression ratios than
+can be achieved by using a compressing remote shell or a compressing transport
because it takes advantage of the implicit information in the matching data
blocks that are not explicitly sent over the connection. This matching-data
-compression comes at a cost of CPU, though, and can be disabled by repeating
-the bf(-z) option, but only if both sides are at least version 3.1.1.
+compression comes at a cost of CPU, though, and can be disabled by using the
+"zlibx" compresson method instead. This can be selected by repeating the
+bf(-z) option or specifying bf(--compress-choice=zlibx), but it only works if
+both sides of the transfer are at least version 3.1.1.
Note that if your version of rsync was compiled with an external zlib (instead
-of the zlib that comes packaged with rsync) then it will not support the
-old-style compression, only the new-style (repeated-option) compression. In
-the future this new-style compression will likely become the default.
-
-The client rsync requests new-style compression on the server via the
-bf(--new-compress) option, so if you see that option rejected it means that
-the server is not new enough to support bf(-zz). Rsync also accepts the
-bf(--old-compress) option for a future time when new-style compression
-becomes the default.
+of the zlib that comes packaged with rsync) then it will give preference to
+using the "zlibx" algorithm over the "zlib" algorithm since the external zlib
+code doesn't seem to handle the extra compression properly. You can try
+forcing the regular algorithm via bf(--zz=zlib) and be on the lookout for
+transfer failures. If all else fails, disable compression altogether.
+
+Note that if you see an error about an option named bf(--old-compress) or
+bf(--new-compress), this is rsync trying to send the bf(--compress-choice=zlib)
+or bf(--compress-choice=zlibx) option in a backward-compatible manner that more
+rsync versions understand. This error indicates that the older rsync version
+will not allow you to force the compression type.
See the bf(--skip-compress) option for the default list of file suffixes
that will not be compressed.
+dit(bf(--compress-choice=STR, --zz=STR)) This option can be used to override
the
+automatic selection of the compression algorithm that is the default when
+bf(--compress) is used.
+
+Currently the STR can be "zlib", "zlibx", or "none".
+
+The "zlib" algorithm is given preference over "zlibx" if your rsync was
+compiled with the internal zlib code, otherwise that preference is reversed.
+These 2 algorithms are the stame except that "zlibx" does not try to include
+matched data that was not transferred in the compression computations.
+
+If "none" is specified, that is equivalent to using bf(--no-compress).
+
+This option implies bf(--compress) unless "none" was specified.
+
+You can also override the compression negotation using the RSYNC_COMPRESS_LIST
+environment variable by setting it to a space-separated list of compression
+names that you consider acceptable. If no common compress choice is found, the
+client exits with an error. It ignores "auto" and all unknown compression
+names. If the remote rsync is not new enough to handle a compression
+negotiation list, the list is silently ignored unless it contains the string
+"FAIL".
+
dit(bf(--compress-level=NUM)) Explicitly set the compression level to use
(see bf(--compress)) instead of letting it default. If NUM is non-zero,
the bf(--compress) option is implied.
diff --git a/token.c b/token.c
index a5ed6ff0..1fdfa425 100644
--- a/token.c
+++ b/token.c
@@ -29,6 +29,10 @@ extern int module_id;
extern int def_compress_level;
extern char *skip_compress;
+#ifndef Z_INSERT_ONLY
+#define Z_INSERT_ONLY Z_SYNC_FLUSH
+#endif
+
static int compression_level, per_file_default_level;
struct suffix_tree {
@@ -405,7 +409,6 @@ send_deflated_token(int f, int32 token, struct map_struct
*buf, OFF_T offset,
} else if (token != -2 && do_compression == 1) {
/* Add the data in the current block to the compressor's
* history and hash table. */
-#ifndef EXTERNAL_ZLIB
do {
/* Break up long sections in the same way that
* see_deflate_token() does. */
@@ -424,11 +427,6 @@ send_deflated_token(int f, int32 token, struct map_struct
*buf, OFF_T offset,
exit_cleanup(RERR_STREAMIO);
}
} while (toklen > 0);
-#else
- toklen++;
- rprintf(FERROR, "Impossible error in external-zlib code
(1).\n");
- exit_cleanup(RERR_STREAMIO);
-#endif
}
}
@@ -579,7 +577,6 @@ static int32 recv_deflated_token(int f, char **data)
*/
static void see_deflate_token(char *buf, int32 len)
{
-#ifndef EXTERNAL_ZLIB
int r;
int32 blklen;
unsigned char hdr[5];
@@ -617,11 +614,6 @@ static void see_deflate_token(char *buf, int32 len)
exit_cleanup(RERR_STREAMIO);
}
} while (len || rx_strm.avail_out == 0);
-#else
- buf++; len++;
- rprintf(FERROR, "Impossible error in external-zlib code (2).\n");
- exit_cleanup(RERR_STREAMIO);
-#endif
}
/**
--
The rsync repository.
_______________________________________________
rsync-cvs mailing list
rsync-cvs@lists.samba.org
https://lists.samba.org/mailman/listinfo/rsync-cvs
