Author: kotkov Date: Tue Aug 1 12:18:23 2017 New Revision: 1803639 URL: http://svn.apache.org/viewvc?rev=1803639&view=rev Log: fsfs: Introduce new 'compression' config option.
This option allows explicitly specifying the compression algorithm for format 8 repositories. It deprecates the previously used 'compression-level' option. The syntax of the new option is: compression = none | lz4 | zlib | zlib-1 ... zlib-9 See the related discussion in: https://lists.apache.org/thread.html/40650a309d8ff041adbb62e8ffe19cc3990b9098a032db932fabd170@%3Cdev.subversion.apache.org%3E * subversion/libsvn_fs_fs/fs.h (CONFIG_OPTION_COMPRESSION): New. (compression_type_t): New. (fs_fs_data_t): Add field to store the delta compression type. * subversion/libsvn_fs_fs/fs.h (write_config): Revamp the section describing delta compression. (parse_compression_option): New helper function. (read_config): Parse the new 'compression' option when working with newer formats, with a possible fall back to 'compression-level' in case it's specified explicitly. In order to always have appropriate and usable compression settings in ffd, move the part of the code that disables compression when only svndiff0 is supported from ... * subversion/libsvn_fs_fs/transaction.c (txdelta_to_svndiff): ...this function. Adjust this function to select the appropriate svndiff version, depending on the options. * win-tests.py (): Rename 'fsfs_compression_level' to 'fsfs_compression'. (_usage_exit): Adjust usage text. * build/run_tests.py (): Update usage comment. (_init_py_tests): Pass the option as string, not int. (create_parser): Parse the option as string, not int. * subversion/tests/cmdline/svntest/main.py (parse_options): Only allow using the --fsfs-compression option with --server-minor-version >= 10. (_create_parser): Parse the option as string, not int. (run_one): Pass the option as string, not int. (_post_create_repos): Update the code that adjusts fsfs.conf. Modified: subversion/trunk/build/run_tests.py subversion/trunk/subversion/libsvn_fs_fs/fs.h subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c subversion/trunk/subversion/libsvn_fs_fs/transaction.c subversion/trunk/subversion/tests/cmdline/svntest/main.py subversion/trunk/win-tests.py Modified: subversion/trunk/build/run_tests.py URL: http://svn.apache.org/viewvc/subversion/trunk/build/run_tests.py?rev=1803639&r1=1803638&r2=1803639&view=diff ============================================================================== --- subversion/trunk/build/run_tests.py (original) +++ subversion/trunk/build/run_tests.py Tue Aug 1 12:18:23 2017 @@ -33,7 +33,7 @@ [--httpd-version=<version>] [--httpd-whitelist=<version>] [--config-file=<file>] [--ssl-cert=<file>] [--exclusive-wc-locks] [--memcached-server=<url:port>] - [--fsfs-compression=<n>] + [--fsfs-compression=<type>] <abs_srcdir> <abs_builddir> <prog ...> @@ -276,8 +276,8 @@ class TestHarness: cmdline.append('--exclusive-wc-locks') if self.opts.memcached_server is not None: cmdline.append('--memcached-server=%s' % self.opts.memcached_server) - if self.opts.fsfs_compression_level is not None: - cmdline.append('--fsfs-compression=%d' % self.opts.fsfs_compression_level) + if self.opts.fsfs_compression is not None: + cmdline.append('--fsfs-compression=%s' % self.opts.fsfs_compression) self.py_test_cmdline = cmdline @@ -1026,9 +1026,8 @@ def create_parser(): help='Use sqlite exclusive locking for working copies') parser.add_option('--memcached-server', action='store', help='Use memcached server at specified URL (FSFS only)') - parser.add_option('--fsfs-compression', action='store', type='int', - dest="fsfs_compression_level", - help='Set compression level (for fsfs)') + parser.add_option('--fsfs-compression', action='store', type='str', + help='Set compression type (for fsfs)') parser.set_defaults(set_log_level=None) return parser Modified: subversion/trunk/subversion/libsvn_fs_fs/fs.h URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/fs.h?rev=1803639&r1=1803638&r2=1803639&view=diff ============================================================================== --- subversion/trunk/subversion/libsvn_fs_fs/fs.h (original) +++ subversion/trunk/subversion/libsvn_fs_fs/fs.h Tue Aug 1 12:18:23 2017 @@ -118,6 +118,7 @@ extern "C" { #define CONFIG_SECTION_DEBUG "debug" #define CONFIG_OPTION_PACK_AFTER_COMMIT "pack-after-commit" #define CONFIG_OPTION_VERIFY_BEFORE_COMMIT "verify-before-commit" +#define CONFIG_OPTION_COMPRESSION "compression" /* The format number of this filesystem. This is independent of the repository format number, and @@ -298,6 +299,13 @@ typedef struct window_cache_key_t apr_uint64_t item_index; } window_cache_key_t; +typedef enum compression_type_t +{ + compression_type_none, + compression_type_zlib, + compression_type_lz4 +} compression_type_t; + /* Private (non-shared) FSFS-specific data for each svn_fs_t object. Any caches in here may be NULL. */ typedef struct fs_fs_data_t @@ -473,7 +481,10 @@ typedef struct fs_fs_data_t * deltification history after which skip deltas will be used. */ apr_int64_t max_linear_deltification; - /* Compression level to use with txdelta storage format in new revs. */ + /* Compression type to use with txdelta storage format in new revs. */ + compression_type_t delta_compression_type; + + /* Compression level (currently, only used with compression_type_zlib). */ int delta_compression_level; /* Pack after every commit. */ Modified: subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c?rev=1803639&r1=1803638&r2=1803639&view=diff ============================================================================== --- subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c (original) +++ subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c Tue Aug 1 12:18:23 2017 @@ -683,6 +683,60 @@ verify_block_size(apr_int64_t block_size return SVN_NO_ERROR; } +static svn_error_t * +parse_compression_option(compression_type_t *compression_type_p, + int *compression_level_p, + const char *value) +{ + compression_type_t type; + int level; + svn_boolean_t is_valid = TRUE; + + /* compression = none | lz4 | zlib | zlib-1 ... zlib-9 */ + if (strcmp(value, "none") == 0) + { + type = compression_type_none; + level = SVN_DELTA_COMPRESSION_LEVEL_NONE; + } + else if (strcmp(value, "lz4") == 0) + { + type = compression_type_lz4; + level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; + } + else if (strncmp(value, "zlib", 4) == 0) + { + const char *p = value + 4; + + type = compression_type_zlib; + if (*p == 0) + { + level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; + } + else if (*p == '-') + { + p++; + SVN_ERR(svn_cstring_atoi(&level, p)); + if (level < 1 || level > 9) + is_valid = FALSE; + } + else + is_valid = FALSE; + } + else + { + is_valid = FALSE; + } + + if (!is_valid) + return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL, + _("Invalid 'compression' value '%s' in the config"), + value); + + *compression_type_p = type; + *compression_level_p = level; + return SVN_NO_ERROR; +} + /* Read the configuration information of the file system at FS_PATH * and set the respective values in FFD. Use pools as usual. */ @@ -709,8 +763,6 @@ read_config(fs_fs_data_t *ffd, /* Initialize deltification settings in ffd. */ if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT) { - apr_int64_t compression_level; - SVN_ERR(svn_config_get_bool(config, &ffd->deltify_directories, CONFIG_SECTION_DELTIFICATION, CONFIG_OPTION_ENABLE_DIR_DELTIFICATION, @@ -727,14 +779,6 @@ read_config(fs_fs_data_t *ffd, CONFIG_SECTION_DELTIFICATION, CONFIG_OPTION_MAX_LINEAR_DELTIFICATION, SVN_FS_FS_MAX_LINEAR_DELTIFICATION)); - - SVN_ERR(svn_config_get_int64(config, &compression_level, - CONFIG_SECTION_DELTIFICATION, - CONFIG_OPTION_COMPRESSION_LEVEL, - SVN_DELTA_COMPRESSION_LEVEL_DEFAULT)); - ffd->delta_compression_level - = (int)MIN(MAX(SVN_DELTA_COMPRESSION_LEVEL_NONE, compression_level), - SVN_DELTA_COMPRESSION_LEVEL_MAX); } else { @@ -742,7 +786,6 @@ read_config(fs_fs_data_t *ffd, ffd->deltify_properties = FALSE; ffd->max_deltification_walk = SVN_FS_FS_MAX_DELTIFICATION_WALK; ffd->max_linear_deltification = SVN_FS_FS_MAX_LINEAR_DELTIFICATION; - ffd->delta_compression_level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; } /* Initialize revprop packing settings in ffd. */ @@ -816,6 +859,68 @@ read_config(fs_fs_data_t *ffd, { ffd->pack_after_commit = FALSE; } + + /* Initialize compression settings in ffd. */ + if (ffd->format >= SVN_FS_FS__MIN_SVNDIFF2_FORMAT) + { + const char *compression_val; + const char *compression_level_val; + + svn_config_get(config, &compression_val, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_COMPRESSION, NULL); + svn_config_get(config, &compression_level_val, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_COMPRESSION_LEVEL, NULL); + if (compression_val) + { + /* 'compression' option overrides deprecated 'compression-level'. */ + SVN_ERR(parse_compression_option(&ffd->delta_compression_type, + &ffd->delta_compression_level, + compression_val)); + } + else if (compression_level_val) + { + /* Handle the deprecated 'compression-level' option. */ + ffd->delta_compression_type = compression_type_zlib; + SVN_ERR(svn_cstring_atoi(&ffd->delta_compression_level, + compression_level_val)); + ffd->delta_compression_level = + MIN(MAX(SVN_DELTA_COMPRESSION_LEVEL_NONE, + ffd->delta_compression_level), + SVN_DELTA_COMPRESSION_LEVEL_MAX); + } + else + { + /* Nothing specified explicitly, use default settings. */ + ffd->delta_compression_type = compression_type_zlib; + ffd->delta_compression_level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; + } + } + else if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT) + { + apr_int64_t compression_level; + + SVN_ERR(svn_config_get_int64(config, &compression_level, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_COMPRESSION_LEVEL, + SVN_DELTA_COMPRESSION_LEVEL_DEFAULT)); + ffd->delta_compression_type = compression_type_zlib; + ffd->delta_compression_level = + (int)MIN(MAX(SVN_DELTA_COMPRESSION_LEVEL_NONE, compression_level), + SVN_DELTA_COMPRESSION_LEVEL_MAX); + } + else if (ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT) + { + ffd->delta_compression_type = compression_type_zlib; + ffd->delta_compression_level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; + } + else + { + ffd->delta_compression_type = compression_type_none; + ffd->delta_compression_level = SVN_DELTA_COMPRESSION_LEVEL_NONE; + } + #ifdef SVN_DEBUG SVN_ERR(svn_config_get_bool(config, &ffd->verify_before_commit, CONFIG_SECTION_DEBUG, @@ -947,23 +1052,25 @@ write_config(svn_fs_t *fs, "# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16" NL "###" NL "### After deltification, we compress the data to minimize on-disk size." NL -"### This settings control the compression level for this process." NL -"### Revisions with highly compressible data in them may shrink in size" NL -"### if the setting is increased but may take much longer to commit." NL -"### The time taken to uncompress that data again is widely independent" NL -"### of the compression level. Compression will be ineffective if the" NL -"### incoming content is already highly compressed. In that case," NL -"### disabling the compression entirely or using the special value 1" NL -"### (see below) will speed up commits as well as reading the data." NL -"### Repositories with many small compressible files (source code) but" NL -"### also a high percentage of large incompressible ones (artwork) may" NL -"### benefit from compression levels lowered." NL -"### Valid values are 0 to 9 with 9 providing the highest compression" NL -"### ratio and 0 disabling it altogether. Using 1 as the level enables" NL -"### LZ4 compression that provides a decent compression ratio, but" NL -"### performs better with large or incompressible files." NL -"### The default value is 5." NL -"# " CONFIG_OPTION_COMPRESSION_LEVEL " = 5" NL +"### This setting controls the compression algorithm, which will be used in" NL +"### future revisions. It can be used to either disable compression or to" NL +"### select between available algorithms (zlib, lz4). zlib is a general-" NL +"### purpose compression algorithm. lz4 is a fast compression algorithm" NL +"### which should be preferred for repositories with large and, possibly," NL +"### incompressible files. Note that the compression ratio of lz4 is" NL +"### usually lower than the one provided by zlib, but using it can" NL +"### significantly speed up commits as well as reading the data." NL +"### The syntax of this option is:" NL +"### " CONFIG_OPTION_COMPRESSION " = none | lz4 | zlib | zlib-1 ... zlib-9" NL +"### The default value is 'zlib', which is currently equivalent to 'zlib-5'." NL +"# " CONFIG_OPTION_COMPRESSION " = zlib" NL +"###" NL +"### DEPRECATED: The new '" CONFIG_OPTION_COMPRESSION "' option deprecates previously used" NL +"### '" CONFIG_OPTION_COMPRESSION_LEVEL "' option, which was used to configure zlib compression." NL +"### For compatibility with previous versions of Subversion, this option can"NL +"### still be used (and it will result in zlib compression with the" NL +"### corresponding compression level)." NL +"### " CONFIG_OPTION_COMPRESSION_LEVEL " = 0 ... 9 (default is 5)" NL "" NL "[" CONFIG_SECTION_PACKED_REVPROPS "]" NL "### This parameter controls the size (in kBytes) of packed revprop files." NL Modified: subversion/trunk/subversion/libsvn_fs_fs/transaction.c URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/transaction.c?rev=1803639&r1=1803638&r2=1803639&view=diff ============================================================================== --- subversion/trunk/subversion/libsvn_fs_fs/transaction.c (original) +++ subversion/trunk/subversion/libsvn_fs_fs/transaction.c Tue Aug 1 12:18:23 2017 @@ -2168,27 +2168,24 @@ txdelta_to_svndiff(svn_txdelta_window_ha { fs_fs_data_t *ffd = fs->fsap_data; int svndiff_version; - int svndiff_compression_level; - if (ffd->delta_compression_level == 1 && - ffd->format >= SVN_FS_FS__MIN_SVNDIFF2_FORMAT) + if (ffd->delta_compression_type == compression_type_lz4) { + SVN_ERR_ASSERT_NO_RETURN(ffd->format >= SVN_FS_FS__MIN_SVNDIFF2_FORMAT); svndiff_version = 2; - svndiff_compression_level = SVN_DELTA_COMPRESSION_LEVEL_DEFAULT; } - else if (ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT) + else if (ffd->delta_compression_type == compression_type_zlib) { + SVN_ERR_ASSERT_NO_RETURN(ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT); svndiff_version = 1; - svndiff_compression_level = ffd->delta_compression_level; } else { svndiff_version = 0; - svndiff_compression_level = SVN_DELTA_COMPRESSION_LEVEL_NONE; } svn_txdelta_to_svndiff3(handler, handler_baton, output, svndiff_version, - svndiff_compression_level, pool); + ffd->delta_compression_level, pool); } /* Get a rep_write_baton and store it in *WB_P for the representation Modified: subversion/trunk/subversion/tests/cmdline/svntest/main.py URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/tests/cmdline/svntest/main.py?rev=1803639&r1=1803638&r2=1803639&view=diff ============================================================================== --- subversion/trunk/subversion/tests/cmdline/svntest/main.py (original) +++ subversion/trunk/subversion/tests/cmdline/svntest/main.py Tue Aug 1 12:18:23 2017 @@ -1043,14 +1043,14 @@ def _post_create_repos(path, minor_versi shutil.copy(options.config_file, confpath) if options.memcached_server is not None or \ - options.fsfs_compression_level is not None and \ + options.fsfs_compression is not None and \ os.path.exists(confpath): with open(confpath, 'r') as conffile: newlines = [] for line in conffile.readlines(): - if line.startswith('# compression-level ') and \ - options.fsfs_compression_level is not None: - line = 'compression-level = %d\n' % options.fsfs_compression_level + if line.startswith('# compression ') and \ + options.fsfs_compression is not None: + line = 'compression = %s\n' % options.fsfs_compression newlines += line if options.memcached_server is not None and \ line == '[memcached-servers]\n': @@ -1737,8 +1737,8 @@ class TestSpawningThread(threading.Threa args.append('--fsfs-version=' + str(options.fsfs_version)) if options.dump_load_cross_check: args.append('--dump-load-cross-check') - if options.fsfs_compression_level: - args.append('--fsfs-compression=' + str(options.fsfs_compression_level)) + if options.fsfs_compression: + args.append('--fsfs-compression=' + options.fsfs_compression) result, stdout_lines, stderr_lines = spawn_process(command, 0, False, None, *args) @@ -2151,9 +2151,8 @@ def _create_parser(usage=None): help='Use sqlite exclusive locking for working copies') parser.add_option('--memcached-server', action='store', help='Use memcached server at specified URL (FSFS only)') - parser.add_option('--fsfs-compression', action='store', type='int', - dest="fsfs_compression_level", - help='Set compression level (for fsfs)') + parser.add_option('--fsfs-compression', action='store', type='str', + help='Set compression type (for fsfs)') # most of the defaults are None, but some are other values, set them here parser.set_defaults( @@ -2203,9 +2202,9 @@ def parse_options(arglist=sys.argv[1:], if options.fsfs_packing and not options.fsfs_sharding: parser.error("--fsfs-packing requires --fsfs-sharding") - if options.fsfs_compression_level is not None and\ - options.fsfs_compression_level not in range(0, 10): - parser.error("--fsfs-compression must be between 0 and 9") + if options.fsfs_compression is not None and \ + options.server_minor_version < 10: + parser.error("--fsfs-compression requires --server-minor-version=10") if options.server_minor_version not in range(3, SVN_VER_MINOR+1): parser.error("test harness only supports server minor versions 3-%d" Modified: subversion/trunk/win-tests.py URL: http://svn.apache.org/viewvc/subversion/trunk/win-tests.py?rev=1803639&r1=1803638&r2=1803639&view=diff ============================================================================== --- subversion/trunk/win-tests.py (original) +++ subversion/trunk/win-tests.py Tue Aug 1 12:18:23 2017 @@ -113,7 +113,7 @@ def _usage_exit(): print(" --config-file : Configuration file for tests") print(" --fsfs-sharding : Specify shard size (for fsfs)") print(" --fsfs-packing : Run 'svnadmin pack' automatically") - print(" --fsfs-compression=VAL : Set compression level to VAL (for fsfs)") + print(" --fsfs-compression=VAL : Set compression type to VAL (for fsfs)") print(" -q, --quiet : Deprecated; this is the default.") print(" Use --set-log-level instead.") @@ -191,7 +191,7 @@ memcached_server = None memcached_dir = None skip_c_tests = None dump_load_cross_check = None -fsfs_compression_level = None +fsfs_compression = None for opt, val in opts: if opt in ('-h', '--help'): @@ -287,7 +287,7 @@ for opt, val in opts: memcached_dir = val run_memcached = 1 elif opt == '--fsfs-compression': - fsfs_compression_level = int(val) + fsfs_compression = val # Calculate the source and test directory names abs_srcdir = os.path.abspath("") @@ -1113,7 +1113,7 @@ if not test_javahl and not test_swig: opts.memcached_server = memcached_server opts.skip_c_tests = skip_c_tests opts.dump_load_cross_check = dump_load_cross_check - opts.fsfs_compression_level = fsfs_compression_level + opts.fsfs_compression = fsfs_compression th = run_tests.TestHarness(abs_srcdir, abs_builddir, log_file, fail_log_file, opts) old_cwd = os.getcwd()
