This is an automated email from the git hooks/post-receive script. daube-guest pushed a commit to branch master in repository libzstd.
commit 61101f9bb35da85b4728dd90623bbdf43a9f11a1 Merge: 73cb715 6837d46 Author: Kevin Murray <[email protected]> Date: Thu Jul 21 10:47:21 2016 +1000 Merge branch 'upstream' There were many conflicts, this needs review .gitattributes | 10 +- .gitignore | 39 +- .travis.yml | 109 +- Makefile | 154 +- NEWS | 42 +- README.md | 89 +- appveyor.yml | 104 + images/Cspeed4.png | Bin 47376 -> 47294 bytes images/DCspeed5.png | Bin 69388 -> 69278 bytes images/Dspeed4.png | Bin 9499 -> 9927 bytes images/smallData.png | Bin 0 -> 36133 bytes lib/Makefile | 40 +- lib/README.md | 69 +- lib/{ => common}/bitstream.h | 233 +- lib/common/entropy_common.c | 231 + lib/{ => common}/error_private.h | 25 +- lib/{ => common}/error_public.h | 8 +- lib/common/fse.h | 628 +++ lib/common/fse_decompress.c | 331 ++ lib/common/huf.h | 228 + lib/{ => common}/mem.h | 161 +- {programs => lib/common}/xxhash.c | 613 ++- {programs => lib/common}/xxhash.h | 97 +- lib/{ => common}/zbuff.h | 70 +- lib/common/zstd.h | 446 ++ lib/common/zstd_common.c | 91 + lib/common/zstd_internal.h | 238 + lib/compress/.debug/zstd_stats.h | 162 + lib/{fse.c => compress/fse_compress.c} | 547 +-- lib/compress/huf_compress.c | 576 +++ lib/compress/zbuff_compress.c | 329 ++ lib/compress/zstd_compress.c | 2774 ++++++++++++ lib/compress/zstd_opt.h | 1041 +++++ lib/decompress/huf_decompress.c | 894 ++++ lib/decompress/zbuff_decompress.c | 294 ++ lib/decompress/zstd_decompress.c | 1346 ++++++ lib/dictBuilder/divsufsort.c | 1913 ++++++++ lib/dictBuilder/divsufsort.h | 67 + lib/dictBuilder/zdict.c | 1036 +++++ lib/dictBuilder/zdict.h | 120 + lib/fse.h | 295 -- lib/fse_static.h | 336 -- lib/huff0.c | 1728 -------- lib/huff0.h | 97 - lib/huff0_static.h | 139 - lib/legacy/zstd_legacy.h | 72 +- lib/legacy/zstd_v01.c | 4 +- lib/legacy/zstd_v02.c | 4 +- lib/legacy/zstd_v03.c | 364 +- lib/legacy/zstd_v04.c | 361 +- lib/legacy/zstd_v04.h | 2 +- lib/legacy/{zstd_v04.c => zstd_v05.c} | 3319 +++++++------- lib/legacy/zstd_v05.h | 171 + lib/legacy/zstd_v06.c | 4583 ++++++++++++++++++++ lib/legacy/zstd_v06.h | 185 + lib/zbuff.c | 548 --- lib/zstd.h | 149 - lib/zstd_compress.c | 2394 ---------- lib/zstd_decompress.c | 1166 ----- lib/zstd_internal.h | 182 - lib/zstd_static.h | 249 -- programs/.gitignore | 22 + programs/Makefile | 142 +- programs/bench.c | 509 +-- programs/bench.h | 18 +- programs/datagen.c | 179 +- programs/datagencli.c | 80 +- programs/dibio.c | 162 +- programs/dibio.h | 3 +- programs/fileio.c | 460 +- programs/fileio.h | 11 +- programs/fullbench.c | 482 +- programs/fuzzer.c | 897 ++-- programs/legacy/fileio_legacy.c | 247 +- programs/legacy/fileio_legacy.h | 4 +- programs/paramgrill.c | 284 +- programs/playTests.sh | 199 +- programs/roundTripCrash.c | 193 + programs/util.h | 401 ++ programs/zbufftest.c | 515 ++- programs/zstd.1 | 42 +- programs/zstdcli.c | 279 +- projects/.gitignore | 2 + projects/README.md | 9 + projects/VS2008/fullbench/fullbench.vcproj | 449 ++ projects/VS2008/fuzzer/fuzzer.vcproj | 461 ++ projects/VS2008/zstd.sln | 56 + projects/VS2008/zstd/zstd.vcproj | 545 +++ projects/VS2008/zstdlib/zstdlib.vcproj | 495 +++ .../VS2010/datagen/datagen.vcxproj | 351 +- projects/VS2010/datagen/datagen.vcxproj.filters | 26 + .../VS2010}/fullbench/fullbench.vcxproj | 371 +- .../VS2010/fullbench/fullbench.vcxproj.filters | 86 + .../2013 => projects/VS2010}/fuzzer/fuzzer.vcxproj | 379 +- projects/VS2010/fuzzer/fuzzer.vcxproj.filters | 92 + {visual/2013 => projects/VS2010}/zstd.sln | 124 +- {visual/2013 => projects/VS2010}/zstd/zstd.vcxproj | 429 +- projects/VS2010/zstd/zstd.vcxproj.filters | 158 + projects/VS2010/zstdlib/zstdlib.rc | 51 + .../VS2010}/zstdlib/zstdlib.vcxproj | 426 +- projects/VS2010/zstdlib/zstdlib.vcxproj.filters | 95 + projects/build/README.md | 51 + projects/build/build.VS2010.cmd | 7 + projects/build/build.VS2012.cmd | 6 + projects/build/build.VS2013.cmd | 7 + projects/build/build.VS2015.cmd | 7 + projects/build/build.generic.cmd | 52 + projects/cmake/.gitignore | 6 + {contrib => projects}/cmake/CMakeLists.txt | 12 +- .../CMakeModules/AddExtraCompilationFlags.cmake | 0 .../cmake/cmake_uninstall.cmake.in | 0 {contrib => projects}/cmake/lib/CMakeLists.txt | 96 +- projects/cmake/programs/.gitignore | 8 + .../cmake/programs/CMakeLists.txt | 24 +- tests/.gitignore | 4 + tests/Makefile | 41 + tests/README.md | 55 + tests/test-zstd-speed.py | 265 ++ tests/test-zstd-versions.py | 266 ++ visual/2013/fullbench/fullbench.vcxproj.filters | 72 - visual/2013/fuzzer/fuzzer.vcxproj.filters | 90 - visual/2013/zstd/zstd.vcxproj.filters | 150 - visual/2013/zstdlib/resource.h | Bin 812 -> 0 bytes visual/2013/zstdlib/zstdlib.rc | Bin 5204 -> 0 bytes visual/2013/zstdlib/zstdlib.vcxproj.filters | 83 - {programs => zlibWrapper}/.gitignore | 22 +- zlibWrapper/Makefile | 61 + zlibWrapper/README.md | 105 + zlibWrapper/examples/example.c | 610 +++ zlibWrapper/examples/example_original.c | 601 +++ zlibWrapper/zstd_zlibwrapper.c | 981 +++++ .../zstd_zlibwrapper.h | 39 +- 132 files changed, 30412 insertions(+), 14546 deletions(-) diff --cc programs/fileio.c index 028c7db,5e7b26d..befe71d --- a/programs/fileio.c +++ b/programs/fileio.c @@@ -381,10 -367,11 +367,31 @@@ static int FIO_compressFilename_interna /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu =>%6llu bytes, %s) \n", srcFileName, + (double)compressedfilesize/readsize*100, (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); -- return 0; ++ ++/*! FIO_compressFilename_extRess() : ++ * @return : 0 : compression completed correctly, ++ * 1 : missing or pb opening srcFileName ++ */ ++static int FIO_compressFilename_extRess(cRess_t ress, ++ const char* dstFileName, const char* srcFileName, ++ int cLevel) ++{ ++ int result; ++ ++ ress.srcFile = FIO_openSrcFile(srcFileName); ++ if (ress.srcFile==0) return 1; ++ ress.dstFile = FIO_openDstFile(dstFileName); ++ if (ress.dstFile==0) { fclose(ress.srcFile); return 1; } ++ ++ result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); ++ ++ fclose(ress.srcFile); /* no pb to expect : only reading */ ++ if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName); ++ return result; } diff --cc programs/zstd.1 index 27d607f,d7760f7..84ab946 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@@ -71,26 -71,41 +71,58 @@@ It also features a very fast decoder, w .TP .BR \-c ", " --stdout force write to standard output, even if it is the console + +.SH DICTIONARY +.PP +\fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages. +It's possible to train \fBzstd\fR with some samples, the result of which is saved into a file called `dictionary`. +Then during compression and decompression, make reference to the same dictionary. +It will improve compression ratio of small files. +Typical gains range from ~10% (at 64KB) to x5 better (at <1KB). +.TP +.B \--train FILEs + use FILEs as training set to create a dictionary. + The training set should contain a lot of small files (> 100). + and weight typically 100x the target dictionary size + (for example, 10 MB for a 100 KB dictionary) +.TP +.B \-o file + dictionary saved into `file` (default: dictionary) .TP + .BR \-C ", " --check + add integrity check computed from uncompressed data + .TP + .BR \-t ", " --test + Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR. + No files are created or removed. + + .SH DICTIONARY + .PP + \fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages. + It's possible to train \fBzstd\fR with some samples, the result of which is saved into a file called `dictionary`. + Then during compression and decompression, make reference to the same dictionary. + It will improve compression ratio of small files. + Typical gains range from ~10% (at 64KB) to x5 better (at <1KB). + .TP + .B \--train FILEs + use FILEs as training set to create a dictionary. + The training set should contain a lot of small files (> 100). + and weight typically 100x the target dictionary size + (for example, 10 MB for a 100 KB dictionary) + .TP + .B \-o file + dictionary saved into `file` (default: dictionary) + .TP .B \--maxdict # - limit dictionary to specified size (default : 112640) + limit dictionary to specified size (default : 112640) + .TP + .B \--dictID # + A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary. + By default, zstd will create a 4-bytes random number ID. + It's possible to give a precise number instead. + Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, + and an ID < 65536 will only need 2 bytes. This compares favorably to 4 bytes default. + However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries. .TP .B \-s# dictionary selectivity level (default: 9) -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/libzstd.git _______________________________________________ debian-med-commit mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/debian-med-commit
