From fb6d3c2eff7caf22069ae240356400dc4ccbcc5e Mon Sep 17 00:00:00 2001 From: Lucas Holt Date: Tue, 9 Apr 2024 23:55:12 -0400 Subject: [PATCH] cleanup --- sys/contrib/zstd/.gitattributes | 3 - sys/contrib/zstd/CODE_OF_CONDUCT.md | 5 - sys/contrib/zstd/NEWS | 366 --- sys/contrib/zstd/circle.yml | 63 - .../contrib/adaptive-compression/Makefile | 76 - .../contrib/adaptive-compression/README.md | 91 - .../zstd/contrib/adaptive-compression/adapt.c | 1128 --------- .../contrib/adaptive-compression/datagencli.c | 129 -- .../adaptive-compression/test-correctness.sh | 252 -- .../adaptive-compression/test-performance.sh | 59 - sys/contrib/zstd/contrib/cleanTabs | 2 - sys/contrib/zstd/contrib/gen_html/.gitignore | 3 - sys/contrib/zstd/contrib/gen_html/Makefile | 51 - sys/contrib/zstd/contrib/gen_html/README.md | 31 - .../zstd/contrib/gen_html/gen-zstd-manual.sh | 9 - .../zstd/contrib/gen_html/gen_html.cpp | 224 -- sys/contrib/zstd/contrib/meson/README | 3 - sys/contrib/zstd/contrib/meson/meson.build | 142 -- .../zstd/contrib/meson/meson_options.txt | 3 - sys/contrib/zstd/contrib/pzstd/.gitignore | 2 - sys/contrib/zstd/contrib/pzstd/BUCK | 72 - sys/contrib/zstd/contrib/pzstd/ErrorHolder.h | 54 - sys/contrib/zstd/contrib/pzstd/Logging.h | 72 - sys/contrib/zstd/contrib/pzstd/Makefile | 269 --- sys/contrib/zstd/contrib/pzstd/Options.cpp | 439 ---- sys/contrib/zstd/contrib/pzstd/Options.h | 68 - sys/contrib/zstd/contrib/pzstd/Pzstd.cpp | 618 ----- sys/contrib/zstd/contrib/pzstd/Pzstd.h | 150 -- sys/contrib/zstd/contrib/pzstd/README.md | 56 - .../zstd/contrib/pzstd/SkippableFrame.cpp | 30 - .../zstd/contrib/pzstd/SkippableFrame.h | 64 - .../zstd/contrib/pzstd/images/Cspeed.png | Bin 69804 -> 0 bytes .../zstd/contrib/pzstd/images/Dspeed.png | Bin 26335 -> 0 bytes sys/contrib/zstd/contrib/pzstd/main.cpp | 27 - sys/contrib/zstd/contrib/pzstd/test/BUCK | 37 - .../zstd/contrib/pzstd/test/OptionsTest.cpp | 536 ----- .../zstd/contrib/pzstd/test/PzstdTest.cpp | 149 -- .../zstd/contrib/pzstd/test/RoundTrip.h | 86 - .../zstd/contrib/pzstd/test/RoundTripTest.cpp | 86 - sys/contrib/zstd/contrib/pzstd/utils/BUCK | 75 - sys/contrib/zstd/contrib/pzstd/utils/Buffer.h | 99 - .../zstd/contrib/pzstd/utils/FileSystem.h | 94 - sys/contrib/zstd/contrib/pzstd/utils/Likely.h | 28 - sys/contrib/zstd/contrib/pzstd/utils/Range.h | 131 -- .../zstd/contrib/pzstd/utils/ResourcePool.h | 96 - .../zstd/contrib/pzstd/utils/ScopeGuard.h | 50 - .../zstd/contrib/pzstd/utils/ThreadPool.h | 58 - .../zstd/contrib/pzstd/utils/WorkQueue.h | 181 -- .../zstd/contrib/pzstd/utils/test/BUCK | 35 - .../contrib/pzstd/utils/test/BufferTest.cpp | 89 - .../contrib/pzstd/utils/test/RangeTest.cpp | 82 - .../pzstd/utils/test/ResourcePoolTest.cpp | 72 - .../pzstd/utils/test/ScopeGuardTest.cpp | 28 - .../pzstd/utils/test/ThreadPoolTest.cpp | 71 - .../pzstd/utils/test/WorkQueueTest.cpp | 282 --- .../seekable_format/examples/.gitignore | 4 - .../contrib/seekable_format/examples/Makefile | 42 - .../examples/parallel_compression.c | 215 -- .../examples/parallel_processing.c | 194 -- .../examples/seekable_compression.c | 132 -- .../examples/seekable_decompression.c | 138 -- .../contrib/seekable_format/zstd_seekable.h | 184 -- .../zstd_seekable_compression_format.md | 116 - .../seekable_format/zstdseek_compress.c | 366 --- .../seekable_format/zstdseek_decompress.c | 462 ---- sys/contrib/zstd/lib/common/xxhash.h | 3 + .../lib/compress/zstd_compress_internal.h | 3 + sys/contrib/zstd/lib/dictBuilder/zdict.c | 3 + sys/contrib/zstd/lib/freebsd/limits.h | 3 + sys/contrib/zstd/lib/freebsd/zstd_kmalloc.c | 7 + sys/contrib/zstd/tests/.gitignore | 65 - sys/contrib/zstd/tests/Makefile | 467 ---- sys/contrib/zstd/tests/README.md | 90 - sys/contrib/zstd/tests/checkTag.c | 65 - sys/contrib/zstd/tests/datagencli.c | 130 -- sys/contrib/zstd/tests/decodecorpus.c | 1923 ---------------- .../tests/files/huffman-compressed-larger | Bin 143 -> 0 bytes sys/contrib/zstd/tests/fullbench.c | 659 ------ sys/contrib/zstd/tests/fuzz/Makefile | 127 -- sys/contrib/zstd/tests/fuzz/README.md | 96 - .../zstd/tests/fuzz/block_decompress.c | 51 - .../zstd/tests/fuzz/block_round_trip.c | 92 - sys/contrib/zstd/tests/fuzz/default.options | 2 - sys/contrib/zstd/tests/fuzz/fuzz.h | 62 - sys/contrib/zstd/tests/fuzz/fuzz.py | 818 ------- sys/contrib/zstd/tests/fuzz/fuzz_helpers.h | 92 - .../zstd/tests/fuzz/regression_driver.c | 71 - .../zstd/tests/fuzz/simple_decompress.c | 49 - .../zstd/tests/fuzz/simple_round_trip.c | 95 - .../zstd/tests/fuzz/stream_decompress.c | 85 - .../zstd/tests/fuzz/stream_round_trip.c | 162 -- sys/contrib/zstd/tests/fuzz/zstd_helpers.c | 84 - sys/contrib/zstd/tests/fuzz/zstd_helpers.h | 35 - sys/contrib/zstd/tests/fuzzer.c | 1824 --------------- sys/contrib/zstd/tests/gzip/Makefile | 44 - sys/contrib/zstd/tests/gzip/gzip-env.sh | 46 - sys/contrib/zstd/tests/gzip/helin-segv.sh | 31 - sys/contrib/zstd/tests/gzip/help-version.sh | 270 --- sys/contrib/zstd/tests/gzip/hufts.sh | 34 - sys/contrib/zstd/tests/gzip/init.cfg | 5 - sys/contrib/zstd/tests/gzip/init.sh | 616 ----- sys/contrib/zstd/tests/gzip/keep.sh | 51 - sys/contrib/zstd/tests/gzip/list.sh | 31 - sys/contrib/zstd/tests/gzip/memcpy-abuse.sh | 34 - sys/contrib/zstd/tests/gzip/mixed.sh | 68 - .../zstd/tests/gzip/null-suffix-clobber.sh | 35 - sys/contrib/zstd/tests/gzip/stdin.sh | 31 - sys/contrib/zstd/tests/gzip/test-driver.sh | 150 -- sys/contrib/zstd/tests/gzip/trailing-nul.sh | 37 - sys/contrib/zstd/tests/gzip/unpack-invalid.sh | 36 - sys/contrib/zstd/tests/gzip/z-suffix.sh | 30 - sys/contrib/zstd/tests/gzip/zdiff.sh | 48 - sys/contrib/zstd/tests/gzip/zgrep-context.sh | 47 - sys/contrib/zstd/tests/gzip/zgrep-f.sh | 43 - sys/contrib/zstd/tests/gzip/zgrep-signal.sh | 64 - sys/contrib/zstd/tests/gzip/znew-k.sh | 40 - sys/contrib/zstd/tests/invalidDictionaries.c | 61 - sys/contrib/zstd/tests/legacy.c | 224 -- sys/contrib/zstd/tests/longmatch.c | 101 - sys/contrib/zstd/tests/namespaceTest.c | 24 - sys/contrib/zstd/tests/paramgrill.c | 1051 --------- sys/contrib/zstd/tests/playTests.sh | 771 ------- sys/contrib/zstd/tests/poolTests.c | 106 - sys/contrib/zstd/tests/roundTripCrash.c | 241 -- sys/contrib/zstd/tests/seqgen.c | 260 --- sys/contrib/zstd/tests/seqgen.h | 58 - sys/contrib/zstd/tests/symbols.c | 162 -- sys/contrib/zstd/tests/test-zstd-speed.py | 376 --- sys/contrib/zstd/tests/test-zstd-versions.py | 276 --- sys/contrib/zstd/tests/zbufftest.c | 618 ----- sys/contrib/zstd/tests/zstreamtest.c | 2029 ----------------- 131 files changed, 19 insertions(+), 24072 deletions(-) delete mode 100644 sys/contrib/zstd/CODE_OF_CONDUCT.md delete mode 100644 sys/contrib/zstd/NEWS delete mode 100644 sys/contrib/zstd/circle.yml delete mode 100644 sys/contrib/zstd/contrib/adaptive-compression/Makefile delete mode 100644 sys/contrib/zstd/contrib/adaptive-compression/README.md delete mode 100644 sys/contrib/zstd/contrib/adaptive-compression/adapt.c delete mode 100644 sys/contrib/zstd/contrib/adaptive-compression/datagencli.c delete mode 100755 sys/contrib/zstd/contrib/adaptive-compression/test-correctness.sh delete mode 100755 sys/contrib/zstd/contrib/adaptive-compression/test-performance.sh delete mode 100755 sys/contrib/zstd/contrib/cleanTabs delete mode 100644 sys/contrib/zstd/contrib/gen_html/.gitignore delete mode 100644 sys/contrib/zstd/contrib/gen_html/Makefile delete mode 100644 sys/contrib/zstd/contrib/gen_html/README.md delete mode 100755 sys/contrib/zstd/contrib/gen_html/gen-zstd-manual.sh delete mode 100644 sys/contrib/zstd/contrib/gen_html/gen_html.cpp delete mode 100644 sys/contrib/zstd/contrib/meson/README delete mode 100644 sys/contrib/zstd/contrib/meson/meson.build delete mode 100644 sys/contrib/zstd/contrib/meson/meson_options.txt delete mode 100644 sys/contrib/zstd/contrib/pzstd/.gitignore delete mode 100644 sys/contrib/zstd/contrib/pzstd/BUCK delete mode 100644 sys/contrib/zstd/contrib/pzstd/ErrorHolder.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/Logging.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/Makefile delete mode 100644 sys/contrib/zstd/contrib/pzstd/Options.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/Options.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/Pzstd.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/Pzstd.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/README.md delete mode 100644 sys/contrib/zstd/contrib/pzstd/SkippableFrame.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/SkippableFrame.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/images/Cspeed.png delete mode 100644 sys/contrib/zstd/contrib/pzstd/images/Dspeed.png delete mode 100644 sys/contrib/zstd/contrib/pzstd/main.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/test/BUCK delete mode 100644 sys/contrib/zstd/contrib/pzstd/test/OptionsTest.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/test/PzstdTest.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/test/RoundTrip.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/test/RoundTripTest.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/BUCK delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/Buffer.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/FileSystem.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/Likely.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/Range.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/ResourcePool.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/ScopeGuard.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/ThreadPool.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/WorkQueue.h delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/test/BUCK delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/test/BufferTest.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/test/RangeTest.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/test/ResourcePoolTest.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/test/ScopeGuardTest.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp delete mode 100644 sys/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp delete mode 100644 sys/contrib/zstd/contrib/seekable_format/examples/.gitignore delete mode 100644 sys/contrib/zstd/contrib/seekable_format/examples/Makefile delete mode 100644 sys/contrib/zstd/contrib/seekable_format/examples/parallel_compression.c delete mode 100644 sys/contrib/zstd/contrib/seekable_format/examples/parallel_processing.c delete mode 100644 sys/contrib/zstd/contrib/seekable_format/examples/seekable_compression.c delete mode 100644 sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression.c delete mode 100644 sys/contrib/zstd/contrib/seekable_format/zstd_seekable.h delete mode 100644 sys/contrib/zstd/contrib/seekable_format/zstd_seekable_compression_format.md delete mode 100644 sys/contrib/zstd/contrib/seekable_format/zstdseek_compress.c delete mode 100644 sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c create mode 100644 sys/contrib/zstd/lib/freebsd/limits.h delete mode 100644 sys/contrib/zstd/tests/.gitignore delete mode 100644 sys/contrib/zstd/tests/Makefile delete mode 100644 sys/contrib/zstd/tests/README.md delete mode 100644 sys/contrib/zstd/tests/checkTag.c delete mode 100644 sys/contrib/zstd/tests/datagencli.c delete mode 100644 sys/contrib/zstd/tests/decodecorpus.c delete mode 100644 sys/contrib/zstd/tests/files/huffman-compressed-larger delete mode 100644 sys/contrib/zstd/tests/fullbench.c delete mode 100644 sys/contrib/zstd/tests/fuzz/Makefile delete mode 100644 sys/contrib/zstd/tests/fuzz/README.md delete mode 100644 sys/contrib/zstd/tests/fuzz/block_decompress.c delete mode 100644 sys/contrib/zstd/tests/fuzz/block_round_trip.c delete mode 100644 sys/contrib/zstd/tests/fuzz/default.options delete mode 100644 sys/contrib/zstd/tests/fuzz/fuzz.h delete mode 100755 sys/contrib/zstd/tests/fuzz/fuzz.py delete mode 100644 sys/contrib/zstd/tests/fuzz/fuzz_helpers.h delete mode 100644 sys/contrib/zstd/tests/fuzz/regression_driver.c delete mode 100644 sys/contrib/zstd/tests/fuzz/simple_decompress.c delete mode 100644 sys/contrib/zstd/tests/fuzz/simple_round_trip.c delete mode 100644 sys/contrib/zstd/tests/fuzz/stream_decompress.c delete mode 100644 sys/contrib/zstd/tests/fuzz/stream_round_trip.c delete mode 100644 sys/contrib/zstd/tests/fuzz/zstd_helpers.c delete mode 100644 sys/contrib/zstd/tests/fuzz/zstd_helpers.h delete mode 100644 sys/contrib/zstd/tests/fuzzer.c delete mode 100644 sys/contrib/zstd/tests/gzip/Makefile delete mode 100755 sys/contrib/zstd/tests/gzip/gzip-env.sh delete mode 100755 sys/contrib/zstd/tests/gzip/helin-segv.sh delete mode 100755 sys/contrib/zstd/tests/gzip/help-version.sh delete mode 100755 sys/contrib/zstd/tests/gzip/hufts.sh delete mode 100644 sys/contrib/zstd/tests/gzip/init.cfg delete mode 100755 sys/contrib/zstd/tests/gzip/init.sh delete mode 100755 sys/contrib/zstd/tests/gzip/keep.sh delete mode 100755 sys/contrib/zstd/tests/gzip/list.sh delete mode 100755 sys/contrib/zstd/tests/gzip/memcpy-abuse.sh delete mode 100755 sys/contrib/zstd/tests/gzip/mixed.sh delete mode 100755 sys/contrib/zstd/tests/gzip/null-suffix-clobber.sh delete mode 100755 sys/contrib/zstd/tests/gzip/stdin.sh delete mode 100755 sys/contrib/zstd/tests/gzip/test-driver.sh delete mode 100755 sys/contrib/zstd/tests/gzip/trailing-nul.sh delete mode 100755 sys/contrib/zstd/tests/gzip/unpack-invalid.sh delete mode 100755 sys/contrib/zstd/tests/gzip/z-suffix.sh delete mode 100755 sys/contrib/zstd/tests/gzip/zdiff.sh delete mode 100755 sys/contrib/zstd/tests/gzip/zgrep-context.sh delete mode 100755 sys/contrib/zstd/tests/gzip/zgrep-f.sh delete mode 100755 sys/contrib/zstd/tests/gzip/zgrep-signal.sh delete mode 100755 sys/contrib/zstd/tests/gzip/znew-k.sh delete mode 100644 sys/contrib/zstd/tests/invalidDictionaries.c delete mode 100644 sys/contrib/zstd/tests/legacy.c delete mode 100644 sys/contrib/zstd/tests/longmatch.c delete mode 100644 sys/contrib/zstd/tests/namespaceTest.c delete mode 100644 sys/contrib/zstd/tests/paramgrill.c delete mode 100755 sys/contrib/zstd/tests/playTests.sh delete mode 100644 sys/contrib/zstd/tests/poolTests.c delete mode 100644 sys/contrib/zstd/tests/roundTripCrash.c delete mode 100644 sys/contrib/zstd/tests/seqgen.c delete mode 100644 sys/contrib/zstd/tests/seqgen.h delete mode 100644 sys/contrib/zstd/tests/symbols.c delete mode 100755 sys/contrib/zstd/tests/test-zstd-speed.py delete mode 100755 sys/contrib/zstd/tests/test-zstd-versions.py delete mode 100644 sys/contrib/zstd/tests/zbufftest.c delete mode 100644 sys/contrib/zstd/tests/zstreamtest.c diff --git a/sys/contrib/zstd/.gitattributes b/sys/contrib/zstd/.gitattributes index 9eb12c0ef62..6212bd405b4 100644 --- a/sys/contrib/zstd/.gitattributes +++ b/sys/contrib/zstd/.gitattributes @@ -19,6 +19,3 @@ # Windows *.bat text eol=crlf *.cmd text eol=crlf - -# .travis.yml merging -.travis.yml merge=ours diff --git a/sys/contrib/zstd/CODE_OF_CONDUCT.md b/sys/contrib/zstd/CODE_OF_CONDUCT.md deleted file mode 100644 index 0f7ad8bfc17..00000000000 --- a/sys/contrib/zstd/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,5 +0,0 @@ -# Code of Conduct - -Facebook has adopted a Code of Conduct that we expect project participants to adhere to. -Please read the [full text](https://code.fb.com/codeofconduct/) -so that you can understand what actions will and will not be tolerated. diff --git a/sys/contrib/zstd/NEWS b/sys/contrib/zstd/NEWS deleted file mode 100644 index 9e903e6bcb3..00000000000 --- a/sys/contrib/zstd/NEWS +++ /dev/null @@ -1,366 +0,0 @@ -v1.3.4 -perf: faster speed (especially decoding speed) on recent cpus (haswell+) -perf: much better performance associating --long with multi-threading, by @terrelln -perf: better compression at levels 13-15 -cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior) -cli : smoother status report in multi-threading mode -cli : added command --fast=#, for faster compression modes -cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb) -api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode -api : compression levels can be negative, for even more speed -api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime -api : ZSTDMT can accept new compression parameters during compression -api : implemented all advanced dictionary decompression prototypes -build: improved meson recipe, by Shawn Landden (@shawnl) -build: VS2017 scripts, by @HaydnTrigg -misc: all /contrib projects fixed -misc: added /contrib/docker script by @gyscos - -v1.3.3 -perf: faster zstd_opt strategy (levels 16-19) -fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen -cli : fix : content size written in header by default -cli : fix : improved LZ4 format support, by @felixhandte -cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file -api : fix : support large skippable frames, by @terrelln -api : fix : streaming interface was adding a useless 3-bytes null block to small frames -api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown" -build: fix : compilation under rhel6 and centos6, reported by @pixelb -build: added `check` target - -v1.3.2 -new : long range mode, using --long command, by Stella Lau (@stellamplau) -new : ability to generate and decode magicless frames (#591) -changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode -fix : multi-threading compression works with custom allocators -fix : ZSTD_sizeof_CStream() was over-evaluating memory usage -fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22) -fix : 32-bits build can now decode large offsets (levels 21+) -cli : added LZ4 frame support by default, by Felix Handte (@felixhandte) -cli : improved --list output -cli : new : can split input file for dictionary training, using command -B# -cli : new : clean operation artefact on Ctrl-C interruption -cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851) -cli : fix : write file size in header in multiple-files mode -api : added macro ZSTD_COMPRESSBOUND() for static allocation -api : experimental : new advanced decompression API -api : fix : sizeof_CCtx() used to over-estimate -build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819) -build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818) -example : added streaming_memory_usage -license : changed /examples license to BSD + GPLv2 -license : fix a few header files to reflect new license (#825) - -v1.3.1 -New license : BSD + GPLv2 -perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk) -perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760) -cli : improved and fixed --list command, by @ib (#772) -cli : command -vV to list supported formats, by @ib (#771) -build : fixed binary variants, reported by @svenha (#788) -build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718) -API exp : breaking change : ZSTD_getframeHeader() provides more information -API exp : breaking change : pinned down values of error codes -doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz) -new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74) -new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau) -updated : contrib/linux-kernel, by Nick Terrell (@terrelln) - -v1.3.0 -cli : new : `--list` command, by Paul Cruz -cli : changed : xz/lzma support enabled by default -cli : changed : `-t *` continue processing list after a decompression error -API : added : ZSTD_versionString() -API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell -API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter() -API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx() -API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700) -API exp : clarified memory estimation / measurement functions. -API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1 -tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz -new : contrib/seekable_format, demo and API, by Sean Purcell -changed : contrib/linux-kernel, updated version and license, by Nick Terrell - -v1.2.0 -cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable) -cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell -cli : new : zstdmt symlink hardwired to `zstd -T0` -cli : new : command --threads=# (#671) -cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell -cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters -cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell -cli : fix : does not output compressed data on console -cli : fix : ignore symbolic links unless --force specified, -API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument -API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters. -API : improved: ZSTDMT_compressCCtx() reduced memory usage -API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634) -API : fix : src size stored in frame header is controlled at end of frame -API : fix : enforced consistent rules for pledgedSrcSize==0 (#641) -API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate -build: improved cmake script, by @Majlen -build: enabled Multi-threading support for *BSD, by Baptiste Daroussin -tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target. -new : contrib/linux-kernel version, by Nick Terrell - -v1.1.4 -cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski -cli : new : advanced benchmark command --priority=rt -cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 -cli : fix : --rm remains silent when input is stdin -cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski -speed : improved decompression speed in streaming mode for single shot scenarios (+5%) -memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB -arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell -API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize() -API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value) -build : new: meson build system in contrib/meson, by Dima Krasner -build : improved cmake script, by @Majlen -build : added -Wformat-security flag, as recommended by Padraig Brady -doc : new : educational decoder, by Sean Purcell - -v1.1.3 -cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`) -cli : new : experimental target `make zstdmt`, with multi-threading support -cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano. -cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski -cli : fix zstdless on Mac OS-X, by Andrew Janke -cli : fix #232 "compress non-files" -dictBuilder : improved dictionary generation quality, thanks to Nick Terrell -API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental) -API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul -API : new : ZDICT_finalizeDictionary() -API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511) -API : fix : all symbols properly exposed in libzstd, by Nick Terrell -build : support for Solaris target, by Przemyslaw Skibinski -doc : clarified specification, by Sean Purcell - -v1.1.2 -API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init -API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize() -API : zbuff : changed : prototypes now generate deprecation warnings -lib : improved : faster decompression speed at ultra compression settings and 32-bits mode -lib : changed : only public ZSTD_ symbols are now exposed -lib : changed : reduced usage of stack memory -lib : fixed : several corner case bugs, by Nick Terrell -cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski -cli : new : preserve file attributes -cli : new : added zstdless and zstdgrep tools -cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) -cli : fixed : zstdcat -zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski -install : better compatibility with FreeBSD, by Dimitry Andric -source tree : changed : zbuff source files moved to lib/deprecated - -v1.1.1 -New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption -New : doc/zstd_manual.html, by Przemyslaw Skibinski -Improved : slightly better compression ratio at --ultra levels (>= 20) -Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report -Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section) -Added : example/multiple_streaming_compression.c -Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h) -Updated man page -Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets - -v1.1.0 -New : contrib/pzstd, parallel version of zstd, by Nick Terrell -added : NetBSD install target (#338) -Improved : speed for batches of small files -Improved : speed of zlib wrapper, by Przemyslaw Skibinski -Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier -Fixed : CLI -d output to stdout by default when input is stdin (#322) -Fixed : CLI correctly detects console on Mac OS-X -Fixed : CLI supports recursive mode `-r` on Mac OS-X -Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski -Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319) -Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365) -Fixed : zstd-pgo, reported by octoploid (#329) - -v1.0.0 -Change Licensing, all project is now BSD, Copyright Facebook -Small decompression speed improvement -API : Streaming API supports legacy format -API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter() -CLI supports legacy formats v0.4+ -Fixed : compression fails on certain huge files, reported by Jesse McGrew -Enhanced documentation, by Przemyslaw Skibinski - -v0.8.1 -New streaming API -Changed : --ultra now enables levels beyond 19 -Changed : -i# now selects benchmark time in second -Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell -Fixed : speed regression on specific patterns (#272) -Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291) -Fixed : ICC compilation, by Przemyslaw Skibinski - -v0.8.0 -Improved : better speed on clang and gcc -O2, thanks to Eric Biggers -New : Build on FreeBSD and DragonFly, thanks to JrMarino -Changed : modified API : ZSTD_compressEnd() -Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist -Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers -Fixed : large dictionaries (> 384 KB), reported by Ilona Papava -Fixed : checksum correctly checked in single-pass mode -Fixed : combined --test amd --rm, reported by Andreas M. Nilsson -Modified : minor compression level adaptations -Updated : compression format specification to v0.2.0 -changed : zstd.h moved to /lib directory - -v0.7.5 -Transition version, supporting decoding of v0.8.x - -v0.7.4 -Added : homebrew for Mac, by Daniel Cade -Added : more examples -Fixed : segfault when using small dictionaries, reported by Felix Handte -Modified : default compression level for CLI is now 3 -Updated : specification, to v0.1.1 - -v0.7.3 -New : compression format specification -New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. -New : `ZSTD_getDecompressedSize()` -New : OpenBSD target, by Juan Francisco Cantero Hurtado -New : `examples` directory -fixed : dictBuilder using HC levels, reported by Bartosz Taudul -fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte -fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski -modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section) -modified : legacy functions no longer need magic number - -v0.7.2 -fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. -fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner. -fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner. - -v0.7.1 -fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier -fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв -fixed : corruption issue, reported by cj -modified : checksum enabled by default in command line mode - -v0.7.0 -New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski -New : Command `--rm`, to remove source file after successful de/compression -New : Visual build scripts, by Christophe Chevalier -New : Support for Sparse File-systems (do not use space for zero-filled sectors) -New : Frame checksum support -New : Support pass-through mode (when using `-df`) -API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()` -API : create dictionary files from custom content, by Giuseppe Ottaviano -API : support for custom malloc/free functions -New : controllable Dictionary ID -New : Support for skippable frames - -v0.6.1 -New : zlib wrapper API, thanks to Przemyslaw Skibinski -New : Ability to compile compressor / decompressor separately -Changed : new lib directory structure -Fixed : Legacy codec v0.5 compatible with dictionary decompression -Fixed : Decoder corruption error (#173) -Fixed : null-string roundtrip (#176) -New : benchmark mode can select directory as input -Experimental : midipix support, VMS support - -v0.6.0 -Stronger high compression modes, thanks to Przemyslaw Skibinski -API : ZSTD_getFrameParams() provides size of decompressed content -New : highest compression modes require `--ultra` command to fully unleash their capacity -Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner - -v0.5.1 -New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski -Changed : Dictionary builder integrated into libzstd and zstd cli -Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`. -Fix : high compression modes for big-endian platforms -New : zstd cli : `-t` | `--test` command - -v0.5.0 -New : dictionary builder utility -Changed : streaming & dictionary API -Improved : better compression of small data - -v0.4.7 -Improved : small compression speed improvement in HC mode -Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default -fix : bt search bug - -v0.4.6 -fix : fast compression mode on Windows -New : cmake configuration file, thanks to Artyom Dymchenko -Improved : high compression mode on repetitive data -New : block-level API -New : ZSTD_duplicateCCtx() - -v0.4.5 -new : -m/--multiple : compress/decompress multiple files - -v0.4.4 -Fixed : high compression modes for Windows 32 bits -new : external dictionary API extended to buffered mode and accessible through command line -new : windows DLL project, thanks to Christophe Chevalier - -v0.4.3 : -new : external dictionary API -new : zstd-frugal - -v0.4.2 : -Generic minor improvements for small blocks -Fixed : big-endian compatibility, by Peter Harris (#85) - -v0.4.1 -Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben) -removed `zstd.c` - -v0.4.0 -Command line utility compatible with high compression levels -Removed zstdhc => merged into zstd -Added : ZBUFF API (see zstd_buffered.h) -Rolling buffer support - -v0.3.6 -small blocks params - -v0.3.5 -minor generic compression improvements - -v0.3.4 -Faster fast cLevels - -v0.3.3 -Small compression ratio improvement - -v0.3.2 -Fixed Visual Studio - -v0.3.1 : -Small compression ratio improvement - -v0.3 -HC mode : compression levels 2-26 - -v0.2.2 -Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier - -v0.2.1 -Fix : Read errors, advanced fuzzer tests, by Hanno Böck - -v0.2.0 -**Breaking format change** -Faster decompression speed -Can still decode v0.1 format - -v0.1.3 -fix uninitialization warning, reported by Evan Nemerson - -v0.1.2 -frame concatenation support - -v0.1.1 -fix compression bug -detects write-flush errors - -v0.1.0 -first release diff --git a/sys/contrib/zstd/circle.yml b/sys/contrib/zstd/circle.yml deleted file mode 100644 index ed50d59e5d9..00000000000 --- a/sys/contrib/zstd/circle.yml +++ /dev/null @@ -1,63 +0,0 @@ -dependencies: - override: - - sudo dpkg --add-architecture i386 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update - - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - -test: - override: - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c99build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make shortest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi - : - parallel: true - - post: - - echo Circle CI tests finished - - # Longer tests - #- make -C tests test-zstd-nolegacy && make clean - #- pyenv global 3.4.4; make -C tests versionsTest && make clean - #- make zlibwrapper && make clean - #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean - #- make uasan && make clean - #- make asan32 && make clean - #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" - # Valgrind tests - #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean - #- make -C tests valgrindTest && make clean - # ARM, AArch64, PowerPC, PowerPC64 tests - #- make ppctest && make clean - #- make ppc64test && make clean - #- make armtest && make clean - #- make aarch64test && make clean diff --git a/sys/contrib/zstd/contrib/adaptive-compression/Makefile b/sys/contrib/zstd/contrib/adaptive-compression/Makefile deleted file mode 100644 index c64fce95412..00000000000 --- a/sys/contrib/zstd/contrib/adaptive-compression/Makefile +++ /dev/null @@ -1,76 +0,0 @@ - -ZSTDDIR = ../../lib -PRGDIR = ../../programs -ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) - -MULTITHREAD_LDFLAGS = -pthread -DEBUGFLAGS= -g -DZSTD_DEBUG=1 -CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ - -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) -CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ - -Wstrict-prototypes -Wundef -Wformat-security \ - -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls -CFLAGS += $(DEBUGFLAGS) -CFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS) - -all: adapt datagen - -adapt: $(ZSTD_FILES) adapt.c - $(CC) $(FLAGS) $^ -o $@ - -adapt-debug: $(ZSTD_FILES) adapt.c - $(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt - -datagen : $(PRGDIR)/datagen.c datagencli.c - $(CC) $(FLAGS) $^ -o $@ - -test-adapt-correctness: datagen adapt - @./test-correctness.sh - @echo "test correctness complete" - -test-adapt-performance: datagen adapt - @./test-performance.sh - @echo "test performance complete" - -clean: - @$(RM) -f adapt datagen - @$(RM) -rf *.dSYM - @$(RM) -f tmp* - @$(RM) -f tests/*.zst - @$(RM) -f tests/tmp* - @echo "finished cleaning" - -#----------------------------------------------------------------------------- -# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets -#----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) - -ifneq (,$(filter $(shell uname),SunOS)) -INSTALL ?= ginstall -else -INSTALL ?= install -endif - -PREFIX ?= /usr/local -DESTDIR ?= -BINDIR ?= $(PREFIX)/bin - -INSTALL_PROGRAM ?= $(INSTALL) -m 755 - -install: adapt - @echo Installing binaries - @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ - @$(INSTALL_PROGRAM) adapt $(DESTDIR)$(BINDIR)/zstd-adaptive - @echo zstd-adaptive installation completed - -uninstall: - @$(RM) $(DESTDIR)$(BINDIR)/zstd-adaptive - @echo zstd-adaptive programs successfully uninstalled -endif diff --git a/sys/contrib/zstd/contrib/adaptive-compression/README.md b/sys/contrib/zstd/contrib/adaptive-compression/README.md deleted file mode 100644 index 0929b16ba76..00000000000 --- a/sys/contrib/zstd/contrib/adaptive-compression/README.md +++ /dev/null @@ -1,91 +0,0 @@ -### Summary - -`adapt` is a new compression tool targeted at optimizing performance across network connections and pipelines. The tool is aimed at sensing network speeds and adapting compression level based on network or pipe speeds. -In situations where the compression level does not appropriately match the network/pipe speed, compression may be bottlenecking the entire pipeline or the files may not be compressed as much as they potentially could be, therefore losing efficiency. It also becomes quite impractical to manually measure and set an optimalcompression level (which could potentially change over time). - -### Using `adapt` - -In order to build and use the tool, you can simply run `make adapt` in the `adaptive-compression` directory under `contrib`. This will generate an executable available for use. Another possible method of installation is running `make install`, which will create and install the binary as the command `zstd-adaptive`. - -Similar to many other compression utilities, `zstd-adaptive` can be invoked by using the following format: - -`zstd-adaptive [options] [file(s)]` - -Supported options for the above format are described below. - -`zstd-adaptive` also supports reading from `stdin` and writing to `stdout`, which is potentially more useful. By default, if no files are given, `zstd-adaptive` reads from and writes to standard I/O. Therefore, you can simply insert it within a pipeline like so: - -`cat FILE | zstd-adaptive | ssh "cat - > tmp.zst"` - -If a file is provided, it is also possible to force writing to stdout using the `-c` flag like so: - -`zstd-adaptive -c FILE | ssh "cat - > tmp.zst"` - -Several options described below can be used to control the behavior of `zstd-adaptive`. More specifically, using the `-l#` and `-u#` flags will will set upper and lower bounds so that the compression level will always be within that range. The `-i#` flag can also be used to change the initial compression level. If an initial compression level is not provided, the initial compression level will be chosen such that it is within the appropriate range (it becomes equal to the lower bound). - -### Options -`-oFILE` : write output to `FILE` - -`-i#` : provide initial compression level (must within the appropriate bounds) - -`-h` : display help/information - -`-f` : force the compression level to stay constant - -`-c` : force write to `stdout` - -`-p` : hide progress bar - -`-q` : quiet mode -- do not show progress bar or other information - -`-l#` : set a lower bound on the compression level (default is 1) - -`-u#` : set an upper bound on the compression level (default is 22) -### Benchmarking / Test results -#### Artificial Tests -These artificial tests were run by using the `pv` command line utility in order to limit pipe speeds (25 MB/s read and 5 MB/s write limits were chosen to mimic severe throughput constraints). A 40 GB backup file was sent through a pipeline, compressed, and written out to a file. Compression time, size, and ratio were computed. Data for `zstd -15` was excluded from these tests because the test runs quite long. - - - - -
25 MB/s read limit
- -| Compressor Name | Ratio | Compressed Size | Compression Time | -|:----------------|------:|----------------:|-----------------:| -| zstd -3 | 2.108 | 20.718 GB | 29m 48.530s | -| zstd-adaptive | 2.230 | 19.581 GB | 29m 48.798s | - -
- - - - -
5 MB/s write limit
- -| Compressor Name | Ratio | Compressed Size | Compression Time | -|:----------------|------:|----------------:|-----------------:| -| zstd -3 | 2.108 | 20.718 GB | 1h 10m 43.076s | -| zstd-adaptive | 2.249 | 19.412 GB | 1h 06m 15.577s | - -
- -The commands used for this test generally followed the form: - -`cat FILE | pv -L 25m -q | COMPRESSION | pv -q > tmp.zst # impose 25 MB/s read limit` - -`cat FILE | pv -q | COMPRESSION | pv -L 5m -q > tmp.zst # impose 5 MB/s write limit` - -#### SSH Tests - -The following tests were performed by piping a relatively large backup file (approximately 80 GB) through compression and over SSH to be stored on a server. The test data includes statistics for time and compressed size on `zstd` at several compression levels, as well as `zstd-adaptive`. The data highlights the potential advantages that `zstd-adaptive` has over using a low static compression level and the negative imapcts that using an excessively high static compression level can have on -pipe throughput. - -| Compressor Name | Ratio | Compressed Size | Compression Time | -|:----------------|------:|----------------:|-----------------:| -| zstd -3 | 2.212 | 32.426 GB | 1h 17m 59.756s | -| zstd -15 | 2.374 | 30.213 GB | 2h 56m 59.441s | -| zstd-adaptive | 2.315 | 30.993 GB | 1h 18m 52.860s | - -The commands used for this test generally followed the form: - -`cat FILE | COMPRESSION | ssh dev "cat - > tmp.zst"` diff --git a/sys/contrib/zstd/contrib/adaptive-compression/adapt.c b/sys/contrib/zstd/contrib/adaptive-compression/adapt.c deleted file mode 100644 index 49f1d4d5753..00000000000 --- a/sys/contrib/zstd/contrib/adaptive-compression/adapt.c +++ /dev/null @@ -1,1128 +0,0 @@ -/* - * Copyright (c) 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -#include /* fprintf */ -#include /* malloc, free */ -#include /* pthread functions */ -#include /* memset */ -#include "zstd_internal.h" -#include "util.h" - -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define PRINT(...) fprintf(stdout, __VA_ARGS__) -#define DEBUG(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } -#define FILE_CHUNK_SIZE 4 << 20 -#define MAX_NUM_JOBS 2 -#define stdinmark "/*stdin*\\" -#define stdoutmark "/*stdout*\\" -#define MAX_PATH 256 -#define DEFAULT_DISPLAY_LEVEL 1 -#define DEFAULT_COMPRESSION_LEVEL 6 -#define MAX_COMPRESSION_LEVEL_CHANGE 2 -#define CONVERGENCE_LOWER_BOUND 5 -#define CLEVEL_DECREASE_COOLDOWN 5 -#define CHANGE_BY_TWO_THRESHOLD 0.1 -#define CHANGE_BY_ONE_THRESHOLD 0.65 - -#ifndef DEBUG_MODE -static int g_displayLevel = DEFAULT_DISPLAY_LEVEL; -#else -static int g_displayLevel = DEBUG_MODE; -#endif - -static unsigned g_compressionLevel = DEFAULT_COMPRESSION_LEVEL; -static UTIL_time_t g_startTime; -static size_t g_streamedSize = 0; -static unsigned g_useProgressBar = 1; -static unsigned g_forceCompressionLevel = 0; -static unsigned g_minCLevel = 1; -static unsigned g_maxCLevel; - -typedef struct { - void* start; - size_t size; - size_t capacity; -} buffer_t; - -typedef struct { - size_t filled; - buffer_t buffer; -} inBuff_t; - -typedef struct { - buffer_t src; - buffer_t dst; - unsigned jobID; - unsigned lastJobPlusOne; - size_t compressedSize; - size_t dictSize; -} jobDescription; - -typedef struct { - pthread_mutex_t pMutex; - int noError; -} mutex_t; - -typedef struct { - pthread_cond_t pCond; - int noError; -} cond_t; - -typedef struct { - unsigned compressionLevel; - unsigned numJobs; - unsigned nextJobID; - unsigned threadError; - - /* - * JobIDs for the next jobs to be created, compressed, and written - */ - unsigned jobReadyID; - unsigned jobCompressedID; - unsigned jobWriteID; - unsigned allJobsCompleted; - - /* - * counter for how many jobs in a row the compression level has not changed - * if the counter becomes >= CONVERGENCE_LOWER_BOUND, the next time the - * compression level tries to change (by non-zero amount) resets the counter - * to 1 and does not apply the change - */ - unsigned convergenceCounter; - - /* - * cooldown counter in order to prevent rapid successive decreases in compression level - * whenever compression level is decreased, cooldown is set to CLEVEL_DECREASE_COOLDOWN - * whenever adaptCompressionLevel() is called and cooldown != 0, it is decremented - * as long as cooldown != 0, the compression level cannot be decreased - */ - unsigned cooldown; - - /* - * XWaitYCompletion - * Range from 0.0 to 1.0 - * if the value is not 1.0, then this implies that thread X waited on thread Y to finish - * and thread Y was XWaitYCompletion finished at the time of the wait (i.e. compressWaitWriteCompletion=0.5 - * implies that the compression thread waited on the write thread and it was only 50% finished writing a job) - */ - double createWaitCompressionCompletion; - double compressWaitCreateCompletion; - double compressWaitWriteCompletion; - double writeWaitCompressionCompletion; - - /* - * Completion values - * Range from 0.0 to 1.0 - * Jobs are divided into mini-chunks in order to measure completion - * these values are updated each time a thread finishes its operation on the - * mini-chunk (i.e. finishes writing out, compressing, etc. this mini-chunk). - */ - double compressionCompletion; - double writeCompletion; - double createCompletion; - - mutex_t jobCompressed_mutex; - cond_t jobCompressed_cond; - mutex_t jobReady_mutex; - cond_t jobReady_cond; - mutex_t allJobsCompleted_mutex; - cond_t allJobsCompleted_cond; - mutex_t jobWrite_mutex; - cond_t jobWrite_cond; - mutex_t compressionCompletion_mutex; - mutex_t createCompletion_mutex; - mutex_t writeCompletion_mutex; - mutex_t compressionLevel_mutex; - size_t lastDictSize; - inBuff_t input; - jobDescription* jobs; - ZSTD_CCtx* cctx; -} adaptCCtx; - -typedef struct { - adaptCCtx* ctx; - FILE* dstFile; -} outputThreadArg; - -typedef struct { - FILE* srcFile; - adaptCCtx* ctx; - outputThreadArg* otArg; -} fcResources; - -static void freeCompressionJobs(adaptCCtx* ctx) -{ - unsigned u; - for (u=0; unumJobs; u++) { - jobDescription job = ctx->jobs[u]; - free(job.dst.start); - free(job.src.start); - } -} - -static int destroyMutex(mutex_t* mutex) -{ - if (mutex->noError) { - int const ret = pthread_mutex_destroy(&mutex->pMutex); - return ret; - } - return 0; -} - -static int destroyCond(cond_t* cond) -{ - if (cond->noError) { - int const ret = pthread_cond_destroy(&cond->pCond); - return ret; - } - return 0; -} - -static int freeCCtx(adaptCCtx* ctx) -{ - if (!ctx) return 0; - { - int error = 0; - error |= destroyMutex(&ctx->jobCompressed_mutex); - error |= destroyCond(&ctx->jobCompressed_cond); - error |= destroyMutex(&ctx->jobReady_mutex); - error |= destroyCond(&ctx->jobReady_cond); - error |= destroyMutex(&ctx->allJobsCompleted_mutex); - error |= destroyCond(&ctx->allJobsCompleted_cond); - error |= destroyMutex(&ctx->jobWrite_mutex); - error |= destroyCond(&ctx->jobWrite_cond); - error |= destroyMutex(&ctx->compressionCompletion_mutex); - error |= destroyMutex(&ctx->createCompletion_mutex); - error |= destroyMutex(&ctx->writeCompletion_mutex); - error |= destroyMutex(&ctx->compressionLevel_mutex); - error |= ZSTD_isError(ZSTD_freeCCtx(ctx->cctx)); - free(ctx->input.buffer.start); - if (ctx->jobs){ - freeCompressionJobs(ctx); - free(ctx->jobs); - } - free(ctx); - return error; - } -} - -static int initMutex(mutex_t* mutex) -{ - int const ret = pthread_mutex_init(&mutex->pMutex, NULL); - mutex->noError = !ret; - return ret; -} - -static int initCond(cond_t* cond) -{ - int const ret = pthread_cond_init(&cond->pCond, NULL); - cond->noError = !ret; - return ret; -} - -static int initCCtx(adaptCCtx* ctx, unsigned numJobs) -{ - ctx->compressionLevel = g_compressionLevel; - { - int pthreadError = 0; - pthreadError |= initMutex(&ctx->jobCompressed_mutex); - pthreadError |= initCond(&ctx->jobCompressed_cond); - pthreadError |= initMutex(&ctx->jobReady_mutex); - pthreadError |= initCond(&ctx->jobReady_cond); - pthreadError |= initMutex(&ctx->allJobsCompleted_mutex); - pthreadError |= initCond(&ctx->allJobsCompleted_cond); - pthreadError |= initMutex(&ctx->jobWrite_mutex); - pthreadError |= initCond(&ctx->jobWrite_cond); - pthreadError |= initMutex(&ctx->compressionCompletion_mutex); - pthreadError |= initMutex(&ctx->createCompletion_mutex); - pthreadError |= initMutex(&ctx->writeCompletion_mutex); - pthreadError |= initMutex(&ctx->compressionLevel_mutex); - if (pthreadError) return pthreadError; - } - ctx->numJobs = numJobs; - ctx->jobReadyID = 0; - ctx->jobCompressedID = 0; - ctx->jobWriteID = 0; - ctx->lastDictSize = 0; - - - ctx->createWaitCompressionCompletion = 1; - ctx->compressWaitCreateCompletion = 1; - ctx->compressWaitWriteCompletion = 1; - ctx->writeWaitCompressionCompletion = 1; - ctx->createCompletion = 1; - ctx->writeCompletion = 1; - ctx->compressionCompletion = 1; - ctx->convergenceCounter = 0; - ctx->cooldown = 0; - - ctx->jobs = calloc(1, numJobs*sizeof(jobDescription)); - - if (!ctx->jobs) { - DISPLAY("Error: could not allocate space for jobs during context creation\n"); - return 1; - } - - /* initializing jobs */ - { - unsigned jobNum; - for (jobNum=0; jobNumjobs[jobNum]; - job->src.start = malloc(2 * FILE_CHUNK_SIZE); - job->dst.start = malloc(ZSTD_compressBound(FILE_CHUNK_SIZE)); - job->lastJobPlusOne = 0; - if (!job->src.start || !job->dst.start) { - DISPLAY("Could not allocate buffers for jobs\n"); - return 1; - } - job->src.capacity = FILE_CHUNK_SIZE; - job->dst.capacity = ZSTD_compressBound(FILE_CHUNK_SIZE); - } - } - - ctx->nextJobID = 0; - ctx->threadError = 0; - ctx->allJobsCompleted = 0; - - ctx->cctx = ZSTD_createCCtx(); - if (!ctx->cctx) { - DISPLAY("Error: could not allocate ZSTD_CCtx\n"); - return 1; - } - - ctx->input.filled = 0; - ctx->input.buffer.capacity = 2 * FILE_CHUNK_SIZE; - - ctx->input.buffer.start = malloc(ctx->input.buffer.capacity); - if (!ctx->input.buffer.start) { - DISPLAY("Error: could not allocate input buffer\n"); - return 1; - } - return 0; -} - -static adaptCCtx* createCCtx(unsigned numJobs) -{ - - adaptCCtx* const ctx = calloc(1, sizeof(adaptCCtx)); - if (ctx == NULL) { - DISPLAY("Error: could not allocate space for context\n"); - return NULL; - } - { - int const error = initCCtx(ctx, numJobs); - if (error) { - freeCCtx(ctx); - return NULL; - } - return ctx; - } -} - -static void signalErrorToThreads(adaptCCtx* ctx) -{ - ctx->threadError = 1; - pthread_mutex_lock(&ctx->jobReady_mutex.pMutex); - pthread_cond_signal(&ctx->jobReady_cond.pCond); - pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex); - - pthread_mutex_lock(&ctx->jobCompressed_mutex.pMutex); - pthread_cond_broadcast(&ctx->jobCompressed_cond.pCond); - pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex); - - pthread_mutex_lock(&ctx->jobWrite_mutex.pMutex); - pthread_cond_signal(&ctx->jobWrite_cond.pCond); - pthread_mutex_unlock(&ctx->jobWrite_mutex.pMutex); - - pthread_mutex_lock(&ctx->allJobsCompleted_mutex.pMutex); - pthread_cond_signal(&ctx->allJobsCompleted_cond.pCond); - pthread_mutex_unlock(&ctx->allJobsCompleted_mutex.pMutex); -} - -static void waitUntilAllJobsCompleted(adaptCCtx* ctx) -{ - if (!ctx) return; - pthread_mutex_lock(&ctx->allJobsCompleted_mutex.pMutex); - while (ctx->allJobsCompleted == 0 && !ctx->threadError) { - pthread_cond_wait(&ctx->allJobsCompleted_cond.pCond, &ctx->allJobsCompleted_mutex.pMutex); - } - pthread_mutex_unlock(&ctx->allJobsCompleted_mutex.pMutex); -} - -/* map completion percentages to values for changing compression level */ -static unsigned convertCompletionToChange(double completion) -{ - if (completion < CHANGE_BY_TWO_THRESHOLD) { - return 2; - } - else if (completion < CHANGE_BY_ONE_THRESHOLD) { - return 1; - } - else { - return 0; - } -} - -/* - * Compression level is changed depending on which part of the compression process is lagging - * Currently, three theads exist for job creation, compression, and file writing respectively. - * adaptCompressionLevel() increments or decrements compression level based on which of the threads is lagging - * job creation or file writing lag => increased compression level - * compression thread lag => decreased compression level - * detecting which thread is lagging is done by keeping track of how many calls each thread makes to pthread_cond_wait - */ -static void adaptCompressionLevel(adaptCCtx* ctx) -{ - double createWaitCompressionCompletion; - double compressWaitCreateCompletion; - double compressWaitWriteCompletion; - double writeWaitCompressionCompletion; - double const threshold = 0.00001; - unsigned prevCompressionLevel; - - pthread_mutex_lock(&ctx->compressionLevel_mutex.pMutex); - prevCompressionLevel = ctx->compressionLevel; - pthread_mutex_unlock(&ctx->compressionLevel_mutex.pMutex); - - - if (g_forceCompressionLevel) { - pthread_mutex_lock(&ctx->compressionLevel_mutex.pMutex); - ctx->compressionLevel = g_compressionLevel; - pthread_mutex_unlock(&ctx->compressionLevel_mutex.pMutex); - return; - } - - - DEBUG(2, "adapting compression level %u\n", prevCompressionLevel); - - /* read and reset completion measurements */ - pthread_mutex_lock(&ctx->compressionCompletion_mutex.pMutex); - DEBUG(2, "createWaitCompressionCompletion %f\n", ctx->createWaitCompressionCompletion); - DEBUG(2, "writeWaitCompressionCompletion %f\n", ctx->writeWaitCompressionCompletion); - createWaitCompressionCompletion = ctx->createWaitCompressionCompletion; - writeWaitCompressionCompletion = ctx->writeWaitCompressionCompletion; - pthread_mutex_unlock(&ctx->compressionCompletion_mutex.pMutex); - - pthread_mutex_lock(&ctx->writeCompletion_mutex.pMutex); - DEBUG(2, "compressWaitWriteCompletion %f\n", ctx->compressWaitWriteCompletion); - compressWaitWriteCompletion = ctx->compressWaitWriteCompletion; - pthread_mutex_unlock(&ctx->writeCompletion_mutex.pMutex); - - pthread_mutex_lock(&ctx->createCompletion_mutex.pMutex); - DEBUG(2, "compressWaitCreateCompletion %f\n", ctx->compressWaitCreateCompletion); - compressWaitCreateCompletion = ctx->compressWaitCreateCompletion; - pthread_mutex_unlock(&ctx->createCompletion_mutex.pMutex); - DEBUG(2, "convergence counter: %u\n", ctx->convergenceCounter); - - assert(g_minCLevel <= prevCompressionLevel && g_maxCLevel >= prevCompressionLevel); - - /* adaptation logic */ - if (ctx->cooldown) ctx->cooldown--; - - if ((1-createWaitCompressionCompletion > threshold || 1-writeWaitCompressionCompletion > threshold) && ctx->cooldown == 0) { - /* create or write waiting on compression */ - /* use whichever one waited less because it was slower */ - double const completion = MAX(createWaitCompressionCompletion, writeWaitCompressionCompletion); - unsigned const change = convertCompletionToChange(completion); - unsigned const boundChange = MIN(change, prevCompressionLevel - g_minCLevel); - if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) { - /* reset convergence counter, might have been a spike */ - ctx->convergenceCounter = 0; - DEBUG(2, "convergence counter reset, no change applied\n"); - } - else if (boundChange != 0) { - pthread_mutex_lock(&ctx->compressionLevel_mutex.pMutex); - ctx->compressionLevel -= boundChange; - pthread_mutex_unlock(&ctx->compressionLevel_mutex.pMutex); - ctx->cooldown = CLEVEL_DECREASE_COOLDOWN; - ctx->convergenceCounter = 1; - - DEBUG(2, "create or write threads waiting on compression, tried to decrease compression level by %u\n\n", boundChange); - } - } - else if (1-compressWaitWriteCompletion > threshold || 1-compressWaitCreateCompletion > threshold) { - /* compress waiting on write */ - double const completion = MIN(compressWaitWriteCompletion, compressWaitCreateCompletion); - unsigned const change = convertCompletionToChange(completion); - unsigned const boundChange = MIN(change, g_maxCLevel - prevCompressionLevel); - if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) { - /* reset convergence counter, might have been a spike */ - ctx->convergenceCounter = 0; - DEBUG(2, "convergence counter reset, no change applied\n"); - } - else if (boundChange != 0) { - pthread_mutex_lock(&ctx->compressionLevel_mutex.pMutex); - ctx->compressionLevel += boundChange; - pthread_mutex_unlock(&ctx->compressionLevel_mutex.pMutex); - ctx->cooldown = 0; - ctx->convergenceCounter = 1; - - DEBUG(2, "compress waiting on write or create, tried to increase compression level by %u\n\n", boundChange); - } - - } - - pthread_mutex_lock(&ctx->compressionLevel_mutex.pMutex); - if (ctx->compressionLevel == prevCompressionLevel) { - ctx->convergenceCounter++; - } - pthread_mutex_unlock(&ctx->compressionLevel_mutex.pMutex); -} - -static size_t getUseableDictSize(unsigned compressionLevel) -{ - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0); - unsigned const overlapLog = compressionLevel >= (unsigned)ZSTD_maxCLevel() ? 0 : 3; - size_t const overlapSize = 1 << (params.cParams.windowLog - overlapLog); - return overlapSize; -} - -static void* compressionThread(void* arg) -{ - adaptCCtx* const ctx = (adaptCCtx*)arg; - unsigned currJob = 0; - for ( ; ; ) { - unsigned const currJobIndex = currJob % ctx->numJobs; - jobDescription* const job = &ctx->jobs[currJobIndex]; - DEBUG(2, "starting compression for job %u\n", currJob); - - { - /* check if compression thread will have to wait */ - unsigned willWaitForCreate = 0; - unsigned willWaitForWrite = 0; - - pthread_mutex_lock(&ctx->jobReady_mutex.pMutex); - if (currJob + 1 > ctx->jobReadyID) willWaitForCreate = 1; - pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex); - - pthread_mutex_lock(&ctx->jobWrite_mutex.pMutex); - if (currJob - ctx->jobWriteID >= ctx->numJobs) willWaitForWrite = 1; - pthread_mutex_unlock(&ctx->jobWrite_mutex.pMutex); - - - pthread_mutex_lock(&ctx->createCompletion_mutex.pMutex); - if (willWaitForCreate) { - DEBUG(2, "compression will wait for create on job %u\n", currJob); - ctx->compressWaitCreateCompletion = ctx->createCompletion; - DEBUG(2, "create completion %f\n", ctx->compressWaitCreateCompletion); - - } - else { - ctx->compressWaitCreateCompletion = 1; - } - pthread_mutex_unlock(&ctx->createCompletion_mutex.pMutex); - - pthread_mutex_lock(&ctx->writeCompletion_mutex.pMutex); - if (willWaitForWrite) { - DEBUG(2, "compression will wait for write on job %u\n", currJob); - ctx->compressWaitWriteCompletion = ctx->writeCompletion; - DEBUG(2, "write completion %f\n", ctx->compressWaitWriteCompletion); - } - else { - ctx->compressWaitWriteCompletion = 1; - } - pthread_mutex_unlock(&ctx->writeCompletion_mutex.pMutex); - - } - - /* wait until job is ready */ - pthread_mutex_lock(&ctx->jobReady_mutex.pMutex); - while (currJob + 1 > ctx->jobReadyID && !ctx->threadError) { - pthread_cond_wait(&ctx->jobReady_cond.pCond, &ctx->jobReady_mutex.pMutex); - } - pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex); - - /* wait until job previously in this space is written */ - pthread_mutex_lock(&ctx->jobWrite_mutex.pMutex); - while (currJob - ctx->jobWriteID >= ctx->numJobs && !ctx->threadError) { - pthread_cond_wait(&ctx->jobWrite_cond.pCond, &ctx->jobWrite_mutex.pMutex); - } - pthread_mutex_unlock(&ctx->jobWrite_mutex.pMutex); - /* reset compression completion */ - pthread_mutex_lock(&ctx->compressionCompletion_mutex.pMutex); - ctx->compressionCompletion = 0; - pthread_mutex_unlock(&ctx->compressionCompletion_mutex.pMutex); - - /* adapt compression level */ - if (currJob) adaptCompressionLevel(ctx); - - pthread_mutex_lock(&ctx->compressionLevel_mutex.pMutex); - DEBUG(2, "job %u compressed with level %u\n", currJob, ctx->compressionLevel); - pthread_mutex_unlock(&ctx->compressionLevel_mutex.pMutex); - - /* compress the data */ - { - size_t const compressionBlockSize = ZSTD_BLOCKSIZE_MAX; /* 128 KB */ - unsigned cLevel; - unsigned blockNum = 0; - size_t remaining = job->src.size; - size_t srcPos = 0; - size_t dstPos = 0; - - pthread_mutex_lock(&ctx->compressionLevel_mutex.pMutex); - cLevel = ctx->compressionLevel; - pthread_mutex_unlock(&ctx->compressionLevel_mutex.pMutex); - - /* reset compressed size */ - job->compressedSize = 0; - DEBUG(2, "calling ZSTD_compressBegin()\n"); - /* begin compression */ - { - size_t const useDictSize = MIN(getUseableDictSize(cLevel), job->dictSize); - ZSTD_parameters params = ZSTD_getParams(cLevel, 0, useDictSize); - params.cParams.windowLog = 23; - { - size_t const initError = ZSTD_compressBegin_advanced(ctx->cctx, job->src.start + job->dictSize - useDictSize, useDictSize, params, 0); - size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_p_forceMaxWindow, 1); - if (ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) { - DISPLAY("Error: something went wrong while starting compression\n"); - signalErrorToThreads(ctx); - return arg; - } - } - } - DEBUG(2, "finished with ZSTD_compressBegin()\n"); - - do { - size_t const actualBlockSize = MIN(remaining, compressionBlockSize); - - /* continue compression */ - if (currJob != 0 || blockNum != 0) { /* not first block of first job flush/overwrite the frame header */ - size_t const hSize = ZSTD_compressContinue(ctx->cctx, job->dst.start + dstPos, job->dst.capacity - dstPos, job->src.start + job->dictSize + srcPos, 0); - if (ZSTD_isError(hSize)) { - DISPLAY("Error: something went wrong while continuing compression\n"); - job->compressedSize = hSize; - signalErrorToThreads(ctx); - return arg; - } - ZSTD_invalidateRepCodes(ctx->cctx); - } - { - size_t const ret = (job->lastJobPlusOne == currJob + 1 && remaining == actualBlockSize) ? - ZSTD_compressEnd (ctx->cctx, job->dst.start + dstPos, job->dst.capacity - dstPos, job->src.start + job->dictSize + srcPos, actualBlockSize) : - ZSTD_compressContinue(ctx->cctx, job->dst.start + dstPos, job->dst.capacity - dstPos, job->src.start + job->dictSize + srcPos, actualBlockSize); - if (ZSTD_isError(ret)) { - DISPLAY("Error: something went wrong during compression: %s\n", ZSTD_getErrorName(ret)); - signalErrorToThreads(ctx); - return arg; - } - job->compressedSize += ret; - remaining -= actualBlockSize; - srcPos += actualBlockSize; - dstPos += ret; - blockNum++; - - /* update completion */ - pthread_mutex_lock(&ctx->compressionCompletion_mutex.pMutex); - ctx->compressionCompletion = 1 - (double)remaining/job->src.size; - pthread_mutex_unlock(&ctx->compressionCompletion_mutex.pMutex); - } - } while (remaining != 0); - job->dst.size = job->compressedSize; - } - pthread_mutex_lock(&ctx->jobCompressed_mutex.pMutex); - ctx->jobCompressedID++; - pthread_cond_broadcast(&ctx->jobCompressed_cond.pCond); - pthread_mutex_unlock(&ctx->jobCompressed_mutex.pMutex); - if (job->lastJobPlusOne == currJob + 1 || ctx->threadError) { - /* finished compressing all jobs */ - break; - } - DEBUG(2, "finished compressing job %u\n", currJob); - currJob++; - } - return arg; -} - -static void displayProgress(unsigned cLevel, unsigned last) -{ - UTIL_time_t currTime = UTIL_getTime(); - if (!g_useProgressBar) return; - { double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_startTime, currTime) / 1000.0); - double const sizeMB = (double)g_streamedSize / (1 << 20); - double const avgCompRate = sizeMB * 1000 / timeElapsed; - fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate); - if (last) { - fprintf(stderr, "\n"); - } else { - fflush(stderr); - } } -} - -static void* outputThread(void* arg) -{ - outputThreadArg* const otArg = (outputThreadArg*)arg; - adaptCCtx* const ctx = otArg->ctx; - FILE* const dstFile = otArg->dstFile; - - unsigned currJob = 0; - for ( ; ; ) { - unsigned const currJobIndex = currJob % ctx->numJobs; - jobDescription* const job = &ctx->jobs[currJobIndex]; - unsigned willWaitForCompress = 0; - DEBUG(2, "starting write for job %u\n", currJob); - - pthread_mutex_lock(&ctx->jobCompressed_mutex.pMutex); - if (currJob + 1 > ctx->jobCompressedID) willWaitForCompress = 1; - pthread_mutex_unlock(&ctx->jobCompressed_mutex.pMutex); - - - pthread_mutex_lock(&ctx->compressionCompletion_mutex.pMutex); - if (willWaitForCompress) { - /* write thread is waiting on compression thread */ - ctx->writeWaitCompressionCompletion = ctx->compressionCompletion; - DEBUG(2, "writer thread waiting for nextJob: %u, writeWaitCompressionCompletion %f\n", currJob, ctx->writeWaitCompressionCompletion); - } - else { - ctx->writeWaitCompressionCompletion = 1; - } - pthread_mutex_unlock(&ctx->compressionCompletion_mutex.pMutex); - - pthread_mutex_lock(&ctx->jobCompressed_mutex.pMutex); - while (currJob + 1 > ctx->jobCompressedID && !ctx->threadError) { - pthread_cond_wait(&ctx->jobCompressed_cond.pCond, &ctx->jobCompressed_mutex.pMutex); - } - pthread_mutex_unlock(&ctx->jobCompressed_mutex.pMutex); - - /* reset write completion */ - pthread_mutex_lock(&ctx->writeCompletion_mutex.pMutex); - ctx->writeCompletion = 0; - pthread_mutex_unlock(&ctx->writeCompletion_mutex.pMutex); - - { - size_t const compressedSize = job->compressedSize; - size_t remaining = compressedSize; - if (ZSTD_isError(compressedSize)) { - DISPLAY("Error: an error occurred during compression\n"); - signalErrorToThreads(ctx); - return arg; - } - { - size_t const blockSize = MAX(compressedSize >> 7, 1 << 10); - size_t pos = 0; - for ( ; ; ) { - size_t const writeSize = MIN(remaining, blockSize); - size_t const ret = fwrite(job->dst.start + pos, 1, writeSize, dstFile); - if (ret != writeSize) break; - pos += ret; - remaining -= ret; - - /* update completion variable for writing */ - pthread_mutex_lock(&ctx->writeCompletion_mutex.pMutex); - ctx->writeCompletion = 1 - (double)remaining/compressedSize; - pthread_mutex_unlock(&ctx->writeCompletion_mutex.pMutex); - - if (remaining == 0) break; - } - if (pos != compressedSize) { - DISPLAY("Error: an error occurred during file write operation\n"); - signalErrorToThreads(ctx); - return arg; - } - } - } - { - unsigned cLevel; - pthread_mutex_lock(&ctx->compressionLevel_mutex.pMutex); - cLevel = ctx->compressionLevel; - pthread_mutex_unlock(&ctx->compressionLevel_mutex.pMutex); - displayProgress(cLevel, job->lastJobPlusOne == currJob + 1); - } - pthread_mutex_lock(&ctx->jobWrite_mutex.pMutex); - ctx->jobWriteID++; - pthread_cond_signal(&ctx->jobWrite_cond.pCond); - pthread_mutex_unlock(&ctx->jobWrite_mutex.pMutex); - - if (job->lastJobPlusOne == currJob + 1 || ctx->threadError) { - /* finished with all jobs */ - pthread_mutex_lock(&ctx->allJobsCompleted_mutex.pMutex); - ctx->allJobsCompleted = 1; - pthread_cond_signal(&ctx->allJobsCompleted_cond.pCond); - pthread_mutex_unlock(&ctx->allJobsCompleted_mutex.pMutex); - break; - } - DEBUG(2, "finished writing job %u\n", currJob); - currJob++; - - } - return arg; -} - -static int createCompressionJob(adaptCCtx* ctx, size_t srcSize, int last) -{ - unsigned const nextJob = ctx->nextJobID; - unsigned const nextJobIndex = nextJob % ctx->numJobs; - jobDescription* const job = &ctx->jobs[nextJobIndex]; - - - job->src.size = srcSize; - job->jobID = nextJob; - if (last) job->lastJobPlusOne = nextJob + 1; - { - /* swap buffer */ - void* const copy = job->src.start; - job->src.start = ctx->input.buffer.start; - ctx->input.buffer.start = copy; - } - job->dictSize = ctx->lastDictSize; - - ctx->nextJobID++; - /* if not on the last job, reuse data as dictionary in next job */ - if (!last) { - size_t const oldDictSize = ctx->lastDictSize; - memcpy(ctx->input.buffer.start, job->src.start + oldDictSize, srcSize); - ctx->lastDictSize = srcSize; - ctx->input.filled = srcSize; - } - - /* signal job ready */ - pthread_mutex_lock(&ctx->jobReady_mutex.pMutex); - ctx->jobReadyID++; - pthread_cond_signal(&ctx->jobReady_cond.pCond); - pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex); - - return 0; -} - -static int performCompression(adaptCCtx* ctx, FILE* const srcFile, outputThreadArg* otArg) -{ - /* early error check to exit */ - if (!ctx || !srcFile || !otArg) { - return 1; - } - - /* create output thread */ - { - pthread_t out; - if (pthread_create(&out, NULL, &outputThread, otArg)) { - DISPLAY("Error: could not create output thread\n"); - signalErrorToThreads(ctx); - return 1; - } - else if (pthread_detach(out)) { - DISPLAY("Error: could not detach output thread\n"); - signalErrorToThreads(ctx); - return 1; - } - } - - /* create compression thread */ - { - pthread_t compression; - if (pthread_create(&compression, NULL, &compressionThread, ctx)) { - DISPLAY("Error: could not create compression thread\n"); - signalErrorToThreads(ctx); - return 1; - } - else if (pthread_detach(compression)) { - DISPLAY("Error: could not detach compression thread\n"); - signalErrorToThreads(ctx); - return 1; - } - } - { - unsigned currJob = 0; - /* creating jobs */ - for ( ; ; ) { - size_t pos = 0; - size_t const readBlockSize = 1 << 15; - size_t remaining = FILE_CHUNK_SIZE; - unsigned const nextJob = ctx->nextJobID; - unsigned willWaitForCompress = 0; - DEBUG(2, "starting creation of job %u\n", currJob); - - pthread_mutex_lock(&ctx->jobCompressed_mutex.pMutex); - if (nextJob - ctx->jobCompressedID >= ctx->numJobs) willWaitForCompress = 1; - pthread_mutex_unlock(&ctx->jobCompressed_mutex.pMutex); - - pthread_mutex_lock(&ctx->compressionCompletion_mutex.pMutex); - if (willWaitForCompress) { - /* creation thread is waiting, take measurement of completion */ - ctx->createWaitCompressionCompletion = ctx->compressionCompletion; - DEBUG(2, "create thread waiting for nextJob: %u, createWaitCompressionCompletion %f\n", nextJob, ctx->createWaitCompressionCompletion); - } - else { - ctx->createWaitCompressionCompletion = 1; - } - pthread_mutex_unlock(&ctx->compressionCompletion_mutex.pMutex); - - /* wait until the job has been compressed */ - pthread_mutex_lock(&ctx->jobCompressed_mutex.pMutex); - while (nextJob - ctx->jobCompressedID >= ctx->numJobs && !ctx->threadError) { - pthread_cond_wait(&ctx->jobCompressed_cond.pCond, &ctx->jobCompressed_mutex.pMutex); - } - pthread_mutex_unlock(&ctx->jobCompressed_mutex.pMutex); - - /* reset create completion */ - pthread_mutex_lock(&ctx->createCompletion_mutex.pMutex); - ctx->createCompletion = 0; - pthread_mutex_unlock(&ctx->createCompletion_mutex.pMutex); - - while (remaining != 0 && !feof(srcFile)) { - size_t const ret = fread(ctx->input.buffer.start + ctx->input.filled + pos, 1, readBlockSize, srcFile); - if (ret != readBlockSize && !feof(srcFile)) { - /* error could not read correct number of bytes */ - DISPLAY("Error: problem occurred during read from src file\n"); - signalErrorToThreads(ctx); - return 1; - } - pos += ret; - remaining -= ret; - pthread_mutex_lock(&ctx->createCompletion_mutex.pMutex); - ctx->createCompletion = 1 - (double)remaining/((size_t)FILE_CHUNK_SIZE); - pthread_mutex_unlock(&ctx->createCompletion_mutex.pMutex); - } - if (remaining != 0 && !feof(srcFile)) { - DISPLAY("Error: problem occurred during read from src file\n"); - signalErrorToThreads(ctx); - return 1; - } - g_streamedSize += pos; - /* reading was fine, now create the compression job */ - { - int const last = feof(srcFile); - int const error = createCompressionJob(ctx, pos, last); - if (error != 0) { - signalErrorToThreads(ctx); - return error; - } - } - DEBUG(2, "finished creating job %u\n", currJob); - currJob++; - if (feof(srcFile)) { - break; - } - } - } - /* success -- created all jobs */ - return 0; -} - -static fcResources createFileCompressionResources(const char* const srcFilename, const char* const dstFilenameOrNull) -{ - fcResources fcr; - unsigned const stdinUsed = !strcmp(srcFilename, stdinmark); - FILE* const srcFile = stdinUsed ? stdin : fopen(srcFilename, "rb"); - const char* const outFilenameIntermediate = (stdinUsed && !dstFilenameOrNull) ? stdoutmark : dstFilenameOrNull; - const char* outFilename = outFilenameIntermediate; - char fileAndSuffix[MAX_PATH]; - size_t const numJobs = MAX_NUM_JOBS; - - memset(&fcr, 0, sizeof(fcr)); - - if (!outFilenameIntermediate) { - if (snprintf(fileAndSuffix, MAX_PATH, "%s.zst", srcFilename) + 1 > MAX_PATH) { - DISPLAY("Error: output filename is too long\n"); - return fcr; - } - outFilename = fileAndSuffix; - } - - { - unsigned const stdoutUsed = !strcmp(outFilename, stdoutmark); - FILE* const dstFile = stdoutUsed ? stdout : fopen(outFilename, "wb"); - fcr.otArg = malloc(sizeof(outputThreadArg)); - if (!fcr.otArg) { - DISPLAY("Error: could not allocate space for output thread argument\n"); - return fcr; - } - fcr.otArg->dstFile = dstFile; - } - /* checking for errors */ - if (!fcr.otArg->dstFile || !srcFile) { - DISPLAY("Error: some file(s) could not be opened\n"); - return fcr; - } - - /* creating context */ - fcr.ctx = createCCtx(numJobs); - fcr.otArg->ctx = fcr.ctx; - fcr.srcFile = srcFile; - return fcr; -} - -static int freeFileCompressionResources(fcResources* fcr) -{ - int ret = 0; - waitUntilAllJobsCompleted(fcr->ctx); - ret |= (fcr->srcFile != NULL) ? fclose(fcr->srcFile) : 0; - ret |= (fcr->ctx != NULL) ? freeCCtx(fcr->ctx) : 0; - if (fcr->otArg) { - ret |= (fcr->otArg->dstFile != stdout) ? fclose(fcr->otArg->dstFile) : 0; - free(fcr->otArg); - /* no need to freeCCtx() on otArg->ctx because it should be the same context */ - } - return ret; -} - -static int compressFilename(const char* const srcFilename, const char* const dstFilenameOrNull) -{ - int ret = 0; - fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull); - g_streamedSize = 0; - ret |= performCompression(fcr.ctx, fcr.srcFile, fcr.otArg); - ret |= freeFileCompressionResources(&fcr); - return ret; -} - -static int compressFilenames(const char** filenameTable, unsigned numFiles, unsigned forceStdout) -{ - int ret = 0; - unsigned fileNum; - for (fileNum=0; fileNum MAX_UINT */ -static unsigned readU32FromChar(const char** stringPtr) -{ - unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; - if ((**stringPtr=='K') || (**stringPtr=='M')) { - result <<= 10; - if (**stringPtr=='M') result <<= 10; - (*stringPtr)++ ; - if (**stringPtr=='i') (*stringPtr)++; - if (**stringPtr=='B') (*stringPtr)++; - } - return result; -} - -static void help(const char* progPath) -{ - PRINT("Usage:\n"); - PRINT(" %s [options] [file(s)]\n", progPath); - PRINT("\n"); - PRINT("Options:\n"); - PRINT(" -oFILE : specify the output file name\n"); - PRINT(" -i# : provide initial compression level -- default %d, must be in the range [L, U] where L and U are bound values (see below for defaults)\n", DEFAULT_COMPRESSION_LEVEL); - PRINT(" -h : display help/information\n"); - PRINT(" -f : force the compression level to stay constant\n"); - PRINT(" -c : force write to stdout\n"); - PRINT(" -p : hide progress bar\n"); - PRINT(" -q : quiet mode -- do not show progress bar or other information\n"); - PRINT(" -l# : provide lower bound for compression level -- default 1\n"); - PRINT(" -u# : provide upper bound for compression level -- default %u\n", ZSTD_maxCLevel()); -} -/* return 0 if successful, else return error */ -int main(int argCount, const char* argv[]) -{ - const char* outFilename = NULL; - const char** filenameTable = (const char**)malloc(argCount*sizeof(const char*)); - unsigned filenameIdx = 0; - unsigned forceStdout = 0; - unsigned providedInitialCLevel = 0; - int ret = 0; - int argNum; - filenameTable[0] = stdinmark; - g_maxCLevel = ZSTD_maxCLevel(); - - if (filenameTable == NULL) { - DISPLAY("Error: could not allocate sapce for filename table.\n"); - return 1; - } - - for (argNum=1; argNum 1) { - switch (argument[1]) { - case 'o': - argument += 2; - outFilename = argument; - break; - case 'i': - argument += 2; - g_compressionLevel = readU32FromChar(&argument); - providedInitialCLevel = 1; - break; - case 'h': - help(argv[0]); - goto _main_exit; - case 'p': - g_useProgressBar = 0; - break; - case 'c': - forceStdout = 1; - outFilename = stdoutmark; - break; - case 'f': - g_forceCompressionLevel = 1; - break; - case 'q': - g_useProgressBar = 0; - g_displayLevel = 0; - break; - case 'l': - argument += 2; - g_minCLevel = readU32FromChar(&argument); - break; - case 'u': - argument += 2; - g_maxCLevel = readU32FromChar(&argument); - break; - default: - DISPLAY("Error: invalid argument provided\n"); - ret = 1; - goto _main_exit; - } - continue; - } - - /* regular files to be compressed */ - filenameTable[filenameIdx++] = argument; - } - - /* check initial, max, and min compression levels */ - { - unsigned const minMaxInconsistent = g_minCLevel > g_maxCLevel; - unsigned const initialNotInRange = g_minCLevel > g_compressionLevel || g_maxCLevel < g_compressionLevel; - if (minMaxInconsistent || (initialNotInRange && providedInitialCLevel)) { - DISPLAY("Error: provided compression level parameters are invalid\n"); - ret = 1; - goto _main_exit; - } - else if (initialNotInRange) { - g_compressionLevel = g_minCLevel; - } - } - - /* error checking with number of files */ - if (filenameIdx > 1 && (outFilename != NULL && strcmp(outFilename, stdoutmark))) { - DISPLAY("Error: multiple input files provided, cannot use specified output file\n"); - ret = 1; - goto _main_exit; - } - - /* compress files */ - if (filenameIdx <= 1) { - ret |= compressFilename(filenameTable[0], outFilename); - } - else { - ret |= compressFilenames(filenameTable, filenameIdx, forceStdout); - } -_main_exit: - free(filenameTable); - return ret; -} diff --git a/sys/contrib/zstd/contrib/adaptive-compression/datagencli.c b/sys/contrib/zstd/contrib/adaptive-compression/datagencli.c deleted file mode 100644 index bf9601f2097..00000000000 --- a/sys/contrib/zstd/contrib/adaptive-compression/datagencli.c +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - - -/*-************************************ -* Dependencies -**************************************/ -#include "util.h" /* Compiler options */ -#include /* fprintf, stderr */ -#include "datagen.h" /* RDG_generate */ - - -/*-************************************ -* Constants -**************************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define SIZE_DEFAULT ((64 KB) + 1) -#define SEED_DEFAULT 0 -#define COMPRESSIBILITY_DEFAULT 50 - - -/*-************************************ -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned displayLevel = 2; - - -/*-******************************************************* -* Command line -*********************************************************/ -static int usage(const char* programName) -{ - DISPLAY( "Compressible data generator\n"); - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); - DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); - DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", - COMPRESSIBILITY_DEFAULT); - DISPLAY( " -h : display help and exit\n"); - return 0; -} - - -int main(int argc, const char** argv) -{ - unsigned probaU32 = COMPRESSIBILITY_DEFAULT; - double litProba = 0.0; - U64 size = SIZE_DEFAULT; - U32 seed = SEED_DEFAULT; - const char* const programName = argv[0]; - - int argNb; - for(argNb=1; argNb='0') && (*argument<='9')) - size *= 10, size += *argument++ - '0'; - if (*argument=='K') { size <<= 10; argument++; } - if (*argument=='M') { size <<= 20; argument++; } - if (*argument=='G') { size <<= 30; argument++; } - if (*argument=='B') { argument++; } - break; - case 's': - argument++; - seed=0; - while ((*argument>='0') && (*argument<='9')) - seed *= 10, seed += *argument++ - '0'; - break; - case 'P': - argument++; - probaU32 = 0; - while ((*argument>='0') && (*argument<='9')) - probaU32 *= 10, probaU32 += *argument++ - '0'; - if (probaU32>100) probaU32 = 100; - break; - case 'L': /* hidden argument : Literal distribution probability */ - argument++; - litProba=0.; - while ((*argument>='0') && (*argument<='9')) - litProba *= 10, litProba += *argument++ - '0'; - if (litProba>100.) litProba=100.; - litProba /= 100.; - break; - case 'v': - displayLevel = 4; - argument++; - break; - default: - return usage(programName); - } - } } } /* for(argNb=1; argNb tmp -./adapt -otmp.zst tmp -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s2 -g500MB > tmp -./adapt -otmp.zst tmp -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s3 -g250MB > tmp -./adapt -otmp.zst tmp -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s4 -g125MB > tmp -./adapt -otmp.zst tmp -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s5 -g50MB > tmp -./adapt -otmp.zst tmp -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s6 -g25MB > tmp -./adapt -otmp.zst tmp -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s7 -g10MB > tmp -./adapt -otmp.zst tmp -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s8 -g5MB > tmp -./adapt -otmp.zst tmp -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s9 -g500KB > tmp -./adapt -otmp.zst tmp -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -echo -e "\ncorrectness tests -- streaming" -./datagen -s10 -g1GB > tmp -cat tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s11 -g100MB > tmp -cat tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s12 -g10MB > tmp -cat tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s13 -g1MB > tmp -cat tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s14 -g100KB > tmp -cat tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s15 -g10KB > tmp -cat tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -echo -e "\ncorrectness tests -- read limit" -./datagen -s16 -g1GB > tmp -pv -L 50m -q tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s17 -g100MB > tmp -pv -L 50m -q tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s18 -g10MB > tmp -pv -L 50m -q tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s19 -g1MB > tmp -pv -L 50m -q tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s20 -g100KB > tmp -pv -L 50m -q tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s21 -g10KB > tmp -pv -L 50m -q tmp | ./adapt > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -echo -e "\ncorrectness tests -- write limit" -./datagen -s22 -g1GB > tmp -pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s23 -g100MB > tmp -pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s24 -g10MB > tmp -pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s25 -g1MB > tmp -pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s26 -g100KB > tmp -pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s27 -g10KB > tmp -pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -echo -e "\ncorrectness tests -- read and write limits" -./datagen -s28 -g1GB > tmp -pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s29 -g100MB > tmp -pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s30 -g10MB > tmp -pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s31 -g1MB > tmp -pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s32 -g100KB > tmp -pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s33 -g10KB > tmp -pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -echo -e "\ncorrectness tests -- forced compression level" -./datagen -s34 -g1GB > tmp -./adapt tmp -otmp.zst -i11 -f -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s35 -g100MB > tmp -./adapt tmp -otmp.zst -i11 -f -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s36 -g10MB > tmp -./adapt tmp -otmp.zst -i11 -f -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s37 -g1MB > tmp -./adapt tmp -otmp.zst -i11 -f -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s38 -g100KB > tmp -./adapt tmp -otmp.zst -i11 -f -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -./datagen -s39 -g10KB > tmp -./adapt tmp -otmp.zst -i11 -f -zstd -d tmp.zst -o tmp2 -diff -s -q tmp tmp2 -rm tmp* - -echo -e "\ncorrectness tests -- window size test" -./datagen -s39 -g1GB | pv -L 25m -q | ./adapt -i1 | pv -q > tmp.zst -zstd -d tmp.zst -rm tmp* - -echo -e "\ncorrectness tests -- testing bounds" -./datagen -s40 -g1GB | pv -L 25m -q | ./adapt -i1 -u4 | pv -q > tmp.zst -rm tmp* - -./datagen -s41 -g1GB | ./adapt -i14 -l4 > tmp.zst -rm tmp* -make clean diff --git a/sys/contrib/zstd/contrib/adaptive-compression/test-performance.sh b/sys/contrib/zstd/contrib/adaptive-compression/test-performance.sh deleted file mode 100755 index 958cb3cc8cb..00000000000 --- a/sys/contrib/zstd/contrib/adaptive-compression/test-performance.sh +++ /dev/null @@ -1,59 +0,0 @@ -echo "testing time -- no limits set" -./datagen -s1 -g1GB > tmp -time ./adapt -otmp1.zst tmp -time zstd -1 -o tmp2.zst tmp -rm tmp* - -./datagen -s2 -g2GB > tmp -time ./adapt -otmp1.zst tmp -time zstd -1 -o tmp2.zst tmp -rm tmp* - -./datagen -s3 -g4GB > tmp -time ./adapt -otmp1.zst tmp -time zstd -1 -o tmp2.zst tmp -rm tmp* - -echo -e "\ntesting compression ratio -- no limits set" -./datagen -s4 -g1GB > tmp -time ./adapt -otmp1.zst tmp -time zstd -1 -o tmp2.zst tmp -ls -l tmp1.zst tmp2.zst -rm tmp* - -./datagen -s5 -g2GB > tmp -time ./adapt -otmp1.zst tmp -time zstd -1 -o tmp2.zst tmp -ls -l tmp1.zst tmp2.zst -rm tmp* - -./datagen -s6 -g4GB > tmp -time ./adapt -otmp1.zst tmp -time zstd -1 -o tmp2.zst tmp -ls -l tmp1.zst tmp2.zst -rm tmp* - -echo e "\ntesting performance at various compression levels -- no limits set" -./datagen -s7 -g1GB > tmp -echo "adapt" -time ./adapt -i5 -f tmp -otmp1.zst -echo "zstdcli" -time zstd -5 tmp -o tmp2.zst -ls -l tmp1.zst tmp2.zst -rm tmp* - -./datagen -s8 -g1GB > tmp -echo "adapt" -time ./adapt -i10 -f tmp -otmp1.zst -echo "zstdcli" -time zstd -10 tmp -o tmp2.zst -ls -l tmp1.zst tmp2.zst -rm tmp* - -./datagen -s9 -g1GB > tmp -echo "adapt" -time ./adapt -i15 -f tmp -otmp1.zst -echo "zstdcli" -time zstd -15 tmp -o tmp2.zst -ls -l tmp1.zst tmp2.zst -rm tmp* diff --git a/sys/contrib/zstd/contrib/cleanTabs b/sys/contrib/zstd/contrib/cleanTabs deleted file mode 100755 index 215913a90ac..00000000000 --- a/sys/contrib/zstd/contrib/cleanTabs +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -sed -i '' $'s/\t/ /g' ../lib/**/*.{h,c} ../programs/*.{h,c} ../tests/*.c ./**/*.{h,cpp} ../examples/*.c ../zlibWrapper/*.{h,c} diff --git a/sys/contrib/zstd/contrib/gen_html/.gitignore b/sys/contrib/zstd/contrib/gen_html/.gitignore deleted file mode 100644 index 34461142872..00000000000 --- a/sys/contrib/zstd/contrib/gen_html/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# make artefact -gen_html -zstd_manual.html diff --git a/sys/contrib/zstd/contrib/gen_html/Makefile b/sys/contrib/zstd/contrib/gen_html/Makefile deleted file mode 100644 index 63598b8d718..00000000000 --- a/sys/contrib/zstd/contrib/gen_html/Makefile +++ /dev/null @@ -1,51 +0,0 @@ -# ################################################################ -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# ################################################################ - -CXXFLAGS ?= -O3 -CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment -CXXFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(CXXFLAGS) $(LDFLAGS) - -ZSTDAPI = ../../lib/zstd.h -ZSTDMANUAL = ../../doc/zstd_manual.html -LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)` -LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)` -LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)` -LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) -LIBVER := $(shell echo $(LIBVER_SCRIPT)) - - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -else -EXT = -endif - - -.PHONY: default -default: gen_html - -.PHONY: all -all: manual - -gen_html: gen_html.cpp - $(CXX) $(FLAGS) $^ -o $@$(EXT) - -$(ZSTDMANUAL): gen_html $(ZSTDAPI) - echo "Update zstd manual in /doc" - ./gen_html $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL) - -.PHONY: manual -manual: gen_html $(ZSTDMANUAL) - -.PHONY: clean -clean: - @$(RM) gen_html$(EXT) - @echo Cleaning completed diff --git a/sys/contrib/zstd/contrib/gen_html/README.md b/sys/contrib/zstd/contrib/gen_html/README.md deleted file mode 100644 index 63a4caa2506..00000000000 --- a/sys/contrib/zstd/contrib/gen_html/README.md +++ /dev/null @@ -1,31 +0,0 @@ -gen_html - a program for automatic generation of zstd manual -============================================================ - -#### Introduction - -This simple C++ program generates a single-page HTML manual from `zstd.h`. - -The format of recognized comment blocks is following: -- comments of type `/*!` mean: this is a function declaration; switch comments with declarations -- comments of type `/**` and `/*-` mean: this is a comment; use a `

` header for the first line -- comments of type `/*=` and `/**=` mean: use a `

` header and show also all functions until first empty line -- comments of type `/*X` where `X` is different from above-mentioned are ignored - -Moreover: -- `ZSTDLIB_API` is removed to improve readability -- `typedef` are detected and included even if uncommented -- comments of type `/**<` and `/*!<` are detected and only function declaration is highlighted (bold) - - -#### Usage - -The program requires 3 parameters: -``` -gen_html [zstd_version] [input_file] [output_html] -``` - -To compile program and generate zstd manual we have used: -``` -make -./gen_html.exe 1.1.1 ../../lib/zstd.h zstd_manual.html -``` diff --git a/sys/contrib/zstd/contrib/gen_html/gen-zstd-manual.sh b/sys/contrib/zstd/contrib/gen_html/gen-zstd-manual.sh deleted file mode 100755 index 57a8b6ea512..00000000000 --- a/sys/contrib/zstd/contrib/gen_html/gen-zstd-manual.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -LIBVER_MAJOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h` -LIBVER_MINOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h` -LIBVER_PATCH_SCRIPT=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h` -LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT - -echo ZSTD_VERSION=$LIBVER_SCRIPT -./gen_html $LIBVER_SCRIPT ../../lib/zstd.h ./zstd_manual.html diff --git a/sys/contrib/zstd/contrib/gen_html/gen_html.cpp b/sys/contrib/zstd/contrib/gen_html/gen_html.cpp deleted file mode 100644 index 90d5b21a3aa..00000000000 --- a/sys/contrib/zstd/contrib/gen_html/gen_html.cpp +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (c) 2016-present, Przemyslaw Skibinski, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -#include -#include -#include -#include -using namespace std; - - -/* trim string at the beginning and at the end */ -void trim(string& s, string characters) -{ - size_t p = s.find_first_not_of(characters); - s.erase(0, p); - - p = s.find_last_not_of(characters); - if (string::npos != p) - s.erase(p+1); -} - - -/* trim C++ style comments */ -void trim_comments(string &s) -{ - size_t spos, epos; - - spos = s.find("/*"); - epos = s.find("*/"); - s = s.substr(spos+3, epos-(spos+3)); -} - - -/* get lines until a given terminator */ -vector get_lines(vector& input, int& linenum, string terminator) -{ - vector out; - string line; - size_t epos; - - while ((size_t)linenum < input.size()) { - line = input[linenum]; - - if (terminator.empty() && line.empty()) { linenum--; break; } - - epos = line.find(terminator); - if (!terminator.empty() && epos!=string::npos) { - out.push_back(line); - break; - } - out.push_back(line); - linenum++; - } - return out; -} - - -/* print line with ZSTDLIB_API removed and C++ comments not bold */ -void print_line(stringstream &sout, string line) -{ - size_t spos; - - if (line.substr(0,12) == "ZSTDLIB_API ") line = line.substr(12); - spos = line.find("/*"); - if (spos!=string::npos) { - sout << line.substr(0, spos); - sout << "" << line.substr(spos) << "" << endl; - } else { - // fprintf(stderr, "lines=%s\n", line.c_str()); - sout << line << endl; - } -} - - -int main(int argc, char *argv[]) { - char exclam; - int linenum, chapter = 1; - vector input, lines, comments, chapters; - string line, version; - size_t spos, l; - stringstream sout; - ifstream istream; - ofstream ostream; - - if (argc < 4) { - cout << "usage: " << argv[0] << " [zstd_version] [input_file] [output_html]" << endl; - return 1; - } - - version = "zstd " + string(argv[1]) + " Manual"; - - istream.open(argv[2], ifstream::in); - if (!istream.is_open()) { - cout << "Error opening file " << argv[2] << endl; - return 1; - } - - ostream.open(argv[3], ifstream::out); - if (!ostream.is_open()) { - cout << "Error opening file " << argv[3] << endl; - return 1; - } - - while (getline(istream, line)) { - input.push_back(line); - } - - for (linenum=0; (size_t)linenum < input.size(); linenum++) { - line = input[linenum]; - - /* typedefs are detected and included even if uncommented */ - if (line.substr(0,7) == "typedef" && line.find("{")!=string::npos) { - lines = get_lines(input, linenum, "}"); - sout << "
";
-            for (l=0; l

" << endl; - continue; - } - - /* comments of type /**< and /*!< are detected and only function declaration is highlighted (bold) */ - if ((line.find("/**<")!=string::npos || line.find("/*!<")!=string::npos) && line.find("*/")!=string::npos) { - sout << "
";
-            print_line(sout, line);
-            sout << "

" << endl; - continue; - } - - spos = line.find("/**="); - if (spos==string::npos) { - spos = line.find("/*!"); - if (spos==string::npos) - spos = line.find("/**"); - if (spos==string::npos) - spos = line.find("/*-"); - if (spos==string::npos) - spos = line.find("/*="); - if (spos==string::npos) - continue; - exclam = line[spos+2]; - } - else exclam = '='; - - comments = get_lines(input, linenum, "*/"); - if (!comments.empty()) comments[0] = line.substr(spos+3); - if (!comments.empty()) comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/")); - for (l=0; l"; - for (l=0; l

"; - for (l=0; l
" << endl << endl; - } else if (exclam == '=') { /* comments of type /*= and /**= mean: use a

header and show also all functions until first empty line */ - trim(comments[0], " "); - sout << "

" << comments[0] << "

";
-            for (l=1; l
";
-            lines = get_lines(input, ++linenum, "");
-            for (l=0; l
" << endl; - } else { /* comments of type /** and /*- mean: this is a comment; use a

header for the first line */ - if (comments.empty()) continue; - - trim(comments[0], " "); - sout << "

" << comments[0] << "

";
-            chapters.push_back(comments[0]);
-            chapter++;
-
-            for (l=1; l 1)
-                sout << "
" << endl << endl; - else - sout << "
" << endl << endl; - } - } - - ostream << "\n\n\n" << version << "\n\n" << endl; - ostream << "

" << version << "

\n"; - - ostream << "
\n

Contents

\n
    \n"; - for (size_t i=0; i" << chapters[i].c_str() << "\n"; - ostream << "
\n
\n"; - - ostream << sout.str(); - ostream << "" << endl << "" << endl; - - return 0; -} diff --git a/sys/contrib/zstd/contrib/meson/README b/sys/contrib/zstd/contrib/meson/README deleted file mode 100644 index 0b5331e6dbf..00000000000 --- a/sys/contrib/zstd/contrib/meson/README +++ /dev/null @@ -1,3 +0,0 @@ -This Meson project is provided with no guarantee and maintained by Dima Krasner . - -It outputs one libzstd, either shared or static, depending on default_library. diff --git a/sys/contrib/zstd/contrib/meson/meson.build b/sys/contrib/zstd/contrib/meson/meson.build deleted file mode 100644 index 079c045a117..00000000000 --- a/sys/contrib/zstd/contrib/meson/meson.build +++ /dev/null @@ -1,142 +0,0 @@ -project('zstd', 'c', license: 'BSD') - -libm = meson.get_compiler('c').find_library('m', required: true) - -lib_dir = join_paths('..', '..', 'lib') -common_dir = join_paths(lib_dir, 'common') -compress_dir = join_paths(lib_dir, 'compress') -decompress_dir = join_paths(lib_dir, 'decompress') -dictbuilder_dir = join_paths(lib_dir, 'dictBuilder') -deprecated_dir = join_paths(lib_dir, 'deprecated') - -libzstd_srcs = [ - join_paths(common_dir, 'entropy_common.c'), - join_paths(common_dir, 'fse_decompress.c'), - join_paths(common_dir, 'threading.c'), - join_paths(common_dir, 'pool.c'), - join_paths(common_dir, 'zstd_common.c'), - join_paths(common_dir, 'error_private.c'), - join_paths(common_dir, 'xxhash.c'), - join_paths(compress_dir, 'fse_compress.c'), - join_paths(compress_dir, 'huf_compress.c'), - join_paths(compress_dir, 'zstd_compress.c'), - join_paths(compress_dir, 'zstd_fast.c'), - join_paths(compress_dir, 'zstd_double_fast.c'), - join_paths(compress_dir, 'zstd_lazy.c'), - join_paths(compress_dir, 'zstd_opt.c'), - join_paths(compress_dir, 'zstd_ldm.c'), - join_paths(compress_dir, 'zstdmt_compress.c'), - join_paths(decompress_dir, 'huf_decompress.c'), - join_paths(decompress_dir, 'zstd_decompress.c'), - join_paths(dictbuilder_dir, 'cover.c'), - join_paths(dictbuilder_dir, 'divsufsort.c'), - join_paths(dictbuilder_dir, 'zdict.c'), - join_paths(deprecated_dir, 'zbuff_common.c'), - join_paths(deprecated_dir, 'zbuff_compress.c'), - join_paths(deprecated_dir, 'zbuff_decompress.c') -] - -libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)] - -legacy = get_option('legacy_support') -if legacy == '0' - legacy = 'false' -endif -if legacy != 'false' - if legacy == 'true' - legacy = '1' - endif - #See ZSTD_LEGACY_SUPPORT of programs/README.md - message('Enabling legacy support back to version 0.' + legacy) - legacy_int = legacy.to_int() - if legacy_int > 7 - legacy_int = 7 - endif - libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=' + legacy] - - legacy_dir = join_paths(lib_dir, 'legacy') - libzstd_includes += [include_directories(legacy_dir)] - if legacy_int <= 1 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v01.c') - endif - if legacy_int <= 2 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v02.c') - endif - if legacy_int <= 3 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v03.c') - endif - if legacy_int <= 4 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v04.c') - endif - if legacy_int <= 5 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v05.c') - endif - if legacy_int <= 6 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v06.c') - endif - if legacy_int <= 7 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v07.c') - endif -else - libzstd_cflags = [] -endif - -if get_option('multithread') - message('Enabling multi-threading support') - add_global_arguments('-DZSTD_MULTITHREAD', language: 'c') - libzstd_deps = [dependency('threads')] -else - libzstd_deps = [] -endif - -libzstd = library('zstd', - libzstd_srcs, - include_directories: libzstd_includes, - c_args: libzstd_cflags, - dependencies: libzstd_deps, - install: true, - soversion: '1', - ) - -programs_dir = join_paths('..', '..', 'programs') - -zstd = executable('zstd', - join_paths(programs_dir, 'bench.c'), - join_paths(programs_dir, 'datagen.c'), - join_paths(programs_dir, 'dibio.c'), - join_paths(programs_dir, 'fileio.c'), - join_paths(programs_dir, 'zstdcli.c'), - include_directories: libzstd_includes, - c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'], - link_with: libzstd, - install: true) - -tests_dir = join_paths('..', '..', 'tests') -datagen_c = join_paths(programs_dir, 'datagen.c') -test_includes = libzstd_includes + [include_directories(programs_dir)] - -fullbench = executable('fullbench', - datagen_c, join_paths(tests_dir, 'fullbench.c'), - include_directories: test_includes, - link_with: libzstd) -test('fullbench', fullbench) - -fuzzer = executable('fuzzer', - datagen_c, join_paths(tests_dir, 'fuzzer.c'), - include_directories: test_includes, - link_with: libzstd) -test('fuzzer', fuzzer) - -if target_machine.system() != 'windows' - paramgrill = executable('paramgrill', - datagen_c, join_paths(tests_dir, 'paramgrill.c'), - include_directories: test_includes, - link_with: libzstd, - dependencies: libm) - test('paramgrill', paramgrill) - - datagen = executable('datagen', - datagen_c, join_paths(tests_dir, 'datagencli.c'), - include_directories: test_includes, - link_with: libzstd) -endif diff --git a/sys/contrib/zstd/contrib/meson/meson_options.txt b/sys/contrib/zstd/contrib/meson/meson_options.txt deleted file mode 100644 index 99845c8aa51..00000000000 --- a/sys/contrib/zstd/contrib/meson/meson_options.txt +++ /dev/null @@ -1,3 +0,0 @@ -option('multithread', type: 'boolean', value: false) -option('legacy_support', type: 'string', value: '4', - description: 'True or false, or 7 to 1 for v0.7+ to v0.1+.') diff --git a/sys/contrib/zstd/contrib/pzstd/.gitignore b/sys/contrib/zstd/contrib/pzstd/.gitignore deleted file mode 100644 index 84e68fb07d5..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# compilation result -pzstd diff --git a/sys/contrib/zstd/contrib/pzstd/BUCK b/sys/contrib/zstd/contrib/pzstd/BUCK deleted file mode 100644 index d04eeedd8a1..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/BUCK +++ /dev/null @@ -1,72 +0,0 @@ -cxx_library( - name='libpzstd', - visibility=['PUBLIC'], - header_namespace='', - exported_headers=[ - 'ErrorHolder.h', - 'Logging.h', - 'Pzstd.h', - ], - headers=[ - 'SkippableFrame.h', - ], - srcs=[ - 'Pzstd.cpp', - 'SkippableFrame.cpp', - ], - deps=[ - ':options', - '//contrib/pzstd/utils:utils', - '//lib:mem', - '//lib:zstd', - ], -) - -cxx_library( - name='options', - visibility=['PUBLIC'], - header_namespace='', - exported_headers=['Options.h'], - srcs=['Options.cpp'], - deps=[ - '//contrib/pzstd/utils:scope_guard', - '//lib:zstd', - '//programs:util', - ], -) - -cxx_binary( - name='pzstd', - visibility=['PUBLIC'], - srcs=['main.cpp'], - deps=[ - ':libpzstd', - ':options', - ], -) - -# Must run "make googletest" first -cxx_library( - name='gtest', - srcs=glob([ - 'googletest/googletest/src/gtest-all.cc', - 'googletest/googlemock/src/gmock-all.cc', - 'googletest/googlemock/src/gmock_main.cc', - ]), - header_namespace='', - exported_headers=subdir_glob([ - ('googletest/googletest/include', '**/*.h'), - ('googletest/googlemock/include', '**/*.h'), - ]), - headers=subdir_glob([ - ('googletest/googletest', 'src/*.cc'), - ('googletest/googletest', 'src/*.h'), - ('googletest/googlemock', 'src/*.cc'), - ('googletest/googlemock', 'src/*.h'), - ]), - platform_linker_flags=[ - ('android', []), - ('', ['-lpthread']), - ], - visibility=['PUBLIC'], -) diff --git a/sys/contrib/zstd/contrib/pzstd/ErrorHolder.h b/sys/contrib/zstd/contrib/pzstd/ErrorHolder.h deleted file mode 100644 index 829651c5961..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/ErrorHolder.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include -#include -#include -#include - -namespace pzstd { - -// Coordinates graceful shutdown of the pzstd pipeline -class ErrorHolder { - std::atomic error_; - std::string message_; - - public: - ErrorHolder() : error_(false) {} - - bool hasError() noexcept { - return error_.load(); - } - - void setError(std::string message) noexcept { - // Given multiple possibly concurrent calls, exactly one will ever succeed. - bool expected = false; - if (error_.compare_exchange_strong(expected, true)) { - message_ = std::move(message); - } - } - - bool check(bool predicate, std::string message) noexcept { - if (!predicate) { - setError(std::move(message)); - } - return !hasError(); - } - - std::string getError() noexcept { - error_.store(false); - return std::move(message_); - } - - ~ErrorHolder() { - assert(!hasError()); - } -}; -} diff --git a/sys/contrib/zstd/contrib/pzstd/Logging.h b/sys/contrib/zstd/contrib/pzstd/Logging.h deleted file mode 100644 index 16a63932c0a..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/Logging.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include -#include - -namespace pzstd { - -constexpr int ERROR = 1; -constexpr int INFO = 2; -constexpr int DEBUG = 3; -constexpr int VERBOSE = 4; - -class Logger { - std::mutex mutex_; - FILE* out_; - const int level_; - - using Clock = std::chrono::system_clock; - Clock::time_point lastUpdate_; - std::chrono::milliseconds refreshRate_; - - public: - explicit Logger(int level, FILE* out = stderr) - : out_(out), level_(level), lastUpdate_(Clock::now()), - refreshRate_(150) {} - - - bool logsAt(int level) { - return level <= level_; - } - - template - void operator()(int level, const char *fmt, Args... args) { - if (level > level_) { - return; - } - std::lock_guard lock(mutex_); - std::fprintf(out_, fmt, args...); - } - - template - void update(int level, const char *fmt, Args... args) { - if (level > level_) { - return; - } - std::lock_guard lock(mutex_); - auto now = Clock::now(); - if (now - lastUpdate_ > refreshRate_) { - lastUpdate_ = now; - std::fprintf(out_, "\r"); - std::fprintf(out_, fmt, args...); - } - } - - void clear(int level) { - if (level > level_) { - return; - } - std::lock_guard lock(mutex_); - std::fprintf(out_, "\r%79s\r", ""); - } -}; - -} diff --git a/sys/contrib/zstd/contrib/pzstd/Makefile b/sys/contrib/zstd/contrib/pzstd/Makefile deleted file mode 100644 index 40531e21653..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/Makefile +++ /dev/null @@ -1,269 +0,0 @@ -# ################################################################ -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# ################################################################ - -# Standard variables for installation -DESTDIR ?= -PREFIX ?= /usr/local -BINDIR := $(DESTDIR)$(PREFIX)/bin - -ZSTDDIR = ../../lib -PROGDIR = ../../programs - -# External program to use to run tests, e.g. qemu or valgrind -TESTPROG ?= -# Flags to pass to the tests -TESTFLAGS ?= - -# We use gcc/clang to generate the header dependencies of files -DEPFLAGS = -MMD -MP -MF $*.Td -POSTCOMPILE = mv -f $*.Td $*.d - -# CFLAGS, CXXFLAGS, CPPFLAGS, and LDFLAGS are for the users to override -CFLAGS ?= -O3 -Wall -Wextra -CXXFLAGS ?= -O3 -Wall -Wextra -pedantic -CPPFLAGS ?= -LDFLAGS ?= - -# Include flags -PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. -GTEST_INC = -isystem googletest/googletest/include - -PZSTD_CPPFLAGS = $(PZSTD_INC) -PZSTD_CCXXFLAGS = -PZSTD_CFLAGS = $(PZSTD_CCXXFLAGS) -PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS) -std=c++11 -PZSTD_LDFLAGS = -EXTRA_FLAGS = -ALL_CFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS) $(PZSTD_CFLAGS) -ALL_CXXFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CXXFLAGS) $(PZSTD_CXXFLAGS) -ALL_LDFLAGS = $(EXTRA_FLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS) - - -# gtest libraries need to go before "-lpthread" because they depend on it. -GTEST_LIB = -L googletest/build/googlemock/gtest -LIBS = - -# Compilation commands -LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -lpthread -o $@ -CC_COMMAND = $(CC) $(DEPFLAGS) $(ALL_CFLAGS) -c $< -o $@ -CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $< -o $@ - -# Get a list of all zstd files so we rebuild the static library when we need to -ZSTDCOMMON_FILES := $(wildcard $(ZSTDDIR)/common/*.c) \ - $(wildcard $(ZSTDDIR)/common/*.h) -ZSTDCOMP_FILES := $(wildcard $(ZSTDDIR)/compress/*.c) \ - $(wildcard $(ZSTDDIR)/compress/*.h) -ZSTDDECOMP_FILES := $(wildcard $(ZSTDDIR)/decompress/*.c) \ - $(wildcard $(ZSTDDIR)/decompress/*.h) -ZSTDPROG_FILES := $(wildcard $(PROGDIR)/*.c) \ - $(wildcard $(PROGDIR)/*.h) -ZSTD_FILES := $(wildcard $(ZSTDDIR)/*.h) \ - $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \ - $(ZSTDPROG_FILES) - -# List all the pzstd source files so we can determine their dependencies -PZSTD_SRCS := $(wildcard *.cpp) -PZSTD_TESTS := $(wildcard test/*.cpp) -UTILS_TESTS := $(wildcard utils/test/*.cpp) -ALL_SRCS := $(PZSTD_SRCS) $(PZSTD_TESTS) $(UTILS_TESTS) - - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -else -EXT = -endif - -# Standard targets -.PHONY: default -default: all - -.PHONY: test-pzstd -test-pzstd: TESTFLAGS=--gtest_filter=-*ExtremelyLarge* -test-pzstd: clean googletest pzstd tests check - -.PHONY: test-pzstd32 -test-pzstd32: clean googletest32 all32 check - -.PHONY: test-pzstd-tsan -test-pzstd-tsan: LDFLAGS=-fuse-ld=gold -test-pzstd-tsan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge* -test-pzstd-tsan: clean googletest tsan check - -.PHONY: test-pzstd-asan -test-pzstd-asan: LDFLAGS=-fuse-ld=gold -test-pzstd-asan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge* -test-pzstd-asan: clean asan check - -.PHONY: check -check: - $(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS) - $(TESTPROG) ./utils/test/RangeTest$(EXT) $(TESTFLAGS) - $(TESTPROG) ./utils/test/ResourcePoolTest$(EXT) $(TESTFLAGS) - $(TESTPROG) ./utils/test/ScopeGuardTest$(EXT) $(TESTFLAGS) - $(TESTPROG) ./utils/test/ThreadPoolTest$(EXT) $(TESTFLAGS) - $(TESTPROG) ./utils/test/WorkQueueTest$(EXT) $(TESTFLAGS) - $(TESTPROG) ./test/OptionsTest$(EXT) $(TESTFLAGS) - $(TESTPROG) ./test/PzstdTest$(EXT) $(TESTFLAGS) - -.PHONY: install -install: PZSTD_CPPFLAGS += -DNDEBUG -install: pzstd$(EXT) - install -d -m 755 $(BINDIR)/ - install -m 755 pzstd$(EXT) $(BINDIR)/pzstd$(EXT) - -.PHONY: uninstall -uninstall: - $(RM) $(BINDIR)/pzstd$(EXT) - -# Targets for many different builds -.PHONY: all -all: PZSTD_CPPFLAGS += -DNDEBUG -all: pzstd$(EXT) - -.PHONY: debug -debug: EXTRA_FLAGS += -g -debug: pzstd$(EXT) tests roundtrip - -.PHONY: tsan -tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC -tsan: PZSTD_LDFLAGS += -fsanitize=thread -tsan: debug - -.PHONY: asan -asan: EXTRA_FLAGS += -fsanitize=address -asan: debug - -.PHONY: ubsan -ubsan: EXTRA_FLAGS += -fsanitize=undefined -ubsan: debug - -.PHONY: all32 -all32: EXTRA_FLAGS += -m32 -all32: all tests roundtrip - -.PHONY: debug32 -debug32: EXTRA_FLAGS += -m32 -debug32: debug - -.PHONY: asan32 -asan32: EXTRA_FLAGS += -m32 -asan32: asan - -.PHONY: tsan32 -tsan32: EXTRA_FLAGS += -m32 -tsan32: tsan - -.PHONY: ubsan32 -ubsan32: EXTRA_FLAGS += -m32 -ubsan32: ubsan - -# Run long round trip tests -.PHONY: roundtripcheck -roundtripcheck: roundtrip check - $(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS) - -# Build the main binary -pzstd$(EXT): main.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a - $(LD_COMMAND) - -# Target that depends on all the tests -.PHONY: tests -tests: EXTRA_FLAGS += -Wno-deprecated-declarations -tests: $(patsubst %,%$(EXT),$(basename $(PZSTD_TESTS) $(UTILS_TESTS))) - -# Build the round trip tests -.PHONY: roundtrip -roundtrip: EXTRA_FLAGS += -Wno-deprecated-declarations -roundtrip: test/RoundTripTest$(EXT) - -# Use the static library that zstd builds for simplicity and -# so we get the compiler options correct -$(ZSTDDIR)/libzstd.a: $(ZSTD_FILES) - CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a - -# Rules to build the tests -test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \ - Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a - $(LD_COMMAND) - -test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB) -test/%Test$(EXT): LIBS += -lgtest -lgtest_main -test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o \ - SkippableFrame.o $(ZSTDDIR)/libzstd.a - $(LD_COMMAND) - -utils/test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB) -utils/test/%Test$(EXT): LIBS += -lgtest -lgtest_main -utils/test/%Test$(EXT): utils/test/%Test.o - $(LD_COMMAND) - - -GTEST_CMAKEFLAGS = - -# Install googletest -.PHONY: googletest -googletest: PZSTD_CCXXFLAGS += -fPIC -googletest: - @$(RM) -rf googletest - @git clone https://github.com/google/googletest - @mkdir -p googletest/build - @cd googletest/build && cmake $(GTEST_CMAKEFLAGS) -DCMAKE_CXX_FLAGS="$(ALL_CXXFLAGS)" .. && $(MAKE) - -.PHONY: googletest32 -googletest32: PZSTD_CCXXFLAGS += -m32 -googletest32: googletest - -.PHONY: googletest-mingw64 -googletest-mingw64: GTEST_CMAKEFLAGS += -G "MSYS Makefiles" -googletest-mingw64: googletest - -.PHONY: clean -clean: - $(RM) -f *.o pzstd$(EXT) *.Td *.d - $(RM) -f test/*.o test/*Test$(EXT) test/*.Td test/*.d - $(RM) -f utils/test/*.o utils/test/*Test$(EXT) utils/test/*.Td utils/test/*.d - $(RM) -f $(PROGDIR)/*.o $(PROGDIR)/*.Td $(PROGDIR)/*.d - $(MAKE) -C $(ZSTDDIR) clean - @echo Cleaning completed - - -# Cancel implicit rules -%.o: %.c -%.o: %.cpp - -# Object file rules -%.o: %.c - $(CC_COMMAND) - $(POSTCOMPILE) - -$(PROGDIR)/%.o: $(PROGDIR)/%.c - $(CC_COMMAND) - $(POSTCOMPILE) - -%.o: %.cpp - $(CXX_COMMAND) - $(POSTCOMPILE) - -test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC) -test/%.o: test/%.cpp - $(CXX_COMMAND) - $(POSTCOMPILE) - -utils/test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC) -utils/test/%.o: utils/test/%.cpp - $(CXX_COMMAND) - $(POSTCOMPILE) - -# Dependency file stuff -.PRECIOUS: %.d test/%.d utils/test/%.d - -# Include rules that specify header file dependencies --include $(patsubst %,%.d,$(basename $(ALL_SRCS))) diff --git a/sys/contrib/zstd/contrib/pzstd/Options.cpp b/sys/contrib/zstd/contrib/pzstd/Options.cpp deleted file mode 100644 index d9b216b4295..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/Options.cpp +++ /dev/null @@ -1,439 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "Options.h" -#include "util.h" -#include "utils/ScopeGuard.h" - -#include -#include -#include -#include -#include -#include -#include - -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || \ - defined(__CYGWIN__) -#include /* _isatty */ -#define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \ - defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ -#include /* isatty */ -#define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#else -#define IS_CONSOLE(stdStream) 0 -#endif - -namespace pzstd { - -namespace { -unsigned defaultNumThreads() { -#ifdef PZSTD_NUM_THREADS - return PZSTD_NUM_THREADS; -#else - return std::thread::hardware_concurrency(); -#endif -} - -unsigned parseUnsigned(const char **arg) { - unsigned result = 0; - while (**arg >= '0' && **arg <= '9') { - result *= 10; - result += **arg - '0'; - ++(*arg); - } - return result; -} - -const char *getArgument(const char *options, const char **argv, int &i, - int argc) { - if (options[1] != 0) { - return options + 1; - } - ++i; - if (i == argc) { - std::fprintf(stderr, "Option -%c requires an argument, but none provided\n", - *options); - return nullptr; - } - return argv[i]; -} - -const std::string kZstdExtension = ".zst"; -constexpr char kStdIn[] = "-"; -constexpr char kStdOut[] = "-"; -constexpr unsigned kDefaultCompressionLevel = 3; -constexpr unsigned kMaxNonUltraCompressionLevel = 19; - -#ifdef _WIN32 -const char nullOutput[] = "nul"; -#else -const char nullOutput[] = "/dev/null"; -#endif - -void notSupported(const char *option) { - std::fprintf(stderr, "Operation not supported: %s\n", option); -} - -void usage() { - std::fprintf(stderr, "Usage:\n"); - std::fprintf(stderr, " pzstd [args] [FILE(s)]\n"); - std::fprintf(stderr, "Parallel ZSTD options:\n"); - std::fprintf(stderr, " -p, --processes # : number of threads to use for (de)compression (default:%d)\n", defaultNumThreads()); - - std::fprintf(stderr, "ZSTD options:\n"); - std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel); - std::fprintf(stderr, " -d, --decompress : decompression\n"); - std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n"); - std::fprintf(stderr, " -f, --force : overwrite output without prompting, (de)compress links\n"); - std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n"); - std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n"); - std::fprintf(stderr, " -h, --help : display help and exit\n"); - std::fprintf(stderr, " -V, --version : display version number and exit\n"); - std::fprintf(stderr, " -v, --verbose : verbose mode; specify multiple times to increase log level (default:2)\n"); - std::fprintf(stderr, " -q, --quiet : suppress warnings; specify twice to suppress errors too\n"); - std::fprintf(stderr, " -c, --stdout : force write to standard output, even if it is the console\n"); -#ifdef UTIL_HAS_CREATEFILELIST - std::fprintf(stderr, " -r : operate recursively on directories\n"); -#endif - std::fprintf(stderr, " --ultra : enable levels beyond %i, up to %i (requires more memory)\n", kMaxNonUltraCompressionLevel, ZSTD_maxCLevel()); - std::fprintf(stderr, " -C, --check : integrity check (default)\n"); - std::fprintf(stderr, " --no-check : no integrity check\n"); - std::fprintf(stderr, " -t, --test : test compressed file integrity\n"); - std::fprintf(stderr, " -- : all arguments after \"--\" are treated as files\n"); -} -} // anonymous namespace - -Options::Options() - : numThreads(defaultNumThreads()), maxWindowLog(23), - compressionLevel(kDefaultCompressionLevel), decompress(false), - overwrite(false), keepSource(true), writeMode(WriteMode::Auto), - checksum(true), verbosity(2) {} - -Options::Status Options::parse(int argc, const char **argv) { - bool test = false; - bool recursive = false; - bool ultra = false; - bool forceStdout = false; - bool followLinks = false; - // Local copy of input files, which are pointers into argv. - std::vector localInputFiles; - for (int i = 1; i < argc; ++i) { - const char *arg = argv[i]; - // Protect against empty arguments - if (arg[0] == 0) { - continue; - } - // Everything after "--" is an input file - if (!std::strcmp(arg, "--")) { - ++i; - std::copy(argv + i, argv + argc, std::back_inserter(localInputFiles)); - break; - } - // Long arguments that don't have a short option - { - bool isLongOption = true; - if (!std::strcmp(arg, "--rm")) { - keepSource = false; - } else if (!std::strcmp(arg, "--ultra")) { - ultra = true; - maxWindowLog = 0; - } else if (!std::strcmp(arg, "--no-check")) { - checksum = false; - } else if (!std::strcmp(arg, "--sparse")) { - writeMode = WriteMode::Sparse; - notSupported("Sparse mode"); - return Status::Failure; - } else if (!std::strcmp(arg, "--no-sparse")) { - writeMode = WriteMode::Regular; - notSupported("Sparse mode"); - return Status::Failure; - } else if (!std::strcmp(arg, "--dictID")) { - notSupported(arg); - return Status::Failure; - } else if (!std::strcmp(arg, "--no-dictID")) { - notSupported(arg); - return Status::Failure; - } else { - isLongOption = false; - } - if (isLongOption) { - continue; - } - } - // Arguments with a short option simply set their short option. - const char *options = nullptr; - if (!std::strcmp(arg, "--processes")) { - options = "p"; - } else if (!std::strcmp(arg, "--version")) { - options = "V"; - } else if (!std::strcmp(arg, "--help")) { - options = "h"; - } else if (!std::strcmp(arg, "--decompress")) { - options = "d"; - } else if (!std::strcmp(arg, "--force")) { - options = "f"; - } else if (!std::strcmp(arg, "--stdout")) { - options = "c"; - } else if (!std::strcmp(arg, "--keep")) { - options = "k"; - } else if (!std::strcmp(arg, "--verbose")) { - options = "v"; - } else if (!std::strcmp(arg, "--quiet")) { - options = "q"; - } else if (!std::strcmp(arg, "--check")) { - options = "C"; - } else if (!std::strcmp(arg, "--test")) { - options = "t"; - } else if (arg[0] == '-' && arg[1] != 0) { - options = arg + 1; - } else { - localInputFiles.emplace_back(arg); - continue; - } - assert(options != nullptr); - - bool finished = false; - while (!finished && *options != 0) { - // Parse the compression level - if (*options >= '0' && *options <= '9') { - compressionLevel = parseUnsigned(&options); - continue; - } - - switch (*options) { - case 'h': - case 'H': - usage(); - return Status::Message; - case 'V': - std::fprintf(stderr, "PZSTD version: %s.\n", ZSTD_VERSION_STRING); - return Status::Message; - case 'p': { - finished = true; - const char *optionArgument = getArgument(options, argv, i, argc); - if (optionArgument == nullptr) { - return Status::Failure; - } - if (*optionArgument < '0' || *optionArgument > '9') { - std::fprintf(stderr, "Option -p expects a number, but %s provided\n", - optionArgument); - return Status::Failure; - } - numThreads = parseUnsigned(&optionArgument); - if (*optionArgument != 0) { - std::fprintf(stderr, - "Option -p expects a number, but %u%s provided\n", - numThreads, optionArgument); - return Status::Failure; - } - break; - } - case 'o': { - finished = true; - const char *optionArgument = getArgument(options, argv, i, argc); - if (optionArgument == nullptr) { - return Status::Failure; - } - outputFile = optionArgument; - break; - } - case 'C': - checksum = true; - break; - case 'k': - keepSource = true; - break; - case 'd': - decompress = true; - break; - case 'f': - overwrite = true; - forceStdout = true; - followLinks = true; - break; - case 't': - test = true; - decompress = true; - break; -#ifdef UTIL_HAS_CREATEFILELIST - case 'r': - recursive = true; - break; -#endif - case 'c': - outputFile = kStdOut; - forceStdout = true; - break; - case 'v': - ++verbosity; - break; - case 'q': - --verbosity; - // Ignore them for now - break; - // Unsupported options from Zstd - case 'D': - case 's': - notSupported("Zstd dictionaries."); - return Status::Failure; - case 'b': - case 'e': - case 'i': - case 'B': - notSupported("Zstd benchmarking options."); - return Status::Failure; - default: - std::fprintf(stderr, "Invalid argument: %s\n", arg); - return Status::Failure; - } - if (!finished) { - ++options; - } - } // while (*options != 0); - } // for (int i = 1; i < argc; ++i); - - // Set options for test mode - if (test) { - outputFile = nullOutput; - keepSource = true; - } - - // Input file defaults to standard input if not provided. - if (localInputFiles.empty()) { - localInputFiles.emplace_back(kStdIn); - } - - // Check validity of input files - if (localInputFiles.size() > 1) { - const auto it = std::find(localInputFiles.begin(), localInputFiles.end(), - std::string{kStdIn}); - if (it != localInputFiles.end()) { - std::fprintf( - stderr, - "Cannot specify standard input when handling multiple files\n"); - return Status::Failure; - } - } - if (localInputFiles.size() > 1 || recursive) { - if (!outputFile.empty() && outputFile != nullOutput) { - std::fprintf( - stderr, - "Cannot specify an output file when handling multiple inputs\n"); - return Status::Failure; - } - } - - g_utilDisplayLevel = verbosity; - // Remove local input files that are symbolic links - if (!followLinks) { - std::remove_if(localInputFiles.begin(), localInputFiles.end(), - [&](const char *path) { - bool isLink = UTIL_isLink(path); - if (isLink && verbosity >= 2) { - std::fprintf( - stderr, - "Warning : %s is symbolic link, ignoring\n", - path); - } - return isLink; - }); - } - - // Translate input files/directories into files to (de)compress - if (recursive) { - char *scratchBuffer = nullptr; - unsigned numFiles = 0; - const char **files = - UTIL_createFileList(localInputFiles.data(), localInputFiles.size(), - &scratchBuffer, &numFiles, followLinks); - if (files == nullptr) { - std::fprintf(stderr, "Error traversing directories\n"); - return Status::Failure; - } - auto guard = - makeScopeGuard([&] { UTIL_freeFileList(files, scratchBuffer); }); - if (numFiles == 0) { - std::fprintf(stderr, "No files found\n"); - return Status::Failure; - } - inputFiles.resize(numFiles); - std::copy(files, files + numFiles, inputFiles.begin()); - } else { - inputFiles.resize(localInputFiles.size()); - std::copy(localInputFiles.begin(), localInputFiles.end(), - inputFiles.begin()); - } - localInputFiles.clear(); - assert(!inputFiles.empty()); - - // If reading from standard input, default to standard output - if (inputFiles[0] == kStdIn && outputFile.empty()) { - assert(inputFiles.size() == 1); - outputFile = "-"; - } - - if (inputFiles[0] == kStdIn && IS_CONSOLE(stdin)) { - assert(inputFiles.size() == 1); - std::fprintf(stderr, "Cannot read input from interactive console\n"); - return Status::Failure; - } - if (outputFile == "-" && IS_CONSOLE(stdout) && !(forceStdout && decompress)) { - std::fprintf(stderr, "Will not write to console stdout unless -c or -f is " - "specified and decompressing\n"); - return Status::Failure; - } - - // Check compression level - { - unsigned maxCLevel = - ultra ? ZSTD_maxCLevel() : kMaxNonUltraCompressionLevel; - if (compressionLevel > maxCLevel || compressionLevel == 0) { - std::fprintf(stderr, "Invalid compression level %u.\n", compressionLevel); - return Status::Failure; - } - } - - // Check that numThreads is set - if (numThreads == 0) { - std::fprintf(stderr, "Invalid arguments: # of threads not specified " - "and unable to determine hardware concurrency.\n"); - return Status::Failure; - } - - // Modify verbosity - // If we are piping input and output, turn off interaction - if (inputFiles[0] == kStdIn && outputFile == kStdOut && verbosity == 2) { - verbosity = 1; - } - // If we are in multi-file mode, turn off interaction - if (inputFiles.size() > 1 && verbosity == 2) { - verbosity = 1; - } - - return Status::Success; -} - -std::string Options::getOutputFile(const std::string &inputFile) const { - if (!outputFile.empty()) { - return outputFile; - } - // Attempt to add/remove zstd extension from the input file - if (decompress) { - int stemSize = inputFile.size() - kZstdExtension.size(); - if (stemSize > 0 && inputFile.substr(stemSize) == kZstdExtension) { - return inputFile.substr(0, stemSize); - } else { - return ""; - } - } else { - return inputFile + kZstdExtension; - } -} -} diff --git a/sys/contrib/zstd/contrib/pzstd/Options.h b/sys/contrib/zstd/contrib/pzstd/Options.h deleted file mode 100644 index f4f2aaa499c..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/Options.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#define ZSTD_STATIC_LINKING_ONLY -#include "zstd.h" -#undef ZSTD_STATIC_LINKING_ONLY - -#include -#include -#include - -namespace pzstd { - -struct Options { - enum class WriteMode { Regular, Auto, Sparse }; - - unsigned numThreads; - unsigned maxWindowLog; - unsigned compressionLevel; - bool decompress; - std::vector inputFiles; - std::string outputFile; - bool overwrite; - bool keepSource; - WriteMode writeMode; - bool checksum; - int verbosity; - - enum class Status { - Success, // Successfully parsed options - Failure, // Failure to parse options - Message // Options specified to print a message (e.g. "-h") - }; - - Options(); - Options(unsigned numThreads, unsigned maxWindowLog, unsigned compressionLevel, - bool decompress, std::vector inputFiles, - std::string outputFile, bool overwrite, bool keepSource, - WriteMode writeMode, bool checksum, int verbosity) - : numThreads(numThreads), maxWindowLog(maxWindowLog), - compressionLevel(compressionLevel), decompress(decompress), - inputFiles(std::move(inputFiles)), outputFile(std::move(outputFile)), - overwrite(overwrite), keepSource(keepSource), writeMode(writeMode), - checksum(checksum), verbosity(verbosity) {} - - Status parse(int argc, const char **argv); - - ZSTD_parameters determineParameters() const { - ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0); - params.fParams.contentSizeFlag = 0; - params.fParams.checksumFlag = checksum; - if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) { - params.cParams.windowLog = maxWindowLog; - params.cParams = ZSTD_adjustCParams(params.cParams, 0, 0); - } - return params; - } - - std::string getOutputFile(const std::string &inputFile) const; -}; -} diff --git a/sys/contrib/zstd/contrib/pzstd/Pzstd.cpp b/sys/contrib/zstd/contrib/pzstd/Pzstd.cpp deleted file mode 100644 index 1eb4ce14cf1..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/Pzstd.cpp +++ /dev/null @@ -1,618 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "Pzstd.h" -#include "SkippableFrame.h" -#include "utils/FileSystem.h" -#include "utils/Range.h" -#include "utils/ScopeGuard.h" -#include "utils/ThreadPool.h" -#include "utils/WorkQueue.h" - -#include -#include -#include -#include -#include -#include - -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _O_BINARY */ -# include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); } -#else -# include /* isatty */ -# define SET_BINARY_MODE(file) -#endif - -namespace pzstd { - -namespace { -#ifdef _WIN32 -const std::string nullOutput = "nul"; -#else -const std::string nullOutput = "/dev/null"; -#endif -} - -using std::size_t; - -static std::uintmax_t fileSizeOrZero(const std::string &file) { - if (file == "-") { - return 0; - } - std::error_code ec; - auto size = file_size(file, ec); - if (ec) { - size = 0; - } - return size; -} - -static std::uint64_t handleOneInput(const Options &options, - const std::string &inputFile, - FILE* inputFd, - const std::string &outputFile, - FILE* outputFd, - SharedState& state) { - auto inputSize = fileSizeOrZero(inputFile); - // WorkQueue outlives ThreadPool so in the case of error we are certain - // we don't accidently try to call push() on it after it is destroyed - WorkQueue> outs{options.numThreads + 1}; - std::uint64_t bytesRead; - std::uint64_t bytesWritten; - { - // Initialize the (de)compression thread pool with numThreads - ThreadPool executor(options.numThreads); - // Run the reader thread on an extra thread - ThreadPool readExecutor(1); - if (!options.decompress) { - // Add a job that reads the input and starts all the compression jobs - readExecutor.add( - [&state, &outs, &executor, inputFd, inputSize, &options, &bytesRead] { - bytesRead = asyncCompressChunks( - state, - outs, - executor, - inputFd, - inputSize, - options.numThreads, - options.determineParameters()); - }); - // Start writing - bytesWritten = writeFile(state, outs, outputFd, options.decompress); - } else { - // Add a job that reads the input and starts all the decompression jobs - readExecutor.add([&state, &outs, &executor, inputFd, &bytesRead] { - bytesRead = asyncDecompressFrames(state, outs, executor, inputFd); - }); - // Start writing - bytesWritten = writeFile(state, outs, outputFd, options.decompress); - } - } - if (!state.errorHolder.hasError()) { - std::string inputFileName = inputFile == "-" ? "stdin" : inputFile; - std::string outputFileName = outputFile == "-" ? "stdout" : outputFile; - if (!options.decompress) { - double ratio = static_cast(bytesWritten) / - static_cast(bytesRead + !bytesRead); - state.log(INFO, "%-20s :%6.2f%% (%6" PRIu64 " => %6" PRIu64 - " bytes, %s)\n", - inputFileName.c_str(), ratio * 100, bytesRead, bytesWritten, - outputFileName.c_str()); - } else { - state.log(INFO, "%-20s: %" PRIu64 " bytes \n", - inputFileName.c_str(),bytesWritten); - } - } - return bytesWritten; -} - -static FILE *openInputFile(const std::string &inputFile, - ErrorHolder &errorHolder) { - if (inputFile == "-") { - SET_BINARY_MODE(stdin); - return stdin; - } - // Check if input file is a directory - { - std::error_code ec; - if (is_directory(inputFile, ec)) { - errorHolder.setError("Output file is a directory -- ignored"); - return nullptr; - } - } - auto inputFd = std::fopen(inputFile.c_str(), "rb"); - if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) { - return nullptr; - } - return inputFd; -} - -static FILE *openOutputFile(const Options &options, - const std::string &outputFile, - SharedState& state) { - if (outputFile == "-") { - SET_BINARY_MODE(stdout); - return stdout; - } - // Check if the output file exists and then open it - if (!options.overwrite && outputFile != nullOutput) { - auto outputFd = std::fopen(outputFile.c_str(), "rb"); - if (outputFd != nullptr) { - std::fclose(outputFd); - if (!state.log.logsAt(INFO)) { - state.errorHolder.setError("Output file exists"); - return nullptr; - } - state.log( - INFO, - "pzstd: %s already exists; do you wish to overwrite (y/n) ? ", - outputFile.c_str()); - int c = getchar(); - if (c != 'y' && c != 'Y') { - state.errorHolder.setError("Not overwritten"); - return nullptr; - } - } - } - auto outputFd = std::fopen(outputFile.c_str(), "wb"); - if (!state.errorHolder.check( - outputFd != nullptr, "Failed to open output file")) { - return nullptr; - } - return outputFd; -} - -int pzstdMain(const Options &options) { - int returnCode = 0; - SharedState state(options); - for (const auto& input : options.inputFiles) { - // Setup the shared state - auto printErrorGuard = makeScopeGuard([&] { - if (state.errorHolder.hasError()) { - returnCode = 1; - state.log(ERROR, "pzstd: %s: %s.\n", input.c_str(), - state.errorHolder.getError().c_str()); - } - }); - // Open the input file - auto inputFd = openInputFile(input, state.errorHolder); - if (inputFd == nullptr) { - continue; - } - auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); }); - // Open the output file - auto outputFile = options.getOutputFile(input); - if (!state.errorHolder.check(outputFile != "", - "Input file does not have extension .zst")) { - continue; - } - auto outputFd = openOutputFile(options, outputFile, state); - if (outputFd == nullptr) { - continue; - } - auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); }); - // (de)compress the file - handleOneInput(options, input, inputFd, outputFile, outputFd, state); - if (state.errorHolder.hasError()) { - continue; - } - // Delete the input file if necessary - if (!options.keepSource) { - // Be sure that we are done and have written everything before we delete - if (!state.errorHolder.check(std::fclose(inputFd) == 0, - "Failed to close input file")) { - continue; - } - closeInputGuard.dismiss(); - if (!state.errorHolder.check(std::fclose(outputFd) == 0, - "Failed to close output file")) { - continue; - } - closeOutputGuard.dismiss(); - if (std::remove(input.c_str()) != 0) { - state.errorHolder.setError("Failed to remove input file"); - continue; - } - } - } - // Returns 1 if any of the files failed to (de)compress. - return returnCode; -} - -/// Construct a `ZSTD_inBuffer` that points to the data in `buffer`. -static ZSTD_inBuffer makeZstdInBuffer(const Buffer& buffer) { - return ZSTD_inBuffer{buffer.data(), buffer.size(), 0}; -} - -/** - * Advance `buffer` and `inBuffer` by the amount of data read, as indicated by - * `inBuffer.pos`. - */ -void advance(Buffer& buffer, ZSTD_inBuffer& inBuffer) { - auto pos = inBuffer.pos; - inBuffer.src = static_cast(inBuffer.src) + pos; - inBuffer.size -= pos; - inBuffer.pos = 0; - return buffer.advance(pos); -} - -/// Construct a `ZSTD_outBuffer` that points to the data in `buffer`. -static ZSTD_outBuffer makeZstdOutBuffer(Buffer& buffer) { - return ZSTD_outBuffer{buffer.data(), buffer.size(), 0}; -} - -/** - * Split `buffer` and advance `outBuffer` by the amount of data written, as - * indicated by `outBuffer.pos`. - */ -Buffer split(Buffer& buffer, ZSTD_outBuffer& outBuffer) { - auto pos = outBuffer.pos; - outBuffer.dst = static_cast(outBuffer.dst) + pos; - outBuffer.size -= pos; - outBuffer.pos = 0; - return buffer.splitAt(pos); -} - -/** - * Stream chunks of input from `in`, compress it, and stream it out to `out`. - * - * @param state The shared state - * @param in Queue that we `pop()` input buffers from - * @param out Queue that we `push()` compressed output buffers to - * @param maxInputSize An upper bound on the size of the input - */ -static void compress( - SharedState& state, - std::shared_ptr in, - std::shared_ptr out, - size_t maxInputSize) { - auto& errorHolder = state.errorHolder; - auto guard = makeScopeGuard([&] { out->finish(); }); - // Initialize the CCtx - auto ctx = state.cStreamPool->get(); - if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) { - return; - } - { - auto err = ZSTD_resetCStream(ctx.get(), 0); - if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) { - return; - } - } - - // Allocate space for the result - auto outBuffer = Buffer(ZSTD_compressBound(maxInputSize)); - auto zstdOutBuffer = makeZstdOutBuffer(outBuffer); - { - Buffer inBuffer; - // Read a buffer in from the input queue - while (in->pop(inBuffer) && !errorHolder.hasError()) { - auto zstdInBuffer = makeZstdInBuffer(inBuffer); - // Compress the whole buffer and send it to the output queue - while (!inBuffer.empty() && !errorHolder.hasError()) { - if (!errorHolder.check( - !outBuffer.empty(), "ZSTD_compressBound() was too small")) { - return; - } - // Compress - auto err = - ZSTD_compressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer); - if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) { - return; - } - // Split the compressed data off outBuffer and pass to the output queue - out->push(split(outBuffer, zstdOutBuffer)); - // Forget about the data we already compressed - advance(inBuffer, zstdInBuffer); - } - } - } - // Write the epilog - size_t bytesLeft; - do { - if (!errorHolder.check( - !outBuffer.empty(), "ZSTD_compressBound() was too small")) { - return; - } - bytesLeft = ZSTD_endStream(ctx.get(), &zstdOutBuffer); - if (!errorHolder.check( - !ZSTD_isError(bytesLeft), ZSTD_getErrorName(bytesLeft))) { - return; - } - out->push(split(outBuffer, zstdOutBuffer)); - } while (bytesLeft != 0 && !errorHolder.hasError()); -} - -/** - * Calculates how large each independently compressed frame should be. - * - * @param size The size of the source if known, 0 otherwise - * @param numThreads The number of threads available to run compression jobs on - * @param params The zstd parameters to be used for compression - */ -static size_t calculateStep( - std::uintmax_t size, - size_t numThreads, - const ZSTD_parameters ¶ms) { - (void)size; - (void)numThreads; - return size_t{1} << (params.cParams.windowLog + 2); -} - -namespace { -enum class FileStatus { Continue, Done, Error }; -/// Determines the status of the file descriptor `fd`. -FileStatus fileStatus(FILE* fd) { - if (std::feof(fd)) { - return FileStatus::Done; - } else if (std::ferror(fd)) { - return FileStatus::Error; - } - return FileStatus::Continue; -} -} // anonymous namespace - -/** - * Reads `size` data in chunks of `chunkSize` and puts it into `queue`. - * Will read less if an error or EOF occurs. - * Returns the status of the file after all of the reads have occurred. - */ -static FileStatus -readData(BufferWorkQueue& queue, size_t chunkSize, size_t size, FILE* fd, - std::uint64_t *totalBytesRead) { - Buffer buffer(size); - while (!buffer.empty()) { - auto bytesRead = - std::fread(buffer.data(), 1, std::min(chunkSize, buffer.size()), fd); - *totalBytesRead += bytesRead; - queue.push(buffer.splitAt(bytesRead)); - auto status = fileStatus(fd); - if (status != FileStatus::Continue) { - return status; - } - } - return FileStatus::Continue; -} - -std::uint64_t asyncCompressChunks( - SharedState& state, - WorkQueue>& chunks, - ThreadPool& executor, - FILE* fd, - std::uintmax_t size, - size_t numThreads, - ZSTD_parameters params) { - auto chunksGuard = makeScopeGuard([&] { chunks.finish(); }); - std::uint64_t bytesRead = 0; - - // Break the input up into chunks of size `step` and compress each chunk - // independently. - size_t step = calculateStep(size, numThreads, params); - state.log(DEBUG, "Chosen frame size: %zu\n", step); - auto status = FileStatus::Continue; - while (status == FileStatus::Continue && !state.errorHolder.hasError()) { - // Make a new input queue that we will put the chunk's input data into. - auto in = std::make_shared(); - auto inGuard = makeScopeGuard([&] { in->finish(); }); - // Make a new output queue that compress will put the compressed data into. - auto out = std::make_shared(); - // Start compression in the thread pool - executor.add([&state, in, out, step] { - return compress( - state, std::move(in), std::move(out), step); - }); - // Pass the output queue to the writer thread. - chunks.push(std::move(out)); - state.log(VERBOSE, "%s\n", "Starting a new frame"); - // Fill the input queue for the compression job we just started - status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead); - } - state.errorHolder.check(status != FileStatus::Error, "Error reading input"); - return bytesRead; -} - -/** - * Decompress a frame, whose data is streamed into `in`, and stream the output - * to `out`. - * - * @param state The shared state - * @param in Queue that we `pop()` input buffers from. It contains - * exactly one compressed frame. - * @param out Queue that we `push()` decompressed output buffers to - */ -static void decompress( - SharedState& state, - std::shared_ptr in, - std::shared_ptr out) { - auto& errorHolder = state.errorHolder; - auto guard = makeScopeGuard([&] { out->finish(); }); - // Initialize the DCtx - auto ctx = state.dStreamPool->get(); - if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) { - return; - } - { - auto err = ZSTD_resetDStream(ctx.get()); - if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) { - return; - } - } - - const size_t outSize = ZSTD_DStreamOutSize(); - Buffer inBuffer; - size_t returnCode = 0; - // Read a buffer in from the input queue - while (in->pop(inBuffer) && !errorHolder.hasError()) { - auto zstdInBuffer = makeZstdInBuffer(inBuffer); - // Decompress the whole buffer and send it to the output queue - while (!inBuffer.empty() && !errorHolder.hasError()) { - // Allocate a buffer with at least outSize bytes. - Buffer outBuffer(outSize); - auto zstdOutBuffer = makeZstdOutBuffer(outBuffer); - // Decompress - returnCode = - ZSTD_decompressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer); - if (!errorHolder.check( - !ZSTD_isError(returnCode), ZSTD_getErrorName(returnCode))) { - return; - } - // Pass the buffer with the decompressed data to the output queue - out->push(split(outBuffer, zstdOutBuffer)); - // Advance past the input we already read - advance(inBuffer, zstdInBuffer); - if (returnCode == 0) { - // The frame is over, prepare to (maybe) start a new frame - ZSTD_initDStream(ctx.get()); - } - } - } - if (!errorHolder.check(returnCode <= 1, "Incomplete block")) { - return; - } - // We've given ZSTD_decompressStream all of our data, but there may still - // be data to read. - while (returnCode == 1) { - // Allocate a buffer with at least outSize bytes. - Buffer outBuffer(outSize); - auto zstdOutBuffer = makeZstdOutBuffer(outBuffer); - // Pass in no input. - ZSTD_inBuffer zstdInBuffer{nullptr, 0, 0}; - // Decompress - returnCode = - ZSTD_decompressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer); - if (!errorHolder.check( - !ZSTD_isError(returnCode), ZSTD_getErrorName(returnCode))) { - return; - } - // Pass the buffer with the decompressed data to the output queue - out->push(split(outBuffer, zstdOutBuffer)); - } -} - -std::uint64_t asyncDecompressFrames( - SharedState& state, - WorkQueue>& frames, - ThreadPool& executor, - FILE* fd) { - auto framesGuard = makeScopeGuard([&] { frames.finish(); }); - std::uint64_t totalBytesRead = 0; - - // Split the source up into its component frames. - // If we find our recognized skippable frame we know the next frames size - // which means that we can decompress each standard frame in independently. - // Otherwise, we will decompress using only one decompression task. - const size_t chunkSize = ZSTD_DStreamInSize(); - auto status = FileStatus::Continue; - while (status == FileStatus::Continue && !state.errorHolder.hasError()) { - // Make a new input queue that we will put the frames's bytes into. - auto in = std::make_shared(); - auto inGuard = makeScopeGuard([&] { in->finish(); }); - // Make a output queue that decompress will put the decompressed data into - auto out = std::make_shared(); - - size_t frameSize; - { - // Calculate the size of the next frame. - // frameSize is 0 if the frame info can't be decoded. - Buffer buffer(SkippableFrame::kSize); - auto bytesRead = std::fread(buffer.data(), 1, buffer.size(), fd); - totalBytesRead += bytesRead; - status = fileStatus(fd); - if (bytesRead == 0 && status != FileStatus::Continue) { - break; - } - buffer.subtract(buffer.size() - bytesRead); - frameSize = SkippableFrame::tryRead(buffer.range()); - in->push(std::move(buffer)); - } - if (frameSize == 0) { - // We hit a non SkippableFrame, so this will be the last job. - // Make sure that we don't use too much memory - in->setMaxSize(64); - out->setMaxSize(64); - } - // Start decompression in the thread pool - executor.add([&state, in, out] { - return decompress(state, std::move(in), std::move(out)); - }); - // Pass the output queue to the writer thread - frames.push(std::move(out)); - if (frameSize == 0) { - // We hit a non SkippableFrame ==> not compressed by pzstd or corrupted - // Pass the rest of the source to this decompression task - state.log(VERBOSE, "%s\n", - "Input not in pzstd format, falling back to serial decompression"); - while (status == FileStatus::Continue && !state.errorHolder.hasError()) { - status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead); - } - break; - } - state.log(VERBOSE, "Decompressing a frame of size %zu", frameSize); - // Fill the input queue for the decompression job we just started - status = readData(*in, chunkSize, frameSize, fd, &totalBytesRead); - } - state.errorHolder.check(status != FileStatus::Error, "Error reading input"); - return totalBytesRead; -} - -/// Write `data` to `fd`, returns true iff success. -static bool writeData(ByteRange data, FILE* fd) { - while (!data.empty()) { - data.advance(std::fwrite(data.begin(), 1, data.size(), fd)); - if (std::ferror(fd)) { - return false; - } - } - return true; -} - -std::uint64_t writeFile( - SharedState& state, - WorkQueue>& outs, - FILE* outputFd, - bool decompress) { - auto& errorHolder = state.errorHolder; - auto lineClearGuard = makeScopeGuard([&state] { - state.log.clear(INFO); - }); - std::uint64_t bytesWritten = 0; - std::shared_ptr out; - // Grab the output queue for each decompression job (in order). - while (outs.pop(out)) { - if (errorHolder.hasError()) { - continue; - } - if (!decompress) { - // If we are compressing and want to write skippable frames we can't - // start writing before compression is done because we need to know the - // compressed size. - // Wait for the compressed size to be available and write skippable frame - SkippableFrame frame(out->size()); - if (!writeData(frame.data(), outputFd)) { - errorHolder.setError("Failed to write output"); - return bytesWritten; - } - bytesWritten += frame.kSize; - } - // For each chunk of the frame: Pop it from the queue and write it - Buffer buffer; - while (out->pop(buffer) && !errorHolder.hasError()) { - if (!writeData(buffer.range(), outputFd)) { - errorHolder.setError("Failed to write output"); - return bytesWritten; - } - bytesWritten += buffer.size(); - state.log.update(INFO, "Written: %u MB ", - static_cast(bytesWritten >> 20)); - } - } - return bytesWritten; -} -} diff --git a/sys/contrib/zstd/contrib/pzstd/Pzstd.h b/sys/contrib/zstd/contrib/pzstd/Pzstd.h deleted file mode 100644 index 79d1fcca265..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/Pzstd.h +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include "ErrorHolder.h" -#include "Logging.h" -#include "Options.h" -#include "utils/Buffer.h" -#include "utils/Range.h" -#include "utils/ResourcePool.h" -#include "utils/ThreadPool.h" -#include "utils/WorkQueue.h" -#define ZSTD_STATIC_LINKING_ONLY -#include "zstd.h" -#undef ZSTD_STATIC_LINKING_ONLY - -#include -#include -#include - -namespace pzstd { -/** - * Runs pzstd with `options` and returns the number of bytes written. - * An error occurred if `errorHandler.hasError()`. - * - * @param options The pzstd options to use for (de)compression - * @returns 0 upon success and non-zero on failure. - */ -int pzstdMain(const Options& options); - -class SharedState { - public: - SharedState(const Options& options) : log(options.verbosity) { - if (!options.decompress) { - auto parameters = options.determineParameters(); - cStreamPool.reset(new ResourcePool{ - [this, parameters]() -> ZSTD_CStream* { - this->log(VERBOSE, "%s\n", "Creating new ZSTD_CStream"); - auto zcs = ZSTD_createCStream(); - if (zcs) { - auto err = ZSTD_initCStream_advanced( - zcs, nullptr, 0, parameters, 0); - if (ZSTD_isError(err)) { - ZSTD_freeCStream(zcs); - return nullptr; - } - } - return zcs; - }, - [](ZSTD_CStream *zcs) { - ZSTD_freeCStream(zcs); - }}); - } else { - dStreamPool.reset(new ResourcePool{ - [this]() -> ZSTD_DStream* { - this->log(VERBOSE, "%s\n", "Creating new ZSTD_DStream"); - auto zds = ZSTD_createDStream(); - if (zds) { - auto err = ZSTD_initDStream(zds); - if (ZSTD_isError(err)) { - ZSTD_freeDStream(zds); - return nullptr; - } - } - return zds; - }, - [](ZSTD_DStream *zds) { - ZSTD_freeDStream(zds); - }}); - } - } - - ~SharedState() { - // The resource pools have references to this, so destroy them first. - cStreamPool.reset(); - dStreamPool.reset(); - } - - Logger log; - ErrorHolder errorHolder; - std::unique_ptr> cStreamPool; - std::unique_ptr> dStreamPool; -}; - -/** - * Streams input from `fd`, breaks input up into chunks, and compresses each - * chunk independently. Output of each chunk gets streamed to a queue, and - * the output queues get put into `chunks` in order. - * - * @param state The shared state - * @param chunks Each compression jobs output queue gets `pushed()` here - * as soon as it is available - * @param executor The thread pool to run compression jobs in - * @param fd The input file descriptor - * @param size The size of the input file if known, 0 otherwise - * @param numThreads The number of threads in the thread pool - * @param parameters The zstd parameters to use for compression - * @returns The number of bytes read from the file - */ -std::uint64_t asyncCompressChunks( - SharedState& state, - WorkQueue>& chunks, - ThreadPool& executor, - FILE* fd, - std::uintmax_t size, - std::size_t numThreads, - ZSTD_parameters parameters); - -/** - * Streams input from `fd`. If pzstd headers are available it breaks the input - * up into independent frames. It sends each frame to an independent - * decompression job. Output of each frame gets streamed to a queue, and - * the output queues get put into `frames` in order. - * - * @param state The shared state - * @param frames Each decompression jobs output queue gets `pushed()` here - * as soon as it is available - * @param executor The thread pool to run compression jobs in - * @param fd The input file descriptor - * @returns The number of bytes read from the file - */ -std::uint64_t asyncDecompressFrames( - SharedState& state, - WorkQueue>& frames, - ThreadPool& executor, - FILE* fd); - -/** - * Streams input in from each queue in `outs` in order, and writes the data to - * `outputFd`. - * - * @param state The shared state - * @param outs A queue of output queues, one for each - * (de)compression job. - * @param outputFd The file descriptor to write to - * @param decompress Are we decompressing? - * @returns The number of bytes written - */ -std::uint64_t writeFile( - SharedState& state, - WorkQueue>& outs, - FILE* outputFd, - bool decompress); -} diff --git a/sys/contrib/zstd/contrib/pzstd/README.md b/sys/contrib/zstd/contrib/pzstd/README.md deleted file mode 100644 index 84d94581583..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Parallel Zstandard (PZstandard) - -Parallel Zstandard is a Pigz-like tool for Zstandard. -It provides Zstandard format compatible compression and decompression that is able to utilize multiple cores. -It breaks the input up into equal sized chunks and compresses each chunk independently into a Zstandard frame. -It then concatenates the frames together to produce the final compressed output. -Pzstandard will write a 12 byte header for each frame that is a skippable frame in the Zstandard format, which tells PZstandard the size of the next compressed frame. -PZstandard supports parallel decompression of files compressed with PZstandard. -When decompressing files compressed with Zstandard, PZstandard does IO in one thread, and decompression in another. - -## Usage - -PZstandard supports the same command line interface as Zstandard, but also provides the `-p` option to specify the number of threads. -Dictionary mode is not currently supported. - -Basic usage - - pzstd input-file -o output-file -p num-threads -# # Compression - pzstd -d input-file -o output-file -p num-threads # Decompression - -PZstandard also supports piping and fifo pipes - - cat input-file | pzstd -p num-threads -# -c > /dev/null - -For more options - - pzstd --help - -PZstandard tries to pick a smart default number of threads if not specified (displayed in `pzstd --help`). -If this number is not suitable, during compilation you can define `PZSTD_NUM_THREADS` to the number of threads you prefer. - -## Benchmarks - -As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia). - -Compression Speed vs Ratio with 4 Threads | Decompression Speed with 4 Threads -------------------------------------------|----------------------------------- -![Compression Speed vs Ratio](images/Cspeed.png "Compression Speed vs Ratio") | ![Decompression Speed](images/Dspeed.png "Decompression Speed") - -The test procedure was to run each of the following commands 2 times for each compression level, and take the minimum time. - - time pzstd -# -p 4 -c silesia.tar > silesia.tar.zst - time pzstd -d -p 4 -c silesia.tar.zst > /dev/null - - time pigz -# -p 4 -k -c silesia.tar > silesia.tar.gz - time pigz -d -p 4 -k -c silesia.tar.gz > /dev/null - -PZstandard was tested using compression levels 1-19, and Pigz was tested using compression levels 1-9. -Pigz cannot do parallel decompression, it simply does each of reading, decompression, and writing on separate threads. - -## Tests - -Tests require that you have [gtest](https://github.com/google/googletest) installed. -Set `GTEST_INC` and `GTEST_LIB` in `Makefile` to specify the location of the gtest headers and libraries. -Alternatively, run `make googletest`, which will clone googletest and build it. -Run `make tests && make check` to run tests. diff --git a/sys/contrib/zstd/contrib/pzstd/SkippableFrame.cpp b/sys/contrib/zstd/contrib/pzstd/SkippableFrame.cpp deleted file mode 100644 index 769866dfc81..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/SkippableFrame.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "SkippableFrame.h" -#include "mem.h" -#include "utils/Range.h" - -#include - -using namespace pzstd; - -SkippableFrame::SkippableFrame(std::uint32_t size) : frameSize_(size) { - MEM_writeLE32(data_.data(), kSkippableFrameMagicNumber); - MEM_writeLE32(data_.data() + 4, kFrameContentsSize); - MEM_writeLE32(data_.data() + 8, frameSize_); -} - -/* static */ std::size_t SkippableFrame::tryRead(ByteRange bytes) { - if (bytes.size() < SkippableFrame::kSize || - MEM_readLE32(bytes.begin()) != kSkippableFrameMagicNumber || - MEM_readLE32(bytes.begin() + 4) != kFrameContentsSize) { - return 0; - } - return MEM_readLE32(bytes.begin() + 8); -} diff --git a/sys/contrib/zstd/contrib/pzstd/SkippableFrame.h b/sys/contrib/zstd/contrib/pzstd/SkippableFrame.h deleted file mode 100644 index 60deed0405b..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/SkippableFrame.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include "utils/Range.h" - -#include -#include -#include -#include - -namespace pzstd { -/** - * We put a skippable frame before each frame. - * It contains a skippable frame magic number, the size of the skippable frame, - * and the size of the next frame. - * Each skippable frame is exactly 12 bytes in little endian format. - * The first 8 bytes are for compatibility with the ZSTD format. - * If we have N threads, the output will look like - * - * [0x184D2A50|4|size1] [frame1 of size size1] - * [0x184D2A50|4|size2] [frame2 of size size2] - * ... - * [0x184D2A50|4|sizeN] [frameN of size sizeN] - * - * Each sizeX is 4 bytes. - * - * These skippable frames should allow us to skip through the compressed file - * and only load at most N pages. - */ -class SkippableFrame { - public: - static constexpr std::size_t kSize = 12; - - private: - std::uint32_t frameSize_; - std::array data_; - static constexpr std::uint32_t kSkippableFrameMagicNumber = 0x184D2A50; - // Could be improved if the size fits in less bytes - static constexpr std::uint32_t kFrameContentsSize = kSize - 8; - - public: - // Write the skippable frame to data_ in LE format. - explicit SkippableFrame(std::uint32_t size); - - // Read the skippable frame from bytes in LE format. - static std::size_t tryRead(ByteRange bytes); - - ByteRange data() const { - return {data_.data(), data_.size()}; - } - - // Size of the next frame. - std::size_t frameSize() const { - return frameSize_; - } -}; -} diff --git a/sys/contrib/zstd/contrib/pzstd/images/Cspeed.png b/sys/contrib/zstd/contrib/pzstd/images/Cspeed.png deleted file mode 100644 index aca4f663ea2e98b6df7c1344b51af99757549d8a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69804 zcmeFZWn5I<_Xi5dAi{&7f~ce-B8}1@VbG~`BOwivLk%!s)7^r^&|T7?G}0XdC?$== zAbt0M0nhKbFYlZC`M(fh_MCI}Uh&;)eb?IOt-P!l0p2w{3=9kc3GqiyFfg#cU|^gv z#Ki%hcwiW|g8wjWpNKuc$ZjW}1Am-*F0N*afkAo^{SVV9t{Yq;j3M#pzM>=MBJx}e zxnb<#kMYsq<~xziGC>l!61X;Aw{Jee!t}iHIw3oouEjZ*Dc?-_hhyn{wmYkQ0EwCw zp>o}(?ge>j;VD1Ti`3*2;#iTRjb9AzZVesoD5uIon&ZlU-r^<}_LrcET>SqSq;>^kp9W?N|LaQX zw-}fOH+dEx{O{C>aO1$(>LxF<=zk1~-w>{0I(NSOzZU(84ve|NWWE1uH9@#UY*L<| zS^h04v>;Z`5wYRx#P#U?*Wmvx@&Er?O#W(i_piAH;b*EBwQyOE8hewn7AwdJa5_Ch z-3WLk41-nfuZ33^rl&LPHoEUMa`^ot+s{_mmBamWbXDOg85S-zktQbAv)UzoQ$Gx> z>lckJEQ(ZgV13$7(#X|UTNCF$_&J$XQE^XxKhV`6nZLmZ(SNN+}IOW?j(M!qU+H1^Jxm-}@R?EE=bzWJ4=2TYc6sGAk?}5h_L#fDw1Mif1+6}sorn1ES-c9W81LpfKP2A@P;hSP+&K-8P|7G&Z)247j)9U|2Z9?h zZPp>ricN%DoEvEf@ zwd4MRj@8aNz3v@Jy!_tY-ob$?6lXNs(+7oDqY{$MWD}_*wi%z-LP!Ur$2Jhgkdcv* zrCFiFQh~kI&z2MG?uT3MOZrs1s&(V!^oXe4USahA_Y)+=8(7~LX zdp@a-YZpNbC2XS$H5Dv}aBWFbNEs%rT78MRZB;gR3mOvZc5b4?y#~|#csru5L2GW* zNs$5*8&73|wEE{uGNvA@AJuE2IfFh|*>O0ObyNkno8_YVQ@VB@-NeqP%{A${Rg#)4 z72W`8EqqKvE+@3VVIKR=MpRrF|I6WLp*^oGo$4`)59ym};x8HFJ+>@LkTYEJJ*c^r zPfKR%uP}gt2k-+QE-gqlK$|`doK)MfHWr2O>JU0~9oJPeul36EO@3^u{#sNaFDg13 zX#$U8v2CGG`RHBJ81`Xy7qwI%wD*;8+1a>^Jzg}+gn)&4C!1A=&%YnhMQX*m59D7| zy*JIetVp#qup{1+BDdurMpc<#c~4zl-3&9$T;@{ugI;xii}o1aF8vC+u{`w@naysy zT9=i|0gC2?y%pzya%pe1##==$pI;FyKNyI5mo#TmCgDlBAB12s;dq55qRv*CpWhvO z;5-v)RMx|w?=QibBO2ADK!bfYIsswxdPJ`5+>6UZ1L7BDO4ofjAKVO1LZy+@YMn{D zl@nL7p3>8pH!EJUQlZbzE9(%oQ{lA!Yzu)SMqwYadl12Ei%>CF_pjbxvO;dX%kHV| zCowlMxr3CDoBQU?>Nj@pdfx&=7T&f&KL@|eT)&a$&TVUk5>cXg^==o1707hg#=Rmj zn*NJc*H$|p64v`NDPSBfm#;W5!~!m$Sy}rxU77T9{oU_z&K-?^0p?57=UUv40? zvOA-r{gqEyCz})N*-*aW*t4xJe&x+>KK1dv8SX?lc~(ktd_pnfocEUwHxxS*#ti*z z=6(-s2A11a?z_*=6L5#hCeL3kwz1Dme0O-SwthIoh_deiEHrdAe|Z}vi&4T zMNz-1Ia0|gRwa>;G40m;2D{HlD6KcoMaidENNgdqpw9}g+I2~S3=`?j=xB75bC?hpBSBj;yt}VFT zepf8O=7Oe->-U zQBe2_gox&jZsfke&keM6x0aEXkx@2R%N-Qc>AHuF#*94++WA@uv`s>St6ms?aPSTI zx#fNc?MWDSqsiWw7B%ZJ7D%9#3LAV&_x`(|YbUAn{>&keK{Zk&jzqpIm3g}BMkije zy`9}hZfbd9Ca&AleOeZ~AZS>e=`k*v^cNZ{l+iIS3fLobTRj6sGR3mrF?r-urD|Fj zE$T)uj#_JFE%%mlak5w8hII9gRqvKp+j@DnoH)yFh%IXspU zJq5#HfMvJK{aQ9HFXzKLzlM#LClhLO1Xhj@3$5~6rSWQPl4`q2OnM;Rb56Zl+E06EIPS_D8$$9a|F2ba zdft8F>Xq)Bk;xd(zUgh?r8-H_a9mnQ+s6w_BcO}53kvVsvgSD-w>vsJYDOEq@^mv0 zYt0I_pt1h#6-{LE9hJE4Iqg~P6vA0s{iLx;Z7ruq=m?<645W}*)s$0hzQ6n4q_wx(f>dUI(i4nF)wBZ%<71@bgX`a=ay*%aXOXlrm<%xMeo}eJ znGg>31&S3%u?=}xsq*RrOzyt>I;xj~J@ulnHitUgxGagIS`PPSg;sFJ0GtVqPx$yL zQDj%`n2wjT&!2~-M`Kaw>~x2ZvhI#;bMig9&%x}RJ*U?i&gzfDzG0{~SGfYqA*<=J z^-b+2H7@GF4>NJ$a|(6O4QX>zFSRj@37^mEg74&~(HG#kCPj0Y%Jr<-RbZVT|JlGj zYIdI{fp|(f?C#v8^e#)!Br&l-IjdKG)9uGLqS4s_opWF?NxZsmHYnpOBUS|I>+Sp<1c^~ZPuT7!iihmArv=hz zOVZ^)zv%i8&C|)a%aHq~)TDBYaHVF2)+NmazpcH?DrX+Ac)!;?#T+21Grj6e z-w?jC<8GmB(r9JJxvX-;Q_pyYlW13!b#Be>^W!IsM z)SiuWvz(e2cuxqEqCh9b4@i0TWa2wDBzMt&Vc;4YybI^pr$Hu(WQPX%GQ1-o`sXs+ zdX;h6%SzO1PYdm}{+nys3y*7QS>-1o`+8e09s4c)f?2TXf|>WS+STigmaWP{v^cF5wvLIHIP-7!B%w@`0uZ&P z1Y9evC5T!2_$i3{UW;6bKl>biEO*ajh444sS|TmFNS2S;9w9-Z(kAJ8=4+nDMlsf+ zb}cqsIT?et^wbCs`9%sllGGmY)cN`O!AyDqe`dNoEqW5RM+hJ@J-I~cs-SE&UxxMW z^$wvGL4mDNTcn$^@I$yWtBEh8Z3qyat@G6={R=Qq)@@xz0yrGe^$BK~0+9fW@My|j3{wT4)p8IkN= ziIg?b9h2+(UvEKFIMhXI*_q!Q)|?^y2$kK-Iox4|CW)F;+o#%YZ(1W@;d9n^e~4b9 zb&30@FRT{DC~rMtQBn$5K|T`haStVqQKlB9ci=;48%>ry;mwmsJW~l!f@#cky$ul` zVj1;=>hK3uSC>*=qhklB=PKAEpL?<54K_LF3g<(=Jg2nXeBW949Kxswh178%6(wq= zX(0}MtYrvJ->wBxD?Y^Xr)Y1pI)QB>cgK~dE9|g0i_40Rz8@7`gf>195`+q{qEfmW zG+k$0eC1~OsFsgVLYv?580BLD%8w2`xr;{mKo2aUs2d^6g(-v70sUmxnC9@`^lv|r zmr`(Cqcy>f2F*l)=}hW- z2GcMSvF_k?8b698Wm^{wpWEpT$sG*wU0^7#sGC)Z-z{vT&-OMGQIBC+LFbPZR}i54P|${QE9QZv+cfqc0O%nFJ>dF4*QCw zv)8`@Pu~%sd?A?V2TW9$2;_~Byn$49EL`G34^p;~JpI@FpJ@8hp5`S&D_F(cqolJS~QXt9tpLlxVV~V2k*j~7DSAYf8UXYcQ<<>`z;__8| zf*}e;_Tnrr_HxQMOmUfgkAw4>aEk?{yi$m5!9H#PS&@>hm zP3xERx>HDIMTKic%%t?;E9Yd=6lYF0O{msY19F*}Nw{cQNA z9gnTRj--ldH5?$y|2^9w?bg267}>cXrhp6Oy5*ktfw|~veRJYWZ3YinV{WT=BM%7$h|60m_+3mB z5fi4w2vR2RR-QU<=~0{Q_pANXJ*;>eS0`l}H~OE3lt9nm_%f}FM$NH{E8Jz5iR3YG zW}!&oGM@UTD>TNLv+%^pj=M+E{VIEm^Yr|n^!Opz8iw{po2Fgvsf>kqgTVQPl81+y zs>^!^8V}(yycj^!EOXpYS`8GVyp!d{q1wH1O@^x1n2Od4bGMPwvG`aaC&XYTONmQ$>_hPaeg(AT^I-j_ zeLpF%)4B?7DLj1W#h>4mVfHPt-?cl0i?-wT9l0`CjLEp6bIj}^K)=2%^)BQp;G1Fw zGedM3pD#g7IY@fQv>ZU<5{-D49CWud`xS#qkLl)iLK0_-or*TANqPde$OwM673u01baoU^md_TA~d}7qbJtJjo_`r=h?O^*V_7z&_{}Yr%1k24=g6ANgLsHkUL}>&rc6?1EafmdkQ>TIzY_&058i$dI;V z>R&ZDZr3BzMXhK4^xD&exiIbS-VCa}iL+<=y%msCSxKxBl51+ zTWl0M$TPuKUUar88=2S~&{-*#z8^YRG@%zAZAoR+4dv58G|s&`FIiUlVn@g98a=&k z(j3A$?jLR^1y+0HJv1?7JytxJU*M!wAI3jui=Wij#BwZ+}u}cw@k^sfS{*TbwP;(wcX)9R#_0c zOr7%h=}s3#XGM9HlTxSEP-Rh5;hS~9W11e(8Jod^KiJh2;<)lj)-<(Grt)LFAkX(hKCxk_!11)M9*2uhSiY)X7PvH218N*n8 zIa)bPe3^Wy1Ea@2-(AONd$wLCnVhX&Af@ob?zJXz>Jxr^nj52X=5s5DJirK-ueC5D zqe;KR5^K`3F_F>+xzT~f>1kMAWukTuF`Y~~5|syLZ{pNlv@LOjOgG;_!i*r z?mnKp0*EwSZ0y2YZNN+IxiJde_MNv{Q~G_?g^gbuQB83GE-K`tFLvbEO9h?eD4{z zE&@_r$=^z!)9+HYnBSUUAw!qbvNgXT;5@rQY1=24H;dZ&%x}ctZ{_%zY}BK%a%*e> zM~j4K1EJd@va1AT6<<(!1_JxL-Anu{6_MmA@#{kSJwlQFY2C!akSD!cQFxirqNy;` z>Y$RDaA=hxm0|K$%5qhDl5Qm8(Jh2sPDb6NzRp6-&f^*dGo@|XBCQonB)vjntsM;x z@|=RilCn#m1FU&~fX$A8{taw9{O&VPKoYK!HMUfM&yQo$}H` z_?*j)yqLOy_1O;S!iR~9VRyhx-h5`g+eijWuL?@kS({%LxHZ%K%By9grDIAUMm5De zY*tHUEOES4uVc38GizK4Yo70z!%EkRs+9(CFX^0hX(8_XZOX0THkzZ5wh5P*_wynR zU#KX1z)>A;O+^@v?>pETj*nDg`62a6r>#=IP%g0)XZPZFij1!7JD={U4n>tA=tIW% z1jek#5&<8P9xiJ9z4c`{hsa9#_DJkE*9`T!5fhlIP zwf?m&WR_Ho%C};xS{tnAU1)t(-^yHzAn14gN3Ir;-JKn=L{$p@T6%78LD*sEB&mnyL{8`o2 zR0M#sf(x6RZ%GpWAK5p3sM)I85^z~-rn6}9x$3aLIWW5_P3$Yo1G)0_a=Tkf-zkg3 zz``VP{Q8n~RCfrtS5`_2U3^yjFz!?dFw(tcD%02hG8tjT_?(;$x1FB2VKt*zLv$u+ zIeQK`vb?HF9uFiE++g7fCEFz#=w8+ z6L(58zM^8Z3X}t*MMLUuAu)t|1akmo9DPS$bdlvfjYka|xRT&fz~_HsMN?6YCj=iS zMsM%u4`pTLXN}~xUp!!-ati-?4*MHRejqnM4wognyVdD5}Djs>!+auh)|NP6E z(0}R@1B~>=;!8QAXinY$j5g^2!1}){|4%~yA6kN>*w>hti>EDbxq^>7$#dR})>LI8f3D>l&;8DO4?X&t4fJP$YoNAc#Mea3uNRUTvZ zX7rYQgc~KY(|}Q>S8Zu9|KolI+RQ~IJ?JB8Xxp4lc?~AEG)?QCr(qrkUJ||kS`Zk7 zcM*ZQ^R@YyOQ$A@o92OdddrV#JyD)=nid^~C0+-mOO9S%x>L~x(G$!pwJ=k_D5V<) zp#w#65rFLsl2i9jI4j!M-g%VNxHG&=4(1MBYeFZ+Nem#)P@-Qu3h_?HHDK%onw_{^ z7Abmq)8D?%mVvlC{Vpk(d-_518}v=6!A&4H^Z#|df!GAW`mH3*&Trd&^s}_+v>^a3 z&7s~TrhnnceF$IK?xIyrvFgo>S2_Gk60?gx0X6*4UPIsCCslav&3oVAe@*hmllZKR zjwveSWs>L=Uv7uoOHq~QFphVMnT$8Mfp?Hw!1Q)dz-oH?X+NX2+?0<% zX>YmGwlmLW^BIN3a&d2#8~BNf@Ph4~dV)r{=F8caXaN8j{`nD!Z*K9u(2B<_jAC)o z7N6es@=Ep0=#ay=Me9h?7WCFsgA34}?7q3|%ahmP5|`D?R8lM>*+$BDkqG6G%canY zd(RpMA6W4%J;Z=xfzVr7I95X8l#tQE1&M7=yx~Pjg+BOc^2o=N*>Mk!S%aWGaLwGDh&ie_m-}tpO(y2dCljA z=*334#ry$1LoVdGAQ9f>Z3G=iR zmmxO--|T;h^xUq&_w5R%;I_@&Uw10VN+{$h^QA+}`8u#@#uwU21g9@N57~*;9rC9f zH}to>M$MG4XqFVJ-T?i0q{6r!Pc7c6MkST50$f{WZ>u;tRpX zuy^b7BbA?M5+fTf0k??{0GlMq6RYmar+vs9O8~a-&@)egvV%z#_7+^0E%axKeOfPq z-I{Rw-hB+XCur(Xey%0yby_Z8@$EKIBLsw&1@`>I2`^(0%dWW`UhTaYhmv_R;S$}@cq|w5PSmkjfK@V1jU`*0UR@Mc zRxAwFwp%hBwjbL?Gj2UkT)r_y-!%Xx2>RNzfDe4wVgrr@?DJ2A^n}=W`Fl~$cu8L| zr-O*UGTg{hc z(}YgyiQfV9e=sL8jP`}U4>X|8T^A0AwVey&EZQm)3nz}&vTtYNw^)AhkK`l~T5)V> z#fb`a$q!n>!D=k>68=CB9@ji28&uB;yuawaED-ra^ z#1O6}3+VDIHLE4${_=%T!+KQOy3xoT_6CG1))4v|pGd%kA7N6hr!K_QfrmR34Q&#d z7qq{BSeu`|rnkERvFCC(4YhLPLT_?UfuUS3rc9|ku{vrZQ}}Fg$uN|1^qBzc!oX8k z!?#+yGrU|zPYhGFjM1C`5hK|Dl)a8S$bMfcjETRu+2@6db04&Xy~YU}s!M;eJE}1= zc;ONw-p4);blN4Z12nPojQq4bv1R{_Lo+z72-(|&ni{EW;Bf(FK<}lO5gm&@8?8J% zsxjgM4p@Hm+Wxj-tn2ZChHka_gDs>aB&GKYXy{aijbe0%c#J-{qe0H2W&*3;YekRn zq%{G6sKPhC_Uu~|Fg!+M)T_i>L`>_SkNTIG|hfZXReMJ{==Ofav;N%D6$M@u>#MMik&64nP4)6!{2IF@RYI2aT)Cx3Cu z)~8DyjPx$Z3~`y!HJ*T*vXOp8aD;3(taaWx$4 z)RjbRaOPdU#fH9-tPe$vi$y$YA|`?Mo1(573A#fqB=*YcLHAN;T-eA!?E2p+9B%f} z&ehLZo>g-A{P|+grIf*BI=SN4+~m2O(NS&XWQcyI9u8?=CK}btqk<_jfY@6D>lGb~ zVLu%en&8`Qp+@8e7V{gqxWxRk7+8ztn4(SS27I99jW%ns`` z{F#;ILh1+IkUod)psUK-Q9BfO6VI+=qra^y4&3G8#Xxiu(O-YL;siesuUmF?jzg9@ zxNl4)*NdPI*bzf(`VkISMZTi(HQ)@mkXJbQG-IR7gT)+M>9GEp-yG>zY zo%l;4!so5!B>FlJVM~B_(+#0dPUj3h*u{$rxnwF|ALX;%4o z8&3{Q4o;4@Y7+fW^WEvlr2B%SBPl~7Nu{hSgW!uUtgSG3p+T(V$FXn>(21uHyV2=P!4-lkwj1#!v?u`x-L(&F z(ZRPsd#Lx^beF}^hDCIR7YK*Os4en@Ou+(p;6WSsFNS>;UI7{Ji?kG9Gum57@rEM=5~+*>`b zQIg2az-D5ernMwpd(X5b;kWUKv(KGRJ@EBke-XN9u}VuPQ`{;Wt5+`~@<%W*AehAQv%SAj@M!vYs`2F| z<8Eco!Uc-+c!yiidk+Y19Bhz3f6sKvAIx-b4&U??O$ND@Tar9t>1teVVe#FkI1?mB z+1CAxASk_e9EV|GZKH|g>K(7)zgse}K{#$^mQ~)0Q2yuP^4&yc^|JV37M;7FTco@b zN6>K}uy1$Z$!j@ssTWQ%huGpSj|_h^Rj^%|Xv}iOlq< zQT}(_j(N#=Zm}quYV7HPL!pS7;D0FNzd1_G3xoMNpDi^jk=N8+5;z6jW>m}979+W; zf^xKvB1XV?5=|)ZW5hlpIq6Z<)sXjNeO6mSKhn{DK~g!D)a)>nrjYqD_4aSs)Pj)s zvX5Ir*h%#Vyv9OS?mG;;5_ECs_$sWX-1$raUj&+owFp z+3swpKo>H*w--Zw4+~8VA{|C7P~B3jB4jX7mK$soMBO11a$Bu2H!@1mvaToTA9w#L zo7IICL&u;$-4eQtK=)68!zy8e$mR7*vOKlfm8fCNks%M^59nBPgAxHdwL1I1KDhN& zxbCkH#%!)X*8K=su2`3aDh?S5?W%DLBPROyx5ll64!1j&CaDg8#`9cziR$uq>jejK zCf5p!iiTL-e&AgTWd`m*)SL`G_lTcf(xqx1bLa9Vsa|E`9~vu zJaQQ2F1J!J4$O!-qfRYh(KGSgl7La04;m(ih(Ce~^dv&JJ(hpqz4Lt9#|uVYu`4oG z&OIJlp!3eZ++ure+Im+~O3DHhhOI{&!IMB1^|ENzg!`nuCee0w51DZbRebJWGWMqA zMw?w3n;v^(L9RRTOFA{uRgE0!9;QZLvT#)C&d!c`U%FE9^REx0xvjK^8*M?*cL7(i z#(c8D_vswza3maKKn%zDI`0cJnwu1{_t{Ii?h4~~UDu?#KEKEj-BJ{IOBHS~K6Eip zr*hEfZ25U^V{MthO4#GN^p^9G){k80M^>4jSa=~w(CdPx(5+lXM!E&N;tn?Gwg$IA zbMF~Hr6KfrE0Yh0&B1gd8*@EbNl6C6f?Ob;hYLf#eCoVjc;Vz$wI($&r?u}ej&NJx zuPCl&&SK0rw0(!HdEnL;HcXseL+JkVq045UipAlZiH zN{Mw)*L29zLD=L}A2|2i=14{#Kzl8;_muxf-KEL3H7qzAE;i^ZyPkLx`y!-be99Vj zvt>2P%*#xd4Kv%f)?knYGn+8ms}}ns7fvw!=n`-qR4^TMXp)s%j7&{Q+dNK{t4B7Q z-___rw(o61xr^8i%cGIGwCmx(ax?ko7i!>V(5KMKH~TeYptTj>6TF|WC_;yNDY(b? zm!X7Lcpb)^W=j{sL&L*0pwJSF!Q5Knr1Gz?$QD6;9zk|bYWtDFqj(H#5Yl7X-H`da zi-R3!5pPjo&_btdH}s>fnwG`O{l?_=Z=Qxj0*uMm**?~MtM^9IN0fO_gA-pC8imGz zE-9@t(Od*>@}J(L8zvDRgS@jb!MTm2T|vVsNsZ{{{DXs)!!Zi`VoPKh=Jt`BVVk~C z`vAo`m7c*z@i+Kv8C7zu@GEl|vsN7O`A?=;r{(7T$^HGxF6fgb6$Q`Q*0{u|ik=OXRBbYAl-Z^A_4R=|gb=_4huK)@@Hx5|T~Ota z=3td0!u19qoV&rXbfT~Tc-^R1kv#cd{F72so1Q6#kC3uPnSq_XWy+|oIPg-Zah_po z*w7imd%9p|uU+(fxm?gE-&VkhnbFOLieclMQ9;>X2R8=w{Y&w;bwH^tFFLWq-Vc3N z5#4sXFgIsi!pH^%g{qN}@dFg*vLa;AeTvr zX>D3_LTkl2ymCEur4WI*#U0P-v^IqaEmfo?Cep~$s|R#()GMdu(%A+G=$^^K808$$ z)CXN-ot?ViFru=8!ozr`%Wr2tp!3u+mXz8Pxc@?-Mq@%Kv_IvT+pT#H;V}K8@j^&V z&21r^o_Yp}X`8B_A`>0PguhFK>npsM4*y`nMetwsZs8h`mqu3OYn=^(i7o z6YSOynzyeW4PgO_k%>)9MC9ZbL4a&T&UtTJG^r=6*kG3Y)PJ(>|+0EURMu0rg=OaicoCPAhJq^*6yUE+&z@v?r zNF8%I=<0BG%GVR3A+GWe4WU;>eHW!uok_YeTb+!5YdEHv1l8Df_Ed_a{Br=pjJBm$ zfO+5wA=*8;F6sa*&}Ps1E%W82rFY5fnZUYOcr2e#kBB6LuT|I<-1_}408DHf)|!wn zqhz)?Cxb^o-hJ>iq|v01Ju1->v4KDlbAc`W6t&w7!40F`ddiAkbxkLb!7HvCXHSX% z`;mvPnlRrxQ#2Ylu>b{h@V}WqNsSAeVqv#FvAse1|9X$9QjFUCdn?rG=!sxU19++7 z^O-*ZFCM}b;2VyF+DG^OM77uR7o3FB%bf ziHyreA-F!dHQxLqi|s|bi^*C`GoyRmPY5Cb z;UMsdbK36cDK9~RCsW=f%;AK{0B-;otulCi_?F!xGwrEIUv9L!m*GFlS5(n>+Dh7Y zv-K02Mge~C1l=-om*m)ok3IqH3MrP^g2t%uP#7%Mx;~uePChMu#!}`>PKSjkE#>h+ zl9F+k56ApN`Wf-#r5fA^YPd{9k9Be%>Av*l(G65@#%nCYxA5GHM#egE}#$2Hx zyd2^_9pqH(3=7U0>NOujvK2jxL_9S$xLTXPvDb_~xknv<)~yZI$!*a?_%fiMg@@uw z$O}Q-X~loO&i@Ldt>_zOe&>YBzH=cm*EB9el_7P?q#&WNsoh%it(M~_gJ{)yl@K0s?IWx5d57Un=8MNq zcLGkS$+0SB;}_qC6Mlr}!Y~)K%Y_a8|ufgIN`47*%MX@qfhMo z7VvE%`0|+Bu2%ui;mUin+%ro*;Q7|F&lf!>O_yAanDf(ZunZyH(;R2O1-H=*31ldX z1Mm0Mz`z4ZzAw&|^s#aduMBVw*7{X_43`FREOhEDj}$RH0rXcW;LnUO z-LvEn_AO=3G}fUJf*(d%1WodpBXuljv0J{GJ09`1MABgDwTs-QN5*)Z1=qa61$E(% z&i-4fGT-nKAWvAT*tG%dHKBB+|kJ%VGUP?8bAaXM^lq$?6$r#kjy z2(7%PeAb6MzYOt6DHDu<_gyJxf2!|(6GUOPzv$zy7j<8#ecXxe|GVwIkk-}EPZ9X) z!?G~H1XZUjU{h-%H)c7$^&I-uF=E~oCvJ@`JR$6ovu`a`An?Pc0Pr*S6eq>^9vn1B z-2oNyAxqRT=E|?4Wmn~GLVbKEa0lTl*k!R74BSVC_5Ato=iBqEw?!&l4ex#vJ_L&8 z){SewJUbWNu&3eQ^LMj^dtP|M^EeL_GDd>iLt2HZZ94f}IF{}ThK3Rk(c3nRd~>?A z=$EoZ;FR*EY#2|YD)m+s(eXA;O>`HS#Px;eQGX*2B3F3q*|O2OFgZ&;T>^{BDgyPX z(XVS?AbqG;G<2#;PH&@Uo9Oiy%ak^~aN3BOdXRUdVyVf_9RUTL4e$^6wktgje{2k1 zgG;{HS!Qr{sX3f*Fl;|3ODar$_iEpCab2XW(~Rd^E+dm zv<#P@&6ZMD*E9gEwXIlCERRdf9Ov;;!LdE`dxc!WLQe2K!VOsE(dDD{yK&E#ZAx>B z8cSjzJq?3TjLN3TV(pt7K&9o^{zn}^21H+Kovirh@gT?~HeOmR5VA0F5Dn%WK~z#y zZKufGw(X~l{w`gbS)6vM1-731n+nKB*OdyBgqti23Jv{~|9n{1T8 z7=;AjB^{TQd4&Uh6?c7NfCC2>%t-&W{_B?pN4l5%ZuVduxSZd~k#gt_xw9;u-!m(d z`y+I3{hmQMy6*wlNs|KRfoOY=-v{--7uOOSFU%KISJ-q?$P_=7E4QLS_%IgKw?`RB z#nOlOWH7q^UAZVIR}Ot2vVHbPba1Q--LA;4gg)Bo| z?PUfcA|h|q9DgIT6Z(Boa(teqf1wszPPet{ywS}e%-zW_ul{eNKknWk;lGo2>6x#s zmq7~8LQrTpAAV@y@1SP|a8J`P(#QAz&cYo3yKE?^w+%(fv=vvT^)S0B%Ovy5*lZ_6 zCE+ixSS{tz#d08^+!;RuHBn9hKX!J?Y4&4%XgQC1TKv147PSH<;j81 zhcD8JJIw68hfhcpP;KcY=@`lGT`clvNy|FEHI>>}HmSb4+h{i--+Nm5+=D z8pr`LZt^tVLP#2E<@e0^K`*bw88?2>)yjWFrF|Tp5Gev%zQ4pf|69CZ3B|t}l^7p6 z30m)pF3itSrB&I z%-$MG7w8aDewZro2Zw8ju-Ygpi|f_Cxc7{mUqGq7 zHqZUsD&@UczM`gEALTWdmbmjSwDY~d*>Yt3;7sLGar0G?;=XfXymt!jE7bAIq#Hd!{stH# zSunM)i%|7HO7V$6wACqaS@!RheP~wBPDO*S#6oX>-|P%j%x_NBXhSul zz#>&~277V_w3KWsGztklxt`*eml#7O z(BG%5F7e~5wW0SjMcS-jPeZ&3L%#X8jI+N2hf{CTI@bSGB``GxYA(em@2P}JTN{)@ zcxv+;Tnyb)Hj{FNtJgYgMg8d6W^kHgX7yt&nuA31>3PDv=@XjUTYn`rM7ZczK}>9r zaQ=zbvFPGCTc61ITkr(f!S68Jg5rKMzs=%pZgFM9o{UxnW(l}cS9_bACtDJF7>W)e`mZxG(#sa{{B5pW4!Ttx}AT;qJdr(fs+x^p-GBemsG4TO>N^9@Xi5D*j zhbQM=b}zJ;HmvVo^0_NT3-ZlaNv^d&kLCp(MPasjvVY_+y@PH>H`Vw|-W6OqG^eZK969%<)Soky>vO3_azFLD z6yS)4K(ppqf+%`b#Jw_G+TXi6QPWwl;_XiSqjyXkJ>`1*X0`G$sT+GiE7mgFh`08Z zj7abIbaNb^sA107wbeIwWYR{R;zyig<3W~gE|Qn#ww#(L@tTt4W`-R?4G|2Wcu0jM zl3UM0F2!={kD2WO9NtblrW%g6#(cDkuH4Qz>aXy(G`0~_qyHEpI}b81im_|@Im|UV z!fF||11*XQZ!aCLAru&>5f^j{Vj3k(`e}!@iT}~GHqY7(W}-sBp5aJ||C0>H#{+tV zX(;3oXVus_8+w0Hy=`BP1^-9p9rtBYe5g2!OK2`780w$yHZG6}*Y{CjK zs!Q4VMpex^t9%Qme^Hbr!y;`{#2B8d$AcLz0V_KyRXvE_>tQpluae4KT0AL{M1vP z|9&eZZYY?zygj$?FQvi*FTdgO-W*U29kMj4Jy7h9ka_ln}H3$ooc`SkQkD>q*>7uQ|A zqx>Xh{_df*$m(B8ikc9tZu&l*Jml9+o7~}TF8qs@bn>W@Eao!|{8l!}T6WzAF8eV* z9bpo^L*DY>KFSX?1zo+y#&o;4jt^f7Oe;Em`1#PS0x;NybWGTP1DC&)Iamv zHavt^(Lqf$p$X=%j|?Q@JJdoJhXnRf2C(X-l|f3C^st-!lc}cVP3J<|($n6I{8cDu zl|%Q~sqRad{g&DpG;ic$@ELP=zYNFBc*&pf!bvMJY&>k>%j*#Z{Ekv7xUfPi{J7dXdbX)!6XeEe?uQA$t>Pn)yp-yiZhr@LO zhui|zc5`X9yx85cYvmIkC6onTx!I_~;Bf?t>pVdp1zFKhrbTLWu)u<~^H7y0|J6+KlpZ|xhw~mYI`@)3l-&<6Rt>tNA+6xPFJG*c!4>gw#?D0< z5pr!ny5vYn5 zCHjEe2zCsV{xx5C93y>5?HX`|rwLI&SAhIk=WgvJP*$HKhD~of>KOa)N-fTd?hbEO znD!0j2fP&uRtUO`+BV+Y_)K5_ZA-?nvE)7|m4ku_+llcZ=%2$h_`#Bvg}QC$L~)zY z&a(VSmFmNw3)B zi~SV+4Hl#(XCAhGUnJ^@NCF50!2oMa|8_2yUDqDq&*iy;Y|c{S@esE3GZ&>B$fh&l zs1tPjJ~V#%`niT_)niFfi=W>VIA$g!oR$Y_jQP2_7$DXHcjzxDgM`|MZ*>CN*zDr) z3eqE<(XnxHpcL`(qw4m^VxVoVetvOVoBDRwQ>7UEP2yQbBZw4n8K)u1`ZKtKqC=~y4`z46zDG(|k$3aw1_Lc|uWv--0l_9b1wK;cMJy`}XsHLkiLY3E{)PjlVHelES^K9x)pnd*ROJ zE$Nj&Y6%)Tyn=2kML{jvAnqo+KE5##Spg}S0)fP&TlqjwRx6_&BxU0OZ^4c|&j01) zL}^H?lkH;Wy(=m&iIcYx%U9o*btN~Hj?(;w(rE6D6n*=)Q6?=fFHab6d?;o0@wv>) zx;iCkm*RS9NNly&z`U%|WBMZ-=Kp`FEXc@@li^0cOp<*X8cC(A9wMly6}r%c-mY26 zq)*Y)oi(O_SfF4Dsd5@z0$0x9s=RWmZ|)yeW4#N`9HQUU8~p$dYQ50 zhy^7-pF$>1KPR$5IIm$J%Ks2r>!hw_Sk0lq=S5OD>if*wd3ssbb__ty4yf1+rzL@` zjL-YS@B}us14sg(t#|bG=?zN+ik}B`O=erN;+zNCP^N)Ak(FIzUBFxyRl>wbp(LrBHUD? z_$;K7UjG7Szs58b{W==ZIXVx;ZkkVj@z@?2T6lY`ziQs}V$lE@XJHxd+b8hb04u2C z`c!lA6VLOnsi$vr5Bx2KMxjV|1f)c&AFKb|M^P!kCz?4r?pzzubMVvce?`7!Zu~H> zGDF0U8$g^>TI@`0m&&`)mN31*`!Bw}rY!l5k^&2YRDf!pf6=(>*V4Mt{IeA&>rRaH zcV?VP7;auFqlia=W)b4mAyk)w`^-tOL zCL}>klXu0A{$R=8ll?4P=iVF}uo)r+I`7Bb)E16CJ)-D_|aB2O|wAV^K zN!}nLyx@{lb@_jxWumA4B*5T_qo67S>}ih)WcolvXjt-Ac9MeGP6ej9<529;(sr%m zlUO)s&&29nXbk2Y)X4Wcczq0fdtQ_us{G)7j@L90vpQmMhM=Sj8crt2T0RbSYezZW z#wNh|ST3vnGOdsux(reI$&xjh4y+Q5Zdxl0U*8X_QSNo>rK6ySUaN$$zUZKku-?0j$mY<3BwERj^8`#7q2J z17rLi%jscxQ(wIZna1Md6E+ZO%lqU%851}|ph*mNjP+ODTy%)k$uxkt^n}N6{IRNo z<2>8VR3qL}E%0N8ZvqG~8;Hq+$dic2I2VKmRiW10?=AL&-7Afq#&Dm3vU^nU!k?DX~NS z5?9Y11q|r+1@AsyKW9H&@SOHZ-ER@c zki<q{KG(8)~p|bazu6=TRZK)K=!?5TEW&J ze8{^)QWk?1*OVxEZh_|DW&XQ+s5}L_e&7%lS~9;4O8{RN-qWGIn0uU3Wzz&L8iYq~$Tn6K{m(hL3Cts9wp zcI`(T?up?{5ywmJyKjYP`VCHgXq;u^mHD^n1E~bCfWGMu!@q)4b7XQRU7usIdTo8S ze%};IiTk3M*-Fdw468%=Km+!Csz=Sggn48a)ZG~_;eGJKnQ2mO0deB!f)V(xg-{;l z@c;6zCVN<>FbiVzLYXBq{+u4@e8y~p44|NT7Z9MtIx8Iia6~yKXk9nb<=|9*T z%qSu3SU0WQw&Zwt^lz<`lhFp8jC4G*+w|bne_~#`udwAhc2r(@Fz(@6AY1zljV>6I zM_HGzP1aJXg)%4q&y_w1B0?BR0*xc1C@)DL{4m`h?QL(qwD%oji8lJ&((+yx&J2pz z_qxCPCyVoIQ2!9DUOk-tH_<+Xq=E=+cNrcn94y7vNEMiAm|0nzX5)tQ_CL)@=OEt4BW2F2gWQPGhZLujTY)ll9x&z|!&~1rx?R%|ALX!k zAD#N*q)&3T+CTe^bLla7>bj^9il0Bct}WoUU-j*@{EPk3Cau8p4@;s;!S)FCeemI0FjByLk=@n71Fgn9Up{2O8kd+pXjNR+m2fJ~f?zRN z`9)G@ZL6HWw!G|iS$IK5HhzNdN>9EGYBNDj(A3SXE_Ka8^A`aHTroIO#mOmuS4lts zlbs?>^~e}J<@?k_>jyk0&|_>9kh@?8OMj_8=5N(J87^Mfi_7(#NIy53$!o83EkiL8 zPxxjeJ5on05}lVeG+ZQ$2rND;+j@3hS^Cdbx4e{=o_<+JaV@5=(7X`D+JzI~gZ z6i`+0{u>SR@#7!<6Fqe$28!mEDX*^|*yaaJq7s;)2R7{^foZRcVfDLXI{Cq{mxvgt zjWUdePS+ss)U2ihkCAV2)1tyZhg*?0{uE$Z3Po?qx{j-8CCS{Prl#f>66zf)b0`B! zA`lIpUr-Rgc{%A?S|mUb*2F|n2M-%^grp?zGuel_H+NXwa)y1q%uHnA7CDNEj&Wu4 znN5yCH~t$g`?|WbYcPey5o}hpmThgORirU|@MD z&dMVPCexZW52wfENjZ_n98WLcvfr9f431d_?d^wqf^&xMDzkVkGx3?3N!0DB+$>M_(^p)j}3u3mYg268Q#)b>`^19(Z z3rqisFzT95tC7rE>7C?k%Ae;H*t#P$GI`cTU_bTb7S`qHC!QQ1^XByu-_+Y>}3wW&J`zZ-MX*pbhOLJ?n?y;C8#=>Q;e3C`G2R72e z{I?%uOs5$&FzEgbCmCac(YMO)|FGPiL{t-kVYhGJzL}bNl=KzA%1-dxEkMEvoQ%gv zQeig93o7R$-ppEB_C&O%= zL}NIj!BSXRPFy+o&leZQrE{@!*4p+D>J+`-Wkr5kMQp~r`XOiYMiQt6yRiLPh(n~c zA~(&GXfXf~l8e(^`esGV8R~uhM$9r@J>JbeP!mb@`nk|#@FnDYx zul!aG!*3S{_E&>>!*_m()?KD!$DsW^612a|FDy)S8OWS!q~y4xP9u-L}8%Da*_-~A4g-^>I?mGLk#ZN;@a@@j*RsG0CtET*0>YPsM#sS*-cs<^fD zA1ZG`?(wns`hoA5@gjl8uF3JS{Gn|5np%8$b4$FCC=PO7;+4R2jNQOWpWFJ;C3|oR zR7kP>w68C-K_~ZM={HGn6)pz$_BjB{AVrkFetq@g#fu5zHtG#4cg`{LBR6i2glzx( zUDp~?xc=b&`bU!8z21NvXHQ=dTSP<6N^!T+v)x)=*@Ij9>C;XkA@xUJz`m{`bfzf! zd2SObK;ugRPAW--^EfH??<2@KgGXSKKFjjw)91{9Vf}Bx3doa+OUNT+BR3*?8si}^ z=Mfo00=;TTbspwZF(9onkY^G^vac|^t})YWK~abhm-L#*db>>Oz9tdsF7MUS5Su6=-_(L z%5`&!f{{du%b&QG?<)XLl9a?6u)jUFKJRO3nf~qDx1G7SDuCe#hI_s0P(0XVRm>Lp zunNhfKWmb%-$2%+i3XJa>$e|%4&?<6?ClZaBTt1W7YJ+IyMqN-+8eL>% zrh23f{O6n3l@MXAi-9~zIMQrPKhvv^9<`mD;E23i$(P_naIJK`T#%6LjA+0=9ix5n zy8y9kV6iw_yG3vb%KiB9V=xuZQ<$pi6M&?ii*uqlpDunCJoja?b|+vNk$XHhVPiSf z261m*7Z2Lb9C&5}DYo*%@G>p7pxpTAgfy2W;FK97K-4niY(O zow#7wS6MNsW~A3M6Qkf%=8*@p^){1Auz=Z~kNC5mCz%E+Tv9s}Ry2hS->CpBLS09v z!W`NNU~-PZm3`&81;T5-`LDNhI5EeE!2C+$EyKLg~ss%Q|x$noOg~E>WTj*Vm8J zmM2Imf;OiuFq$C?%+6wFVR6`4`r6js9^OInc|QKA#S<7-L7n1%dM?3>3Lp*| zy?1>Dq|68ra5#Jpnp*WTI=akn-Pw=pQ1oi*1C{sW@pn%w@HXs%1xKD_l5r$Npl3oV zz;+tgEzUj*eDCb6e((4|Q16x^n=vxcIs9Z)x<(dVSI-%Y-tAK|Z|kY!M&7@nHe2o1e=%K#&yZuv1C>KXsQr1J$x*Ij#u&27GP zP?4yu^RvSE7~`}IJS?F zb8EW;&0~v(J+XrWj-dl;;!nRKVPJHAc%LtVN_Vd>Irx{3+fzb@fpgy-cQ+0)3`STnh1kEKC z5=tJgpZs zr`!4WERc}R&CfeMx|N~2X%UZxXZeFRC(r=H)s%Q0@K#Rn={yL8UcyoSE=Qqy|4}2~ z&Hq}M-7Dqg(CBaPY-8FvP%DpHcjc9u|Mmx~1RWFA5Z4luw?GKDwzd*+jSnfk3#LvY zL!c{)MLvjtuK0p?X)n9X7oU73C(qkg-d3tka2KTAO6$!l?J-`3XF zPlrRswLvM6IB46;uHc}Ij*3bH(@J9MW1J|z+8RP}?c?M`=#{?^a!qR!tXFs~M2XAl zs#r&u+|4N{k28<>WE-}KU<&-3;d+Wh#mC=EHL3OWYEb2gI`PH?3@-~rmZ}imTuCbg zmt2b#?1u{3={M7gnlo+lCsBol;K7D=ZV$et4_Nztxt%SM>2XFn=C@ltMEdpEN#(#A z=h`*>i_+4RBjgfIdGmQCY08>c$b4@Ae56Ef+PGSD(I?M8vJ*PGm7fwPDI@g|T_^a1-(Y(L}_C$t|8)9MB@ZKdMoIa2Sm zP0ZelM#ZHSCf|OetLA;)h9HCa`$`XPVH

<^Cy&%i!i4)Eca=wTuqIGA0*ebIMyM z6{p`Jwv@cx>x2{Z(_*fUIn~#O%lz^32gYp+3Cw)pQq-w)khc|Hvb&bS4)eQS-F60- zi%->kk~PoousJ!7w@&pP{dw@g!yPQ^?k%uvLH={k0i6$YZkR^v3P{DwryA#ev0KYr zmN-_n`sH%_X>{+R=1J(F{r(vE7O397VEqQl7%f#AoWK^g@+;e$-o4!`U0uti?)hQ1 z=Iy4D`{zU4-0yZi(>DDskpH{}B!xQR0A23Gn}&{J3ct-sb_W)0$D~G+geM5+B}E0^ zdR%Pz=NlhgZJ7162Ytd&OZb(Ci`s;&!yu?anYcd#@zCYZ9L^6^IKewW-gyJD)zb~r zvbx{a*P3Vb%I5^A-io)0GpGMtny&f3O8}zP&`_FPFk-741ZsXYlbh42h2O20=ZPU7 zT9p9k*_T~o0+qg~l7n%CZU_B{;_N=u+oCeCtzMSxxVNGx1~f#O+V$g)|8}R~%l|vS z1nN?_#$u76O-zEA)e~`v-F@+XED+*6kwy8zsTRENn&Yg{QgjQvBb4|m)+|I34K8jU zvF!V#88DKdC)J?7pz}9tRpA^at@d7oQejUZ4EeZPV3-5R4khiMnr=WlzL-4cv{Bhk ztvD3k_H;K6-aw=j^EHY4&-wfbZ-HBtAqOE&Y?R7{b4i`Sew4fo6_UTA&ZFGprE%ta zRvMWpgN6lJ+8B{&rf;4#>r9l{+h0~Fe%>2&gUIVlU?!$~Dd_2~HU{0RIu#>+OO2F=DSE1Q z5Aib2*nn%+9%-9|es%Q3J+I2OtKT z#!6C+IKf*Wn2ThSl)#de{CWz4i#{e7s zSH(RW{qGr$xIpXob0yCiSpd-F=!ml1Q>Lm10j8&1H2;53#nnuJS{YZp$ji%ngI7+DFa=LlwC#B66bMsasf6|9vnVO$?eD(1 zoSz(Lp2gcTzkKSj*VG4-=9Z#uo8wAGRlj3yD$b=!04r^N(bcir-CDDEbS!ny)z!Ti zu;mGfBJf|h&_IKH2W9u$Txo~Fi%>~ycv4&Di%rnezstSuBElp2*XQmCRF6DP{d;)3 z`7GcuLdNBqQ`4Z3Nf;3YDeoO4qnt8_e&|G35z29Q3=K~vAAa30#?_EoXE}9C94J?X z=p#Gr6Y^CC&j!@gkJ?gY&|sG_{}(@-u4mC4$_xU<9uU;i-`1t`g65Y z3pLpmpj^Oyp;2kk4SP9o#i|8~fMwW~Y$Lm0o;Sf%l<)6KQ+>#3h&p?v)V4Fl`KiI} z+m@*q3}(KZj+c?@F)-LSFL(eZ$ILouxR9{o%B4zu=YkkqPveW^xh?}8;V%HSWQ%3} zlZ1kOm;ul#G|YW-XQk8O$G2uBjBthHV7hH;fIgS%^PBGO?$2pN^X@^>hmcMv@ct@a zXz~VO`MhHSdRiG6`6u^*-;s@8_H11x&EzinUq7ItNej)U1@o~VvPt>@No8wz5;qUe zkcKFxt?f7mM+s7hw_8S4EDhaMF+y*_}Ug^(|xAGQ*W>Q~zHs85z&w zD#J>oj|W1&Ir{lVnIqNSIO@HU;$nRJA>sNRceDchIN5(fU_%V+1aE^WDp=|-($%Gw zSI}lSBW|7SNtBNr_0|yIV>vHR1U4LffFk@_XvRE%P{HG@5>Pk-$TV-6oYV{D9@7}sKS0v1%U{Oftr0PvuBg^w$hfWs|?+e?mb8~Z~V@6zn zrz-Bfj6+4v#04G$^o@m75Hwxq*Z7v zP0FMVsU^=LDR71Zz>MM_oA^YRy0?Inq7#UUm-EUA1jBf@dnt|G=H6xx$+q!m##;n^ z2Xjt~y+7b*SKvZ;$t)UV$E0l@Wv>B8y-IU8&10eRgzfJzXb7tR;fk8TfERmq{Pcc77{HC|I` z@7{Gy?{?Rnx@aGR&8-C-tBl>KRH~~`>t9Hqf+xTXE&~WGjvR1B>*}drkL-B5;hq<3 zOZ3o|0IPZ_LI2Cj*n(HxiGO1*G69PRrpdM@;jFT^=lcuS(QbVsWp#ik#4lfMg(8Bn zrL#}YICp{kY<&G_iKBbt1Mi zMuIJnz>w0_)-?-7G7C!lC_WE%`AZSHcPI;HqE+5*fVZ+d zB}VfM({W(dN}?CN*U@!(K@;K9tAJWH52jb3V7_+#TU^h6YXP)1F(~IjBNK^k0Iv)L zf5DIuFHk+w<;)?4gZbW6+05$^t|WCFNB4sN8>qFYJF`EGr>T1CrZ4>E8M&LB?VXxp z-JDfBH@&XZ%)8dqWGf>fdF(xjH))o3sWK6sBVmiGo{EJxiE)dn7s|F5Ycym@F0yB>&d>LIv ziNWqWYbH}iwI@pJh}3PQv-{4Rg@kNcJkBVEHMr4U)oZ}NSHqBl-H&u8Yn#*h7AWC@ z57W3@qlQZTRIgNT7<%HbE2!mO0tJ|BJjkGH*B%2oC4z|+EsLFoH-W%xo&>?1(({@1}x84j=RS~ za)M#;&))++DJTO2e3Gag-*}|AH1nUNhfy7(O!~R3hr9S+f#ge4{FiS)r8NMY|45l% zNCS#hW>fL0sf8WYR^=6uVPPc?ZKi6e6SUCc*UYzv`aKtG*X?_61YV6ejN$= zp2b?f^)|94?90yPFc!bUN(kS&@n)766jNxTGUbxChJj4)IaLc{d_?lYI*>y45W@cRO~(^HQ;Z- zwo=zMdcWa)VDMXJ#V52sg;)NN;QVtN$WP3otsA)L?-lEx$&xf@tL}UzYt~fBuZ_1r z!EVF!^fJwNzOe)na5(a4e#BItL@qXKkAu9B=%p?-_d-B{|F#eS287z8Hq3dge@S6 z=Q%`l*(bUnQVN00lYp!&%(%k2%)5HQtmzEHo7VdY>a`r$~AX4I>WrH)CUR^YeRW zx-)7uu553mm~bFdAXwvd6DXU6_#hKhvlsja+fr{1Iuc$Co}fw;JEz-Ar%fodPxN(m zk3-wU!{I;T=hD!7TMHG89%*Qde%D z{M{g|Ck7hhzBjnZ0S2SLf(BvxsT%qER_%dV(8geJ0)mMtUVb{s!xJF#8zzM-(~*-K zU^H12HGU*;n=lIVfHF5lB@e|Ct0^#j*UF_~(Qu*BmT`s-IJeqYpDqID#(&`4wl)0L z025>+h)H;RYPiuz<^|vQtxc8LKMIl{L^J@C|Dce{50*)N4?QBTRlHNsj6Da*J$1gs zZZA|9I!M&7^Bh`l;@Gczq3B$AAU)k;w(~oZ$1Es5)M)49%-dl<4;EqJ; zPI|oJYS$Ga1$oPTTBRo{g&=yEXI2#kNO3n0gm>cdEMPvSxUn>}Ui_GRK*&@?`?K;_ zUFN@+@(?nR)8c^v;l_sFY3J!38^tnO&kp}`MUtSjmD*djr0&2NuA12cg*ReeF*J#d z2qzmH9E1wjdZwoNNvwg=>zCaa-NAyK<(+;4Vqs?}8}Zr>^V72vs+MfNOJLb4v^~<( z%4_|&+F|%lZw)Zc@h3zt)gPv$AFz^C9t&7El@E@4@#1d34IDnC9LXyC+2t@?8wCiW zxu2^{FTQ?8J+XfpKx{`-jX@YXLr&C_nneiM!kjWNbw25_4f~X|PiHl*Iql!24~TsK zFa2v*%>qEszF=|*fsP8;Lwsgt7>fvz6hB&v>p;h)rY`2s>rnDIVMhcs@T{2R7E_ba z3oAyowQZvzf+C7VE3@$mdtfS?@Jaxz*!qUyN89$?G|1lVUa_cH=AH z;qhP|wp7e1qv*?*FP{{qE!sdTkp=1%774@~XW;Q`)B-F(t^GRFK;WCkJq8Uc2VBRc zx6giT4Why%%SDAgSE&d6jr|xyU?kDJ1PAl&gxxv0>(SYStVHKxY1SFv0_7TMjHs3h zJ}lJEYDhRCX8oEL*W6BlY4s|`eD5>;%Z_+4uJo-FZzMoj@u0N%A*NyKu$pIf*^v@O z!?+jtuAn682Np0*MH`C001BfPDSWDHd$HbSF$!3;NkUbt8yqo2jdzI5ECDK;!m;R$&17ccsxN@h{GOlSIK=v%z zqeB)UpWX8oGRf}M0hG2WBg2&*1Sy~tM4Ya09^vl^1ewFu-g?e z_+&Qk%v3Yi#S8Lpbrvzre|HfuHSZeExYWFY2odygnyE^rxdw&ARy!!)Rs@c63*bTy zy!#CPv7hoyD<@r3DKQb=t32_xTZ6I|aQKeXXbtR)wDT9BZss{;mu=UcaM5?;>(khm zFVjJ4U<#Mb-y9qoviH6_Ru^DYjmVc?vg2O_v%x{p8VXq;by;vNO^04V0lc3mf)5P&5;Jqlw4qP4+0gO9OA-=Ij@{??O|NTNV6YTGGBL%;=>Z%JwuxK?dBOlv zT-Q#@NwgVDZ&L1vR(*c*YqBXEo`P1efl$#bO|)f&1o$1zU@Y0FW&`%SNFu1snY!%V zg;#)#t_>fGj){3K(W%fw|7H}B4@RZF%78?k-fIDHF#`CP=6GX$tw+u-Dn6MyZ;TNQ zi%C>{hYkDhUm*7`BQyjYgpzn?+ekD_Gb?`ZdbeRglHP?Cxwv9?jZAjG>WNMrPjulGUKV$=d%SwU@vZs<35skISldLFrW{hql-Rx=8LTXmwS+`Jc6SH^X6^vrz))6Chv|&yelkfMhFWN6Opz=~IZL2> zuPrJrSs9$(MRAVq zKq=gDDJcuJBan_B<$K8ax|nVwVMab67n)1)TPX5~MbnXKrc^D2!m{i_D|KEo?;Ss7 zr}JnJK&WEW3O_o|*49*h-W~)5N6)Ws;M|XAl+TZz`4^iyL^?ua;@N#|Yy2StJ~*`X z7Mzq@PfyRN%sv*N41{s4cil>|vogm#@A7A+H5{$l?3$CFQAMsQDDdOYGGMg2@nVBS zY;he;Tv=)9Js^BYJR;_y8y$^3|D8k0|GhVLzK=G*ES@>nZNkyqNh7hKnuU9|FYPq+ znoS<@NwCP#+!^Qd)@iiEXl1+^#}Zg`b`8s zW8$5AF457oztg!t11xHp2O1Q`=RrV>7vr=we-{bNXI><-5(wL`9+VtX#}wX_^^Mh( zqF`JSYlXJ-x|jX$WnHKkL;VA=g6Jd(JfM(S%|IYpCmyYt%+yBd&1|$*9*h=40CiG~ zV1{I)*vek6`EHIkizd^sALfaI>CmwfneyouOo>Tz-Fc^|6&B@O%vyDTeZB0G8O%a1Oa{D_g z1nd2&8onuZ_^ATwKJ;V)4?_Z9Js9T|u;60wWQhZ?MMB#1L9?VcnNgDS$o5uX!289J z^#``vpSmlijt~p-*HgxSBk~8~1xU950qC?m!tulE(})e)c+UPGf?*et5-W2B@mKc= zt$XqnViF9vhc;Ptiqo)ZsaFbc6!xaQhY0Amb4F(AR{*Z)Hzq-OZ}${HD5GjDp6HwW zw|=&o-vCDG<^W85Kx2K2sBOZAoM5teGWcWjhaD(3y;8Xnrr}+CJ3ZMZe6-qZBa@`c ze@bQ1?;qd`tg<_~aP8`CpxTO)&8u6EEgWbat0wjdc1u*|UwFkN>hqCDCxx}@jA~;t zAkH>kNe`pqotwx9`bnaWNc%#91A<_wRhw2_&=p40F57g4}$hG2Lqfl5hMz( z6}5p-ca;C;$oLn62%#4_2X|p8lyi#ep#C9+ z8BP3ObJBADoptLmKeZ1jN{;Y&@;0~L9$T5GDlQN<?F1Z@RtirH_M16a5#r z&CMiSLZ%`+XM49@#r|=kj5B^nw1h2jSL^*@(pu2TkjKJUNd^u~crf+L5qP_xyd!zL z=Cg`)n|`5_Il5}L>+N)v{QoZhqdN_g?t_WmXqe4$vU5=B_`D+B7RW8kkh^;ARZU=0 zj9&Ea3W2m~y5OMS$jG-%% z9HNyNJ6B!{4-aqq^r;JEn3JwDP`w01L^FhnvA%Zcu{S74;OqhuhI!QR|^BqyfRGEgoz^45BE>~3Yf!gz9p%%wI z!V#3Gyz`X-Oe)-a4aAG3k9fywA&DMq}yN-+LAYOialJqV!VEhGtXhOiOQ zre9&EBXV6izjx7mWSTB+GL!q>;3gNq(s%#V*uY8oNCv@O03g;*4IrY&J^?svZOEZ; zuyvo7hUOQe4^WBJofbgLRv@VZLVv!~g*_(Q$r4Q?e#~pdhf7jaqP0-yAAU73xg(@N zI=IBUr3SQsuI0M~U~J0=fvB-JnoY8>a_Te?&Tf|P8Xh*GJ0<>MJLNFhWvKjgylohW z{*lt`co+kLgcY<>WUQSl;-z!BJGl7@ttR@7v;2RpBe-{aa?LcTbrrF{2qOC6iH()i zeY~!Vd*bfSwDS7f3A%s4{aup=yUFu@BQ($Rae5iMdO+Rl-SF{ALyWLA{%!QPq2MRe z+&EcGvF>L7ZRPzo(Z4fHkS9;P2UZ>bdQVN_DjdZYe!|I5BUl+tj62^CjDs?=ZA*Wo zw)iS{Y`fBQDCP`dJ*9IxE?oF}ua-Ict5xf#I`aQREl_c!j%O3+qq+()(C*z=Up)#c zZh*;7N{*3Z;1(9vzCrD_H7g5-1j#l1Q_MKnON{y;>ApKTJw^4QAQ<*!&n@j3 zG^(i959666m(%*gMTLIK-Vao>ufN)s~w*pP7}__S$~d1bd_;CMKps zkCp2$nDC)MMK{-Y5ns?fN9eUjRPF9Hu~c)}{xhH#Y_I1HfCNRDj02==U4MUnQI1!1 zKrJ1d_7e!_P_Vvueuu6z`ca_o;IzQaKB;^;?Ybb1FFheoCzmVK6gOS}(gXpa?Wkx; zE6C8P;+oqIhktbg1+}kFF^`H)meB#T%)`Axffb=8d&O^_#*iryY$IEV zKL5jLVhqtj0;;+()WQJpZA-e-k{6^XTi;khv+6VzRDp#0J+fhaGRGC9ub z?k>tp^1`6*ozh#dDhHcZmW3Et3<~J@y6=iyr2}TUOO2G>HaDTa@a9U32u4reajaR) zwe)3nO73?wYOUL}Qt+4UK8d^*cES#vg#&IVH1VCpBeqC`XDs%w6=eKMItfbW9%^O! z#)WOZ+2$^~$0oems9{DHtZYPe2YITU09O=P=5iosv;p@WTj&0ObK zair#{h6HdlAivTgyac8H0CgTU74PZk*_EP| zNF-V9Hx=jFUFT!^Gz@Y5?vImXW_oGH$q-$xeh+*Z4EwHh#`g&rE&#x&!Ar##^@HcH zmtYDRV0`rLMGs7AKTg}LhUko;cJm)?ZO*m+YYzN46XD zay_qXXpeJ-DETWfLynUzFE2}i#+e5;B<~}p*7WjzwbDAfAIOs*ac;hqSjz7P**|)4t0|FBrl1Lo4j;<*#w@5$b zqg-VowzJ$S(rb99apls4WI>g=R-eF^X0FI@dHb|kGCAwyJX9raZp9_LQg}z}`H^8%Q9ss5k~*0XqCCr65sz2;CWVRs?aeFJMr5VT39M zyZ|QbJB7Ag1tl_^&#b13q*xK5GXgMSdp|s5A`Cuqu2_t4V=-|>0u_k)TIciZz(9P+ zzMRT9a|M||k==*SDYuOd(6?&G&fLc0i%xM#lr_-AXtB?w>ZECSjGkj$A$^hLxpr@< zT?U_KU=_aG<3tW>VYxyxHoK*ymP%ecDsWOOeZMkE|F8KGJSVvYTr)Wj8<^j>I{hp5 zQMp5nO;I!k0J~dcwRHk3oTuwpc;?r#{p_r=$f1(2+r|6vrt~&*ZRK)fnPV~H)BKsq zUbX8hJTK~D!Vb&P?Pv)#Fl!+JaHKLoNH);H>kaZ~oD%~8Rv>D#a)tkskzz6cw9?&8+>Aso0|HDl3GChc$KRrx4L$#U- zF~C~3BMnPhbIQ8azq{0~2{h%i2}~PbDz3D0kGVU^2YL&GkK5faP|Vjt1Y?{iOOzN8 z0qg_fjUp9_6S+bgy(d_=YdY+?=1Y$1e*1PjRo=vne%^(n9Wx{=(ec8Xjrab%e`i^l zLt&W4l9NM0cUeKL6Rxtnlf%asJ^U0w(LCOEiv5oK>0h;uhHmO%w-z>VyQtk)hQr(? zv&p>M3Wf;JZzM*>Z-q2Lu@tK*P~tHQwa0_1V&`5Ib;9n9n(}8LF=+6JkCq#ZC^)Rc0@-p(?+E5b%JlgI@PKJvQE8f&aS|P*3paFcd)TQ*vqM; zZs@R8+1Kq!42^3ux9tVti@Ai@mv3d}nHK|6GqdP)FuGo%Zx%KG>uPPQ>)T4~XGF!! zvb_D8S^c2OVdazTrLv6Q%t_YdhL_wLAPMcBc$y9@Zj5eSdX`t@20jU)Y_OTthvsU31aF}kBfb-MsU$_+l67XlN&020_^9n8Y-9?;!=p3e~jhL)c4ERXapuiq^ z`t)f$%*qtkk#JE10(74+a1?-XHkU+dmLA*&OfR2h>(s2L^lYC)nFXxjiJ&KPtn*^@nt-vJ~Obc4s1v z*$VAqO?(UI*!L@(AHuVOVoQSVi7?Qqaa8`oadx%R%KH+oxx^CDH+G$Nl3UI^tJmr6 z@5lLk(hMwCu>l<;#A6r$R@}-We0ssPR+74lt!6FRm0(a<3I;?aPf+r!Fu1m(Lkd5- zA2P#9IzkKi6ogxNa>ITP&Hm0+_~lPGn#A)4tz{GRWB8uNcBO)ZaF29hi@&lhG@r9g z8#viESh_k^F;meHnWaj2kf0j8pj%PRF>U2iP1%+sN`X zajvl&dC0aZq8R$rBc2QPDb;)t$P62J9oqbkhIi2Ef(5rPk=sQ1h=2Z0kGOA<@oMZKqV{hoFi}!R;Tvm^iHNW23f$SxC`iwAL_z~0>!D`10Y zt^sq44(EWU9W22>`aX1Qa$n3s?MPEW{N4j*$yh0yM~~qFh31`YifpoW06;E?W!NW& z0AG`&@C;vU(YCSQYpNawz!*=1zLMH~&WMos`(x}BC$)3P!8C+OgKMF;HGtdhyO=qrLND~;POkGo;q8=E%NSG4bJ zU8~3Md=2!|6wMh^@}>AC0eh2_ClZtulM4Wq2Y9NM=}sYe?}abP-lK0JpbBwQ=nd^h ziwxIYB^!81W&g%igDdoH@iy<#|+QXoOF(U9AQwW;+{EZdHOU=nX=J)b=DbT^{0_X!km2E`A301gBs6CD|VfLU$*=t@ObkGGT z$zlMr#I_&31LsY-pRU*>Seh}DT>XPE&!$%JK&6m|HOH=R4wPOT)WUPl>`phl=5C&s z6i`7tiA{9;9$csOeI)nrge4W(Sy#zwUHuWaCtp4@e$Z2Pv79su={!^LvGkJr)QPuC zA9VA^2Mn91>2*y|PQ^^Bng*CDgK91MvA`Zzo+6jU!@tAUL&$qaMUnA9bO?bnElKK} zbZ`CM@~eTk+nW^lni1h=7fDKqoJJmCc)>F#T=`%LdHWd&4EDRChNfmI3QjZ?+1_g- z!)}en%X2cuKs?>`yd-NE$HKnyE6Br#0Aw>ry`!UaR^tZsk4vT~RjR9r4c&O{*`fzF zBSJ~fH9AXdLl3s0W0#>UjVkQfite3qtEcTRt#=6q-p!hi8Q^NouDnHy1^ZtPlntOD z>%|38(B0+dW9A(oz*ltpo?-L%mh`LT4K+S1tAHpw114ELWDnf9yNyQEv1v9Cw&n{X zw~}?J4}zE^!-&AV4W)AKMyeyX;3&`la<8|XKlVb^r87WKq-;F9=?#OR%y zj3IYP40=g`fSXQP;Bu8het2JA(Rkg8oBlh&b>+Sdp=`WXjw`n*pX=+C|0CIic) z0FB3@PpAcDRdI<9?6P7B8EAsWnk{&@U+llC?!B$0jE#tGsdAWH`r2e;R-{iA-#o-* zqi|Hf0ztrS!5-?vs!a;UbhGtQK@|I!`i>ND>YEk&Sb12(<8Fc8IucZwl}#8kr^DcK zl?kQttmK#R2R2iO;Zm^vVXI`}4`=`2m@1C|#Bu^9WSj+cECJ=N&kG~f1hRb?3`T?> zLYW8pe3`Bb6?L!jcuuXj+_K-S-Cqj5Beh|o z$lgxQCI!83u6p<)QtQG&k^WcZ3YMlVEVkTJC}s=#tatJ@qUxo8u;OzX4cH437_oK? z0f8eaF*N*-1=hAYbsrGn*$O#R)j%-ZL+F0W?FC|{j9oR-y}_XG!N>hj8YMUzqnBI_ zdT%asN`Xp&c5yl9V6HnV==A-Y5oFvEa@TG|cijGOpi*vmpuSk%#)8lOHzJ07!z6Qm zDGeem9F$Q&jV@rz1!3aI5C@@q)d<38gSusAmt-BQ(Z4;mMIx^6ss^>kUEndjQk6Rt zNhz#_viiZ;^GhOaIzQxf$VrRNjqf-n)_y4lOMkv3&Wu8k`i45IN5V^OlQ*xaX^*2` zes%h|5MxI32Z2YvVc7r^``!RG0aN!@Zp8?jrSV2$MA+=?Y!yf?{2Vbi{_wYV<5}(% zBg*|3dyE{UV6i~Efr54M_1RhQ5j;E6g}x@kjYP*`kqo>OHePW>ClyYpEW z-0-XpJZR?gXS}Pvz3d^Cvhr>V7QFMP%jy=d1>&ZpPg|U@DSt_>r4Ce0?z+L7{JJ;S zfJtHp) zZ7Y?dj1xwLMlL=D!ljw8#v&g?l%U(NNi&E~EoiEBDKf79X>tR7CrZ2jhrO%*s;fuJk?M~P=%9ri%z-OonVu4+u^{qMZRWED{064vq7 z3%91BrDYHhKw*#eoP$^0n3CCZPj4+sQisTJvM1y$tJ`b_gFoOR6+{fx3Yu*>tYkq^ zeT7Fj0CmbeoBA^b#y2t@fB{&n+;b)|M|b3A<{OkAvfi+A{><2*R-q;TOX-UV5%K-! z-XN~sfjTxo0|DUjun?Hr?`3SztoEpE+iHKPyz>1(7Ec}R^f|t|^*Z5KbWn3dD!!Z* z!<<=Y3ajKrJA715**`LOUlgfco8Naw)RR#0Hf{{S;{4S8CQVz#ir20>BLXmPJ=1JF zXKvj3AZt)foS0k7X7hN1lJZ8_zq@b_{k{YvYF7uP1Zm2J1AhdebbuxfG>zm%m=+fn zrWF<{0`qOKCRtdvT7JYiyia^m=&?i>{Q@_hILs0 zVPbtOQ+j?WGKOh#cs#rYrlOXlvmjZ1+AP$&rp0U=@J0Sr^P zcabRO+Gbyfg7;!DJbir=7P8WHEl#>@&szEQ zuGz%*5BDP`81ANKI2^W$QO)49Bm)~~dbYbe^Q{nnckx_;Yf#Z}a zOC456M(Cy-``T1a=6=(>9iD5q&Nrz(&M#PvFqlg7v`qV@>H@s>9%i1ydv$s^Yw&XH zSHKvh`&eB6Q)S(pn>pw35ZsiFv9?QJ4!l6E(yI>e8_7t#o?dm=eq_7BeL-gNix{Ww zQ&)vw+FpNnCVSv?2QdFC>878ucioZx7*;cRIoOiF>tn09jEd{2UXJGvLW)509V&%B-1 z>|A3l_yds**ND}-`;8(vWYsD5GAJX9=c|RWo&-op%I)owW2e$XR0+R+j|a40uKZQ2 zXO9Ktz#Z%A=E&B2Ys}VLD}~L20){4DF*Pti)^)euiBW7DVoCbQSTj%X@!0<6zm)es z=DCvqhQ(h$*Hg)mcOv_m?2hXoDE7ekGNl6$(B5o}Q=~>&^Hwju>;AmvRX7&zi*N#o z0VBd%!ls(;TF#U+uQ;Hij21k##IQbm#|uG@6C;LqzfEdqcbcgm^OzU~`gca8UiIxw z3~8bZxXIxmqWHAhv*~g$NA-3xiFT!mEvLr$%T@!=xBBeBBE35^+y7_~T4OQCl+jUv zdlVmEKPQz~ehv^kKz4($Zl{TUKqGolB1?@+`L^nO(^KIqSf%FT4znB8!FKzudnUZG zrJdC)Xs2(xO+N)Zo(bjM&X?x6_1``G_aVZ7L}FCwM|R5uO)dQ<3LVQdEh{s!qYSkq zGKvc30Et@G*z+>bCqA;vzJce>4WE}{=>Jw-(%h@KK&J8e{U!0Et-$_oTjRE5BVV&O z+;m?VMoDZaX`)Mnoay+19=-niTRu?su9|ySMjFdM4>VF5a<@x#XSl;ZW&lE_?RuE(yk~KtAU(HPI!J_=JWO z2W8LIs|;`yIfL#%QKZ`m?bX(=uU&|3TjQMi3+HrVib3;HXg zp7*=-y7aS`A76+Jzrlh}slm!GN_{c;5XuHlX}XTP{W4%*ucenwJhE@<5^E`4Rp#FE z`{eMWfBEP2f%O;Ah*x_#mUNoKBwZpYov#Fr+8St>7HnWF(q!cy)OLvq#idBQ+tT;L_%n9byDBKAYDgob?_tIGwgt%(3rL+uH&;YMVQlRVWM1S zvC;~7^W-QlKWTg)5+2{%G;dPH5uL@#&uKt*uzGFRruT&U!2sQ#6~e4hSx& zepnj^Wy9s*x#hxcHE{{s@4zt7Fh(#m!;X|tZ7j2l2WhYC)g7Ze)@8zYXA%Q=0)D%0 zs30A=dVWOJ#!&6D5pWzv#zkh)njTz|G691h*tpeMnMH*U=poN>4owF>@86kmno^M* zF0$eC*H?uwtGjznvsbtO+`VBCDo`CA?6_t8y;z7!1lPd`czyr>T!tE-?e%m|GG`ZZ)bht6onxbuzPShW!0}vxfC-)D1BEB+*ePN>!Fp zuJ{Vk{8YWMQk;gWUehyfuPsuqVuDH&?3Y=F5my;dfrv?+uf3E9p$uV;9XK~4zT7vW zAU;#G0T^8XOCO^zdH7Ps-wa32#5C_6dwunb9**ZDj>2wpivg@c6A*J`XDL>pdk890 ze#1oXIEi+6-T?ceRn|FrpSeJAiReUJ%)r?7&q+z6CEk_W8~e8bFri!B(Ny1X&8{&R za1AV{S%4_ymOq0e_72`MxNfj9KthI?IJV}<@es6ivIq@&IsGJsMJ@Gv;VMfhI`Usk z04&!-ezcO&?7Fu&4E+4s&d$!h1}R;e@GVn}awrnJDw(Vt4T7xjhYb3hfXR^T&Ld0v zV)^T0QxsWbYq*Q|rg$_HsbyF~S`k|FOwCK;0vj}UVM}h-1Q&mC7CUb|FNN}*?DV>T zDty52Wjsj*sv}jj9Kt{ce=kwVECM`wfnAf^`#y?)e6e@!j=A!1E(`DN$W#ECz{xQX zS-gfc{EE z+Q?9D75Waug)?jtH_*Xa4f5@eB_$`{IdTM@Sc+0o{*V;;+R(?>o@#XD==(nc@x)ad zP|5qz>GPG;(Nrd2P61N!hAX5Qu=SEHc12f~DdLCfOA+H4Ad)}pt8g!)y{)^wy{yR_ zTdXw!-j^P!*nytqN=jTMv@?s0r?sPHUZ|O zM8M-o#$*2sA_V|q3L9FfHv{->WH9uVpdM42kGF*nJ*-}PoK(lD->^aB_T370_?tHL z0xyMm*cvRPX`gx^c6Bqb>C~$9!@Z1yXYCZ#j4Em$-Q@}j?>jG-iN`ueO<%u0XH%46 z(|=;-vU0n5W^dx5X$w_K3~J-vv-X_<8(dQKV^?aGZFiU!UuR*9g8M-_AV zB9%24C#d}fR9NE^Oxwl+M^Po9Xb$%wbVoN?6ZbHzKJ1I%=PwGTQ`-a#5*15hRFJbU zr6|1#`0}P~itnw&IngDQCy%a|Ut5zM7n%$a3m>V!Yx)^Zc z1n*!;g3e6PsnrLXNU0MZ?V!OUA(lzuz~1A8(~nNqoL${U(imSEYG1vE`;EXvyE?H+ zS54F$pRvHmlqWdUaqp@)|JurctC5N0_ZR<4&w#k%62xh>xmo3$5sg5AZv0HJ`l3&W zmd>NIL7KO|>)%y&$ML&j&Gnm5md(LOj&<4aDVb@iiXvl^kjn55D`QQ*CV7m{5R z2|p3vbLa^-AlkaU_3=#S?RnEL?{8yKb&RTrM`1SX%#ZFw&m{dK93_B&37#=yJ#uM8iHbz}XP8B$83BR|;s0^YX_ zPypO1O|M@b1d3WgbAmE5&ch|rrNI)<8=@GI+p>s55m#!!&Grh!Pz3fT1#65mC zc*T9F4F{a2i{J$-j4vrGBSRd3f}o)F;Y8~eXkT3vsc}NH=kiaxmMbU>_=hLlhDLoD zj89N&l`qEKmCVrvu^QKIr^P8pmunXYQ0K=jJS)!@is}>uEQfosqopgN(_xlxRipP=g`nGu(HVD;o;#H z9rtVVi)Pjs=Lm2e(&5F`i+kMZMU{5)TOI6%`Vu4N(VHAE`k8}j$_a?s)}}ravA+NG z21r=`_{6{8GE)XpP~KeQ?}q()EDPq9a<)qOXk1_E2-l&*t1v}YZ57QlV188(j_qV- zX4C1n#4)h`{jbnJD2ae(-CeIKb5T*z@~MI0FoscsmQr?yp0t*7OmvJxLK71+>1}P8 z`m)LD(N|(1=IT1eB|jT><|Ceh%(^`&O?|AYiHF+_?B?sit1Spp(jK-7DUw$zOTk!I zwBUGw6T$Kt=Hge$(v$R5jdd-aF4zkBa<}Re8|^USGm@6CEwudl(YzcCjJAO=x3Fzy zQ>|DJkERdeIYLOw_cQ_B*nkZmE=U01TOgN7S}(gZZ4i@w+kXU=WI8*sk!~A0{@WDq z3f+IZsP!$nVm_6Ku*guQWFjrq{!5{c(2lqkAv}RT?;95?jAZ1IYlK5;V>62klWUc^ ziFR5hh`E0*HT*U$r7q_z;wBq&*S}lBT|-N?!~r-P3z@yW~Kv*1i0838CZ-b zt#j#XUq=fx$zrtHL?~=T%!mh6=jbYa+5of*HJ8!kB5%K>)wDnQ&^Iy>q%|Ds!(^Ik z*@#n{X)lK1m0Gj)rV}Zw;+|q_<5%KpXEyRZEX)D>?EK;pAQt9GDEb6zToqm_R@=U3 z%x4wY9An377{aPltVSsqZw(h#@t}8L^k8f4;&-VlG(nc^#|)DLOZ>$ay8OU*Sm;*F z6#lh^-VHWGO$3jrqTx6?8u|$~W3;oKy5ojYm?XCLQB~-4bL4f%nty(S*b^xFoXZ=q zA0RcZiZp813XVxF`*@qIU^JP-U3?BdjT9bW=+~RrDxj0z`OVZ?I-7JuQLW_Hbp{{w zy$lqn!qn>y!V8&Y1gZ_B=NR8gj#5h9>eYmPR}h8nhx)cwc>D&LFL{-^2`P6G_6c=< z_3qivxMP9TAFZI6OUGk>Z0B=g@oM3UznS=3>FCm8mf8cXzAVAfkbUIclH)?#6JXbz zE+;=s{+H=cvf`ql6~Ilh6F%83ut=y(99g977&wj76|tKmE_-95pJ3S+mZZF?=6{e9 zFsGK2nynKp@@cutBOuT3r{zFPw8KJqueZ8Web}{o%}(CjRn#$u9$%wJd-R20&yIT4MWpeQUO)wb46z?2F`I0 zER2Y=D~sFuvEfpSYh3m!uj8PQ1GMbF zu5ZoqW$&J>oT+=aVE=CP>_103_X+*A+p_#W55RZm0MwO?7eLE?<{Vw|0>?;~?ppl2 zn*~HEqektSw`|79LllIBgnpbvcWlPfx{}^4qS%)E?C);HiI5#{Ry`OM`o&#=?I%WN;#? zD$Mp`D_c9;kLDE}Z{4?i&2A%U_wQ`V=4jL1_g*vHXZL@W^oaH_;lwx5tYvs)0%AhCXvDDwcGJu8q1%qcqy|?EPvy{Z6xW9(^7ZN(O4Z%fYX=?T? zCYS80X&sSF$L#ra<&wYbEY+IjY&EBrgMX3N>G z=NMa{XE@qIge!y~u)*Zn)pqKhZ)^J-#hpN;;*9&VK$DBEC0tAS6OXP$aECD(R{K^3Q06Est(q23B?da|Jv3h{9{&AHl3-x9eXA*1KP-Se zR%O`khXvI6p571h2UhpRORrdfuOQ>;4!_^+=->RXsscK?k-2zr!B{{?Sp;*bgX7$l z3YnV`snR4$(CMj>KxZ>n>t=udNw_sM@T~F8g8y1pPx7+TX5Pv00(l zTPt9h#n1iMou?GkEdobH+0T0JwQ-wvsFmwWZNnxTS4n9g3;1L5pYG(jO?+VAINFd} zD=@Y{edM!tHC=B^gvD_29PCjzc^Cba&dikfDLnNtW9E*qgS~w+wpEOl z&5-*q0h;!86WFOwN30<#IK2AXtgrvox}HU9{ur-~m$K?EO(~{Je7y>%>}`<+j9z*o zogmKqXdpWS8f5cb27IcF{=C<^1IQNRsh75q%4-zOE!tlRJPv${MXas6DP(3@heDPy@$^ zkm5GjOSB)Y+rF1_tE6eRud7QI9MSkybcv11drZd-H|d$IJL$aqqx)_$KH}n#9!p$h zUzR!ZAt6O@kj`E09T@go9jWGvvpPNUTiRSuxxyfkep`4l->U<)R<-*hsz(c@V}KtL zs=6%X_I{rH5GCK_OCDKAV-dc~Xs&5JbC^>)a>hCYrN9MEU{SR|Se#mr z+IBDzcU{7F$r)E%UI-o5qWGeya-V`3`=MTH@9PdX=1LmK@W|w4t#WxC;{Rf+@gm@i zU1nFjQm0vFJlt|oB=!pf9$i6AVrF$XA?oWW!K+jzN9h@!1rW`dRkm>c7AWh|L31&Q zi7d!6A$c~DXxJhePl&<8vp}!$@&Jas%XLy}Th>0h8q`>I6rG-6C}|iCQlMY0o3#(K zO;0)gjW%Mc+GQR-o2`wwg<&QXpR|1`5*xM~Rk*s&aF1y0*q1ArO%x?#W@46OQE+zt z7-o+_=tQXA%yFIgGdh`NwrTC%xM$fo#|gG!b;wF?fBvn4bUjsrKmN8X1G?99LX&i< zL2LT==EC9dStiZ2cNku2f}$(Ig;PkRBX+p|&~6(0p{4QV@T=XNc4}wFE~^FpZn@lu zUGyfVZT6S@f9H}lxHlG6^C{a=n<$BFSapi61CwjzBY*zYko1Ivg#308_y78pG!Af% z(3y%S9aoU;wC~ao54N6a`v_KIGwSTL&3dC zS!$z~{RljIpOoECS8dkk({;EDp(;Pj3W0jPpRz~0k*jFe_n1P4dh*-pA5pn%D>}^y z%a9Ac2p6)m(onp7&$<&CjXKqDiSxQ?7v8|ywx~vAOqGGGy-`S9%Q7U==DtB@SGi{(vLdh*vQGS0#gFHHizXH21i$LMOa z=X>bDkM@9OpW@@pV+S>zu(KKB7MNOC$y)yhK|Ode(%A7rD}N*pA%DqqWaB*hd?ZS$ zu4J_2#m)QIe=%v#b8vO&)(i``jlmcQ5*(e`PdOkuuI~Hza>AFv#7fLy@}R%()2{ho zHvu=(+2N`dK)3vW$`Xc|di-wCqh1iFAbRqJ)%Zw6-iaAZxI3CnxFHca5q`91ZSVy6 zuwA491^hac(kZCF=r*M}H%FMzGTUzpExojATG z|HvbbQ1h@kbYwK2nzYd>vN7h{T3_kcP&6P-e0G#+eBl3$m_JaF)b+p2Sh-SS`Zbq7 zM80WjmA7tS^r1iB*Rj5bdc7Pj+HZb6>tJCLK;X!!sQ%djS5@N2Nm(L`NKM(7lv%SQ zZ|N8*?H(|#FMM4Z^12FJIjOAzXNe*J_wkt9W93%(j_?WHt zNPf@xB*0FwsHsxP4uU)T*kM`qyPI;8-i%H-6ohFFs?42PNY9!K;3u*(KFnbCy%!X9 z)lj$X#SCMx8b0DnPw4hug>Z#4o7W`_77E)-7=8C%-fHZU>4Be&VZR^pdBDqEjpd?@ zW#O|6rK|yhfuc(Ug1RQy_8?|#;5yB-4PtWr#K6H`qBy{GHakN@9|zlC?y5>H>x7<- zqZtw28B>&~O)Y}Nt02>Ia!|Z|b}+1wV6v~%(;3l)WOXQ>eL&R|NJxq zl-yhBW5dhwOxzcX`%ddm7?yk4sTy8L{$>ogi#lMEFbda8eT6mCq}t0`g!m^oqz@8O zS7xTqoD0vI5fEQ$khXL@)+NI#=Sz zhXJp4>b`ncHU!nnuH4>{|7n&fj_zf)q#~5X43H|2eiHUuDKbS`(FlQ z6xE8)njHp~BEVxat?pceBd)93_691bPsD23^@fhi_ohDT>)WkRJ-Z!wIbg^G4%iO` z;E!*pR!t2T?sQh?gqp19Uw-r3kY!cDGO&h$h@q+`Si>V&el&me2`QJeq1nV?kq|0T z>-Jf{O{@=gS}5zW<~6tyfoj!r4|Ut4yJ2;r>vr;n!@q?El(L%O=2brFJ=4kc`^{|$fwOOsR{U;-iuD{kVBju2X~03Jfk@SQJvff6p4XqB zL1w_XY{6=4erAA^{D@o?nm#WSkX#`}<+Hmry*4Z5WCo=;zb;Ic2Kar4m5o&lWsM&g zMcL9Tac(HG;b7-f`qNO)q8CL9r0Ejw|f;iUfnB5yj*VmFIq|1XKs2l z^rAb!%SmL$gbG-z}S;Rr#J*jNUw4IQ( z03SAeFmi!y;8I_vr;Wz$NdB_YW%$6w9EsM24t2Rfyrf?5@c?GwN~PfcW+oVgLGC;o zv;){Q*r7fawJ8SjZ$;p(Ep@+YztR-6!??>cs^#CQ+;HEJXpLam;OGS8Jj)P+nc=KG z&|1_3n9ff71_lP0IzPX`?iD7|3T+Ie&>7~3ogVj%jeU^j-K=jZR*hsgl`nB%z-WPP zkMMBiP6UVst)&sLucGott2ZnAl5RVA)P{O(Of^SVP7VFM8{+3NrTGtLt!lFYjS0W| z_c{t`ppc-GF|g*4>pCv(42Z0@EJguE<)Ebw#?&5IH>=SsT)Iq8?zWbZt!1qT|6Jkg z)T(>(9E7LK0t#myH_}+(IefK=MtDOAR=p=?mSo?LPonm}GPJMO!CRk!&)ZStFd?N2?Sn9t=#w>r)xdt>FpU8~)(w z2a%TRY9N(1AQQD7cxc*Mo8#kZkSDx?F2)10)d1*~KGfH4!c=*(Iq9 zv98QyfGL)NjMOd$XxnJTP^*&kfS&7Kpg*KwU@#+oU;$z^IO1|-Si5ulI5h%%N}p;d(aA601#E6l?=J~ zhR_rUS4q;w*vkWBiYYfLl8nEB&OyNH$DDK>9*cg!<-F?xPim~|fDBv`(5q@6 zjto|XSAB0_F{&+iLL1~G2oBj-jK6;UOjD9zA0ZEsE3n2B5+U2PL1lcMr*5~0akZ0Z}RIWx4nT(mN ze0CsyHu3pQ%d?-CO$`r`YUvBT!_P!j?}2__5A3)Jmds|*cx4e0>$04_nvUpZ$X@)o zsSAI`W{h|mZvh;vp}Ea&Ha8rv*H>m*;Qc3#i(g;;yzW93DbbbL7@<}snYiOw+9NXZ zn`lK7JxVHCVLUuMJ&!Mv^^LA;mq0>)!+U`0E`RWZ^Lm;w9D%6IiY0wAjAfvnN{v=7Ucd%!O+x3eC!z9=*3dnI7q!cnuvM;ly+M~oR0@mvXcRf1Zv^nc{us} zwTF~sz*D9hJ6DG3k-}Yoch4*#REjIVK@4&O_Vp8@4`V2VYLX>Ctty6t2{{0qJ_hsJ`lzDphj8%EYtVr%-Uv z9Sg+8Ab&QX19OMx$I1>!-0eEikxS3W$;U90phVYas;H(uO3!9+D*7e3%VL7Z*5*I7 z#;D(-)!lOh`ktUE_?mcc-#zi4JMhnqBl{HG4c@`Le~#ea-K6{Tb9i6~ zRN`B|2bmaHp(5aG|C`YNcIba5_ao;2-x&gK>3>e)e{TEl>*as<5On|k&nf)RDf}H% z{lBrC$-Q8)-(Ir?b;mD5Xdj@)NCgtd?0TR7LerAGHTr;qoqcEQdCWOD44{r>1ZNT= z&l+4p)hajV!`)n1OitN6g57%J)2et=AVpk&!0GowP_n5~xfSTKle83N9IHk@2})>( z3jw1_PT}|kbPW0Htae0aBR%+S#&3&BfvHQqNF-pAX2pIX{B330k3||tM@Wy@_evGS zoiYo&vcOY~=Co8Vv9QT{>L~~T>Xj8kzeS*4f7KX8TZUI1_UER8BuX&{Sy{;kQB8>h z?8B+wQ|;CbzGf()Z`lNZ?4nLhGV0`LNaAc5JCF|87!^u%)im2c))1+s0A#FqDmiIs zWpmLvdh1pJ`>^CDHQu5`%}IB}yS4SJ`+_>?>;6^GEpX8G-yX%YpzVw3+%K-+8ar>U zFaEXU%2(wH$s=!@WWwgolynh7({sbrPm2&_n{7@q#Y+d)dZci(krYYhIFyjl0eBq>jv@TS_aat(FWdgGahyHqFHQx?j3S@* z1$4t3X+EB6>}`necNH2hs2GDo@^jhKEa8zAj36IE>RcmLc27+Kbc&A}@;b0WK{dYJ z5~RLUez=^e04S<>zW#-8Mq1a8uPTJzk21_oz-I{m4_^BFAHfRQL7Od;_lG=OK`kIuVaH4Hz0oEDSO;ADl@A`aaZ9puRWR1Kj~6 z&?n+0dr^ay>2LxV@1FsNIIZmawi*C{&afcVr;i2H*Qa9heLyyOVB?*Xarb+rg9UyokKhnN^|7tpUK(zI_MxLh@Cv zF;>fpwPKL2T4m_6b8@BvtKriK)E%ZrehB6qwJ6w0mvO&0>Ik5KWS~xPyK-46R~*m? z$^a%TF}5r_phC3-tU4#u0My5=18W`U%lerlLa%x7+jk*>P9&C>aNdf>i0hb(6?0F`E4hH!BOYFSgr(Ub+v;#;f zJbhyMx1+Eu9iN@eB%E6iIaO`$i!@`_EM;vNGX#Qpi{-8N(a^Op04ror_LANJW&_ok zjlfMagr}xX2+RkPaxpgqg2Tp!7II2t7yMfx6fmdwJ78jWO-=fLd(Z*4Mn?lsxi&H8 zGb{XK+QFbiBP=;VDdfy1Le?1{37%cpXv==F)PJ5m#UYF@*h^XJ&(A@SbHj((-@sOA z1U}$vVkGJuKf#fIPk`^GtPoE%U>(Z`y_0jyI#s+t89lz5(exM8b%iS~Mo5Qp-QPIY znFcx*C)yUBu5<%O5&$$9=96F3Ue3L8;q(kN7BKfirQq_`I$%&b9lHF`RcOuLz zH!=9#{l3%s*bdM%i6DC1enjx?G*O5A(WNVD@YEuAL$0wGLOftVDm-gIDye4)8CTV_ z2lPM$Ajvt20^?5U$|X65bXgSJH7Tm0>;q#B`FC>51@ZSOdL(@r+zeO za%?p>G{LTq_6O9CgG&hW95KKq+{(;3as*6MredSYwZsnv94NxQYpM97-5ek^vlww4 zwXCr80O}UKKp5mbup_wx*`4yiN@n+|^AwBWoh@BNszAx*5b&eK=2RUn7fyN{rGVa- z^ig4)S#sZ_X)rMv9KIddly`tU{q7I&87OdOS`^4upl{yZ;dp^f$zIp z-1pr9p_nw#$~zDza0Q%HJvWU%1YsHo>?Gi%KwFc-%h^{YcpMAhJdYp`0PG+Swik>E zeD|?8oI726v|ZPf(x(JH;uNpt%MzjYpAO81?VM6Y&~2C>$Cl0pOXOESIEY}{oY>Xc za8Vfnw9~EFN%P|vpQ;xd_1*|dUCP?^RQtqjX=No1*q`*d(tzfigGL}113374K9+NLLT+ z6qi&ot$|ON0$6kj=>NI;#;aK`i3%LbL7?>jes(qayGk;1jWw+LixJ1$OQ#FOUNWDmU*O<^>)PQu zE5noGCURHB^82DdM6=J%+X!PO@vR`nXfha6v7h!UtzIZx$!;t~!BZo17a^dV_QQ)O zcU3vN?$1*2pvR$(7YcM;VJeC{qJOHrkr|bfDgzjg4X&C^~I%IBH`K zdO`E;md*(lgrR5dT`H`;p1JCG&^oOVNqj9xAe(U_xp8*9ef09He92v3E$HmkG_axp&C_ZU-p5>Z94Pl-^VAlsODq3iJyH4Q z#I)!6YsAmwwg#erX*dkTXDFz{i}>$6K)-dsRk~RY{GE|d{B$&mn6n^*exAw~Ini)Y zOQuOvsUg8mlhZ_lj@4XzbPzBf8PSmSR?VXnW=VXu$wjZz!TDWB>r(xsKk+Nq zHnyhcGA&RSvVz+Zp-k&4)9(IJa>;RVe+ii3?s(PtbS+-SJLu0hU!;Hk46z%F?+IJF zu^lxDoZ3(dH*1KZPw4c?8DxkrVGg+96in0`1fM=*Lk(plzriL-WpVq8J$x)={J}=# z?!)|hhTKNu*!86E{8*|Ux<1_(tcvkFqH~F!qj$l5>Z`j9U7glBk}U|FOF64~vRvr> z>j^KfxJhoE0O7FeElN06Tcg4Q*tgdd&(XrL<|T?4a8aif{C&4`#UxukTzpCao@rM3 z9|DQ5?eVa|?|Jx>*F?|VMZM4*HceQ^n4Wayee4W+s~P&UM>ayjuH_}HL&kqZ!>m|K zj3Cunhu+u24qB!+gg=#dvl!@)yeC$o4wC@}u~k$?GMHQQ9xD=hY>WC99uw$DVd6`v zab?a8Gz2SPv>uX(vvt|d>2RU5vB9qR`6W~U4{!0+&pVbfkaz3ZvI|bbUz0(AV066@ z+`Sz{6V>kJ)V^O-x{BNpA~ih;+j;2E8E}A2 zEsT>->_Jp#L?vw93s^q6H9AmS#vLDaVm91pv4@)8^X)%jRuIVDw60!YMZILf6Uxxu z%a3r#Gzprd9lw?*e;I9qMZCLBiQ%RlrHF4ela>OnAf3k+w?oBxf#x;v!s^EI3X&bg-RX-g5O)t)>O93~w zyPsAGUE{vP2QJ!Snc}2v7#=Sh+$o1`N5|m$yh$gPOCv@bAcIzzOPonhXDhtlL!hD= zqz!1scQRBe@A*>*@Hbo>BWA|8R!w{>8#}FsW+T&}fNQ`V8-8CJZZZhuPwu*JM^CnSu++1zN-&q*Aq{W`VXtwRw>FitOwcFggKul!AK_QOO+Li; zrX6ZHdh?RyI*c@tR{!{_Py8_aBDre%i|$F`Q@0+X?cJ+5HPW8E$9Ut z*v|Mxeg8y-_XV8EFLys(PrC|#8AoiIeWA@3v6y=6anAhw{Oah*_ZkK~;xxf4aXw#_ zHypF4-7j6T>%&uTMxfJIKN-Eq zO}HIFAl6N)E?DWUXu046+sE4>e&m={CSzVfKe*KI3i~L1@V2daW<#VciB#uQv__Ju zD;l`M%Jd--U6adMy`IBecWjh?g<$?pYz+y&oHwd#c^AL_=g%-)0Q$CjL1c+fvU zpBZ=(l;^5$Ia~Y+{cs0hD=YN+ztuKDxpg1NlT)lb!hWmk1%LPN66SiIvnT)V&F;QTMgyZv8!k8rkbmp1QoaNGu*P ztNt(SkOJXTh6S1IKSl$pdARN7lSL`q3tyx#SFGmS1JsS6qnh*n1eAp(he(SRp3#wrfGOE7+;gG!JK$10LRk79sSi)Od*rr@i zAbk-BlIk8%7Ie{VRhKN!q&sn^<~&X^1V8|GKAK9Pt|i$WF2whvbOA(-uyP=llc8-k zTLJ!bF94?Gn@Q2qz!k!nxvi{WsWdz>_+)IsrrH{uJX4UeWj=@G8lb}`r$)i&Vvl%e zqYt5wh8jB9kUTjk1u}5y&(!W$32)g0&F1!iOg0(RVeQ23e?sO%{k;FI+Z5Dg6Obei zR!r6m4s?N2w>t#{#$VK39lr%dsMj8?=Gg<}!Ew~e&=3yY>5%ipmd?)f9F9BPLD~b5 z%Mc!ad>OzI`JQdSPexjk1m~W=+4#`?a0jZ1(B4K2DlT6~D*749f+=Y9 zLRB}68$fP#WT!jOqfovB$K%`Uodp3@S$-_bYUxbC5?~wecx8N0oRbUq3_CiiftarG zlmCSn!522ZU!JtyzgHXN| zCH&ggKz4K26liiT0lY=J+h874vPl8W`Mg*orS*tKpB<27^ZJuLDi-hHC^&+uZze&A zp1&REIAUt*$5i|bh*3#F9^Yke*m$r_Z18b3#s_;h6;#a4BC6yGY z6h$~MuLRn$Zs~(vhZwY~6?2^` zz@&<^N`|1S8taV1a$pO|&@BdurPH6TiT!u?ujdAkQt^WTrxcLJ=%p}%OLnqAuW7?$zv74tkh_BtAatwTmkkW^ zw}=|MLs(xW@hR-w@LTJy%QAtA#qwbqctE|FA%lkEACvnP8H^`_zf7EjvgK8wBRNsB zeu;q4;&EVVqXznWiwXF-Hf|V%RL?$w`^%|yA0R?(*s80=qgG4cB|_~GXvT)`dhkqp zQ|C_!?yQ3x*CQB6+$Us}$Xq#Dy=mcnUw;TxJND*tXX;z!EIz(5Isx`ERj*FlWlzeZ z3e@gBuaj74kB*S4sRDVd(I5d@VCg#0guio@f}a-xMwURT;aYM~sedhNQI2R^qXD-z z0w5Smz%*MsZqmPf5_#?%?rpKBj}_l2j-sGsVkKtKPC5gQcw^YsO|VNQ;Nj-0Ezxa| zQGMfa@-tzVkgN3^EsXkH@7?e;eJQ&ES9x!^hE>#I`F>(s>{2f5#JrgPm1<>5DG^l0 zmpd^4 zMmJf!8!L!JWA)j}v-Ih-FnXxo6V>fgHM}qHben%b-%)}(Ju*I; z7W0bxUQe5E8AR*F?y+tc*P62n_=wuQ5*l~s(GX@wN$+kxFnaztOtqso10no^X5=nS z0qQFbED}Nl7lii?NpMWO&q=mz5Y(of&SUr$(=V&nGc%`|R4sNB3yB((nqcr#58bG{ zMO~_R=Z!>PL9W<$gIA}K){N_K@>UR3-Xubl0Br1hvhV6)5sF{gIvqO#UN^Sd!v?X= zi9oO+!>I2VOAvR9MzMvO3&4QyHL3ptU}NE`x})*6DNo8?0%wpeTX5%_&keh9B?-2% z4G47Gds+)La38E3+-Q2rlHKXCSD+?Sv^b70#Jzn39)-oDfXuNn-8+)e6&;&Ubdjx$t` zoVEzttEnm#L_T2lx+3CR(+X;n5$y>Ce36IZcrSg#ma4a7Hj<$ezOCl-5V=sfnCO?^5Lw4KFX?wqYw+#)*@Nj_kPu6T(8}aodE;*>yb*DH zDHZR-+F@Y_ozQ=;0ICkqYL9mWL91OUxU=Qcr(zwGBM6<4d~Mdvomyg=NF=!9vbdBn z!l`5iZ`7@A>8e=-az|6W#IMbqC>FIYiluMyb}rkBm$`y`hm;lhHVbb2wQ#;yB2ZQ9 zUF+3%Wm$>)(mRZ;uMI!)dF?bLZ{j`lWe0fpDu>HobC2_f-s2o)6$?BnZ9*ycGvb5` z+pyhW^@ERnFTxO_7FH&TN4<9F3r>>l3&PmOSS?5C_W@9sKsw<`r@w5d%7%HXDM<*NH^(7>-*B8U1@sOk6r0os~RBa>^L6#1eH)Yx0`b z&hBlFj@=E^>BjK4fkibUwA#gfB6wjAS{+r+!LCXH1A+x-Y&XqC~W zdbW0%%C1vM+U!5lg!l{)iz-pwgkrvU7PuRo5&5@81|5sQvJR?U!l6YH@8e+aUJ~2j2#5+|scIA~VS~2Y zbG7#wNbT!uh^e52>X-nsOBo?-z#6wK#?7Yx!k1O-@f~FC`XU_Escy z|K=;+opR-}py&$K&hZkrJ>?}ptGhcHj!)K>}rJ^XhMo$(Tt}{8>^>kHgoi4INv}c{L zv(`hy!x531ANP3|8r*Lw&viykJep-bQ02eKm!S4D{kQqNLEkW+FWa`_syl?nP`GYP zeVJ#Z%VX~sf#Y5NL04qB5^8Yi7e*dcdfWb|#DOK=8f;T>Ofj7`iRty+!!;uvfWS^s z*po4vZjz#zLPK+Y0msAF1lcr<>ZhA-!<#+Svca!`@n)(5bNen2|2(34G12O!?6hP5 z`q_4DZat$iGupGIYzd&|A0(T?&eSr@(hm!X-Ld0*qT}3cQMaDf7b23@V(;Y|H>-M) z#$r(oc_;7K$-31QF0F=;NU-YIdMcixv`)*HDjw4+MXLM_tIf_9<#^w>ear`zr5Ph_ zeOvcIRW=Yd9a#iP6c%^1A(9Y8D{Sr!k;ZwT-N&I%A)jlvZwhzdfdidkx>)_V@@EL zGj1llK`}z>sG+1Cx-Aj2V)7+3*@3eW%Zg_s1zPg;+iMFqbReG-0DXtoZDGGMwMsiL zg|Xc^i*7VNBhh)gLS$l};} zDCpEWQo=OusW>K?84b|G#+1(IT;#r@k`x!xZtOU+#NhOx?I#^f@okPob{8w_jO`+a z*huWWhEN9v>b#FQ?i7C*4>|%tMQ?i00RtAg5J|Hh<&8z?)KU^E8!@MUnsnChcarxi zARz@XB~%P6ZCvQBN&)yc43L%}(~GYadj(Om7CzOM+U1_eXQ2%<={b;UAvFjFCMaB) zuJEhcGnEvRN;M^x^5o*o6t`s8nPqc(7w2R^btStl$+DK^E9hV)FL-tfsOIV@Y+3~2 z)=+hoO9gGAQaEatY}euxbBRp2CM~Eai~U~hw9^CQU1Pazcq4`6)`j4pxS0EO+!K#J zw=zl2HvU>OHajSqr4b`1OGhVEn_O{EnokJ2uLTZ-URg`#5vfIN2K_2q|IfmW8Q<8U z{i}!o_qY%UHxmX@m*in*T8|$d67Np8W9Lo2QomE}ckjuYOd0_nY^O^uq(Ju}l;&tm zuY;);1}mRb)eKc}nxlKe^$X^60b|n5Kj`lr`b!~~`t#jpTU;;TnBR{&(EGJPmuvlS&_6-%eJ<8`5L;^N cvzWZHB0(f` z`-Ykm@f7YrDQ(C0?)+@>88+s3GSbfz)s9jd(o!1Kev*3TlX&^{7WZX;nVPhNvd7j( zWG+AELIpsw}LeBQDe@IACo)oqp_pKnKJbtoqTt($7DYXO<@xTA^ zj68fX_h+Hm5PIRo-fxIV7|y?c`0J)-A|HbW&vReGe{+zOe2`}F66N1_kRbAwr(yGI zDP&Bz{`WKC`m@w*UP@jcFA_YAPr-{wtj+xK!oS}Pxoe07{EhfX0l_OZ@Dr1)$!8vI zBzUX@3H*)p>eripUR;8$4PN9*wiE}|pHC$s`TsBXKRR+8k!ECx|C-gFbMHO2U0g?_ zOtNnJQ8;v{Yih4AjNGdBRTAGWbYCBd>@2ZCUOw@rvb)l?=nue|r>BmK1F=;MDl{;v6WT@ljBX%$a6=>Ke>;5ON`!_qw0&)-lo^6c^Rp%XH7QWqI?>fIW6x{1WM)~_&v!lzm zHv-e%2Ur>$S!j+*usuS8J(b5U@vLr%HD)ut^Gwlo&I_f=a-E~!qOGQX7R@AdrZn+Z z_ibNCr5uVhznXS(C#j~`eRHMQvWE}foo6_Hwl0X-Vzzp>imN2Zwdf&>uzdrj_VpE$ zpou(#&(H5PnT$JJx=qG2uBDT2}ij4mkMT?j+J|~x)c!TvuZce*t4^}NvTfD zY$5#XaEhqw%Ig*7beXX88reEUgO(zruMHFX7xxpB@~OhAzJfo8mrIT_^W0gClb#+6 zVivL7*;-%eHVp280i5z@kA1HYEm$TY#(BoZPUYU(+^>W_8;s#x?WS=g>H)pJ)BSdj z@e`g9TG@FJZhIYHBPZkKlZ=PhbiiS{da_IUCpRcF$tJt1cQ93-RV{j^dJ0fo6ezat z_#68yeio)LM-YjM_}M)B&Y0poU?l{O`D42q>{ zXTw5dGA*gEr6pUpq!`9}r2K^r3PT*lgKg(MV(zcr#fEy8zFO(BoL_9!EA>iUZc5jt za%5pbhX_+J{v+w>g}DHQ*~-;eGfz6|$wa}33GvzLdB5(o#QLiTx3KoLfGn zsXStx8$z}nI&WpMgPI)3cYW28s$CeV71+H6D`~z0$^ursv_?I~(kYc*6CNTctXFeJ4*Abh`VOvd)4P>a}R7T0sdIlct0 z`}bG!&Wf3|PutbKV3_e#uMY^i=DsuG^nFz#F?=%b&D5BBqSX~D@83|Wc#lYSDo>G6 zeA--^7{ao+t@M~!FZne_a7kN9yWhg(Mfl8jx2Gweaa4}4vNsH?s^}1v_BOsWL880N z(BXgUQiIARyWQ>8^3Yp8>)RG&mEvo)REe*CRA}lue_o!}l(=r(Nb5S8o~l=EQr*{U zQ*9NgXXk(T#HE@Ph3E%VqDvvYO5*6+rlpBCtM2#rL)kvGwDHnx@O88d2zx8jA0m4u zW`^b$uH*HXr&JALk+0IDSfD#=&Lo!Z`S6mL(&x1p$9`M!&H9xt^{6LDm>D}>YQ3(Y zIWPZkGfS+}JoJSEOi{km!WVW06uplO`(g2&`QYHRp?k4Txxu0<-3=EROle;ksVGls z;gynSp69>S30U@|D{SN6e&bNbU~;ygN!i(cYmG7zb2GEW_g-t!6>O>MI_--Itzd-> z);8ND4_qygShb8Y`kiVcgKZv$v^NcCwQSlhO*CAFx=+#{3bUp&NHwU(GU>$Gc;&XVc3Ua?^Oi^1Fe zAazZu+^gl|+7jFiRkQDPKGTM56aq-u+@AHE_0cFQe(O9MWlhRNR>WAY#ob!HiJP6` z^<0-z<0{m#*jXv5F6$Vl)J6xH&ubFJciO~UOdCjNzHaAmi3MKI< zGU818o=YfzVPYmF(t1`c)SLIfAfoRS>K;7%Z@HfW;Bs2j@v0LAL%xm$u@)D8Q;k5fo z>Qut|&DX}>)@X#erW?v|=QE;ph|_bHzxzk)P=?8*x7%`JQuE(o-@a9wmE91Ji1WDi zMi`U9L?*eb>ZgD`FTrju?9k4~!c>DN{Tz38@+Sk?QZ0>*l@fc2?1p4cynW+^)Y4rA z_s%q9^u+G~-k`VF#~fSJu5avi*`nUo2kOjY?6RdOmsNSE)xzE7>&IR^X=^u0;i>5` zDcz}Wr&D0RMlp%GKJKY7aLPQ5JVh33tD?!{!2F3!p4Mx>8<(aUs!*`Z)@hgzknNWac(B3*7rA)LLck?Ukm5?@Bq!ocoM z3FVyBixa0BAkwckA3xF2 zaDMV;(!D!L&&MCj;RBe-&N_&zrVpu2D)uTEU+YNsAXZTkI*wwS3{EX*5$h#W5xypS zZemx3xB}}mQA!b%mKbIgM_a)aPo*D{&SNHgSyT4BS68ls%Q$KGfw}FC<)#vR&_GTs zT?PG^?HMJ{Ul}|%)%V>pOHG_I6c#hFH%LN%YIjQ7<&AqIWW$n9DR5_;ly5CcFWj)` zs(7xOv$Iq@2Z{0X{7X%zLlnIS(@vUaugGL`ZK&LBO=HQEVI3D6RHrOc+A!!mY#&~= zyFDxIViXCzgzkF$!v3U}hSed%$z^3IXoS%dugoR-X~I*yXsirHbZ8`aN6W9=_a?w*Yp-e*%(S{G!J<0UI|qvWN|gzuM< zD*P_jiBDvUwDtJkV%F^oImex*BNe^)PSF{&w#{EVLldKSz}Ip1-Kpp5`sWVIos<`@ zQx(++z%O!YvFM+>HxqW>HW_2{* z3y7EM%@yg0MJ$ogTJ}(mD!(7INn)v3<9;n&8%>qs+qN85{xa6x*>}-hBG)#Wu4XuL zNit#pyZm|7_mz{Qfm*fE_S9RnY)7}JZAN*+?)JcLnf1-}Y;+q%m{7kke`V>U%*dD7 znO9XT?JlWkjEcf!?7JCxVVODa3Z3EQ%$A1GGbp^92cGAzp-JZ%Q@TV~T3=4ChYMDb z*%)cs4c_$9!z+uXVNkW}0IFv-vT2s?Evzr&HHCSJxSNNh;+!|sm1O91_Iuir&*;ij z^E8U33_VJT-(IaYpD>uZ$Jv<2xi8sBtkn@jGMcrYS$kxS6&ubc}Q~OIuY_7xqX5#R^7lU7JCT@A?TLM6wiSy;I z_17_V-|04~j@!6!tvVb%IJ@Ya`GGN$(oaVhb2(mW_%3xa&B>y(8%&PtR7dBV#(Ki7<9d{4RTXS48_N{nTyA>C+cYf9q z{*_L*yXQkxWc)irS=_I1`;+5mb>iqY;#Rqr7`M~-RXFPd6-V1qlDKbMy!Oe;D@G+z zUy%{0i&SE-nLM8~qdLi#Oe4}!$-4e!Y_Uc18WJUqk$xtm&mq5|)o#2}ik7(KMYSTg z%Gzvh%#(HwwYW|HOl&a1K)S%cWM5TLQT&z|^#C+W73vX<_Gq4>lk|eg=2_EeOny3g zbe}SNeR)oH-l{f8<1Mve=CbT}+PwR6nRbeVHuw4$Ma{wVA=T^!it7S>3D}2ZN_`YM znty#I9j5F#hl4eAkBNNKA)OtxkhzMw*sJJd|o!!WDqJt)|^ zEgUr@wV8rH6Rt@gQLn#-$4eVW_)Y4wE4UcllzqTz*fg=4{yuI*Cs<6CpCW2fTZWxx z!&1in*hER>WS-9J7C}vp9kviDSQ-4k^CuGzu2uDJ+&kisFwgya+W7BY#2NZLeN65otnZw*;mK)XI|=EYNGAzwz+j9jAimwLV%mVf-m-}X_~wz zeS_M3c%m{Fg=|$YR;SKjQ)Pc-2M2>}E&5>J<6~<>walwmOoKYQz>B_Fwh9Es5Sn0SE+WPuIfk--^+n-&na~4(>qZa zP3trqF%28Xo4eiRlPO=LpL*O_#2L}P&SWjvSzX>84Ym?Tqb}IHtrO{IuS&|yPX**R zJ2jbmF3I`$ENrnTyy5r1n1N;pucvZkiaMH4pU(2g+`Y@BdJ>)VW36Sy$D#UaF!Ggj zYCNN7LY@;5yK$Od4AUT9wsBwjA&nDUI>jJjAIEx3nRSpc?^Mq_Ug>jWvmUQYyNx>{ zYSGv6Hm2{UKQ-kh9N4}iz$lG<7iN8ESjpa}SA#OSdOpA=`A{?OI3}ZRz7x+&qCsg` zLDbKy`?^Bsd>dZIFq55vbbI{5M)H>f{QWmu{IdTwqZQSYf=;TJs;Pw6nT4$1dPcVv zP<>kNi62zB;G~Dwq%6vr^i*FBx=T-knV6%0P5&UNHjKmbzRmqtN&>l3F+W7v+Y@64 zh9axCjE$f{b1!O2VruZamhAF%op|4pco;g8VW3O2^E`9C`F-(RgQiceq$+g4P^kxv zJU<)H>x!aIds;O}M~)wr^xWMXz|n{KC@s&o)2mP2Ib`nNN^KyEl~H+^b_dOYGgfzh z-f?;?Oeb@z;o|N_SG8W`5pks(m34;I)5X1~n>Yqn>1!@jH;#G!?zG$ZveWN6i^;r8 zlt9@cD}3Pui-37JGRmRoGxl;$o&}smlV(}NA5zo;SzV_tkJ0x2&fwvf#g4Z+KIN3} zzdh2jN3twO^X1)q=TNI47K6N7`fi8Rfr&WuorE&aUiG!+}lfOx*@%$G*^q z9PPI;__1_9RNOvn^KGFyWeR;-ZXwn?6QicGoME$kzs{dsuSbXHOkpKd*vVvfKJ{1r zQpzKL2cyN*oX({%=cI}fT^saWkG+#o^RFD2p*5&=kmM=Z_`a6=cR5KW>-dZsLP~pq zk1S`pU$;BV5Uo5^ojIa=#>BMr=Jw-^P8attTx*)EM~hrC@K0EGO+zT1=mvLx zQF%T@&rxu6Bqj|?czf+?>#ygQ{-$V)e2o1u?sEZWrGh4LnbmcsJ&X3-US8ji){zzt zdvd1z((_fEV_f;K7bM5U-#ksyK=%-3 zSV{cC13S_5a$m&^d%KLMK|;$-otKA|$OV4inXA*R-Bw5rQcV}Jy3#SiFZ^OB&R)fC zbGoD)ea!xT;*MiGnba<;Vcm;^Ga3o{nd*AG^^A*X_vvL%MiJQnA2*cH&^{~A1ytH$ z=H{%HlU8!Byj*pG0dyE(`lK4R!@@Ab_4>%it@AesPQe9Iz(+t)E5@xcG=Gv6SIU^-E+`nXu zXn)K}cMb44CF-Ukms?e{t>Cm}zis5&BDz`{a6!O6ZewaP!ApXBi6Ze0GmGm__6V6? zWr+-4JNIj*0HhP}sqSRS6dDF$tg6@hCG9=1o+q2(^m2* zw_85LxV|krs!gwx*`dWfEnK!lA8W21<20#QOL6CSjA^n;Jv9D}VUu>|)15KCtR8Fk z9Y3U;V#a(m=RdveIlili``vz%(MG&z`*ylXODvCA?`tpp}W7`YE zc>ey~{_hF5rgt`H9og%vs(ermg_S;?zs6Es`b?Q>&njihvAaVjy8fl=l|nv#zSn!l7SndXY|ZyJBIp{tBR) z29Z9`&cMrKo+JugPqcVRCTQ-LmwkGs>Ex4Si#GsXA_bu0RU89E$YP5o(< zibwZunUAw?E~R0Xh6}XME_K<{o936k% zd4WUYo)W!t#734Zl}amGDyZ~k`>i^Cj;OD~&+(=6oAEqP`PxmSlvVOE%Fxt3=-FCH zcwhfk>R_9R-PG8NpnYU>{)$-DYZ_tO6^%9nub)WRl8<9K&ylSC{%*uf9{yg!qZW0L z-uqj!wwJKtl}nk@z|!C37~JQ?V{rKXh_cZi4Zyn(V!Z{2HEB8eM}oM?Awk6YK{ly@ ze2tQLXgNljV~eESb`2jQid@oqPJ(sK{!6|-X)8xOWug7$2r0um_MsY#0W}$I7bTce z8D4js0ioj*?VS4{gB0K626UU%U3`j@1=7aBMG;jB055`?NHq~!+M2? z(_DO0=Gg5EJ#PM;wbu#4^C(B8NI9dYlH+W@Rm=6o7Yc8${%F2Bbf_HY;Uqq;29Js4 zNXyoDGQ}9YhEOg-_aZ__FpFs7!pzU2Cg2$jAB{B^0W+?ZEE|!fnX4~o)IiOpI@-IK zY_uqu{Z?o%IIi)AED~>}#(9`iiEq7+;MVPI|BxeXxv|j1JH6b6E2g;QzDNtU2w@Z7WMuPFQW_F7xGCZ6@W(|cV^{<_|S9Rhugnz~_~ z$fK{+)?s5JRd;u18x`PEbABNoNc)vB@!g&bjcmp!hrt?R*Wdo^md$Ukg%Wwx`j8cy z{y5DfX!&}12G={%k>B)+yWmS0dvA^RzHCr`URhQ()H!+Yt(ux$(yHmU(4@0xq=O2H z_wCmLX=sQdoJ*63(@C=qi@{^JHdkkZFA8BM)tZ``3V2Z1+bI*0sfsZ`4%3!$or`WU z9wpTAh`rz1cxyh4mDK?~NLLYQQ47$>gCkTnPM*++gQezk8$f3myRLBde%>f zlAcHY{)OW={;Wri7ZTBeRuo~C4bG3M$i(Pw1Zpg<^xEitd~$dibe7<(o5U`bb2(8S zKY=vY>xbKDe5L#o3dE#4>>)afkAfD{XUnCM%I52slB8b!Fcp|&VvXJ!zp`_!$1M+k zVQTZG7}G_|^@iX-gK^i1SZ9#ZYz3yVoKi*Lu@Q(BNw0D=vfrd*#01lK%U?c;Rg)Er zYx>Gc@H7(09mFqu^s83;U9ImlQ1X=4Gz;Ui&g64RXbLd|abR;LIdYW2$J3I!C_{W=suhazb3EK$;$?h zxwZ3$pY6L@yK;+4)JQ!;?J_3j2Y%}%$u{T@?VyN^KgdOucg~d0inUz|!w#7PLD#2{ zh;W{1pW(xcsBQN0x1B}})zR2RVmE9M(XwyX=S=fZrSO;EB2AgU-ue><4Iad-HDhj& z{M9kACgp>DRPxs;bf^~h2GH)7SAf=|q}U$0KQrQe@hDWPm(Pua2p)F}lDW`iRORix zn;(P3eKGxx!@fV?%q9dCbCi_Ic} z4pUkp4+p&jtbVpDUK^pc=*o#fR;zh?+I{0jw_d!oGwmmv?)SBzR+^}$Di-BdZ8Xx3 zQ!xv-D~fL~z3-wKK|sU5$GYa=8j;u#_IsMeS4n!*uuf}-=y_!oM+UMLmvQ>XkY7rV zyOJ%8G({>EaxCu64>ftRL|swMQE{{4NVv90e*w^|wE4O+Fivx~7RBSF-e5Vxp=`Y5)dS&@IAfr2>mTVn2 zVI=sa_8IEu$DPO~L6!_8o!wZMXrq0PbS@gHip-9`EIQjMM120dj+ewq@4KXPjK-rN zga%H10L@Dwdq$|=DQS74t$zQJlct#(*>-1IZ}ZI)uA_^zZe?F(-3GE~rBi0`?&7!9TM|K=m z09o0Xj!UC(23KBwuL?3}Xtc<>v6IwZB(gD)-8@=z;8%yZ$LQCn9AgitkPdT$H@dP@ zptk2kD}Vy&+v6NaFufWFAMK1?fW)3s+<;dM;$11}veOIeXOoWM^yvU1c@QOL@8@1a^0_MZkQ}#W^ zkv&`MizrWL6sa7cI|g2yp$6bV&+^lOJj16&rawM_3usWMe>v;GKjw(8VVr0Ou*M^A ztW?~!iFaXB!(mFr&WocxkRmKxSEl^iWkCPM9gc5|Xd1`)hey&AM9n+Ski4U_?_|QJ zz>^U?)wMZWZ9|W;55?nS!7QeiFj*m@37s=vPC zL77t?zxa%F_DxRb8jaHRANro#m`a5U)BTmbvCg9m(N}E~MNX9h?hDom#?HTBxKcOT z9D~Qsj$6NH^wl@MsG1^w*3)(@AnP^ZctAC6Ao6HSQOL1l6tp~{E(6IU4?g|8`J4We zn094l_us({A@@EI!an;kO;Llelt4y-Mnv2vS%D;3ZY_csYjmyW25YTDe>cvU_EOt)u;-RxUq?oV=Z{3m91*2BF?V_P zV_>LAz|8jJFt{HmgNWxM=suFm1Xas~j4z+8zv~H)k?6*F(zY%=Tc*V~$OiOLDfAw9 zH`DxOsu-@>z55)?@rfs6XI0pCyfs;_Jxei0IEp7xebD2>LH1IQjn7fM=7tq}AWUNL zy@b^MIc$j91b8CsTJk*goqqXs%N*prE_(XWy_+{`%ky*4)%~bKoie+TnwJlt(m2kR zSoZvm4G=-M*Qq$EhSNmT3f7ocT?gn^+Zlj%7R?YHoEr%t(7Z{3SR@LXBx zB3Vg9BOdUaeDaHnd)~9VSG`|UJS%!w9ts(vN#C~Gvo(K{7b%D6|iV~@B-{s~-P zSbA%&YvkyeZLUmH(hAJW1>Q~ialbW`Ha;`@MO{GG+N@Z%e&x@~EY;K?SDqO+50UEx z6FZ13hpx|&j(xUIc@0}z0AdwE;yxcE-t&Ud>cPVB7eUt*OZ<-{X`?(ChA5v^@X}z- zKF9FppVBA5{d3~pFAfd5H@W)mKiu~{&>Y3faY4zCf{vom;nS9J)2p7&Y~3?$LYCe4 z9*usrb<{)>C~htk%aq0Qvg4r7#+bzt!(r}pFORuj(O1phs6e38J|_(HEhVylwL@UanFf{H9%qVF{DJjm$Eg=rnjhY}cHHq>;EBs0UR~6-`(#cRF23~rmPWEH zosbO}KO6?!S9>6ea5x={jtpaP@RU4izQ%6e1L0m|*T45Q!&zRer>~#c=hX3B%3fKn z)Oz%t30eDOVxq*(CmJPt?%`x**U1C90h*MI#rdU+5p`Ah3VX@_>(K^)6L*Rpd}DN7 z{3bNLy*eO-^H<+)rfbMF?hhuS`PyRf=W1fod<8Z&;)?o#a>rcvwO>zpUTY3umR)Cm z7G=hOzW~brfw({uQtCTlHSyA;sb=4@1gA)bo)lAU^4#@++prChxe>*M(&yE+wMk}} zW4a9Ie}B$#4dglBVR{Lcc)oIyR9U=O|AIUL;9;90G5&zv#~54*7NpQnldAPmrY;+? ztj$|xdnlFqjue|3_Z(=9$m*l9B-rk=CC% z9y6e*kgD?4EKr*uY?rAgxx$3Z{Y@rcq-O)CJ#)0=e}v@=w0(!eOJ`5xA3VokOkq_1Igq4 z8x?DU-?swr6SDgWid^1QmQ>y;LTqD2!q7#syS3O-9xV`K*LWdC8i2`5K{Qx36aYYo zx~xo@Me4hlwKlyx`UlMrd0!$o(jywEbam7P|6%1)tK3XN-av1OjZKkxNB++ui}EoT z&7ZIY5eF0EIaz%zWD}bWC@P8-R({HI2mz{X5cGaQIS6;Ue@5!XrJ@wix!MCW4-}!O zJn)_oFmJyUc=HiBi*x`SQhZg~Pw-_QI6`cRE2J$uv!W(0n|^2O%C7=5pl+F6veiHp z&SAEHTI#K`oExMFy((z@boBLO(QhHP|J3Jv6*j|tYw-?XK2V_g4;*Hj^nKXH@BQxI zi5XJtS%egO_i>L<0Vkk5^h_T)AOGH4&1@le;BTQv2$6fkCFr9LWl|GbjU`UQ-);*N zv~m(`A4&PFPsju=?m#C`|0Z2Nim$WU)3eTpx9}w8-V+|nunm!{b!2Mxf`bladJFey zVyd8}7#b#87wzMJz77`aMG|OWglzizzyuh?QN8cLk?w(=6(ON%Y4MFvWI^%mf~hng zLca&)h?AUSQT|2tX8}kQQyA&7u0a!}4bhj#0)Z@q$!35i7c>6-lzBgQNuL2!jDmVr z&+EGQXWNAE>@>`b+dCs~dStBu9ij#F)IQbSO%>u@=dl=*D-XWDuXJ5yxg2+pV0?|j z5Qq+opeI(;^*465JMUw-33T)zGZD&VQz# z9#0BxF7Xq49oa>2-^VDnAs&NoR7=mfa^54z5Ae`^g#KcF^!0K|&J|#S-UE8aCY-S; z{qAQh(&3{tE$cKXWYKT@j0yz;XD3DpC z6Kct^9yhp^)%f>>oXw!PG$PNhuP+Na&gga7RI4P*c4lhwGVvMRQDcj<6sy*c-McGw zh=eJL6P*Umm{!8(XW1iQKDoX7$<0@)6OrRdo{M>u+)XC@+08wMcvDxGx709BxBaIQ$;i`&xPgz_VXg_7D<5B_el?UiXu)m({%AM`spz$(VB}KPvX{pmP=BMl)6F6ThQuf__(#-X)iRSvE}JrA!}C0Wlv36`-0TO zjDBd)1px&Mg(t^il&4|I8bb^|&(it7QGF1?Rn|D{sPDj*xiz#EJf)gd_Q}8l{H%g* zQPsi)5IyeUE9|r7RPpU}3rv@Tz}a`RNvs#2vP!jzvq$rm2C3;sF7@2wG&YI~Ik;uw z(Lf<@D<|vK68-Y!0}=QNFKv#Enz)A_4z6_S+LV>&)oJ_$b`8~a9P=psZoypPAJ~x1djw~wn$!vIe6VXd%1Eej$tlmlWel?%su6aU+t&{*ik-P zwJ9&;*~&|bRBl)xy+A|`75v3IbAMtYmDRrY{fyoxoksPLyl7UTLif5?fj$U;=s7+3 zbn37I5vwb^2JZWp0srM7k(oRA7%lf?F17)n+3-X9(6Qd}bP^X|%kr4RI;nFn_yaR} zEZf(aJUkp2T^X6DHX@{RzcR1L!q%K$G;_pm5JHhFaF68-$`?Em>Rl&1FK5QlZ)OE( zQBw35$>*;dbhWtdjbtB?MyzYCRWugXER0hEdxh&f1%!#dqk-sjVcThX@4A~Coe#x1 z<)_>Td(04Sk_6av@JlhPN9EVlfxTasBAf{F9v@Mk!&&~!99uN$%^wG<3THpPQJ`zC zA}GhYY_!~S$?oi(4>d0*@4?=N^GFJztllM#eh}g@7N$FwY$m2&pik1 zT2bZ7PYa|M>diL}U{cA4&bhPLO1FH_hZ2!Vt|{N2I=lDE)|8JSV*qN~o^h-$`1eMT zY)IG|#g^L0vTz4bwemxnwI|nOnG2{?ck8b6u6v|@$TBh(Xs2Ify+BpK)|0OY@BRNb#H8&2hVZd-I}o)Fo?oguv>15zItrrz66sNkWA04Oa2qDZKS!;~q| zBkv*q^@0bS&xx0Vu&PSZLk-Whz5)dA^YRj)Ivp3=Wt9jKL^LuBtJA$*2+O1k=KrIZ zJd+j{SA2F!-{&mYrXp$}Mzc3?)W~Qrd5bT%tK!D5AUX}OA-9X_fDH>sIcg53ar4E| zZvpKJQ1OtO6uSCPnuZjf#`O>lcy^IR)WyDP2C>N`rDWP{S(V_93j_-N6Hm|*Zz29d z9u@9Gg4wfvKTh-RO@ue3Aw1LsHWxXgAqH0QUFzjY4)Q0d$}Rs##J6(kd&7E{CgU zY5nk6DTwhnHO0O@~+Lv_9y3e|HCd<)t zY395Igjm}wwrQ#Qy&AN}v@Jr;gsqnpNB~XB@|W1wwRfGXz|^;3GOyvY3bu?^r#7o| zbxU~6g$Z;JYNSOtG#D*lG5QwmJVGa8KOQpOSJuYjG)N3xN1U1b8eres9Up;M!fnb% za7rqS9sV;PkgTACqR->>ngxRQf|n=kV?Yskf+h%Ec?6z^M)x3WcgbfIFxP^n-O%Kp zqkRlUTmJ*XUyB0uBxurnGE*h_IcBQ+2T$daO4qEY);R41b7hLXmFy+KNCpc3v!;x_ zl0eO24XwuS#h+tz$NlnP*-u#cp$wfdd4jd)LI*0#=ko1 zE3=Euf1;UD1T3_ii^{}y!b?}Cb+E9g4%i!}{|!eg{sWF?-c9g<%4 z&)jv%R|e}iYt0^G|8je0wYr4i%7g9-=fxl=x1IGdv)+9ur;K+*`n8a2n%@FDpK|$E z$7g97_eIBR0D^;5_kG6kf>N|9b%%dPT1fYFEFt0Us3Y0LT zJmm5?KufOt2p?t>*u<9hL}ea;J!}@FTOpMR2M0U+Mte-+ij>6GKW#}SO#0?@PQi}T z_DSH7PDd0N)*o?9$k-^@Jifdo2xyWsbyhqM&7vSDh^`ok3X+hapstC0=TKTKNyM77 zkyuF1My&{{_|rtnK%Ks)RID>ktih`ii*HV}{e^kjenB;*`)kydX;9&V`Yv6FQBS-) zxXi(k?k5%{whkj<>iU%c#T`cUOt2xoWx^T5-nE<-=l3J7pggQR72m2w`eyp!F{z@w z$i1;E`TA$)}~m#i)8OLEGI%*Z>|jtMBnQ z`Z#}dPca6#!%zQryhm;VQzS2%rOOp~M(bE9w|~%i-w+gsATO+c>*IqoLF4C>?lU7Q z6S5@}D*NSV{B!p`>wrYfP1~dRzxkN?p?aiA_-&n{R^{g0Wh^z6j_1j`_4G)x@ZOtJ zH@G0ffn@13U;o-JIy0mx+D=4t-U&GipnMh^eRje}9oHa3U!rgKWKRbpX`?O=M9!We`NN4jnD+nfp%o{X_ zu7@Gmt3{Aj{-QD<6^_h&c$0RMw!Orni%0qRC;`h!?Si0(jxH2^<|{7EW?aQY!c!JL z+}HH@CD5p~K~AM;taBrH%bQ#X;|i`PJ=RrVc7(eFc(7?T(fsLTv4f(8^{2@8UgnMp z8|XC0xpiK%h#`AktA`o(AfETzweL|dleg3&tLK(&G{4!)CQyxx%ST@6BGZr;u<#EY2t!0!L4sL+swwW6>_(RI403$!K099;!LjbodS|3DNkFZu@E_xEZ~Rf2F77_WV^Uii%)B&@m5`HyTt(m{GB(%Jct z*Fk^oAEJ(|E`|eQnL*ba_8R(la*~7-eot%%1MYsj`27FnYUCzWraXhXmg~^gquP3^Zz^Zu#No{>e;%8 zMdWx%{wrSejF0PxoFlwnEM@<4Kn<8E5(QIvu0vafkmhAxsOm#s0lvn9gON+OIO{OW z_3jZRL3Deg^WH3a)!g^CiGF=16sDNT{ZF9O6x7EoN$SuX!}EB<900I9q4oeWL++y}~RNPU9N3r4-5wU(pCdFpGL5maBfAt!&LXLCokP#kAW2|3TyH&7CB|-rz5VJwVrLe`zjLc zQ3mYu%Cl)i_}M=89oA@?bBr)1f1)iln#3{Y#aY0+=l%=tIxX8G1}O#NLY@2)n}Hb; zGqOdnhZf55lzjpO8rMmI0|G-7GTREUPy^5t#LgCjsl^5`N-?tk{M)K=n=rWA=!;&` z_OI3MsA`2v!hCq&JUHj9%E7~T#!KSW!GB5O98f!M(|%gTceRKv=_i$t@=|mZ+NY$( zx-^oFnZ#f;`o3{U1Rw5?=uukSUi+`8(MA>%vTa&#*H&~>?PYDb&a3tXal~?QIfM7s zRDmMrR$z+HqMP>iqx^Lr{GAbuOx)oV6{%MTqiQ@ypJvt2mLS0S!UeNpWOg49pYDXV zLD;$^;$_+=I0oZGd+;_?6b#O>-OmKN>KRRq5Y5~+uwuJQ$s}8fw zUk~c$AmDNA^bv%BO*0+;d|QTluGX5%f8lujb*mP1_+(dV=LX1EC4UL)=-my?HfBvk zS%*UilF6iVWd!5Uye)f4mE6g@;-e6~@H?RwY=bf{T|4jaxtKaNZN1#6#s5|>YG9ZB zGAdViWc1Ws6{rjMY2UoeKho<^ci4)x__V4%Bn38mlATzlb?G>UDiOvEWt_5sZjD@i z_j*yyfs$PGzL)EgY=Cbui;R02#1WEdjRnUS@HTy0X736Np{>bum4$W*$8iZUh~adM z2=)fpNx{c%L4&0rWUCSx4)quNL$C1_H)4X!L)>3t8{=m-J90gYr+XJ^tea1kO>z9q zYlI~IP1y;(7*WP^qU z4!ha3L6kA*Y)Qyb4S^~JP&qG69WmZBfJ(3#BdcaoFl^lCZeV>5t!qJYxVjegpD!lz zK7@$y$cE3&xsOyLi%@#1==fL2665I*Q6%2@f8#z ztA>zM%GvYCnFwZ&-+=P;6Z*H)n56kG(B)1zw-8F~`8Sa0AX$ZT2CmJ5Nn{KUJ>RaV zId-FNz@gVlxy^|n?Mieb={yc8WT!hm+-uv=Kf%2~AX`Zy_mYYyM9_$g!DEn|NB~78 z2CEUmel?;9Jn`Vd$~GI0>V79u03|}ufp3-&8gn8;1%q0AfYt3gb}BO@b{p_yc2Kfc8#@)49LBJY3w z2DDzi)z`m1KGlulv+t#v$dk(3dyD5VLOEa^+NF-6245KpvU-N4*?kYBk0a^YlwAaA~jA* zDj{7Zx$$Vy_cYJzN#BjWD1WiuyY>?jRk_NhAS+yDx@QHX?NhHPZG4G*Vs1L$Fw+~2MA zXDGZ+B2O2G({qp$UouZiC4BjDCIYv|Dm_HL7wP16t3BL5a2nFSd3^or-l}OhhKw}O z@ZXME6IiE-yb}*T{(ZW$8`TF2$;1r;IK|clkeMV`R+;_{jDM%t`Y=r|5NesJP$G39 zdrr5?@gi7wR0o$=y_e#yar`AtPk($$<(NIgNTr|j4R$3oL$hHB{=K}YD<(g2jssQz zo?B`;jUow#0TvMsC(L4l2e4__W%;=1BGT`0;b;wdsXTy%==37 zn8Dl!K%4$bs?z{+R+E(>ctyx^)_~dO$&ICf?Ih=P4iI=K?4*&21Jj`?kb=RwTwEGi z4;P^Up%YS4e!%f(=St`g8^9RWnkY;J7;w&*cVKHAKs}#n&I3|154hnnYw=r84qXmZ zpHN19bTP7-elpHWocFHm3JVp065%GiR#BbEiNZjO_Uam>%ap zJ6iO&=P=6IJjLps?&rA6!2*LEYjpVJ)$f2R2RHESVZ8a534u76=WI6ZjV}4?O_vUgAuiK9K(l>+N-GMUdMQ~xhJ)8^{ zh{`2G1Hq%r^^1^fe@cb~@*9;4kN6Ug}H7gH1yR(;4|xzYr8>E*R_Zd*c6;Xk9p zp)+_6Yy$hce&=*yPqZJH6>*&5M}5lJ89NtI_zi^mUd+ffARo^}t9!99Ram6NrQ}UT zy!UB8h6keaUcRz=VqiEWuV`YB`mzq?|>AiiQiss$>p42?G{GR84fDL`9D-x)#_BS<8mMlH>CNw^f=jUw-YCMuJd(Z0zjg)}O4!x^5&zGjFurxC|A@!{fbJNdy2< z=(Xp%LGoAu!+5!b2S}|m26r)XW#fqf7uysi)bwMV(j>){Wd+fRm`haNFTqpBdtqYE z>M4bcHOV z1laBg;{8#7%439rJs=Z_9pLwnByl2J1W!u9(&d)G3`1Um{IC3P&yK=S?IzQ?X#i}^ z;=68|CZL?!J4#m0VDWZ&M|$?ma^5G8k$=AQ68lVTdqB6y+*ni3){nBYBe0IWsLmp- zZjfI*c<^-%l*0tz!lE#CCi9`D|(z8Zz zY5p`zjX7QtT)t7wm9BgEpN>fc6fkjs6vk~WdJ$dKN4DAsXO#X~*KB{O#(&#Tdcf5l zltzg=w+vcs36ONRrIN3mrXlDysF6y&o}Xq;tcq+GD%# z)+`mvbyMTU$G^93eLhD(Sl+!zk`U;1uscy_B)|u$)D#XeLhf11=fJf<;l%nn4R8!wUu_f(u3KU)hbKFlvB4q)Exsy{p!w5>f zT`;oo5jk)YT9)j~kBue|5X%vOWNE}92|P>>*otR2u&c8J1I^bL<%LKBvCV08`q}ev z3fp_22zq^~#7>mSSvZYfxbWwd!E#IA&@DCxwEnyDf=R@11{TCZu=Ysu=%{m5)VWtr zq(o2UH}Mp}>8-DIhW@c(r(B|L{~XlM0>E+K>}gVOPk_bBar>7^^V=v6CIVS=JC9}|{<{;KKA&)YlK{Xp1fU7MOOdi%K8 zcp&OnQP$62oj*a}op{-B?|f;v8sN(bEx$;9+lZKmCJ-3vDFv2_TB6}ZE0~B@p7`*$FS8FGC`Mb{%2MWD1@ zn^>>!GYA?6z|-q4pm^;GTDz@o?4-zidmy0FiVvx|?&gmgg$Ymf^X@T@`hQCC zY09L&g5DZ;+kbFZ(!f}!=W!-^3dv2ALCWOEk%wlOTOR4KJl-{kqfdiUN-}c?PS)6m zpLx*2*~?MVqkA}6?yf<(>>M0hp%JcUe~f_?z`D?PpS+-ZbfrquYIgOx#WKu_rOJwm zdz6-EVIs!P)+ofDZG`|JZk3qO$G<&^$Pw+`z|)LrIgzx{(?|}Z)rVV2(Jpv-o8Xed+|A-ud2j7=*E)8kdN{fN;p;9{WY)dB- z>l{O4&oAtS8nCX_=S`vH-?ha%*}(VK8325iL{n-&r%a8CO@zx&;41?560NX3D zhuwc>_nW6$tQTaf8yDukw;d8AKrZcyG~3UKZ^~X9jdg?FdM9PuRa?szZBgv0V<$pZ zJQs6&QnCliK?nST6cNY+6FuGaaL?WZIZ-(D0n5TyD2EoJFyHVjm3kEnR>S-8$89A}HT#%g7VGegi!?Bu39;FruOTkcTYY?QQS)?TCxG zrT`M*$sHJsXlMhNRG!Z+)fA~Dhz(fjs=n#9m~T%QPNcy>^8e+69`^sfTt7N=K{%JC z_*Axnej)XpOxBeF#PtGhQ44gOtZri{C2yA2(@P;LCeHZrP{X z5G7Rux?>MWr0~Z}pzbUL7_0^P8b4j7_pqkjur@`GQ1P0y>1Kz*AJ0I0C7?eTECS7H zD-?oK49#E&rig3E$D}pKs}`)8DPm!h^=>v1lh8>}D$S_X^#KggyQ?bVHGquFVA@=R zNm>_zY=CVDp@KUo!JVR8=w|^Z3vtUB3@xm7=v*S2 zYZ(yv+ko{i0G$E@D(*I@?0Y4Fb>rc~nh98L0U)ga*^JtR^rZ$0N}BgE06)0ji?5tG zfV9|epbdR2c`9mlB$4Ge>BD@x&=8E6ceRJX0pbk=MmRzZkjp5NJQOuF1;yo2N04r7 z;o=otPx6aUVNw&L?1Sh=&~tYIQZ*0HEpY2RcCT07x~6dRc#=2TiE3(a!Z|cippYzb zB+fi#fsSd_rbAI$Rh5w5^URX;Skpy_h7!mQgtIpf`sVfuSGDaBH-pm)VvQ1uHNhPG z1Hne5Wwb8K_7|BaiI72P=f@U762Iw&q$>UDD)=r!p^X ze^pI48iE&)mOiIysz;=x0Prqg{wXzXad|b_Ep*aT)s0-CS*Kd+3Pk7t4FWhBSNL+N zQMR^R?E+avHAVFkQ6vr2HjSXjy03Ul$+aJc<+R#$9f((PU8LXM^Cnq0nRj%4+%N$8 zk2L^>AzEqbsgcKV&TusrA`J_w*<7+zHp!IRx#x1>V)>B{-Ig}3+t*}*-IDd<3B#T| zz{Ntc=>#Ll8lHW`no}p@MRlACl{$;%_|87V>$=7PO z2nF!bUd{0BIavYZ+5SBlvjQ(;29mYW4eXN+bWnKA7rTbrFD9k_>T}xQ;I309{!KUC zXMllKV&MA+t%flRtq3cLv-6AS zW(qFt;Tn165r=+n=pOdFWUN({d#^L(6gJ$K7SaHDgM&vx25 zpK_dRdZTfAP&XCfqo1L>Bd~Sht}K{Iyb)QO;AC<+lr#rpwIajbd_sNb+HfKjd-&su zXj=D&gq%2s(5aW{=J+aC;0)@#1S|2zJ4Iv;4GO2dklB!Ml~8eQVL{Os0BTE%-<|p^ z#e$X)vWbrEfR<&y0T^Pi+9`GW;ptCsp0)Wmlf3w6WD4XAb1>vnyJM-Wt*?yxcbaRg z+xGayBPM@ku{&Grb9<-o5dT9MV*Hw1Ci9dWk>cLt9I@l=QuDyQ#?hg~xkO#FJUsF& z_wIS*zRi=4Sy}I@TPsDxrfima2JKj0(4cUkIotoeZJv||p(6xp(lZQKr&ZoBJ2{s5 z6H*tfRpe@R_U=jYFB-m*KF#@v<-~-3ym{w9=gI#3!Dh)fAE_wMf0q*;xyms^!pJEG zGrkHes?@Iw`Pu4Y=}w<$L6#=R%Evy_^+Q{mq&jBC&o&M}$=l98p-pQpSaJVKzU`TW zkNY&tDs|#h>X7{l?mSA9m?N$J=)+5Kz~w!Vs(vbYfE25~yl5SaKwpPt zV7QunmjPslI`sw7QzNjeoI%!;hwi81qgo%V?QOUpND6QHbjJOZeDf?ep*P3f1pa#6 zg3jL}F-DiPuaZP6j=Fuo0`pb5EyzDOzW8fyMZ{rYQjEI}t!gl-FDX}q09|+;_dOfG zj=%|T2dNZR=PF55Bf;;9O$hS${xg1TY_@D~tvcqR-C~Kg9<$l;YUNQqHL=T{_Gmwt z-bRq2Vit&rOG@L2*LB>h0v}wB`uZ8C>0b3OYq@{$_s2ij9E_S7lfemJ5_|f pQ%ftJInO=Pm*Hvg*8yQhk811c_TYJD@K0(mmS(m)>89>y{sO6Bn*jg- diff --git a/sys/contrib/zstd/contrib/pzstd/main.cpp b/sys/contrib/zstd/contrib/pzstd/main.cpp deleted file mode 100644 index b93f043b16b..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/main.cpp +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "ErrorHolder.h" -#include "Options.h" -#include "Pzstd.h" - -using namespace pzstd; - -int main(int argc, const char** argv) { - Options options; - switch (options.parse(argc, argv)) { - case Options::Status::Failure: - return 1; - case Options::Status::Message: - return 0; - default: - break; - } - - return pzstdMain(options); -} diff --git a/sys/contrib/zstd/contrib/pzstd/test/BUCK b/sys/contrib/zstd/contrib/pzstd/test/BUCK deleted file mode 100644 index 6d3fdd3c269..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/test/BUCK +++ /dev/null @@ -1,37 +0,0 @@ -cxx_test( - name='options_test', - srcs=['OptionsTest.cpp'], - deps=['//contrib/pzstd:options'], -) - -cxx_test( - name='pzstd_test', - srcs=['PzstdTest.cpp'], - deps=[ - ':round_trip', - '//contrib/pzstd:libpzstd', - '//contrib/pzstd/utils:scope_guard', - '//programs:datagen', - ], -) - -cxx_binary( - name='round_trip_test', - srcs=['RoundTripTest.cpp'], - deps=[ - ':round_trip', - '//contrib/pzstd/utils:scope_guard', - '//programs:datagen', - ] -) - -cxx_library( - name='round_trip', - header_namespace='test', - exported_headers=['RoundTrip.h'], - deps=[ - '//contrib/pzstd:libpzstd', - '//contrib/pzstd:options', - '//contrib/pzstd/utils:scope_guard', - ] -) diff --git a/sys/contrib/zstd/contrib/pzstd/test/OptionsTest.cpp b/sys/contrib/zstd/contrib/pzstd/test/OptionsTest.cpp deleted file mode 100644 index e601148255d..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/test/OptionsTest.cpp +++ /dev/null @@ -1,536 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "Options.h" - -#include -#include - -using namespace pzstd; - -namespace pzstd { -bool operator==(const Options &lhs, const Options &rhs) { - return lhs.numThreads == rhs.numThreads && - lhs.maxWindowLog == rhs.maxWindowLog && - lhs.compressionLevel == rhs.compressionLevel && - lhs.decompress == rhs.decompress && lhs.inputFiles == rhs.inputFiles && - lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite && - lhs.keepSource == rhs.keepSource && lhs.writeMode == rhs.writeMode && - lhs.checksum == rhs.checksum && lhs.verbosity == rhs.verbosity; -} - -std::ostream &operator<<(std::ostream &out, const Options &opt) { - out << "{"; - { - out << "\n\t" - << "numThreads: " << opt.numThreads; - out << ",\n\t" - << "maxWindowLog: " << opt.maxWindowLog; - out << ",\n\t" - << "compressionLevel: " << opt.compressionLevel; - out << ",\n\t" - << "decompress: " << opt.decompress; - out << ",\n\t" - << "inputFiles: {"; - { - bool first = true; - for (const auto &file : opt.inputFiles) { - if (!first) { - out << ","; - } - first = false; - out << "\n\t\t" << file; - } - } - out << "\n\t}"; - out << ",\n\t" - << "outputFile: " << opt.outputFile; - out << ",\n\t" - << "overwrite: " << opt.overwrite; - out << ",\n\t" - << "keepSource: " << opt.keepSource; - out << ",\n\t" - << "writeMode: " << static_cast(opt.writeMode); - out << ",\n\t" - << "checksum: " << opt.checksum; - out << ",\n\t" - << "verbosity: " << opt.verbosity; - } - out << "\n}"; - return out; -} -} - -namespace { -#ifdef _WIN32 -const char nullOutput[] = "nul"; -#else -const char nullOutput[] = "/dev/null"; -#endif - -constexpr auto autoMode = Options::WriteMode::Auto; -} // anonymous namespace - -#define EXPECT_SUCCESS(...) EXPECT_EQ(Options::Status::Success, __VA_ARGS__) -#define EXPECT_FAILURE(...) EXPECT_EQ(Options::Status::Failure, __VA_ARGS__) -#define EXPECT_MESSAGE(...) EXPECT_EQ(Options::Status::Message, __VA_ARGS__) - -template -std::array makeArray(Args... args) { - return {{nullptr, args...}}; -} - -TEST(Options, ValidInputs) { - { - Options options; - auto args = makeArray("--processes", "5", "-o", "x", "y", "-f"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected = {5, 23, 3, false, {"y"}, "x", - true, true, autoMode, true, 2}; - EXPECT_EQ(expected, options); - } - { - Options options; - auto args = makeArray("-p", "1", "input", "-19"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected = {1, 23, 19, false, {"input"}, "", - false, true, autoMode, true, 2}; - EXPECT_EQ(expected, options); - } - { - Options options; - auto args = - makeArray("--ultra", "-22", "-p", "1", "-o", "x", "-d", "x.zst", "-f"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected = {1, 0, 22, true, {"x.zst"}, "x", - true, true, autoMode, true, 2}; - EXPECT_EQ(expected, options); - } - { - Options options; - auto args = makeArray("--processes", "100", "hello.zst", "--decompress", - "--force"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected = {100, 23, 3, true, {"hello.zst"}, "", true, - true, autoMode, true, 2}; - EXPECT_EQ(expected, options); - } - { - Options options; - auto args = makeArray("x", "-dp", "1", "-c"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected = {1, 23, 3, true, {"x"}, "-", - false, true, autoMode, true, 2}; - EXPECT_EQ(expected, options); - } - { - Options options; - auto args = makeArray("x", "-dp", "1", "--stdout"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected = {1, 23, 3, true, {"x"}, "-", - false, true, autoMode, true, 2}; - EXPECT_EQ(expected, options); - } - { - Options options; - auto args = makeArray("-p", "1", "x", "-5", "-fo", "-", "--ultra", "-d"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected = {1, 0, 5, true, {"x"}, "-", - true, true, autoMode, true, 2}; - EXPECT_EQ(expected, options); - } - { - Options options; - auto args = makeArray("silesia.tar", "-o", "silesia.tar.pzstd", "-p", "2"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected = {2, - 23, - 3, - false, - {"silesia.tar"}, - "silesia.tar.pzstd", - false, - true, - autoMode, - true, - 2}; - EXPECT_EQ(expected, options); - } - { - Options options; - auto args = makeArray("x", "-p", "1"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("x", "-p", "1"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - } -} - -TEST(Options, GetOutputFile) { - { - Options options; - auto args = makeArray("x"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0])); - } - { - Options options; - auto args = makeArray("x", "y", "-o", nullOutput); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0])); - } - { - Options options; - auto args = makeArray("x.zst", "-do", nullOutput); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0])); - } - { - Options options; - auto args = makeArray("x.zst", "-d"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ("x", options.getOutputFile(options.inputFiles[0])); - } - { - Options options; - auto args = makeArray("xzst", "-d"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ("", options.getOutputFile(options.inputFiles[0])); - } - { - Options options; - auto args = makeArray("xzst", "-doxx"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ("xx", options.getOutputFile(options.inputFiles[0])); - } -} - -TEST(Options, MultipleFiles) { - { - Options options; - auto args = makeArray("x", "y", "z"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected; - expected.inputFiles = {"x", "y", "z"}; - expected.verbosity = 1; - EXPECT_EQ(expected, options); - } - { - Options options; - auto args = makeArray("x", "y", "z", "-o", nullOutput); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected; - expected.inputFiles = {"x", "y", "z"}; - expected.outputFile = nullOutput; - expected.verbosity = 1; - EXPECT_EQ(expected, options); - } - { - Options options; - auto args = makeArray("x", "y", "-o-"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("x", "y", "-o", "file"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("-qqvd12qp4", "-f", "x", "--", "--rm", "-c"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - Options expected = {4, 23, 12, true, {"x", "--rm", "-c"}, - "", true, true, autoMode, true, - 0}; - EXPECT_EQ(expected, options); - } -} - -TEST(Options, NumThreads) { - { - Options options; - auto args = makeArray("x", "-dfo", "-"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("x", "-p", "0", "-fo", "-"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("-f", "-p", "-o", "-"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } -} - -TEST(Options, BadCompressionLevel) { - { - Options options; - auto args = makeArray("x", "-20"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("x", "--ultra", "-23"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("x", "--1"); // negative 1? - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } -} - -TEST(Options, InvalidOption) { - { - Options options; - auto args = makeArray("x", "-x"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } -} - -TEST(Options, BadOutputFile) { - { - Options options; - auto args = makeArray("notzst", "-d", "-p", "1"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ("", options.getOutputFile(options.inputFiles.front())); - } -} - -TEST(Options, BadOptionsWithArguments) { - { - Options options; - auto args = makeArray("x", "-pf"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("x", "-p", "10f"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("x", "-p"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("x", "-o"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("x", "-o"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } -} - -TEST(Options, KeepSource) { - { - Options options; - auto args = makeArray("x", "--rm", "-k"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(true, options.keepSource); - } - { - Options options; - auto args = makeArray("x", "--rm", "--keep"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(true, options.keepSource); - } - { - Options options; - auto args = makeArray("x"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(true, options.keepSource); - } - { - Options options; - auto args = makeArray("x", "--rm"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(false, options.keepSource); - } -} - -TEST(Options, Verbosity) { - { - Options options; - auto args = makeArray("x"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(2, options.verbosity); - } - { - Options options; - auto args = makeArray("--quiet", "-qq", "x"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(-1, options.verbosity); - } - { - Options options; - auto args = makeArray("x", "y"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(1, options.verbosity); - } - { - Options options; - auto args = makeArray("--", "x", "y"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(1, options.verbosity); - } - { - Options options; - auto args = makeArray("-qv", "x", "y"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(1, options.verbosity); - } - { - Options options; - auto args = makeArray("-v", "x", "y"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(3, options.verbosity); - } - { - Options options; - auto args = makeArray("-v", "x"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(3, options.verbosity); - } -} - -TEST(Options, TestMode) { - { - Options options; - auto args = makeArray("x", "-t"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(true, options.keepSource); - EXPECT_EQ(true, options.decompress); - EXPECT_EQ(nullOutput, options.outputFile); - } - { - Options options; - auto args = makeArray("x", "--test", "--rm", "-ohello"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(true, options.keepSource); - EXPECT_EQ(true, options.decompress); - EXPECT_EQ(nullOutput, options.outputFile); - } -} - -TEST(Options, Checksum) { - { - Options options; - auto args = makeArray("x.zst", "--no-check", "-Cd"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(true, options.checksum); - } - { - Options options; - auto args = makeArray("x"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(true, options.checksum); - } - { - Options options; - auto args = makeArray("x", "--no-check", "--check"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(true, options.checksum); - } - { - Options options; - auto args = makeArray("x", "--no-check"); - EXPECT_SUCCESS(options.parse(args.size(), args.data())); - EXPECT_EQ(false, options.checksum); - } -} - -TEST(Options, InputFiles) { - { - Options options; - auto args = makeArray("-cd"); - options.parse(args.size(), args.data()); - EXPECT_EQ(1, options.inputFiles.size()); - EXPECT_EQ("-", options.inputFiles[0]); - EXPECT_EQ("-", options.outputFile); - } - { - Options options; - auto args = makeArray(); - options.parse(args.size(), args.data()); - EXPECT_EQ(1, options.inputFiles.size()); - EXPECT_EQ("-", options.inputFiles[0]); - EXPECT_EQ("-", options.outputFile); - } - { - Options options; - auto args = makeArray("-d"); - options.parse(args.size(), args.data()); - EXPECT_EQ(1, options.inputFiles.size()); - EXPECT_EQ("-", options.inputFiles[0]); - EXPECT_EQ("-", options.outputFile); - } - { - Options options; - auto args = makeArray("x", "-"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } -} - -TEST(Options, InvalidOptions) { - { - Options options; - auto args = makeArray("-ibasdf"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("- "); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("-n15"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("-0", "x"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - } -} - -TEST(Options, Extras) { - { - Options options; - auto args = makeArray("-h"); - EXPECT_MESSAGE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("-H"); - EXPECT_MESSAGE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("-V"); - EXPECT_MESSAGE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("--help"); - EXPECT_MESSAGE(options.parse(args.size(), args.data())); - } - { - Options options; - auto args = makeArray("--version"); - EXPECT_MESSAGE(options.parse(args.size(), args.data())); - } -} diff --git a/sys/contrib/zstd/contrib/pzstd/test/PzstdTest.cpp b/sys/contrib/zstd/contrib/pzstd/test/PzstdTest.cpp deleted file mode 100644 index 5c7d6631080..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/test/PzstdTest.cpp +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "Pzstd.h" -extern "C" { -#include "datagen.h" -} -#include "test/RoundTrip.h" -#include "utils/ScopeGuard.h" - -#include -#include -#include -#include -#include - -using namespace std; -using namespace pzstd; - -TEST(Pzstd, SmallSizes) { - unsigned seed = std::random_device{}(); - std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed); - std::mt19937 gen(seed); - - for (unsigned len = 1; len < 256; ++len) { - if (len % 16 == 0) { - std::fprintf(stderr, "%u / 16\n", len / 16); - } - std::string inputFile = std::tmpnam(nullptr); - auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); - { - static uint8_t buf[256]; - RDG_genBuffer(buf, len, 0.5, 0.0, gen()); - auto fd = std::fopen(inputFile.c_str(), "wb"); - auto written = std::fwrite(buf, 1, len, fd); - std::fclose(fd); - ASSERT_EQ(written, len); - } - for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) { - for (unsigned level = 1; level <= 4; level *= 4) { - auto errorGuard = makeScopeGuard([&] { - std::fprintf(stderr, "# threads: %u\n", numThreads); - std::fprintf(stderr, "compression level: %u\n", level); - }); - Options options; - options.overwrite = true; - options.inputFiles = {inputFile}; - options.numThreads = numThreads; - options.compressionLevel = level; - options.verbosity = 1; - ASSERT_TRUE(roundTrip(options)); - errorGuard.dismiss(); - } - } - } -} - -TEST(Pzstd, LargeSizes) { - unsigned seed = std::random_device{}(); - std::fprintf(stderr, "Pzstd.LargeSizes seed: %u\n", seed); - std::mt19937 gen(seed); - - for (unsigned len = 1 << 20; len <= (1 << 24); len *= 2) { - std::string inputFile = std::tmpnam(nullptr); - auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); - { - std::unique_ptr buf(new uint8_t[len]); - RDG_genBuffer(buf.get(), len, 0.5, 0.0, gen()); - auto fd = std::fopen(inputFile.c_str(), "wb"); - auto written = std::fwrite(buf.get(), 1, len, fd); - std::fclose(fd); - ASSERT_EQ(written, len); - } - for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { - for (unsigned level = 1; level <= 4; level *= 4) { - auto errorGuard = makeScopeGuard([&] { - std::fprintf(stderr, "# threads: %u\n", numThreads); - std::fprintf(stderr, "compression level: %u\n", level); - }); - Options options; - options.overwrite = true; - options.inputFiles = {inputFile}; - options.numThreads = std::min(numThreads, options.numThreads); - options.compressionLevel = level; - options.verbosity = 1; - ASSERT_TRUE(roundTrip(options)); - errorGuard.dismiss(); - } - } - } -} - -TEST(Pzstd, DISABLED_ExtremelyLargeSize) { - unsigned seed = std::random_device{}(); - std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed); - std::mt19937 gen(seed); - - std::string inputFile = std::tmpnam(nullptr); - auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); - - { - // Write 4GB + 64 MB - constexpr size_t kLength = 1 << 26; - std::unique_ptr buf(new uint8_t[kLength]); - auto fd = std::fopen(inputFile.c_str(), "wb"); - auto closeGuard = makeScopeGuard([&] { std::fclose(fd); }); - for (size_t i = 0; i < (1 << 6) + 1; ++i) { - RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen()); - auto written = std::fwrite(buf.get(), 1, kLength, fd); - if (written != kLength) { - std::fprintf(stderr, "Failed to write file, skipping test\n"); - return; - } - } - } - - Options options; - options.overwrite = true; - options.inputFiles = {inputFile}; - options.compressionLevel = 1; - if (options.numThreads == 0) { - options.numThreads = 1; - } - ASSERT_TRUE(roundTrip(options)); -} - -TEST(Pzstd, ExtremelyCompressible) { - std::string inputFile = std::tmpnam(nullptr); - auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); - { - std::unique_ptr buf(new uint8_t[10000]); - std::memset(buf.get(), 'a', 10000); - auto fd = std::fopen(inputFile.c_str(), "wb"); - auto written = std::fwrite(buf.get(), 1, 10000, fd); - std::fclose(fd); - ASSERT_EQ(written, 10000); - } - Options options; - options.overwrite = true; - options.inputFiles = {inputFile}; - options.numThreads = 1; - options.compressionLevel = 1; - ASSERT_TRUE(roundTrip(options)); -} diff --git a/sys/contrib/zstd/contrib/pzstd/test/RoundTrip.h b/sys/contrib/zstd/contrib/pzstd/test/RoundTrip.h deleted file mode 100644 index c6364ecb422..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/test/RoundTrip.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include "Options.h" -#include "Pzstd.h" -#include "utils/ScopeGuard.h" - -#include -#include -#include -#include - -namespace pzstd { - -inline bool check(std::string source, std::string decompressed) { - std::unique_ptr sBuf(new std::uint8_t[1024]); - std::unique_ptr dBuf(new std::uint8_t[1024]); - - auto sFd = std::fopen(source.c_str(), "rb"); - auto dFd = std::fopen(decompressed.c_str(), "rb"); - auto guard = makeScopeGuard([&] { - std::fclose(sFd); - std::fclose(dFd); - }); - - size_t sRead, dRead; - - do { - sRead = std::fread(sBuf.get(), 1, 1024, sFd); - dRead = std::fread(dBuf.get(), 1, 1024, dFd); - if (std::ferror(sFd) || std::ferror(dFd)) { - return false; - } - if (sRead != dRead) { - return false; - } - - for (size_t i = 0; i < sRead; ++i) { - if (sBuf.get()[i] != dBuf.get()[i]) { - return false; - } - } - } while (sRead == 1024); - if (!std::feof(sFd) || !std::feof(dFd)) { - return false; - } - return true; -} - -inline bool roundTrip(Options& options) { - if (options.inputFiles.size() != 1) { - return false; - } - std::string source = options.inputFiles.front(); - std::string compressedFile = std::tmpnam(nullptr); - std::string decompressedFile = std::tmpnam(nullptr); - auto guard = makeScopeGuard([&] { - std::remove(compressedFile.c_str()); - std::remove(decompressedFile.c_str()); - }); - - { - options.outputFile = compressedFile; - options.decompress = false; - if (pzstdMain(options) != 0) { - return false; - } - } - { - options.decompress = true; - options.inputFiles.front() = compressedFile; - options.outputFile = decompressedFile; - if (pzstdMain(options) != 0) { - return false; - } - } - return check(source, decompressedFile); -} -} diff --git a/sys/contrib/zstd/contrib/pzstd/test/RoundTripTest.cpp b/sys/contrib/zstd/contrib/pzstd/test/RoundTripTest.cpp deleted file mode 100644 index 36af0673ae6..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/test/RoundTripTest.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -extern "C" { -#include "datagen.h" -} -#include "Options.h" -#include "test/RoundTrip.h" -#include "utils/ScopeGuard.h" - -#include -#include -#include -#include -#include - -using namespace std; -using namespace pzstd; - -namespace { -string -writeData(size_t size, double matchProba, double litProba, unsigned seed) { - std::unique_ptr buf(new uint8_t[size]); - RDG_genBuffer(buf.get(), size, matchProba, litProba, seed); - string file = tmpnam(nullptr); - auto fd = std::fopen(file.c_str(), "wb"); - auto guard = makeScopeGuard([&] { std::fclose(fd); }); - auto bytesWritten = std::fwrite(buf.get(), 1, size, fd); - if (bytesWritten != size) { - std::abort(); - } - return file; -} - -template -string generateInputFile(Generator& gen) { - // Use inputs ranging from 1 Byte to 2^16 Bytes - std::uniform_int_distribution size{1, 1 << 16}; - std::uniform_real_distribution<> prob{0, 1}; - return writeData(size(gen), prob(gen), prob(gen), gen()); -} - -template -Options generateOptions(Generator& gen, const string& inputFile) { - Options options; - options.inputFiles = {inputFile}; - options.overwrite = true; - - std::uniform_int_distribution numThreads{1, 32}; - std::uniform_int_distribution compressionLevel{1, 10}; - - options.numThreads = numThreads(gen); - options.compressionLevel = compressionLevel(gen); - - return options; -} -} - -int main() { - std::mt19937 gen(std::random_device{}()); - - auto newlineGuard = makeScopeGuard([] { std::fprintf(stderr, "\n"); }); - for (unsigned i = 0; i < 10000; ++i) { - if (i % 100 == 0) { - std::fprintf(stderr, "Progress: %u%%\r", i / 100); - } - auto inputFile = generateInputFile(gen); - auto inputGuard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); - for (unsigned i = 0; i < 10; ++i) { - auto options = generateOptions(gen, inputFile); - if (!roundTrip(options)) { - std::fprintf(stderr, "numThreads: %u\n", options.numThreads); - std::fprintf(stderr, "level: %u\n", options.compressionLevel); - std::fprintf(stderr, "decompress? %u\n", (unsigned)options.decompress); - std::fprintf(stderr, "file: %s\n", inputFile.c_str()); - return 1; - } - } - } - return 0; -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/BUCK b/sys/contrib/zstd/contrib/pzstd/utils/BUCK deleted file mode 100644 index e757f412070..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/BUCK +++ /dev/null @@ -1,75 +0,0 @@ -cxx_library( - name='buffer', - visibility=['PUBLIC'], - header_namespace='utils', - exported_headers=['Buffer.h'], - deps=[':range'], -) - -cxx_library( - name='file_system', - visibility=['PUBLIC'], - header_namespace='utils', - exported_headers=['FileSystem.h'], - deps=[':range'], -) - -cxx_library( - name='likely', - visibility=['PUBLIC'], - header_namespace='utils', - exported_headers=['Likely.h'], -) - -cxx_library( - name='range', - visibility=['PUBLIC'], - header_namespace='utils', - exported_headers=['Range.h'], - deps=[':likely'], -) - -cxx_library( - name='resource_pool', - visibility=['PUBLIC'], - header_namespace='utils', - exported_headers=['ResourcePool.h'], -) - -cxx_library( - name='scope_guard', - visibility=['PUBLIC'], - header_namespace='utils', - exported_headers=['ScopeGuard.h'], -) - -cxx_library( - name='thread_pool', - visibility=['PUBLIC'], - header_namespace='utils', - exported_headers=['ThreadPool.h'], - deps=[':work_queue'], -) - -cxx_library( - name='work_queue', - visibility=['PUBLIC'], - header_namespace='utils', - exported_headers=['WorkQueue.h'], - deps=[':buffer'], -) - -cxx_library( - name='utils', - visibility=['PUBLIC'], - deps=[ - ':buffer', - ':file_system', - ':likely', - ':range', - ':resource_pool', - ':scope_guard', - ':thread_pool', - ':work_queue', - ], -) diff --git a/sys/contrib/zstd/contrib/pzstd/utils/Buffer.h b/sys/contrib/zstd/contrib/pzstd/utils/Buffer.h deleted file mode 100644 index f69c3b4d9f7..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/Buffer.h +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include "utils/Range.h" - -#include -#include -#include - -namespace pzstd { - -/** - * A `Buffer` has a pointer to a shared buffer, and a range of the buffer that - * it owns. - * The idea is that you can allocate one buffer, and write chunks into it - * and break off those chunks. - * The underlying buffer is reference counted, and will be destroyed when all - * `Buffer`s that reference it are destroyed. - */ -class Buffer { - std::shared_ptr buffer_; - MutableByteRange range_; - - static void delete_buffer(unsigned char* buffer) { - delete[] buffer; - } - - public: - /// Construct an empty buffer that owns no data. - explicit Buffer() {} - - /// Construct a `Buffer` that owns a new underlying buffer of size `size`. - explicit Buffer(std::size_t size) - : buffer_(new unsigned char[size], delete_buffer), - range_(buffer_.get(), buffer_.get() + size) {} - - explicit Buffer(std::shared_ptr buffer, MutableByteRange data) - : buffer_(buffer), range_(data) {} - - Buffer(Buffer&&) = default; - Buffer& operator=(Buffer&&) & = default; - - /** - * Splits the data into two pieces: [begin, begin + n), [begin + n, end). - * Their data both points into the same underlying buffer. - * Modifies the original `Buffer` to point to only [begin + n, end). - * - * @param n The offset to split at. - * @returns A buffer that owns the data [begin, begin + n). - */ - Buffer splitAt(std::size_t n) { - auto firstPiece = range_.subpiece(0, n); - range_.advance(n); - return Buffer(buffer_, firstPiece); - } - - /// Modifies the buffer to point to the range [begin + n, end). - void advance(std::size_t n) { - range_.advance(n); - } - - /// Modifies the buffer to point to the range [begin, end - n). - void subtract(std::size_t n) { - range_.subtract(n); - } - - /// Returns a read only `Range` pointing to the `Buffer`s data. - ByteRange range() const { - return range_; - } - /// Returns a mutable `Range` pointing to the `Buffer`s data. - MutableByteRange range() { - return range_; - } - - const unsigned char* data() const { - return range_.data(); - } - - unsigned char* data() { - return range_.data(); - } - - std::size_t size() const { - return range_.size(); - } - - bool empty() const { - return range_.empty(); - } -}; -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/FileSystem.h b/sys/contrib/zstd/contrib/pzstd/utils/FileSystem.h deleted file mode 100644 index 3cfbe86e507..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/FileSystem.h +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include "utils/Range.h" - -#include -#include -#include -#include - -// A small subset of `std::filesystem`. -// `std::filesystem` should be a drop in replacement. -// See http://en.cppreference.com/w/cpp/filesystem for documentation. - -namespace pzstd { - -// using file_status = ... causes gcc to emit a false positive warning -#if defined(_MSC_VER) -typedef struct ::_stat64 file_status; -#else -typedef struct ::stat file_status; -#endif - -/// http://en.cppreference.com/w/cpp/filesystem/status -inline file_status status(StringPiece path, std::error_code& ec) noexcept { - file_status status; -#if defined(_MSC_VER) - const auto error = ::_stat64(path.data(), &status); -#else - const auto error = ::stat(path.data(), &status); -#endif - if (error) { - ec.assign(errno, std::generic_category()); - } else { - ec.clear(); - } - return status; -} - -/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file -inline bool is_regular_file(file_status status) noexcept { -#if defined(S_ISREG) - return S_ISREG(status.st_mode); -#elif !defined(S_ISREG) && defined(S_IFMT) && defined(S_IFREG) - return (status.st_mode & S_IFMT) == S_IFREG; -#else - static_assert(false, "No POSIX stat() support."); -#endif -} - -/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file -inline bool is_regular_file(StringPiece path, std::error_code& ec) noexcept { - return is_regular_file(status(path, ec)); -} - -/// http://en.cppreference.com/w/cpp/filesystem/is_directory -inline bool is_directory(file_status status) noexcept { -#if defined(S_ISDIR) - return S_ISDIR(status.st_mode); -#elif !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR) - return (status.st_mode & S_IFMT) == S_IFDIR; -#else - static_assert(false, "NO POSIX stat() support."); -#endif -} - -/// http://en.cppreference.com/w/cpp/filesystem/is_directory -inline bool is_directory(StringPiece path, std::error_code& ec) noexcept { - return is_directory(status(path, ec)); -} - -/// http://en.cppreference.com/w/cpp/filesystem/file_size -inline std::uintmax_t file_size( - StringPiece path, - std::error_code& ec) noexcept { - auto stat = status(path, ec); - if (ec) { - return -1; - } - if (!is_regular_file(stat)) { - ec.assign(ENOTSUP, std::generic_category()); - return -1; - } - ec.clear(); - return stat.st_size; -} -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/Likely.h b/sys/contrib/zstd/contrib/pzstd/utils/Likely.h deleted file mode 100644 index 7cea8da2771..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/Likely.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * Compiler hints to indicate the fast path of an "if" branch: whether - * the if condition is likely to be true or false. - * - * @author Tudor Bosman (tudorb@fb.com) - */ - -#pragma once - -#undef LIKELY -#undef UNLIKELY - -#if defined(__GNUC__) && __GNUC__ >= 4 -#define LIKELY(x) (__builtin_expect((x), 1)) -#define UNLIKELY(x) (__builtin_expect((x), 0)) -#else -#define LIKELY(x) (x) -#define UNLIKELY(x) (x) -#endif diff --git a/sys/contrib/zstd/contrib/pzstd/utils/Range.h b/sys/contrib/zstd/contrib/pzstd/utils/Range.h deleted file mode 100644 index 7e2559cc9e3..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/Range.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * A subset of `folly/Range.h`. - * All code copied verbatiam modulo formatting - */ -#pragma once - -#include "utils/Likely.h" - -#include -#include -#include -#include -#include - -namespace pzstd { - -namespace detail { -/* - *Use IsCharPointer::type to enable const char* or char*. - *Use IsCharPointer::const_type to enable only const char*. -*/ -template -struct IsCharPointer {}; - -template <> -struct IsCharPointer { - typedef int type; -}; - -template <> -struct IsCharPointer { - typedef int const_type; - typedef int type; -}; - -} // namespace detail - -template -class Range { - Iter b_; - Iter e_; - - public: - using size_type = std::size_t; - using iterator = Iter; - using const_iterator = Iter; - using value_type = typename std::remove_reference< - typename std::iterator_traits::reference>::type; - using reference = typename std::iterator_traits::reference; - - constexpr Range() : b_(), e_() {} - constexpr Range(Iter begin, Iter end) : b_(begin), e_(end) {} - - constexpr Range(Iter begin, size_type size) : b_(begin), e_(begin + size) {} - - template ::type = 0> - /* implicit */ Range(Iter str) : b_(str), e_(str + std::strlen(str)) {} - - template ::const_type = 0> - /* implicit */ Range(const std::string& str) - : b_(str.data()), e_(b_ + str.size()) {} - - // Allow implicit conversion from Range to Range if From is - // implicitly convertible to To. - template < - class OtherIter, - typename std::enable_if< - (!std::is_same::value && - std::is_convertible::value), - int>::type = 0> - constexpr /* implicit */ Range(const Range& other) - : b_(other.begin()), e_(other.end()) {} - - Range(const Range&) = default; - Range(Range&&) = default; - - Range& operator=(const Range&) & = default; - Range& operator=(Range&&) & = default; - - constexpr size_type size() const { - return e_ - b_; - } - bool empty() const { - return b_ == e_; - } - Iter data() const { - return b_; - } - Iter begin() const { - return b_; - } - Iter end() const { - return e_; - } - - void advance(size_type n) { - if (UNLIKELY(n > size())) { - throw std::out_of_range("index out of range"); - } - b_ += n; - } - - void subtract(size_type n) { - if (UNLIKELY(n > size())) { - throw std::out_of_range("index out of range"); - } - e_ -= n; - } - - Range subpiece(size_type first, size_type length = std::string::npos) const { - if (UNLIKELY(first > size())) { - throw std::out_of_range("index out of range"); - } - - return Range(b_ + first, std::min(length, size() - first)); - } -}; - -using ByteRange = Range; -using MutableByteRange = Range; -using StringPiece = Range; -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/ResourcePool.h b/sys/contrib/zstd/contrib/pzstd/utils/ResourcePool.h deleted file mode 100644 index a6ff5ffc5db..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/ResourcePool.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include -#include -#include -#include -#include - -namespace pzstd { - -/** - * An unbounded pool of resources. - * A `ResourcePool` requires a factory function that takes allocates `T*` and - * a free function that frees a `T*`. - * Calling `ResourcePool::get()` will give you a new `ResourcePool::UniquePtr` - * to a `T`, and when it goes out of scope the resource will be returned to the - * pool. - * The `ResourcePool` *must* survive longer than any resources it hands out. - * Remember that `ResourcePool` hands out mutable `T`s, so make sure to clean - * up the resource before or after every use. - */ -template -class ResourcePool { - public: - class Deleter; - using Factory = std::function; - using Free = std::function; - using UniquePtr = std::unique_ptr; - - private: - std::mutex mutex_; - Factory factory_; - Free free_; - std::vector resources_; - unsigned inUse_; - - public: - /** - * Creates a `ResourcePool`. - * - * @param factory The function to use to create new resources. - * @param free The function to use to free resources created by `factory`. - */ - ResourcePool(Factory factory, Free free) - : factory_(std::move(factory)), free_(std::move(free)), inUse_(0) {} - - /** - * @returns A unique pointer to a resource. The resource is null iff - * there are no avaiable resources and `factory()` returns null. - */ - UniquePtr get() { - std::lock_guard lock(mutex_); - if (!resources_.empty()) { - UniquePtr resource{resources_.back(), Deleter{*this}}; - resources_.pop_back(); - ++inUse_; - return resource; - } - UniquePtr resource{factory_(), Deleter{*this}}; - ++inUse_; - return resource; - } - - ~ResourcePool() noexcept { - assert(inUse_ == 0); - for (const auto resource : resources_) { - free_(resource); - } - } - - class Deleter { - ResourcePool *pool_; - public: - explicit Deleter(ResourcePool &pool) : pool_(&pool) {} - - void operator() (T *resource) { - std::lock_guard lock(pool_->mutex_); - // Make sure we don't put null resources into the pool - if (resource) { - pool_->resources_.push_back(resource); - } - assert(pool_->inUse_ > 0); - --pool_->inUse_; - } - }; -}; - -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/ScopeGuard.h b/sys/contrib/zstd/contrib/pzstd/utils/ScopeGuard.h deleted file mode 100644 index 31768f43d22..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/ScopeGuard.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include - -namespace pzstd { - -/** - * Dismissable scope guard. - * `Function` must be callable and take no parameters. - * Unless `dissmiss()` is called, the callable is executed upon destruction of - * `ScopeGuard`. - * - * Example: - * - * auto guard = makeScopeGuard([&] { cleanup(); }); - */ -template -class ScopeGuard { - Function function; - bool dismissed; - - public: - explicit ScopeGuard(Function&& function) - : function(std::move(function)), dismissed(false) {} - - void dismiss() { - dismissed = true; - } - - ~ScopeGuard() noexcept { - if (!dismissed) { - function(); - } - } -}; - -/// Creates a scope guard from `function`. -template -ScopeGuard makeScopeGuard(Function&& function) { - return ScopeGuard(std::forward(function)); -} -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/ThreadPool.h b/sys/contrib/zstd/contrib/pzstd/utils/ThreadPool.h deleted file mode 100644 index 8ece8e0da4e..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/ThreadPool.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include "utils/WorkQueue.h" - -#include -#include -#include -#include - -namespace pzstd { -/// A simple thread pool that pulls tasks off its queue in FIFO order. -class ThreadPool { - std::vector threads_; - - WorkQueue> tasks_; - - public: - /// Constructs a thread pool with `numThreads` threads. - explicit ThreadPool(std::size_t numThreads) { - threads_.reserve(numThreads); - for (std::size_t i = 0; i < numThreads; ++i) { - threads_.emplace_back([this] { - std::function task; - while (tasks_.pop(task)) { - task(); - } - }); - } - } - - /// Finishes all tasks currently in the queue. - ~ThreadPool() { - tasks_.finish(); - for (auto& thread : threads_) { - thread.join(); - } - } - - /** - * Adds `task` to the queue of tasks to execute. Since `task` is a - * `std::function<>`, it cannot be a move only type. So any lambda passed must - * not capture move only types (like `std::unique_ptr`). - * - * @param task The task to execute. - */ - void add(std::function task) { - tasks_.push(std::move(task)); - } -}; -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/WorkQueue.h b/sys/contrib/zstd/contrib/pzstd/utils/WorkQueue.h deleted file mode 100644 index 1d14d922c64..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/WorkQueue.h +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#pragma once - -#include "utils/Buffer.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace pzstd { - -/// Unbounded thread-safe work queue. -template -class WorkQueue { - // Protects all member variable access - std::mutex mutex_; - std::condition_variable readerCv_; - std::condition_variable writerCv_; - std::condition_variable finishCv_; - - std::queue queue_; - bool done_; - std::size_t maxSize_; - - // Must have lock to call this function - bool full() const { - if (maxSize_ == 0) { - return false; - } - return queue_.size() >= maxSize_; - } - - public: - /** - * Constructs an empty work queue with an optional max size. - * If `maxSize == 0` the queue size is unbounded. - * - * @param maxSize The maximum allowed size of the work queue. - */ - WorkQueue(std::size_t maxSize = 0) : done_(false), maxSize_(maxSize) {} - - /** - * Push an item onto the work queue. Notify a single thread that work is - * available. If `finish()` has been called, do nothing and return false. - * If `push()` returns false, then `item` has not been moved from. - * - * @param item Item to push onto the queue. - * @returns True upon success, false if `finish()` has been called. An - * item was pushed iff `push()` returns true. - */ - bool push(T&& item) { - { - std::unique_lock lock(mutex_); - while (full() && !done_) { - writerCv_.wait(lock); - } - if (done_) { - return false; - } - queue_.push(std::move(item)); - } - readerCv_.notify_one(); - return true; - } - - /** - * Attempts to pop an item off the work queue. It will block until data is - * available or `finish()` has been called. - * - * @param[out] item If `pop` returns `true`, it contains the popped item. - * If `pop` returns `false`, it is unmodified. - * @returns True upon success. False if the queue is empty and - * `finish()` has been called. - */ - bool pop(T& item) { - { - std::unique_lock lock(mutex_); - while (queue_.empty() && !done_) { - readerCv_.wait(lock); - } - if (queue_.empty()) { - assert(done_); - return false; - } - item = std::move(queue_.front()); - queue_.pop(); - } - writerCv_.notify_one(); - return true; - } - - /** - * Sets the maximum queue size. If `maxSize == 0` then it is unbounded. - * - * @param maxSize The new maximum queue size. - */ - void setMaxSize(std::size_t maxSize) { - { - std::lock_guard lock(mutex_); - maxSize_ = maxSize; - } - writerCv_.notify_all(); - } - - /** - * Promise that `push()` won't be called again, so once the queue is empty - * there will never any more work. - */ - void finish() { - { - std::lock_guard lock(mutex_); - assert(!done_); - done_ = true; - } - readerCv_.notify_all(); - writerCv_.notify_all(); - finishCv_.notify_all(); - } - - /// Blocks until `finish()` has been called (but the queue may not be empty). - void waitUntilFinished() { - std::unique_lock lock(mutex_); - while (!done_) { - finishCv_.wait(lock); - } - } -}; - -/// Work queue for `Buffer`s that knows the total number of bytes in the queue. -class BufferWorkQueue { - WorkQueue queue_; - std::atomic size_; - - public: - BufferWorkQueue(std::size_t maxSize = 0) : queue_(maxSize), size_(0) {} - - void push(Buffer buffer) { - size_.fetch_add(buffer.size()); - queue_.push(std::move(buffer)); - } - - bool pop(Buffer& buffer) { - bool result = queue_.pop(buffer); - if (result) { - size_.fetch_sub(buffer.size()); - } - return result; - } - - void setMaxSize(std::size_t maxSize) { - queue_.setMaxSize(maxSize); - } - - void finish() { - queue_.finish(); - } - - /** - * Blocks until `finish()` has been called. - * - * @returns The total number of bytes of all the `Buffer`s currently in the - * queue. - */ - std::size_t size() { - queue_.waitUntilFinished(); - return size_.load(); - } -}; -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/test/BUCK b/sys/contrib/zstd/contrib/pzstd/utils/test/BUCK deleted file mode 100644 index a5113cab6b0..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/test/BUCK +++ /dev/null @@ -1,35 +0,0 @@ -cxx_test( - name='buffer_test', - srcs=['BufferTest.cpp'], - deps=['//contrib/pzstd/utils:buffer'], -) - -cxx_test( - name='range_test', - srcs=['RangeTest.cpp'], - deps=['//contrib/pzstd/utils:range'], -) - -cxx_test( - name='resource_pool_test', - srcs=['ResourcePoolTest.cpp'], - deps=['//contrib/pzstd/utils:resource_pool'], -) - -cxx_test( - name='scope_guard_test', - srcs=['ScopeGuardTest.cpp'], - deps=['//contrib/pzstd/utils:scope_guard'], -) - -cxx_test( - name='thread_pool_test', - srcs=['ThreadPoolTest.cpp'], - deps=['//contrib/pzstd/utils:thread_pool'], -) - -cxx_test( - name='work_queue_test', - srcs=['RangeTest.cpp'], - deps=['//contrib/pzstd/utils:work_queue'], -) diff --git a/sys/contrib/zstd/contrib/pzstd/utils/test/BufferTest.cpp b/sys/contrib/zstd/contrib/pzstd/utils/test/BufferTest.cpp deleted file mode 100644 index fbba74e8262..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/test/BufferTest.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "utils/Buffer.h" -#include "utils/Range.h" - -#include -#include - -using namespace pzstd; - -namespace { -void deleter(const unsigned char* buf) { - delete[] buf; -} -} - -TEST(Buffer, Constructors) { - Buffer empty; - EXPECT_TRUE(empty.empty()); - EXPECT_EQ(0, empty.size()); - - Buffer sized(5); - EXPECT_FALSE(sized.empty()); - EXPECT_EQ(5, sized.size()); - - Buffer moved(std::move(sized)); - EXPECT_FALSE(sized.empty()); - EXPECT_EQ(5, sized.size()); - - Buffer assigned; - assigned = std::move(moved); - EXPECT_FALSE(sized.empty()); - EXPECT_EQ(5, sized.size()); -} - -TEST(Buffer, BufferManagement) { - std::shared_ptr buf(new unsigned char[10], deleter); - { - Buffer acquired(buf, MutableByteRange(buf.get(), buf.get() + 10)); - EXPECT_EQ(2, buf.use_count()); - Buffer moved(std::move(acquired)); - EXPECT_EQ(2, buf.use_count()); - Buffer assigned; - assigned = std::move(moved); - EXPECT_EQ(2, buf.use_count()); - - Buffer split = assigned.splitAt(5); - EXPECT_EQ(3, buf.use_count()); - - split.advance(1); - assigned.subtract(1); - EXPECT_EQ(3, buf.use_count()); - } - EXPECT_EQ(1, buf.use_count()); -} - -TEST(Buffer, Modifiers) { - Buffer buf(10); - { - unsigned char i = 0; - for (auto& byte : buf.range()) { - byte = i++; - } - } - - auto prefix = buf.splitAt(2); - - ASSERT_EQ(2, prefix.size()); - EXPECT_EQ(0, *prefix.data()); - - ASSERT_EQ(8, buf.size()); - EXPECT_EQ(2, *buf.data()); - - buf.advance(2); - EXPECT_EQ(4, *buf.data()); - - EXPECT_EQ(9, *(buf.range().end() - 1)); - - buf.subtract(2); - EXPECT_EQ(7, *(buf.range().end() - 1)); - - EXPECT_EQ(4, buf.size()); -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/test/RangeTest.cpp b/sys/contrib/zstd/contrib/pzstd/utils/test/RangeTest.cpp deleted file mode 100644 index 755b50fa6e8..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/test/RangeTest.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "utils/Range.h" - -#include -#include - -using namespace pzstd; - -// Range is directly copied from folly. -// Just some sanity tests to make sure everything seems to work. - -TEST(Range, Constructors) { - StringPiece empty; - EXPECT_TRUE(empty.empty()); - EXPECT_EQ(0, empty.size()); - - std::string str = "hello"; - { - Range piece(str.begin(), str.end()); - EXPECT_EQ(5, piece.size()); - EXPECT_EQ('h', *piece.data()); - EXPECT_EQ('o', *(piece.end() - 1)); - } - - { - StringPiece piece(str.data(), str.size()); - EXPECT_EQ(5, piece.size()); - EXPECT_EQ('h', *piece.data()); - EXPECT_EQ('o', *(piece.end() - 1)); - } - - { - StringPiece piece(str); - EXPECT_EQ(5, piece.size()); - EXPECT_EQ('h', *piece.data()); - EXPECT_EQ('o', *(piece.end() - 1)); - } - - { - StringPiece piece(str.c_str()); - EXPECT_EQ(5, piece.size()); - EXPECT_EQ('h', *piece.data()); - EXPECT_EQ('o', *(piece.end() - 1)); - } -} - -TEST(Range, Modifiers) { - StringPiece range("hello world"); - ASSERT_EQ(11, range.size()); - - { - auto hello = range.subpiece(0, 5); - EXPECT_EQ(5, hello.size()); - EXPECT_EQ('h', *hello.data()); - EXPECT_EQ('o', *(hello.end() - 1)); - } - { - auto hello = range; - hello.subtract(6); - EXPECT_EQ(5, hello.size()); - EXPECT_EQ('h', *hello.data()); - EXPECT_EQ('o', *(hello.end() - 1)); - } - { - auto world = range; - world.advance(6); - EXPECT_EQ(5, world.size()); - EXPECT_EQ('w', *world.data()); - EXPECT_EQ('d', *(world.end() - 1)); - } - - std::string expected = "hello world"; - EXPECT_EQ(expected, std::string(range.begin(), range.end())); - EXPECT_EQ(expected, std::string(range.data(), range.size())); -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/test/ResourcePoolTest.cpp b/sys/contrib/zstd/contrib/pzstd/utils/test/ResourcePoolTest.cpp deleted file mode 100644 index 6fe145180be..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/test/ResourcePoolTest.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "utils/ResourcePool.h" - -#include -#include -#include - -using namespace pzstd; - -TEST(ResourcePool, FullTest) { - unsigned numCreated = 0; - unsigned numDeleted = 0; - { - ResourcePool pool( - [&numCreated] { ++numCreated; return new int{5}; }, - [&numDeleted](int *x) { ++numDeleted; delete x; }); - - { - auto i = pool.get(); - EXPECT_EQ(5, *i); - *i = 6; - } - { - auto i = pool.get(); - EXPECT_EQ(6, *i); - auto j = pool.get(); - EXPECT_EQ(5, *j); - *j = 7; - } - { - auto i = pool.get(); - EXPECT_EQ(6, *i); - auto j = pool.get(); - EXPECT_EQ(7, *j); - } - } - EXPECT_EQ(2, numCreated); - EXPECT_EQ(numCreated, numDeleted); -} - -TEST(ResourcePool, ThreadSafe) { - std::atomic numCreated{0}; - std::atomic numDeleted{0}; - { - ResourcePool pool( - [&numCreated] { ++numCreated; return new int{0}; }, - [&numDeleted](int *x) { ++numDeleted; delete x; }); - auto push = [&pool] { - for (int i = 0; i < 100; ++i) { - auto x = pool.get(); - ++*x; - } - }; - std::thread t1{push}; - std::thread t2{push}; - t1.join(); - t2.join(); - - auto x = pool.get(); - auto y = pool.get(); - EXPECT_EQ(200, *x + *y); - } - EXPECT_GE(2, numCreated); - EXPECT_EQ(numCreated, numDeleted); -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/test/ScopeGuardTest.cpp b/sys/contrib/zstd/contrib/pzstd/utils/test/ScopeGuardTest.cpp deleted file mode 100644 index 7bc624da79b..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/test/ScopeGuardTest.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "utils/ScopeGuard.h" - -#include - -using namespace pzstd; - -TEST(ScopeGuard, Dismiss) { - { - auto guard = makeScopeGuard([&] { EXPECT_TRUE(false); }); - guard.dismiss(); - } -} - -TEST(ScopeGuard, Executes) { - bool executed = false; - { - auto guard = makeScopeGuard([&] { executed = true; }); - } - EXPECT_TRUE(executed); -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp b/sys/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp deleted file mode 100644 index 703fd4c9ca1..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "utils/ThreadPool.h" - -#include -#include -#include -#include -#include - -using namespace pzstd; - -TEST(ThreadPool, Ordering) { - std::vector results; - - { - ThreadPool executor(1); - for (int i = 0; i < 10; ++i) { - executor.add([ &results, i ] { results.push_back(i); }); - } - } - - for (int i = 0; i < 10; ++i) { - EXPECT_EQ(i, results[i]); - } -} - -TEST(ThreadPool, AllJobsFinished) { - std::atomic numFinished{0}; - std::atomic start{false}; - { - std::cerr << "Creating executor" << std::endl; - ThreadPool executor(5); - for (int i = 0; i < 10; ++i) { - executor.add([ &numFinished, &start ] { - while (!start.load()) { - std::this_thread::yield(); - } - ++numFinished; - }); - } - std::cerr << "Starting" << std::endl; - start.store(true); - std::cerr << "Finishing" << std::endl; - } - EXPECT_EQ(10, numFinished.load()); -} - -TEST(ThreadPool, AddJobWhileJoining) { - std::atomic done{false}; - { - ThreadPool executor(1); - executor.add([&executor, &done] { - while (!done.load()) { - std::this_thread::yield(); - } - // Sleep for a second to be sure that we are joining - std::this_thread::sleep_for(std::chrono::seconds(1)); - executor.add([] { - EXPECT_TRUE(false); - }); - }); - done.store(true); - } -} diff --git a/sys/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp b/sys/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp deleted file mode 100644 index 14cf77304f2..00000000000 --- a/sys/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ -#include "utils/Buffer.h" -#include "utils/WorkQueue.h" - -#include -#include -#include -#include -#include -#include - -using namespace pzstd; - -namespace { -struct Popper { - WorkQueue* queue; - int* results; - std::mutex* mutex; - - void operator()() { - int result; - while (queue->pop(result)) { - std::lock_guard lock(*mutex); - results[result] = result; - } - } -}; -} - -TEST(WorkQueue, SingleThreaded) { - WorkQueue queue; - int result; - - queue.push(5); - EXPECT_TRUE(queue.pop(result)); - EXPECT_EQ(5, result); - - queue.push(1); - queue.push(2); - EXPECT_TRUE(queue.pop(result)); - EXPECT_EQ(1, result); - EXPECT_TRUE(queue.pop(result)); - EXPECT_EQ(2, result); - - queue.push(1); - queue.push(2); - queue.finish(); - EXPECT_TRUE(queue.pop(result)); - EXPECT_EQ(1, result); - EXPECT_TRUE(queue.pop(result)); - EXPECT_EQ(2, result); - EXPECT_FALSE(queue.pop(result)); - - queue.waitUntilFinished(); -} - -TEST(WorkQueue, SPSC) { - WorkQueue queue; - const int max = 100; - - for (int i = 0; i < 10; ++i) { - queue.push(int{i}); - } - - std::thread thread([ &queue, max ] { - int result; - for (int i = 0;; ++i) { - if (!queue.pop(result)) { - EXPECT_EQ(i, max); - break; - } - EXPECT_EQ(i, result); - } - }); - - std::this_thread::yield(); - for (int i = 10; i < max; ++i) { - queue.push(int{i}); - } - queue.finish(); - - thread.join(); -} - -TEST(WorkQueue, SPMC) { - WorkQueue queue; - std::vector results(50, -1); - std::mutex mutex; - std::vector threads; - for (int i = 0; i < 5; ++i) { - threads.emplace_back(Popper{&queue, results.data(), &mutex}); - } - - for (int i = 0; i < 50; ++i) { - queue.push(int{i}); - } - queue.finish(); - - for (auto& thread : threads) { - thread.join(); - } - - for (int i = 0; i < 50; ++i) { - EXPECT_EQ(i, results[i]); - } -} - -TEST(WorkQueue, MPMC) { - WorkQueue queue; - std::vector results(100, -1); - std::mutex mutex; - std::vector popperThreads; - for (int i = 0; i < 4; ++i) { - popperThreads.emplace_back(Popper{&queue, results.data(), &mutex}); - } - - std::vector pusherThreads; - for (int i = 0; i < 2; ++i) { - auto min = i * 50; - auto max = (i + 1) * 50; - pusherThreads.emplace_back( - [ &queue, min, max ] { - for (int i = min; i < max; ++i) { - queue.push(int{i}); - } - }); - } - - for (auto& thread : pusherThreads) { - thread.join(); - } - queue.finish(); - - for (auto& thread : popperThreads) { - thread.join(); - } - - for (int i = 0; i < 100; ++i) { - EXPECT_EQ(i, results[i]); - } -} - -TEST(WorkQueue, BoundedSizeWorks) { - WorkQueue queue(1); - int result; - queue.push(5); - queue.pop(result); - queue.push(5); - queue.pop(result); - queue.push(5); - queue.finish(); - queue.pop(result); - EXPECT_EQ(5, result); -} - -TEST(WorkQueue, BoundedSizePushAfterFinish) { - WorkQueue queue(1); - int result; - queue.push(5); - std::thread pusher([&queue] { - queue.push(6); - }); - // Dirtily try and make sure that pusher has run. - std::this_thread::sleep_for(std::chrono::seconds(1)); - queue.finish(); - EXPECT_TRUE(queue.pop(result)); - EXPECT_EQ(5, result); - EXPECT_FALSE(queue.pop(result)); - - pusher.join(); -} - -TEST(WorkQueue, SetMaxSize) { - WorkQueue queue(2); - int result; - queue.push(5); - queue.push(6); - queue.setMaxSize(1); - std::thread pusher([&queue] { - queue.push(7); - }); - // Dirtily try and make sure that pusher has run. - std::this_thread::sleep_for(std::chrono::seconds(1)); - queue.finish(); - EXPECT_TRUE(queue.pop(result)); - EXPECT_EQ(5, result); - EXPECT_TRUE(queue.pop(result)); - EXPECT_EQ(6, result); - EXPECT_FALSE(queue.pop(result)); - - pusher.join(); -} - -TEST(WorkQueue, BoundedSizeMPMC) { - WorkQueue queue(10); - std::vector results(200, -1); - std::mutex mutex; - std::cerr << "Creating popperThreads" << std::endl; - std::vector popperThreads; - for (int i = 0; i < 4; ++i) { - popperThreads.emplace_back(Popper{&queue, results.data(), &mutex}); - } - - std::cerr << "Creating pusherThreads" << std::endl; - std::vector pusherThreads; - for (int i = 0; i < 2; ++i) { - auto min = i * 100; - auto max = (i + 1) * 100; - pusherThreads.emplace_back( - [ &queue, min, max ] { - for (int i = min; i < max; ++i) { - queue.push(int{i}); - } - }); - } - - std::cerr << "Joining pusherThreads" << std::endl; - for (auto& thread : pusherThreads) { - thread.join(); - } - std::cerr << "Finishing queue" << std::endl; - queue.finish(); - - std::cerr << "Joining popperThreads" << std::endl; - for (auto& thread : popperThreads) { - thread.join(); - } - - std::cerr << "Inspecting results" << std::endl; - for (int i = 0; i < 200; ++i) { - EXPECT_EQ(i, results[i]); - } -} - -TEST(WorkQueue, FailedPush) { - WorkQueue> queue; - std::unique_ptr x(new int{5}); - EXPECT_TRUE(queue.push(std::move(x))); - EXPECT_EQ(nullptr, x); - queue.finish(); - x.reset(new int{6}); - EXPECT_FALSE(queue.push(std::move(x))); - EXPECT_NE(nullptr, x); - EXPECT_EQ(6, *x); -} - -TEST(BufferWorkQueue, SizeCalculatedCorrectly) { - { - BufferWorkQueue queue; - queue.finish(); - EXPECT_EQ(0, queue.size()); - } - { - BufferWorkQueue queue; - queue.push(Buffer(10)); - queue.finish(); - EXPECT_EQ(10, queue.size()); - } - { - BufferWorkQueue queue; - queue.push(Buffer(10)); - queue.push(Buffer(5)); - queue.finish(); - EXPECT_EQ(15, queue.size()); - } - { - BufferWorkQueue queue; - queue.push(Buffer(10)); - queue.push(Buffer(5)); - queue.finish(); - Buffer buffer; - queue.pop(buffer); - EXPECT_EQ(5, queue.size()); - } -} diff --git a/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore b/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore deleted file mode 100644 index df2f9ab0749..00000000000 --- a/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -seekable_compression -seekable_decompression -parallel_processing -parallel_compression diff --git a/sys/contrib/zstd/contrib/seekable_format/examples/Makefile b/sys/contrib/zstd/contrib/seekable_format/examples/Makefile deleted file mode 100644 index 1847aa7e7b3..00000000000 --- a/sys/contrib/zstd/contrib/seekable_format/examples/Makefile +++ /dev/null @@ -1,42 +0,0 @@ -# ################################################################ -# Copyright (c) 2017-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# ################################################################ - -# This Makefile presumes libzstd is built, using `make` in / or /lib/ - -LDFLAGS += ../../../lib/libzstd.a -CPPFLAGS += -I../ -I../../../lib -I../../../lib/common - -CFLAGS ?= -O3 -CFLAGS += -g - -SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c - -.PHONY: default all clean test - -default: all - -all: seekable_compression seekable_decompression parallel_processing - -seekable_compression : seekable_compression.c $(SEEKABLE_OBJS) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ - -seekable_decompression : seekable_decompression.c $(SEEKABLE_OBJS) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ - -parallel_processing : parallel_processing.c $(SEEKABLE_OBJS) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread - -parallel_compression : parallel_compression.c $(SEEKABLE_OBJS) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread - -clean: - @rm -f core *.o tmp* result* *.zst \ - seekable_compression seekable_decompression \ - parallel_processing parallel_compression - @echo Cleaning completed diff --git a/sys/contrib/zstd/contrib/seekable_format/examples/parallel_compression.c b/sys/contrib/zstd/contrib/seekable_format/examples/parallel_compression.c deleted file mode 100644 index 69644d2b3c8..00000000000 --- a/sys/contrib/zstd/contrib/seekable_format/examples/parallel_compression.c +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright (c) 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -#include // malloc, free, exit, atoi -#include // fprintf, perror, feof, fopen, etc. -#include // strlen, memset, strcat -#define ZSTD_STATIC_LINKING_ONLY -#include // presumes zstd library is installed -#include -#if defined(WIN32) || defined(_WIN32) -# include -# define SLEEP(x) Sleep(x) -#else -# include -# define SLEEP(x) usleep(x * 1000) -#endif - -#define XXH_NAMESPACE ZSTD_ -#include "xxhash.h" - -#include "pool.h" // use zstd thread pool for demo - -#include "zstd_seekable.h" - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror("malloc:"); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(3); -} - -static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) -{ - size_t const readSize = fread(buffer, 1, sizeToRead, file); - if (readSize == sizeToRead) return readSize; /* good */ - if (feof(file)) return readSize; /* good, reached end of file */ - /* error */ - perror("fread"); - exit(4); -} - -static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) -{ - size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); - if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ - /* error */ - perror("fwrite"); - exit(5); -} - -static size_t fclose_orDie(FILE* file) -{ - if (!fclose(file)) return 0; - /* error */ - perror("fclose"); - exit(6); -} - -static void fseek_orDie(FILE* file, long int offset, int origin) -{ - if (!fseek(file, offset, origin)) { - if (!fflush(file)) return; - } - /* error */ - perror("fseek"); - exit(7); -} - -static long int ftell_orDie(FILE* file) -{ - long int off = ftell(file); - if (off != -1) return off; - /* error */ - perror("ftell"); - exit(8); -} - -struct job { - const void* src; - size_t srcSize; - void* dst; - size_t dstSize; - - unsigned checksum; - - int compressionLevel; - int done; -}; - -static void compressFrame(void* opaque) -{ - struct job* job = opaque; - - job->checksum = XXH64(job->src, job->srcSize, 0); - - size_t ret = ZSTD_compress(job->dst, job->dstSize, job->src, job->srcSize, job->compressionLevel); - if (ZSTD_isError(ret)) { - fprintf(stderr, "ZSTD_compress() error : %s \n", ZSTD_getErrorName(ret)); - exit(20); - } - - job->dstSize = ret; - job->done = 1; -} - -static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned frameSize, int nbThreads) -{ - POOL_ctx* pool = POOL_create(nbThreads, nbThreads); - if (pool == NULL) { fprintf(stderr, "POOL_create() error \n"); exit(9); } - - FILE* const fin = fopen_orDie(fname, "rb"); - FILE* const fout = fopen_orDie(outName, "wb"); - - if (ZSTD_compressBound(frameSize) > 0xFFFFFFFFU) { fprintf(stderr, "Frame size too large \n"); exit(10); } - unsigned dstSize = ZSTD_compressBound(frameSize); - - - fseek_orDie(fin, 0, SEEK_END); - long int length = ftell_orDie(fin); - fseek_orDie(fin, 0, SEEK_SET); - - size_t numFrames = (length + frameSize - 1) / frameSize; - - struct job* jobs = malloc_orDie(sizeof(struct job) * numFrames); - - size_t i; - for(i = 0; i < numFrames; i++) { - void* in = malloc_orDie(frameSize); - void* out = malloc_orDie(dstSize); - - size_t inSize = fread_orDie(in, frameSize, fin); - - jobs[i].src = in; - jobs[i].srcSize = inSize; - jobs[i].dst = out; - jobs[i].dstSize = dstSize; - jobs[i].compressionLevel = cLevel; - jobs[i].done = 0; - POOL_add(pool, compressFrame, &jobs[i]); - } - - ZSTD_frameLog* fl = ZSTD_seekable_createFrameLog(1); - if (fl == NULL) { fprintf(stderr, "ZSTD_seekable_createFrameLog() failed \n"); exit(11); } - for (i = 0; i < numFrames; i++) { - while (!jobs[i].done) SLEEP(5); /* wake up every 5 milliseconds to check */ - fwrite_orDie(jobs[i].dst, jobs[i].dstSize, fout); - free((void*)jobs[i].src); - free(jobs[i].dst); - - size_t ret = ZSTD_seekable_logFrame(fl, jobs[i].dstSize, jobs[i].srcSize, jobs[i].checksum); - if (ZSTD_isError(ret)) { fprintf(stderr, "ZSTD_seekable_logFrame() error : %s \n", ZSTD_getErrorName(ret)); } - } - - { unsigned char seekTableBuff[1024]; - ZSTD_outBuffer out = {seekTableBuff, 1024, 0}; - while (ZSTD_seekable_writeSeekTable(fl, &out) != 0) { - fwrite_orDie(seekTableBuff, out.pos, fout); - out.pos = 0; - } - fwrite_orDie(seekTableBuff, out.pos, fout); - } - - ZSTD_seekable_freeFrameLog(fl); - free(jobs); - fclose_orDie(fout); - fclose_orDie(fin); -} - -static const char* createOutFilename_orDie(const char* filename) -{ - size_t const inL = strlen(filename); - size_t const outL = inL + 5; - void* outSpace = malloc_orDie(outL); - memset(outSpace, 0, outL); - strcat(outSpace, filename); - strcat(outSpace, ".zst"); - return (const char*)outSpace; -} - -int main(int argc, const char** argv) { - const char* const exeName = argv[0]; - if (argc!=4) { - printf("wrong arguments\n"); - printf("usage:\n"); - printf("%s FILE FRAME_SIZE NB_THREADS\n", exeName); - return 1; - } - - { const char* const inFileName = argv[1]; - unsigned const frameSize = (unsigned)atoi(argv[2]); - int const nbThreads = atoi(argv[3]); - - const char* const outFileName = createOutFilename_orDie(inFileName); - compressFile_orDie(inFileName, outFileName, 5, frameSize, nbThreads); - } - - return 0; -} diff --git a/sys/contrib/zstd/contrib/seekable_format/examples/parallel_processing.c b/sys/contrib/zstd/contrib/seekable_format/examples/parallel_processing.c deleted file mode 100644 index da3477632e2..00000000000 --- a/sys/contrib/zstd/contrib/seekable_format/examples/parallel_processing.c +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/* - * A simple demo that sums up all the bytes in the file in parallel using - * seekable decompression and the zstd thread pool - */ - -#include // malloc, exit -#include // fprintf, perror, feof -#include // strerror -#include // errno -#define ZSTD_STATIC_LINKING_ONLY -#include // presumes zstd library is installed -#include -#if defined(WIN32) || defined(_WIN32) -# include -# define SLEEP(x) Sleep(x) -#else -# include -# define SLEEP(x) usleep(x * 1000) -#endif - -#include "pool.h" // use zstd thread pool for demo - -#include "zstd_seekable.h" - -#define MIN(a, b) ((a) < (b) ? (a) : (b)) - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror("malloc"); - exit(1); -} - -static void* realloc_orDie(void* ptr, size_t size) -{ - ptr = realloc(ptr, size); - if (ptr) return ptr; - /* error */ - perror("realloc"); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(3); -} - -static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) -{ - size_t const readSize = fread(buffer, 1, sizeToRead, file); - if (readSize == sizeToRead) return readSize; /* good */ - if (feof(file)) return readSize; /* good, reached end of file */ - /* error */ - perror("fread"); - exit(4); -} - -static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) -{ - size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); - if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ - /* error */ - perror("fwrite"); - exit(5); -} - -static size_t fclose_orDie(FILE* file) -{ - if (!fclose(file)) return 0; - /* error */ - perror("fclose"); - exit(6); -} - -static void fseek_orDie(FILE* file, long int offset, int origin) { - if (!fseek(file, offset, origin)) { - if (!fflush(file)) return; - } - /* error */ - perror("fseek"); - exit(7); -} - -struct sum_job { - const char* fname; - unsigned long long sum; - unsigned frameNb; - int done; -}; - -static void sumFrame(void* opaque) -{ - struct sum_job* job = (struct sum_job*)opaque; - job->done = 0; - - FILE* const fin = fopen_orDie(job->fname, "rb"); - - ZSTD_seekable* const seekable = ZSTD_seekable_create(); - if (seekable==NULL) { fprintf(stderr, "ZSTD_seekable_create() error \n"); exit(10); } - - size_t const initResult = ZSTD_seekable_initFile(seekable, fin); - if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } - - size_t const frameSize = ZSTD_seekable_getFrameDecompressedSize(seekable, job->frameNb); - unsigned char* data = malloc_orDie(frameSize); - - size_t result = ZSTD_seekable_decompressFrame(seekable, data, frameSize, job->frameNb); - if (ZSTD_isError(result)) { fprintf(stderr, "ZSTD_seekable_decompressFrame() error : %s \n", ZSTD_getErrorName(result)); exit(12); } - - unsigned long long sum = 0; - size_t i; - for (i = 0; i < frameSize; i++) { - sum += data[i]; - } - job->sum = sum; - job->done = 1; - - fclose(fin); - ZSTD_seekable_free(seekable); - free(data); -} - -static void sumFile_orDie(const char* fname, int nbThreads) -{ - POOL_ctx* pool = POOL_create(nbThreads, nbThreads); - if (pool == NULL) { fprintf(stderr, "POOL_create() error \n"); exit(9); } - - FILE* const fin = fopen_orDie(fname, "rb"); - - ZSTD_seekable* const seekable = ZSTD_seekable_create(); - if (seekable==NULL) { fprintf(stderr, "ZSTD_seekable_create() error \n"); exit(10); } - - size_t const initResult = ZSTD_seekable_initFile(seekable, fin); - if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } - - size_t const numFrames = ZSTD_seekable_getNumFrames(seekable); - struct sum_job* jobs = (struct sum_job*)malloc(numFrames * sizeof(struct sum_job)); - - size_t i; - for (i = 0; i < numFrames; i++) { - jobs[i] = (struct sum_job){ fname, 0, i, 0 }; - POOL_add(pool, sumFrame, &jobs[i]); - } - - unsigned long long total = 0; - - for (i = 0; i < numFrames; i++) { - while (!jobs[i].done) SLEEP(5); /* wake up every 5 milliseconds to check */ - total += jobs[i].sum; - } - - printf("Sum: %llu\n", total); - - POOL_free(pool); - ZSTD_seekable_free(seekable); - fclose(fin); - free(jobs); -} - - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - - if (argc!=3) { - fprintf(stderr, "wrong arguments\n"); - fprintf(stderr, "usage:\n"); - fprintf(stderr, "%s FILE NB_THREADS\n", exeName); - return 1; - } - - { - const char* const inFilename = argv[1]; - int const nbThreads = atoi(argv[2]); - sumFile_orDie(inFilename, nbThreads); - } - - return 0; -} diff --git a/sys/contrib/zstd/contrib/seekable_format/examples/seekable_compression.c b/sys/contrib/zstd/contrib/seekable_format/examples/seekable_compression.c deleted file mode 100644 index 9485bf26fc4..00000000000 --- a/sys/contrib/zstd/contrib/seekable_format/examples/seekable_compression.c +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (c) 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -#include // malloc, free, exit, atoi -#include // fprintf, perror, feof, fopen, etc. -#include // strlen, memset, strcat -#define ZSTD_STATIC_LINKING_ONLY -#include // presumes zstd library is installed - -#include "zstd_seekable.h" - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror("malloc:"); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(3); -} - -static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) -{ - size_t const readSize = fread(buffer, 1, sizeToRead, file); - if (readSize == sizeToRead) return readSize; /* good */ - if (feof(file)) return readSize; /* good, reached end of file */ - /* error */ - perror("fread"); - exit(4); -} - -static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) -{ - size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); - if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ - /* error */ - perror("fwrite"); - exit(5); -} - -static size_t fclose_orDie(FILE* file) -{ - if (!fclose(file)) return 0; - /* error */ - perror("fclose"); - exit(6); -} - -static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned frameSize) -{ - FILE* const fin = fopen_orDie(fname, "rb"); - FILE* const fout = fopen_orDie(outName, "wb"); - size_t const buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */ - void* const buffIn = malloc_orDie(buffInSize); - size_t const buffOutSize = ZSTD_CStreamOutSize(); /* can always flush a full block */ - void* const buffOut = malloc_orDie(buffOutSize); - - ZSTD_seekable_CStream* const cstream = ZSTD_seekable_createCStream(); - if (cstream==NULL) { fprintf(stderr, "ZSTD_seekable_createCStream() error \n"); exit(10); } - size_t const initResult = ZSTD_seekable_initCStream(cstream, cLevel, 1, frameSize); - if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } - - size_t read, toRead = buffInSize; - while( (read = fread_orDie(buffIn, toRead, fin)) ) { - ZSTD_inBuffer input = { buffIn, read, 0 }; - while (input.pos < input.size) { - ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; - toRead = ZSTD_seekable_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ - if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_seekable_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } - if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/ - fwrite_orDie(buffOut, output.pos, fout); - } - } - - while (1) { - ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; - size_t const remainingToFlush = ZSTD_seekable_endStream(cstream, &output); /* close stream */ - if (ZSTD_isError(remainingToFlush)) { fprintf(stderr, "ZSTD_seekable_endStream() error : %s \n", ZSTD_getErrorName(remainingToFlush)); exit(13); } - fwrite_orDie(buffOut, output.pos, fout); - if (!remainingToFlush) break; - } - - ZSTD_seekable_freeCStream(cstream); - fclose_orDie(fout); - fclose_orDie(fin); - free(buffIn); - free(buffOut); -} - -static const char* createOutFilename_orDie(const char* filename) -{ - size_t const inL = strlen(filename); - size_t const outL = inL + 5; - void* outSpace = malloc_orDie(outL); - memset(outSpace, 0, outL); - strcat(outSpace, filename); - strcat(outSpace, ".zst"); - return (const char*)outSpace; -} - -int main(int argc, const char** argv) { - const char* const exeName = argv[0]; - if (argc!=3) { - printf("wrong arguments\n"); - printf("usage:\n"); - printf("%s FILE FRAME_SIZE\n", exeName); - return 1; - } - - { const char* const inFileName = argv[1]; - unsigned const frameSize = (unsigned)atoi(argv[2]); - - const char* const outFileName = createOutFilename_orDie(inFileName); - compressFile_orDie(inFileName, outFileName, 5, frameSize); - } - - return 0; -} diff --git a/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression.c b/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression.c deleted file mode 100644 index 9cd23292263..00000000000 --- a/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression.c +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - - -#include // malloc, exit -#include // fprintf, perror, feof -#include // strerror -#include // errno -#define ZSTD_STATIC_LINKING_ONLY -#include // presumes zstd library is installed -#include - -#include "zstd_seekable.h" - -#define MIN(a, b) ((a) < (b) ? (a) : (b)) - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror("malloc"); - exit(1); -} - -static void* realloc_orDie(void* ptr, size_t size) -{ - ptr = realloc(ptr, size); - if (ptr) return ptr; - /* error */ - perror("realloc"); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(3); -} - -static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) -{ - size_t const readSize = fread(buffer, 1, sizeToRead, file); - if (readSize == sizeToRead) return readSize; /* good */ - if (feof(file)) return readSize; /* good, reached end of file */ - /* error */ - perror("fread"); - exit(4); -} - -static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) -{ - size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); - if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ - /* error */ - perror("fwrite"); - exit(5); -} - -static size_t fclose_orDie(FILE* file) -{ - if (!fclose(file)) return 0; - /* error */ - perror("fclose"); - exit(6); -} - -static void fseek_orDie(FILE* file, long int offset, int origin) { - if (!fseek(file, offset, origin)) { - if (!fflush(file)) return; - } - /* error */ - perror("fseek"); - exit(7); -} - - -static void decompressFile_orDie(const char* fname, unsigned startOffset, unsigned endOffset) -{ - FILE* const fin = fopen_orDie(fname, "rb"); - FILE* const fout = stdout; - size_t const buffOutSize = ZSTD_DStreamOutSize(); /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */ - void* const buffOut = malloc_orDie(buffOutSize); - - ZSTD_seekable* const seekable = ZSTD_seekable_create(); - if (seekable==NULL) { fprintf(stderr, "ZSTD_seekable_create() error \n"); exit(10); } - - size_t const initResult = ZSTD_seekable_initFile(seekable, fin); - if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } - - while (startOffset < endOffset) { - size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset); - - if (ZSTD_isError(result)) { - fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n", - ZSTD_getErrorName(result)); - exit(12); - } - fwrite_orDie(buffOut, result, fout); - startOffset += result; - } - - ZSTD_seekable_free(seekable); - fclose_orDie(fin); - fclose_orDie(fout); - free(buffOut); -} - - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - - if (argc!=4) { - fprintf(stderr, "wrong arguments\n"); - fprintf(stderr, "usage:\n"); - fprintf(stderr, "%s FILE START END\n", exeName); - return 1; - } - - { - const char* const inFilename = argv[1]; - unsigned const startOffset = (unsigned) atoi(argv[2]); - unsigned const endOffset = (unsigned) atoi(argv[3]); - decompressFile_orDie(inFilename, startOffset, endOffset); - } - - return 0; -} diff --git a/sys/contrib/zstd/contrib/seekable_format/zstd_seekable.h b/sys/contrib/zstd/contrib/seekable_format/zstd_seekable.h deleted file mode 100644 index 438ac20149f..00000000000 --- a/sys/contrib/zstd/contrib/seekable_format/zstd_seekable.h +++ /dev/null @@ -1,184 +0,0 @@ -#ifndef SEEKABLE_H -#define SEEKABLE_H - -#if defined (__cplusplus) -extern "C" { -#endif - -#include - -static const unsigned ZSTD_seekTableFooterSize = 9; - -#define ZSTD_SEEKABLE_MAGICNUMBER 0x8F92EAB1 - -#define ZSTD_SEEKABLE_MAXFRAMES 0x8000000U - -/* Limit the maximum size to avoid any potential issues storing the compressed size */ -#define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0x80000000U - -/*-**************************************************************************** -* Seekable Format -* -* The seekable format splits the compressed data into a series of "frames", -* each compressed individually so that decompression of a section in the -* middle of an archive only requires zstd to decompress at most a frame's -* worth of extra data, instead of the entire archive. -******************************************************************************/ - -typedef struct ZSTD_seekable_CStream_s ZSTD_seekable_CStream; -typedef struct ZSTD_seekable_s ZSTD_seekable; - -/*-**************************************************************************** -* Seekable compression - HowTo -* A ZSTD_seekable_CStream object is required to tracking streaming operation. -* Use ZSTD_seekable_createCStream() and ZSTD_seekable_freeCStream() to create/ -* release resources. -* -* Streaming objects are reusable to avoid allocation and deallocation, -* to start a new compression operation call ZSTD_seekable_initCStream() on the -* compressor. -* -* Data streamed to the seekable compressor will automatically be split into -* frames of size `maxFrameSize` (provided in ZSTD_seekable_initCStream()), -* or if none is provided, will be cut off whenever ZSTD_seekable_endFrame() is -* called or when the default maximum frame size (2GB) is reached. -* -* Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object -* for a new compression operation. -* `maxFrameSize` indicates the size at which to automatically start a new -* seekable frame. `maxFrameSize == 0` implies the default maximum size. -* `checksumFlag` indicates whether or not the seek table should include frame -* checksums on the uncompressed data for verification. -* @return : a size hint for input to provide for compression, or an error code -* checkable with ZSTD_isError() -* -* Use ZSTD_seekable_compressStream() repetitively to consume input stream. -* The function will automatically update both `pos` fields. -* Note that it may not consume the entire input, in which case `pos < size`, -* and it's up to the caller to present again remaining data. -* @return : a size hint, preferred nb of bytes to use as input for next -* function call or an error code, which can be tested using -* ZSTD_isError(). -* Note 1 : it's just a hint, to help latency a little, any other -* value will work fine. -* -* At any time, call ZSTD_seekable_endFrame() to end the current frame and -* start a new one. -* -* ZSTD_seekable_endStream() will end the current frame, and then write the seek -* table so that decompressors can efficiently find compressed frames. -* ZSTD_seekable_endStream() may return a number > 0 if it was unable to flush -* all the necessary data to `output`. In this case, it should be called again -* until all remaining data is flushed out and 0 is returned. -******************************************************************************/ - -/*===== Seekable compressor management =====*/ -ZSTDLIB_API ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void); -ZSTDLIB_API size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs); - -/*===== Seekable compression functions =====*/ -ZSTDLIB_API size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, unsigned maxFrameSize); -ZSTDLIB_API size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -ZSTDLIB_API size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output); -ZSTDLIB_API size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output); - -/*= Raw seek table API - * These functions allow for the seek table to be constructed directly. - * This table can then be appended to a file of concatenated frames. - * This allows the frames to be compressed independently, even in parallel, - * and compiled together afterward into a seekable archive. - * - * Use ZSTD_seekable_createFrameLog() to allocate and initialize a tracking - * structure. - * - * Call ZSTD_seekable_logFrame() once for each frame in the archive. - * checksum is optional, and will not be used if checksumFlag was 0 when the - * frame log was created. If present, it should be the least significant 32 - * bits of the XXH64 hash of the uncompressed data. - * - * Call ZSTD_seekable_writeSeekTable to serialize the data into a seek table. - * If the entire table was written, the return value will be 0. Otherwise, - * it will be equal to the number of bytes left to write. */ -typedef struct ZSTD_frameLog_s ZSTD_frameLog; -ZSTDLIB_API ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag); -ZSTDLIB_API size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl); -ZSTDLIB_API size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl, unsigned compressedSize, unsigned decompressedSize, unsigned checksum); -ZSTDLIB_API size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output); - -/*-**************************************************************************** -* Seekable decompression - HowTo -* A ZSTD_seekable object is required to tracking the seekTable. -* -* Call ZSTD_seekable_init* to initialize a ZSTD_seekable object with the -* the seek table provided in the input. -* There are three modes for ZSTD_seekable_init: -* - ZSTD_seekable_initBuff() : An in-memory API. The data contained in -* `src` should be the entire seekable file, including the seek table. -* `src` should be kept alive and unmodified until the ZSTD_seekable object -* is freed or reset. -* - ZSTD_seekable_initFile() : A simplified file API using stdio. fread and -* fseek will be used to access the required data for building the seek -* table and doing decompression operations. `src` should not be closed -* or modified until the ZSTD_seekable object is freed or reset. -* - ZSTD_seekable_initAdvanced() : A general API allowing the client to -* provide its own read and seek callbacks. -* + ZSTD_seekable_read() : read exactly `n` bytes into `buffer`. -* Premature EOF should be treated as an error. -* + ZSTD_seekable_seek() : seek the read head to `offset` from `origin`, -* where origin is either SEEK_SET (beginning of -* file), or SEEK_END (end of file). -* Both functions should return a non-negative value in case of success, and a -* negative value in case of failure. If implementing using this API and -* stdio, be careful with files larger than 4GB and fseek. All of these -* functions return an error code checkable with ZSTD_isError(). -* -* Call ZSTD_seekable_decompress to decompress `dstSize` bytes at decompressed -* offset `offset`. ZSTD_seekable_decompress may have to decompress the entire -* prefix of the frame before the desired data if it has not already processed -* this section. If ZSTD_seekable_decompress is called multiple times for a -* consecutive range of data, it will efficiently retain the decompressor object -* and avoid redecompressing frame prefixes. The return value is the number of -* bytes decompressed, or an error code checkable with ZSTD_isError(). -* -* The seek table access functions can be used to obtain the data contained -* in the seek table. If frameIndex is larger than the value returned by -* ZSTD_seekable_getNumFrames(), they will return error codes checkable with -* ZSTD_isError(). Note that since the offset access functions return -* unsigned long long instead of size_t, in this case they will instead return -* the value ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE. -******************************************************************************/ - -/*===== Seekable decompressor management =====*/ -ZSTDLIB_API ZSTD_seekable* ZSTD_seekable_create(void); -ZSTDLIB_API size_t ZSTD_seekable_free(ZSTD_seekable* zs); - -/*===== Seekable decompression functions =====*/ -ZSTDLIB_API size_t ZSTD_seekable_initBuff(ZSTD_seekable* zs, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_seekable_initFile(ZSTD_seekable* zs, FILE* src); -ZSTDLIB_API size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned long long offset); -ZSTDLIB_API size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned frameIndex); - -#define ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE (0ULL-2) -/*===== Seek Table access functions =====*/ -ZSTDLIB_API unsigned ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs); -ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex); -ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex); -ZSTDLIB_API size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, unsigned frameIndex); -ZSTDLIB_API size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, unsigned frameIndex); -ZSTDLIB_API unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long offset); - -/*===== Seekable advanced I/O API =====*/ -typedef int(ZSTD_seekable_read)(void* opaque, void* buffer, size_t n); -typedef int(ZSTD_seekable_seek)(void* opaque, long long offset, int origin); -typedef struct { - void* opaque; - ZSTD_seekable_read* read; - ZSTD_seekable_seek* seek; -} ZSTD_seekable_customFile; -ZSTDLIB_API size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile src); - -#if defined (__cplusplus) -} -#endif - -#endif diff --git a/sys/contrib/zstd/contrib/seekable_format/zstd_seekable_compression_format.md b/sys/contrib/zstd/contrib/seekable_format/zstd_seekable_compression_format.md deleted file mode 100644 index bf3080f7bbe..00000000000 --- a/sys/contrib/zstd/contrib/seekable_format/zstd_seekable_compression_format.md +++ /dev/null @@ -1,116 +0,0 @@ -# Zstandard Seekable Format - -### Notices - -Copyright (c) 2017-present Facebook, Inc. - -Permission is granted to copy and distribute this document -for any purpose and without charge, -including translations into other languages -and incorporation into compilations, -provided that the copyright notice and this notice are preserved, -and that any substantive changes or deletions from the original -are clearly marked. -Distribution of this document is unlimited. - -### Version -0.1.0 (11/04/17) - -## Introduction -This document defines a format for compressed data to be stored so that subranges of the data can be efficiently decompressed without requiring the entire document to be decompressed. -This is done by splitting up the input data into frames, -each of which are compressed independently, -and so can be decompressed independently. -Decompression then takes advantage of a provided 'seek table', which allows the decompressor to immediately jump to the desired data. This is done in a way that is compatible with the original Zstandard format by placing the seek table in a Zstandard skippable frame. - -### Overall conventions -In this document: -- square brackets i.e. `[` and `]` are used to indicate optional fields or parameters. -- the naming convention for identifiers is `Mixed_Case_With_Underscores` -- All numeric fields are little-endian unless specified otherwise - -## Format - -The format consists of a number of frames (Zstandard compressed frames and skippable frames), followed by a final skippable frame at the end containing the seek table. - -### Seek Table Format -The structure of the seek table frame is as follows: - -|`Skippable_Magic_Number`|`Frame_Size`|`[Seek_Table_Entries]`|`Seek_Table_Footer`| -|------------------------|------------|----------------------|-------------------| -| 4 bytes | 4 bytes | 8-12 bytes each | 9 bytes | - -__`Skippable_Magic_Number`__ - -Value : 0x184D2A5E. -This is for compatibility with [Zstandard skippable frames]. -Since it is legal for other Zstandard skippable frames to use the same -magic number, it is not recommended for a decoder to recognize frames -solely on this. - -__`Frame_Size`__ - -The total size of the skippable frame, not including the `Skippable_Magic_Number` or `Frame_Size`. -This is for compatibility with [Zstandard skippable frames]. - -[Zstandard skippable frames]: https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#skippable-frames - -#### `Seek_Table_Footer` -The seek table footer format is as follows: - -|`Number_Of_Frames`|`Seek_Table_Descriptor`|`Seekable_Magic_Number`| -|------------------|-----------------------|-----------------------| -| 4 bytes | 1 byte | 4 bytes | - -__`Seekable_Magic_Number`__ - -Value : 0x8F92EAB1. -This value must be the last bytes present in the compressed file so that decoders -can efficiently find it and determine if there is an actual seek table present. - -__`Number_Of_Frames`__ - -The number of stored frames in the data. - -__`Seek_Table_Descriptor`__ - -A bitfield describing the format of the seek table. - -| Bit number | Field name | -| ---------- | ---------- | -| 7 | `Checksum_Flag` | -| 6-2 | `Reserved_Bits` | -| 1-0 | `Unused_Bits` | - -While only `Checksum_Flag` currently exists, there are 7 other bits in this field that can be used for future changes to the format, -for example the addition of inline dictionaries. - -__`Checksum_Flag`__ - -If the checksum flag is set, each of the seek table entries contains a 4 byte checksum of the uncompressed data contained in its frame. - -`Reserved_Bits` are not currently used but may be used in the future for breaking changes, so a compliant decoder should ensure they are set to 0. `Unused_Bits` may be used in the future for non-breaking changes, so a compliant decoder should not interpret these bits. - -#### __`Seek_Table_Entries`__ - -`Seek_Table_Entries` consists of `Number_Of_Frames` (one for each frame in the data, not including the seek table frame) entries of the following form, in sequence: - -|`Compressed_Size`|`Decompressed_Size`|`[Checksum]`| -|-----------------|-------------------|------------| -| 4 bytes | 4 bytes | 4 bytes | - -__`Compressed_Size`__ - -The compressed size of the frame. -The cumulative sum of the `Compressed_Size` fields of frames `0` to `i` gives the offset in the compressed file of frame `i+1`. - -__`Decompressed_Size`__ - -The size of the decompressed data contained in the frame. For skippable or otherwise empty frames, this value is 0. - -__`Checksum`__ - -Only present if `Checksum_Flag` is set in the `Seek_Table_Descriptor`. Value : the least significant 32 bits of the XXH64 digest of the uncompressed data, stored in little-endian format. - -## Version Changes -- 0.1.0: initial version diff --git a/sys/contrib/zstd/contrib/seekable_format/zstdseek_compress.c b/sys/contrib/zstd/contrib/seekable_format/zstdseek_compress.c deleted file mode 100644 index 4ab666bdbba..00000000000 --- a/sys/contrib/zstd/contrib/seekable_format/zstdseek_compress.c +++ /dev/null @@ -1,366 +0,0 @@ -/* - * Copyright (c) 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -#include /* malloc, free */ - -#define XXH_STATIC_LINKING_ONLY -#define XXH_NAMESPACE ZSTD_ -#include "xxhash.h" - -#define ZSTD_STATIC_LINKING_ONLY -#include "zstd.h" -#include "zstd_errors.h" -#include "mem.h" -#include "zstd_seekable.h" - -#define CHECK_Z(f) { size_t const ret = (f); if (ret != 0) return ret; } - -#undef ERROR -#define ERROR(name) ((size_t)-ZSTD_error_##name) - -#undef MIN -#undef MAX -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define MAX(a, b) ((a) > (b) ? (a) : (b)) - -typedef struct { - U32 cSize; - U32 dSize; - U32 checksum; -} framelogEntry_t; - -struct ZSTD_frameLog_s { - framelogEntry_t* entries; - U32 size; - U32 capacity; - - int checksumFlag; - - /* for use when streaming out the seek table */ - U32 seekTablePos; - U32 seekTableIndex; -} framelog_t; - -struct ZSTD_seekable_CStream_s { - ZSTD_CStream* cstream; - ZSTD_frameLog framelog; - - U32 frameCSize; - U32 frameDSize; - - XXH64_state_t xxhState; - - U32 maxFrameSize; - - int writingSeekTable; -}; - -size_t ZSTD_seekable_frameLog_allocVec(ZSTD_frameLog* fl) -{ - /* allocate some initial space */ - size_t const FRAMELOG_STARTING_CAPACITY = 16; - fl->entries = (framelogEntry_t*)malloc( - sizeof(framelogEntry_t) * FRAMELOG_STARTING_CAPACITY); - if (fl->entries == NULL) return ERROR(memory_allocation); - fl->capacity = FRAMELOG_STARTING_CAPACITY; - - return 0; -} - -size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl) -{ - if (fl != NULL) free(fl->entries); - return 0; -} - -ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag) -{ - ZSTD_frameLog* fl = malloc(sizeof(ZSTD_frameLog)); - if (fl == NULL) return NULL; - - if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) { - free(fl); - return NULL; - } - - fl->checksumFlag = checksumFlag; - fl->seekTablePos = 0; - fl->seekTableIndex = 0; - fl->size = 0; - - return fl; -} - -size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl) -{ - ZSTD_seekable_frameLog_freeVec(fl); - free(fl); - return 0; -} - -ZSTD_seekable_CStream* ZSTD_seekable_createCStream() -{ - ZSTD_seekable_CStream* zcs = malloc(sizeof(ZSTD_seekable_CStream)); - - if (zcs == NULL) return NULL; - - memset(zcs, 0, sizeof(*zcs)); - - zcs->cstream = ZSTD_createCStream(); - if (zcs->cstream == NULL) goto failed1; - - if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(&zcs->framelog))) goto failed2; - - return zcs; - -failed2: - ZSTD_freeCStream(zcs->cstream); -failed1: - free(zcs); - return NULL; -} - -size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs) -{ - if (zcs == NULL) return 0; /* support free on null */ - ZSTD_freeCStream(zcs->cstream); - ZSTD_seekable_frameLog_freeVec(&zcs->framelog); - free(zcs); - - return 0; -} - -size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, - int compressionLevel, - int checksumFlag, - U32 maxFrameSize) -{ - zcs->framelog.size = 0; - zcs->frameCSize = 0; - zcs->frameDSize = 0; - - /* make sure maxFrameSize has a reasonable value */ - if (maxFrameSize > ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE) { - return ERROR(frameParameter_unsupported); - } - - zcs->maxFrameSize = maxFrameSize - ? maxFrameSize - : ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE; - - zcs->framelog.checksumFlag = checksumFlag; - if (zcs->framelog.checksumFlag) { - XXH64_reset(&zcs->xxhState, 0); - } - - zcs->framelog.seekTablePos = 0; - zcs->framelog.seekTableIndex = 0; - zcs->writingSeekTable = 0; - - return ZSTD_initCStream(zcs->cstream, compressionLevel); -} - -size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl, - unsigned compressedSize, - unsigned decompressedSize, - unsigned checksum) -{ - if (fl->size == ZSTD_SEEKABLE_MAXFRAMES) - return ERROR(frameIndex_tooLarge); - - /* grow the buffer if required */ - if (fl->size == fl->capacity) { - /* exponential size increase for constant amortized runtime */ - size_t const newCapacity = fl->capacity * 2; - framelogEntry_t* const newEntries = realloc(fl->entries, - sizeof(framelogEntry_t) * newCapacity); - - if (newEntries == NULL) return ERROR(memory_allocation); - - fl->entries = newEntries; - fl->capacity = newCapacity; - } - - fl->entries[fl->size] = (framelogEntry_t){ - compressedSize, decompressedSize, checksum - }; - fl->size++; - - return 0; -} - -size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output) -{ - size_t const prevOutPos = output->pos; - /* end the frame */ - size_t ret = ZSTD_endStream(zcs->cstream, output); - - zcs->frameCSize += output->pos - prevOutPos; - - /* need to flush before doing the rest */ - if (ret) return ret; - - /* frame done */ - - /* store the frame data for later */ - ret = ZSTD_seekable_logFrame( - &zcs->framelog, zcs->frameCSize, zcs->frameDSize, - zcs->framelog.checksumFlag - ? XXH64_digest(&zcs->xxhState) & 0xFFFFFFFFU - : 0); - if (ret) return ret; - - /* reset for the next frame */ - zcs->frameCSize = 0; - zcs->frameDSize = 0; - - ZSTD_resetCStream(zcs->cstream, 0); - if (zcs->framelog.checksumFlag) - XXH64_reset(&zcs->xxhState, 0); - - return 0; -} - -size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) -{ - const BYTE* const inBase = (const BYTE*) input->src + input->pos; - size_t inLen = input->size - input->pos; - - inLen = MIN(inLen, (size_t)(zcs->maxFrameSize - zcs->frameDSize)); - - /* if we haven't finished flushing the last frame, don't start writing a new one */ - if (inLen > 0) { - ZSTD_inBuffer inTmp = { inBase, inLen, 0 }; - size_t const prevOutPos = output->pos; - - size_t const ret = ZSTD_compressStream(zcs->cstream, output, &inTmp); - - if (zcs->framelog.checksumFlag) { - XXH64_update(&zcs->xxhState, inBase, inTmp.pos); - } - - zcs->frameCSize += output->pos - prevOutPos; - zcs->frameDSize += inTmp.pos; - - input->pos += inTmp.pos; - - if (ZSTD_isError(ret)) return ret; - } - - if (zcs->maxFrameSize == zcs->frameDSize) { - /* log the frame and start over */ - size_t const ret = ZSTD_seekable_endFrame(zcs, output); - if (ZSTD_isError(ret)) return ret; - - /* get the client ready for the next frame */ - return (size_t)zcs->maxFrameSize; - } - - return (size_t)(zcs->maxFrameSize - zcs->frameDSize); -} - -static inline size_t ZSTD_seekable_seekTableSize(const ZSTD_frameLog* fl) -{ - size_t const sizePerFrame = 8 + (fl->checksumFlag?4:0); - size_t const seekTableLen = ZSTD_skippableHeaderSize + - sizePerFrame * fl->size + - ZSTD_seekTableFooterSize; - - return seekTableLen; -} - -static inline size_t ZSTD_stwrite32(ZSTD_frameLog* fl, - ZSTD_outBuffer* output, U32 const value, - U32 const offset) -{ - if (fl->seekTablePos < offset + 4) { - BYTE tmp[4]; /* so that we can work with buffers too small to write a whole word to */ - size_t const lenWrite = - MIN(output->size - output->pos, offset + 4 - fl->seekTablePos); - MEM_writeLE32(tmp, value); - memcpy((BYTE*)output->dst + output->pos, - tmp + (fl->seekTablePos - offset), lenWrite); - output->pos += lenWrite; - fl->seekTablePos += lenWrite; - - if (lenWrite < 4) return ZSTD_seekable_seekTableSize(fl) - fl->seekTablePos; - } - return 0; -} - -size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output) -{ - /* seekTableIndex: the current index in the table and - * seekTableSize: the amount of the table written so far - * - * This function is written this way so that if it has to return early - * because of a small buffer, it can keep going where it left off. - */ - - size_t const sizePerFrame = 8 + (fl->checksumFlag?4:0); - size_t const seekTableLen = ZSTD_seekable_seekTableSize(fl); - - CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_MAGIC_SKIPPABLE_START | 0xE, 0)); - CHECK_Z(ZSTD_stwrite32(fl, output, seekTableLen - ZSTD_skippableHeaderSize, - 4)); - - while (fl->seekTableIndex < fl->size) { - CHECK_Z(ZSTD_stwrite32(fl, output, - fl->entries[fl->seekTableIndex].cSize, - ZSTD_skippableHeaderSize + - sizePerFrame * fl->seekTableIndex + 0)); - - CHECK_Z(ZSTD_stwrite32(fl, output, - fl->entries[fl->seekTableIndex].dSize, - ZSTD_skippableHeaderSize + - sizePerFrame * fl->seekTableIndex + 4)); - - if (fl->checksumFlag) { - CHECK_Z(ZSTD_stwrite32( - fl, output, fl->entries[fl->seekTableIndex].checksum, - ZSTD_skippableHeaderSize + - sizePerFrame * fl->seekTableIndex + 8)); - } - - fl->seekTableIndex++; - } - - CHECK_Z(ZSTD_stwrite32(fl, output, fl->size, - seekTableLen - ZSTD_seekTableFooterSize)); - - if (output->size - output->pos < 1) return seekTableLen - fl->seekTablePos; - if (fl->seekTablePos < seekTableLen - 4) { - BYTE sfd = 0; - sfd |= (fl->checksumFlag) << 7; - - ((BYTE*)output->dst)[output->pos] = sfd; - output->pos++; - fl->seekTablePos++; - } - - CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_SEEKABLE_MAGICNUMBER, - seekTableLen - 4)); - - if (fl->seekTablePos != seekTableLen) return ERROR(GENERIC); - return 0; -} - -size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output) -{ - if (!zcs->writingSeekTable && zcs->frameDSize) { - const size_t endFrame = ZSTD_seekable_endFrame(zcs, output); - if (ZSTD_isError(endFrame)) return endFrame; - /* return an accurate size hint */ - if (endFrame) return endFrame + ZSTD_seekable_seekTableSize(&zcs->framelog); - } - - zcs->writingSeekTable = 1; - - return ZSTD_seekable_writeSeekTable(&zcs->framelog, output); -} diff --git a/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c b/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c deleted file mode 100644 index c67653338ac..00000000000 --- a/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c +++ /dev/null @@ -1,462 +0,0 @@ -/* - * Copyright (c) 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -/* ********************************************************* -* Turn on Large Files support (>4GB) for 32-bit Linux/Unix -***********************************************************/ -#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ -# if !defined(_FILE_OFFSET_BITS) -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# endif -# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ -# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ -# endif -# if defined(_AIX) || defined(__hpux) -# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ -# endif -#endif - -/* ************************************************************ -* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW -***************************************************************/ -#if defined(_MSC_VER) && _MSC_VER >= 1400 -# define LONG_SEEK _fseeki64 -#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# define LONG_SEEK fseeko -#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) -# define LONG_SEEK fseeko64 -#elif defined(_WIN32) && !defined(__DJGPP__) -# include - static int LONG_SEEK(FILE* file, __int64 offset, int origin) { - LARGE_INTEGER off; - DWORD method; - off.QuadPart = offset; - if (origin == SEEK_END) - method = FILE_END; - else if (origin == SEEK_CUR) - method = FILE_CURRENT; - else - method = FILE_BEGIN; - - if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) - return 0; - else - return -1; - } -#else -# define LONG_SEEK fseek -#endif - -#include /* malloc, free */ -#include /* FILE* */ - -#define XXH_STATIC_LINKING_ONLY -#define XXH_NAMESPACE ZSTD_ -#include "xxhash.h" - -#define ZSTD_STATIC_LINKING_ONLY -#include "zstd.h" -#include "zstd_errors.h" -#include "mem.h" -#include "zstd_seekable.h" - -#undef ERROR -#define ERROR(name) ((size_t)-ZSTD_error_##name) - -#define CHECK_IO(f) { int const errcod = (f); if (errcod < 0) return ERROR(seekableIO); } - -#undef MIN -#undef MAX -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define MAX(a, b) ((a) > (b) ? (a) : (b)) - -/* Special-case callbacks for FILE* and in-memory modes, so that we can treat - * them the same way as the advanced API */ -static int ZSTD_seekable_read_FILE(void* opaque, void* buffer, size_t n) -{ - size_t const result = fread(buffer, 1, n, (FILE*)opaque); - if (result != n) { - return -1; - } - return 0; -} - -static int ZSTD_seekable_seek_FILE(void* opaque, S64 offset, int origin) -{ - int const ret = LONG_SEEK((FILE*)opaque, offset, origin); - if (ret) return ret; - return fflush((FILE*)opaque); -} - -typedef struct { - const void *ptr; - size_t size; - size_t pos; -} buffWrapper_t; - -static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n) -{ - buffWrapper_t* buff = (buffWrapper_t*) opaque; - if (buff->size + n > buff->pos) return -1; - memcpy(buffer, (const BYTE*)buff->ptr + buff->pos, n); - buff->pos += n; - return 0; -} - -static int ZSTD_seekable_seek_buff(void* opaque, S64 offset, int origin) -{ - buffWrapper_t* buff = (buffWrapper_t*) opaque; - unsigned long long newOffset; - switch (origin) { - case SEEK_SET: - newOffset = offset; - break; - case SEEK_CUR: - newOffset = (unsigned long long)buff->pos + offset; - break; - case SEEK_END: - newOffset = (unsigned long long)buff->size - offset; - break; - } - if (newOffset > buff->size) { - return -1; - } - buff->pos = newOffset; - return 0; -} - -typedef struct { - U64 cOffset; - U64 dOffset; - U32 checksum; -} seekEntry_t; - -typedef struct { - seekEntry_t* entries; - size_t tableLen; - - int checksumFlag; -} seekTable_t; - -#define SEEKABLE_BUFF_SIZE ZSTD_BLOCKSIZE_MAX - -struct ZSTD_seekable_s { - ZSTD_DStream* dstream; - seekTable_t seekTable; - ZSTD_seekable_customFile src; - - U64 decompressedOffset; - U32 curFrame; - - BYTE inBuff[SEEKABLE_BUFF_SIZE]; /* need to do our own input buffering */ - BYTE outBuff[SEEKABLE_BUFF_SIZE]; /* so we can efficiently decompress the - starts of chunks before we get to the - desired section */ - ZSTD_inBuffer in; /* maintain continuity across ZSTD_seekable_decompress operations */ - buffWrapper_t buffWrapper; /* for `src.opaque` in in-memory mode */ - - XXH64_state_t xxhState; -}; - -ZSTD_seekable* ZSTD_seekable_create(void) -{ - ZSTD_seekable* zs = malloc(sizeof(ZSTD_seekable)); - - if (zs == NULL) return NULL; - - /* also initializes stage to zsds_init */ - memset(zs, 0, sizeof(*zs)); - - zs->dstream = ZSTD_createDStream(); - if (zs->dstream == NULL) { - free(zs); - return NULL; - } - - return zs; -} - -size_t ZSTD_seekable_free(ZSTD_seekable* zs) -{ - if (zs == NULL) return 0; /* support free on null */ - ZSTD_freeDStream(zs->dstream); - free(zs->seekTable.entries); - free(zs); - - return 0; -} - -/** ZSTD_seekable_offsetToFrameIndex() : - * Performs a binary search to find the last frame with a decompressed offset - * <= pos - * @return : the frame's index */ -U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, U64 pos) -{ - U32 lo = 0; - U32 hi = zs->seekTable.tableLen; - - if (pos >= zs->seekTable.entries[zs->seekTable.tableLen].dOffset) { - return zs->seekTable.tableLen; - } - - while (lo + 1 < hi) { - U32 const mid = lo + ((hi - lo) >> 1); - if (zs->seekTable.entries[mid].dOffset <= pos) { - lo = mid; - } else { - hi = mid; - } - } - return lo; -} - -U32 ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs) -{ - return zs->seekTable.tableLen; -} - -U64 ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, U32 frameIndex) -{ - if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE; - return zs->seekTable.entries[frameIndex].cOffset; -} - -U64 ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, U32 frameIndex) -{ - if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE; - return zs->seekTable.entries[frameIndex].dOffset; -} - -size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, U32 frameIndex) -{ - if (frameIndex >= zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge); - return zs->seekTable.entries[frameIndex + 1].cOffset - - zs->seekTable.entries[frameIndex].cOffset; -} - -size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, U32 frameIndex) -{ - if (frameIndex > zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge); - return zs->seekTable.entries[frameIndex + 1].dOffset - - zs->seekTable.entries[frameIndex].dOffset; -} - -static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs) -{ - int checksumFlag; - ZSTD_seekable_customFile src = zs->src; - /* read the footer, fixed size */ - CHECK_IO(src.seek(src.opaque, -(int)ZSTD_seekTableFooterSize, SEEK_END)); - CHECK_IO(src.read(src.opaque, zs->inBuff, ZSTD_seekTableFooterSize)); - - if (MEM_readLE32(zs->inBuff + 5) != ZSTD_SEEKABLE_MAGICNUMBER) { - return ERROR(prefix_unknown); - } - - { BYTE const sfd = zs->inBuff[4]; - checksumFlag = sfd >> 7; - - /* check reserved bits */ - if ((checksumFlag >> 2) & 0x1f) { - return ERROR(corruption_detected); - } - } - - { U32 const numFrames = MEM_readLE32(zs->inBuff); - U32 const sizePerEntry = 8 + (checksumFlag?4:0); - U32 const tableSize = sizePerEntry * numFrames; - U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_skippableHeaderSize; - - U32 remaining = frameSize - ZSTD_seekTableFooterSize; /* don't need to re-read footer */ - { - U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE); - - CHECK_IO(src.seek(src.opaque, -(S64)frameSize, SEEK_END)); - CHECK_IO(src.read(src.opaque, zs->inBuff, toRead)); - - remaining -= toRead; - } - - if (MEM_readLE32(zs->inBuff) != (ZSTD_MAGIC_SKIPPABLE_START | 0xE)) { - return ERROR(prefix_unknown); - } - if (MEM_readLE32(zs->inBuff+4) + ZSTD_skippableHeaderSize != frameSize) { - return ERROR(prefix_unknown); - } - - { /* Allocate an extra entry at the end so that we can do size - * computations on the last element without special case */ - seekEntry_t* entries = (seekEntry_t*)malloc(sizeof(seekEntry_t) * (numFrames + 1)); - const BYTE* tableBase = zs->inBuff + ZSTD_skippableHeaderSize; - - U32 idx = 0; - U32 pos = 8; - - - U64 cOffset = 0; - U64 dOffset = 0; - - if (!entries) { - free(entries); - return ERROR(memory_allocation); - } - - /* compute cumulative positions */ - for (; idx < numFrames; idx++) { - if (pos + sizePerEntry > SEEKABLE_BUFF_SIZE) { - U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE); - U32 const offset = SEEKABLE_BUFF_SIZE - pos; - memmove(zs->inBuff, zs->inBuff + pos, offset); /* move any data we haven't read yet */ - CHECK_IO(src.read(src.opaque, zs->inBuff+offset, toRead)); - remaining -= toRead; - pos = 0; - } - entries[idx].cOffset = cOffset; - entries[idx].dOffset = dOffset; - - cOffset += MEM_readLE32(zs->inBuff + pos); - pos += 4; - dOffset += MEM_readLE32(zs->inBuff + pos); - pos += 4; - if (checksumFlag) { - entries[idx].checksum = MEM_readLE32(zs->inBuff + pos); - pos += 4; - } - } - entries[numFrames].cOffset = cOffset; - entries[numFrames].dOffset = dOffset; - - zs->seekTable.entries = entries; - zs->seekTable.tableLen = numFrames; - zs->seekTable.checksumFlag = checksumFlag; - return 0; - } - } -} - -size_t ZSTD_seekable_initBuff(ZSTD_seekable* zs, const void* src, size_t srcSize) -{ - zs->buffWrapper = (buffWrapper_t){src, srcSize, 0}; - { ZSTD_seekable_customFile srcFile = {&zs->buffWrapper, - &ZSTD_seekable_read_buff, - &ZSTD_seekable_seek_buff}; - return ZSTD_seekable_initAdvanced(zs, srcFile); } -} - -size_t ZSTD_seekable_initFile(ZSTD_seekable* zs, FILE* src) -{ - ZSTD_seekable_customFile srcFile = {src, &ZSTD_seekable_read_FILE, - &ZSTD_seekable_seek_FILE}; - return ZSTD_seekable_initAdvanced(zs, srcFile); -} - -size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile src) -{ - zs->src = src; - - { const size_t seekTableInit = ZSTD_seekable_loadSeekTable(zs); - if (ZSTD_isError(seekTableInit)) return seekTableInit; } - - zs->decompressedOffset = (U64)-1; - zs->curFrame = (U32)-1; - - { const size_t dstreamInit = ZSTD_initDStream(zs->dstream); - if (ZSTD_isError(dstreamInit)) return dstreamInit; } - return 0; -} - -size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, U64 offset) -{ - U32 targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, offset); - do { - /* check if we can continue from a previous decompress job */ - if (targetFrame != zs->curFrame || offset != zs->decompressedOffset) { - zs->decompressedOffset = zs->seekTable.entries[targetFrame].dOffset; - zs->curFrame = targetFrame; - - CHECK_IO(zs->src.seek(zs->src.opaque, - zs->seekTable.entries[targetFrame].cOffset, - SEEK_SET)); - zs->in = (ZSTD_inBuffer){zs->inBuff, 0, 0}; - XXH64_reset(&zs->xxhState, 0); - ZSTD_resetDStream(zs->dstream); - } - - while (zs->decompressedOffset < offset + len) { - size_t toRead; - ZSTD_outBuffer outTmp; - size_t prevOutPos; - if (zs->decompressedOffset < offset) { - /* dummy decompressions until we get to the target offset */ - outTmp = (ZSTD_outBuffer){zs->outBuff, MIN(SEEKABLE_BUFF_SIZE, offset - zs->decompressedOffset), 0}; - } else { - outTmp = (ZSTD_outBuffer){dst, len, zs->decompressedOffset - offset}; - } - - prevOutPos = outTmp.pos; - toRead = ZSTD_decompressStream(zs->dstream, &outTmp, &zs->in); - if (ZSTD_isError(toRead)) { - return toRead; - } - - if (zs->seekTable.checksumFlag) { - XXH64_update(&zs->xxhState, (BYTE*)outTmp.dst + prevOutPos, - outTmp.pos - prevOutPos); - } - zs->decompressedOffset += outTmp.pos - prevOutPos; - - if (toRead == 0) { - /* frame complete */ - - /* verify checksum */ - if (zs->seekTable.checksumFlag && - (XXH64_digest(&zs->xxhState) & 0xFFFFFFFFU) != - zs->seekTable.entries[targetFrame].checksum) { - return ERROR(corruption_detected); - } - - if (zs->decompressedOffset < offset + len) { - /* go back to the start and force a reset of the stream */ - targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, zs->decompressedOffset); - } - break; - } - - /* read in more data if we're done with this buffer */ - if (zs->in.pos == zs->in.size) { - toRead = MIN(toRead, SEEKABLE_BUFF_SIZE); - CHECK_IO(zs->src.read(zs->src.opaque, zs->inBuff, toRead)); - zs->in.size = toRead; - zs->in.pos = 0; - } - } - } while (zs->decompressedOffset != offset + len); - - return len; -} - -size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, U32 frameIndex) -{ - if (frameIndex >= zs->seekTable.tableLen) { - return ERROR(frameIndex_tooLarge); - } - - { - size_t const decompressedSize = - zs->seekTable.entries[frameIndex + 1].dOffset - - zs->seekTable.entries[frameIndex].dOffset; - if (dstSize < decompressedSize) { - return ERROR(dstSize_tooSmall); - } - return ZSTD_seekable_decompress( - zs, dst, decompressedSize, - zs->seekTable.entries[frameIndex].dOffset); - } -} diff --git a/sys/contrib/zstd/lib/common/xxhash.h b/sys/contrib/zstd/lib/common/xxhash.h index 8ebbfdd6261..60fe033a91c 100644 --- a/sys/contrib/zstd/lib/common/xxhash.h +++ b/sys/contrib/zstd/lib/common/xxhash.h @@ -695,6 +695,9 @@ typedef uint64_t XXH64_hash_t; * @see * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. */ +/* Begin FreeBSD - This symbol is needed by dll-linked CLI zstd(1). */ +__attribute__((visibility ("default"))) +/* End FreeBSD */ XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed); /******* Streaming *******/ diff --git a/sys/contrib/zstd/lib/compress/zstd_compress_internal.h b/sys/contrib/zstd/lib/compress/zstd_compress_internal.h index c406e794bdb..bd403bf58d3 100644 --- a/sys/contrib/zstd/lib/compress/zstd_compress_internal.h +++ b/sys/contrib/zstd/lib/compress/zstd_compress_internal.h @@ -1448,6 +1448,9 @@ size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSe /** ZSTD_cycleLog() : * condition for correct operation : hashLog > 1 */ +/* Begin FreeBSD - This symbol is needed by dll-linked CLI zstd(1). */ +ZSTDLIB_API +/* End FreeBSD */ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); /** ZSTD_CCtx_trace() : diff --git a/sys/contrib/zstd/lib/dictBuilder/zdict.c b/sys/contrib/zstd/lib/dictBuilder/zdict.c index 006aba7c9c4..df0f1155fce 100644 --- a/sys/contrib/zstd/lib/dictBuilder/zdict.c +++ b/sys/contrib/zstd/lib/dictBuilder/zdict.c @@ -1047,6 +1047,9 @@ static size_t ZDICT_addEntropyTablesFromBuffer_advanced( * Warning : `samplesBuffer` must be followed by noisy guard band !!! * @return : size of dictionary, or an error code which can be tested with ZDICT_isError() */ +/* Begin FreeBSD - This symbol is needed by dll-linked CLI zstd(1). */ +ZSTDLIB_API +/* End FreeBSD */ static size_t ZDICT_trainFromBuffer_unsafe_legacy( void* dictBuffer, size_t maxDictSize, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, diff --git a/sys/contrib/zstd/lib/freebsd/limits.h b/sys/contrib/zstd/lib/freebsd/limits.h new file mode 100644 index 00000000000..85435679632 --- /dev/null +++ b/sys/contrib/zstd/lib/freebsd/limits.h @@ -0,0 +1,3 @@ +/* This file is in the public domain */ +/* $FreeBSD$ */ +#include diff --git a/sys/contrib/zstd/lib/freebsd/zstd_kmalloc.c b/sys/contrib/zstd/lib/freebsd/zstd_kmalloc.c index a59120e1c24..f6add4862a4 100644 --- a/sys/contrib/zstd/lib/freebsd/zstd_kmalloc.c +++ b/sys/contrib/zstd/lib/freebsd/zstd_kmalloc.c @@ -23,6 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * + * $FreeBSD$ */ #include @@ -30,3 +31,9 @@ #include MALLOC_DEFINE(M_ZSTD, "zstd", "ZSTD Compressor"); + +/* + * Build zstd lib/common/debug.c (single extern variable) while avoiding + * conflict with Xen's debug.c in objdir. + */ +#include "debug.c" diff --git a/sys/contrib/zstd/tests/.gitignore b/sys/contrib/zstd/tests/.gitignore deleted file mode 100644 index 4911b2d62a5..00000000000 --- a/sys/contrib/zstd/tests/.gitignore +++ /dev/null @@ -1,65 +0,0 @@ -# local binary (Makefile) -fullbench -fullbench32 -fuzzer -fuzzer32 -fuzzer-dll -zbufftest -zbufftest32 -zbufftest-dll -zstreamtest -zstreamtest32 -zstreamtest_asan -zstreamtest_tsan -zstreamtest-dll -datagen -paramgrill -paramgrill32 -roundTripCrash -longmatch -symbols -legacy -decodecorpus -pool -poolTests -invalidDictionaries -checkTag -zcat -zstdcat - -# Tmp test directory -zstdtest -speedTest -versionsTest -namespaceTest - -# Local script -startSpeedTest -speedTest.pid - -# Object files -*.o -*.ko - -# Executables -*.exe -*.out -*.app - -# Default result files -dictionary -grillResults.txt -_* -tmp* -*.zst -*.gz -result -out -*.zstd - -# fuzzer -afl - -# Misc files -*.bat -dirTest* diff --git a/sys/contrib/zstd/tests/Makefile b/sys/contrib/zstd/tests/Makefile deleted file mode 100644 index 5b35ad40631..00000000000 --- a/sys/contrib/zstd/tests/Makefile +++ /dev/null @@ -1,467 +0,0 @@ -# ################################################################ -# Copyright (c) 2015-present, Yann Collet, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# ################################################################ -# datagen : Synthetic and parametrable data generator, for tests -# fullbench : Precisely measure speed for each zstd inner functions -# fullbench32: Same as fullbench, but forced to compile in 32-bits mode -# fuzzer : Test tool, to check zstd integrity on target platform -# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode -# paramgrill : parameter tester for zstd -# test-zstd-speed.py : script for testing zstd speed difference between commits -# versionsTest : compatibility test between zstd versions stored on Github (v0.1+) -# zstreamtest : Fuzzer test tool for zstd streaming API -# zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode -# ########################################################################## - -ZSTDDIR = ../lib -PRGDIR = ../programs -PYTHON ?= python3 -TESTARTEFACT := versionsTest - -DEBUGLEVEL ?= 1 -DEBUGFLAGS = -g -DZSTD_DEBUG=$(DEBUGLEVEL) -CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ - -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) -CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ - -Wstrict-prototypes -Wundef -Wformat-security \ - -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) - - -ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) -ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c -ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c - -ZSTD_F1 := $(wildcard $(ZSTD_FILES)) -ZSTD_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdm_,$(ZSTD_F1)) -ZSTD_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdc_,$(ZSTD_OBJ1)) -ZSTD_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdd_,$(ZSTD_OBJ2)) -ZSTD_OBJECTS := $(ZSTD_OBJ3:.c=.o) - -ZSTDMT_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdmt_m_,$(ZSTD_F1)) -ZSTDMT_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdmt_c_,$(ZSTDMT_OBJ1)) -ZSTDMT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdmt_d_,$(ZSTDMT_OBJ2)) -ZSTDMT_OBJECTS := $(ZSTDMT_OBJ3:.c=.o) - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -MULTITHREAD_CPP = -DZSTD_MULTITHREAD -MULTITHREAD_LD = -else -EXT = -MULTITHREAD_CPP = -DZSTD_MULTITHREAD -MULTITHREAD_LD = -pthread -endif -MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD) - -VOID = /dev/null -ZSTREAM_TESTTIME ?= -T90s -FUZZERTEST ?= -T200s -ZSTDRTTEST = --test-large-data -DECODECORPUS_TESTTIME ?= -T30 - -.PHONY: default all all32 allnothread dll clean test test32 test-all versionsTest - -default: fullbench - @echo $(ZSTDMT_OBJECTS) - -all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash - -all32: fullbench32 fuzzer32 zstreamtest32 - -allnothread: MULTITHREAD_CPP= -allnothread: MULTITHREAD_LD= -allnothread: fullbench fuzzer paramgrill datagen decodecorpus - -dll: fuzzer-dll zstreamtest-dll - -zstd: - $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" - -zstd32: - $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" - -zstd-nolegacy: - $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" - -gzstd: - $(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" - -.PHONY: zstd-dll -zstd-dll : - $(MAKE) -C $(ZSTDDIR) libzstd - -.PHONY: zstd-staticLib -zstd-staticLib : - $(MAKE) -C $(ZSTDDIR) libzstd.a - -zstdm_%.o : $(ZSTDDIR)/common/%.c - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ - -zstdc_%.o : $(ZSTDDIR)/compress/%.c - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ - -zstdd_%.o : $(ZSTDDIR)/decompress/%.c - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ - -zstdmt%.o : CPPFLAGS += $(MULTITHREAD_CPP) - -zstdmt_m_%.o : $(ZSTDDIR)/common/%.c - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ - -zstdmt_c_%.o : $(ZSTDDIR)/compress/%.c - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ - -zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ - -fullbench32: CPPFLAGS += -m32 -fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) -fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) -fullbench fullbench32 : DEBUGFLAGS = # turn off assert() for speed measurements -fullbench fullbench32 : $(ZSTD_FILES) -fullbench fullbench32 : $(PRGDIR)/datagen.c fullbench.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -fullbench-lib : zstd-staticLib -fullbench-lib : $(PRGDIR)/datagen.c fullbench.c - $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a - -# note : broken : requires unavailable symbols -fullbench-dll : zstd-dll -fullbench-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -fullbench-dll: $(PRGDIR)/datagen.c fullbench.c -# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll - $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) - -fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) -fuzzer : LDFLAGS += $(MULTITHREAD_LD) -fuzzer32: CFLAGS += -m32 -fuzzer : $(ZSTDMT_OBJECTS) -fuzzer32: $(ZSTD_FILES) -fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -fuzzer-dll : zstd-dll -fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c - $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) - -zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated -zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings -zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated -zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32 -zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -zbufftest-dll : zstd-dll -zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated -zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings -zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c - $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) - -ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c seqgen.c zstreamtest.c -ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES) -ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES) -zstreamtest32 : CFLAGS += -m32 -zstreamtest zstreamtest32 : CPPFLAGS += $(MULTITHREAD_CPP) -zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD) -zstreamtest : $(ZSTDMT_OBJECTS) $(ZSTREAM_PROPER_FILES) -zstreamtest32 : $(ZSTREAMFILES) -zstreamtest zstreamtest32 : - $(CC) $(FLAGS) $^ -o $@$(EXT) - -zstreamtest_asan : CFLAGS += -fsanitize=address -zstreamtest_asan : $(ZSTREAMFILES) - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) - -zstreamtest_tsan : CFLAGS += -fsanitize=thread -zstreamtest_tsan : $(ZSTREAMFILES) - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) - -zstreamtest-dll : zstd-dll -zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll -zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) - $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) - -paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements -paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c - $(CC) $(FLAGS) $^ -lm -o $@$(EXT) - -datagen : $(PRGDIR)/datagen.c datagencli.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) - -longmatch : $(ZSTD_OBJECTS) longmatch.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4 -legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) decodecorpus.c - $(CC) $(FLAGS) $^ -o $@$(EXT) -lm - -symbols : symbols.c zstd-dll -ifneq (,$(filter Windows%,$(OS))) - cp $(ZSTDDIR)/dll/libzstd.dll . - $(CC) $(FLAGS) $< -o $@$(EXT) -DZSTD_DLL_IMPORT=1 libzstd.dll -else - $(CC) $(FLAGS) $< -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so # broken on Mac -endif - -poolTests : poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) - -.PHONY: versionsTest -versionsTest: clean - $(PYTHON) test-zstd-versions.py - -checkTag: checkTag.c $(ZSTDDIR)/zstd.h - $(CC) $(FLAGS) $< -o $@$(EXT) - -clean: - $(MAKE) -C $(ZSTDDIR) clean - @$(RM) -fR $(TESTARTEFACT) - @$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \ - $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ - fullbench$(EXT) fullbench32$(EXT) \ - fullbench-lib$(EXT) fullbench-dll$(EXT) \ - fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ - fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\ - zstreamtest$(EXT) zstreamtest32$(EXT) \ - datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \ - symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \ - decodecorpus$(EXT) checkTag$(EXT) - @echo Cleaning completed - - -#---------------------------------------------------------------------------------- -#make valgrindTest is validated only for Linux, OSX, BSD, Hurd and Solaris targets -#---------------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) -HOST_OS = POSIX - -valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 -valgrindTest: zstd datagen fuzzer fullbench - @echo "\n ---- valgrind tests : memory analyzer ----" - $(VALGRIND) ./datagen -g50M > $(VOID) - $(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi - ./datagen -g80 | $(VALGRIND) $(PRGDIR)/zstd - -c > $(VOID) - ./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID) - ./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp - $(VALGRIND) $(PRGDIR)/zstd -vdf tmp -c > $(VOID) - ./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID) - @rm tmp - $(VALGRIND) ./fuzzer -T1mn -t1 - $(VALGRIND) ./fullbench -i1 - -endif - - -ifneq (,$(filter MSYS%,$(shell uname))) -HOST_OS = MSYS -endif - - -#----------------------------------------------------------------------------- -# make tests validated only for below targets -#----------------------------------------------------------------------------- -ifneq (,$(filter $(HOST_OS),MSYS POSIX)) - -DIFF:=diff -ifneq (,$(filter $(shell uname),SunOS)) -DIFF:=gdiff -endif - -.PHONY: list -list: - @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs - -.PHONY: zstd-playTests -zstd-playTests: datagen - file $(ZSTD) - ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) - -.PHONY: shortest -shortest: ZSTDRTTEST= -shortest: test-zstd - -.PHONY: fuzztest -fuzztest: test-fuzzer test-zstream test-decodecorpus - -.PHONY: test -test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus -ifeq ($(QEMU_SYS),) -test: test-pool -endif - -test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 - -test-all: test test32 valgrindTest test-decodecorpus-cli - -test-zstd: ZSTD = $(PRGDIR)/zstd -test-zstd: zstd zstd-playTests - -test-zstd32: ZSTD = $(PRGDIR)/zstd32 -test-zstd32: zstd32 zstd-playTests - -test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy -test-zstd-nolegacy: zstd-nolegacy zstd-playTests - -test-gzstd: gzstd - $(PRGDIR)/zstd -f README.md test-zstd-speed.py - gzip -f README.md test-zstd-speed.py - cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst - cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz - $(PRGDIR)/zstd -df README.md.gz -o README2.md - $(PRGDIR)/zstd -df README.md.gz test-zstd-speed.py.gz - $(PRGDIR)/zstd -df zstd_gz.zst gz_zstd.gz - $(DIFF) -q zstd_gz gz_zstd - echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst - echo Hello World GZIP | gzip -c - >hello.gz - echo Hello World TEXT >hello.txt - cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz - $(PRGDIR)/zstd -dcf hello.* - $(PRGDIR)/zstd -dcf - zstd - lz4 < tmp | \ - $(ZSTD) -d | \ - cmp - tmp - lz4 < tmp | \ - $(ZSTD_UNLZ4) | \ - cmp - tmp - # zstd -> lz4 - $(ZSTD) --format=lz4 < tmp | \ - lz4 -d | \ - cmp - tmp - $(ZSTD_LZ4) < tmp | \ - lz4 -d | \ - cmp - tmp - # zstd -> zstd - $(ZSTD) --format=lz4 < tmp | \ - $(ZSTD) -d | \ - cmp - tmp - # zstd -> zstd - $(ZSTD) < tmp | \ - $(ZSTD) -d | \ - cmp - tmp - - ./datagen -g384KB | $(ZSTD) --format=lz4 | $(ZSTD) -d > /dev/null - - rm tmp lz4 unlz4 - -endif diff --git a/sys/contrib/zstd/tests/README.md b/sys/contrib/zstd/tests/README.md deleted file mode 100644 index 24a28ab7b9d..00000000000 --- a/sys/contrib/zstd/tests/README.md +++ /dev/null @@ -1,90 +0,0 @@ -Programs and scripts for automated testing of Zstandard -======================================================= - -This directory contains the following programs and scripts: -- `datagen` : Synthetic and parametrable data generator, for tests -- `fullbench` : Precisely measure speed for each zstd inner functions -- `fuzzer` : Test tool, to check zstd integrity on target platform -- `paramgrill` : parameter tester for zstd -- `test-zstd-speed.py` : script for testing zstd speed difference between commits -- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+) -- `zbufftest` : Test tool to check ZBUFF (a buffered streaming API) integrity -- `zstreamtest` : Fuzzer test tool for zstd streaming API -- `legacy` : Test tool to test decoding of legacy zstd frames -- `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations - - -#### `test-zstd-versions.py` - script for testing zstd interoperability between versions - -This script creates `versionsTest` directory to which zstd repository is cloned. -Then all tagged (released) versions of zstd are compiled. -In the following step interoperability between zstd versions is checked. - - -#### `test-zstd-speed.py` - script for testing zstd speed difference between commits - -This script creates `speedTest` directory to which zstd repository is cloned. -Then it compiles all branches of zstd and performs a speed benchmark for a given list of files (the `testFileNames` parameter). -After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits. -If a new commit is found it is compiled and a speed benchmark for this commit is performed. -The results of the speed benchmark are compared to the previous results. -If compression or decompression speed for one of zstd levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted. -If second results are also lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter). - -Additional remarks: -- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel -- Using the script with virtual machines can lead to large variations of speed results -- The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75) -- The script sends e-mails using `mutt`; if `mutt` is not available it sends e-mails without attachments using `mail`; if both are not available it only prints a warning - - -The example usage with two test files, one e-mail address, and with an additional message: -``` -./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60 -``` - -To run the script in background please use: -``` -nohup ./test-zstd-speed.py testFileNames emails & -``` - -The full list of parameters: -``` -positional arguments: - testFileNames file names list for speed benchmark - emails list of e-mail addresses to send warnings - -optional arguments: - -h, --help show this help message and exit - --message MESSAGE attach an additional message to e-mail - --lowerLimit LOWERLIMIT - send email if speed is lower than given limit - --maxLoadAvg MAXLOADAVG - maximum load average to start testing - --lastCLevel LASTCLEVEL - last compression level for testing - --sleepTime SLEEPTIME - frequency of repository checking in seconds -``` - -#### `decodecorpus` - tool to generate Zstandard frames for decoder testing -Command line tool to generate test .zst files. - -This tool will generate .zst files with checksums, -as well as optionally output the corresponding correct uncompressed data for -extra verfication. - -Example: -``` -./decodecorpus -ptestfiles -otestfiles -n10000 -s5 -``` -will generate 10,000 sample .zst files using a seed of 5 in the `testfiles` directory, -with the zstd checksum field set, -as well as the 10,000 original files for more detailed comparison of decompression results. - -``` -./decodecorpus -t -T1mn -``` -will choose a random seed, and for 1 minute, -generate random test frames and ensure that the -zstd library correctly decompresses them in both simple and streaming modes. diff --git a/sys/contrib/zstd/tests/checkTag.c b/sys/contrib/zstd/tests/checkTag.c deleted file mode 100644 index fda3fd1f9df..00000000000 --- a/sys/contrib/zstd/tests/checkTag.c +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2018-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -/* checkTag : validation tool for libzstd - * command : - * $ ./checkTag tag - * checkTag validates tags of following format : v[0-9].[0-9].[0-9]{any} - * The tag is then compared to zstd version number. - * They are compatible if first 3 digits are identical. - * Anything beyond that is free, and doesn't impact validation. - * Example : tag v1.8.1.2 is compatible with version 1.8.1 - * When tag and version are not compatible, program exits with error code 1. - * When they are compatible, it exists with a code 0. - * checkTag is intended to be used in automated testing environment. - */ - -#include /* printf */ -#include /* strlen, strncmp */ -#include "zstd.h" /* ZSTD_VERSION_STRING */ - - -/* validate() : - * @return 1 if tag is compatible, 0 if not. - */ -static int validate(const char* const tag) -{ - size_t const tagLength = strlen(tag); - size_t const verLength = strlen(ZSTD_VERSION_STRING); - - if (tagLength < 2) return 0; - if (tag[0] != 'v') return 0; - if (tagLength <= verLength) return 0; - - if (strncmp(ZSTD_VERSION_STRING, tag+1, verLength)) return 0; - - return 1; -} - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - const char* const tag = argv[1]; - if (argc!=2) { - printf("incorrect usage : %s tag \n", exeName); - return 2; - } - - printf("Version : %s \n", ZSTD_VERSION_STRING); - printf("Tag : %s \n", tag); - - if (validate(tag)) { - printf("OK : tag is compatible with zstd version \n"); - return 0; - } - - printf("!! error : tag and versions are not compatible !! \n"); - return 1; -} diff --git a/sys/contrib/zstd/tests/datagencli.c b/sys/contrib/zstd/tests/datagencli.c deleted file mode 100644 index 4814974e27f..00000000000 --- a/sys/contrib/zstd/tests/datagencli.c +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -/*-************************************ -* Dependencies -**************************************/ -#include "util.h" /* Compiler options */ -#include /* fprintf, stderr */ -#include "datagen.h" /* RDG_generate */ - - -/*-************************************ -* Constants -**************************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define SIZE_DEFAULT ((64 KB) + 1) -#define SEED_DEFAULT 0 -#define COMPRESSIBILITY_DEFAULT 50 - - -/*-************************************ -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned displayLevel = 2; - - -/*-******************************************************* -* Command line -*********************************************************/ -static int usage(const char* programName) -{ - DISPLAY( "Compressible data generator\n"); - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); - DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); - DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", - COMPRESSIBILITY_DEFAULT); - DISPLAY( " -h : display help and exit\n"); - return 0; -} - - -int main(int argc, const char** argv) -{ - unsigned probaU32 = COMPRESSIBILITY_DEFAULT; - double litProba = 0.0; - U64 size = SIZE_DEFAULT; - U32 seed = SEED_DEFAULT; - const char* const programName = argv[0]; - - int argNb; - for(argNb=1; argNb='0') && (*argument<='9')) - size *= 10, size += *argument++ - '0'; - if (*argument=='K') { size <<= 10; argument++; } - if (*argument=='M') { size <<= 20; argument++; } - if (*argument=='G') { size <<= 30; argument++; } - if (*argument=='B') { argument++; } - break; - case 's': - argument++; - seed=0; - while ((*argument>='0') && (*argument<='9')) - seed *= 10, seed += *argument++ - '0'; - break; - case 'P': - argument++; - probaU32 = 0; - while ((*argument>='0') && (*argument<='9')) - probaU32 *= 10, probaU32 += *argument++ - '0'; - if (probaU32>100) probaU32 = 100; - break; - case 'L': /* hidden argument : Literal distribution probability */ - argument++; - litProba=0.; - while ((*argument>='0') && (*argument<='9')) - litProba *= 10, litProba += *argument++ - '0'; - if (litProba>100.) litProba=100.; - litProba /= 100.; - break; - case 'v': - displayLevel = 4; - argument++; - break; - default: - return usage(programName); - } - } } } /* for(argNb=1; argNb -#include -#include -#include -#include -#include - -#include "util.h" -#include "zstd.h" -#include "zstd_internal.h" -#include "mem.h" -#define ZDICT_STATIC_LINKING_ONLY -#include "zdict.h" - -// Direct access to internal compression functions is required -#include "zstd_compress.c" - -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" /* XXH64 */ - -#ifndef MIN - #define MIN(a, b) ((a) < (b) ? (a) : (b)) -#endif - -#ifndef MAX_PATH - #ifdef PATH_MAX - #define MAX_PATH PATH_MAX - #else - #define MAX_PATH 256 - #endif -#endif - -/*-************************************ -* DISPLAY Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 2; - -#define DISPLAYUPDATE(...) \ - do { \ - if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || \ - (g_displayLevel >= 4)) { \ - g_displayClock = UTIL_getTime(); \ - DISPLAY(__VA_ARGS__); \ - if (g_displayLevel >= 4) fflush(stderr); \ - } \ - } while (0) - -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; - -#define CHECKERR(code) \ - do { \ - if (ZSTD_isError(code)) { \ - DISPLAY("Error occurred while generating data: %s\n", \ - ZSTD_getErrorName(code)); \ - exit(1); \ - } \ - } while (0) - -/*-******************************************************* -* Random function -*********************************************************/ -static unsigned RAND(unsigned* src) -{ -#define RAND_rotl32(x,r) ((x << r) | (x >> (32 - r))) - static const U32 prime1 = 2654435761U; - static const U32 prime2 = 2246822519U; - U32 rand32 = *src; - rand32 *= prime1; - rand32 += prime2; - rand32 = RAND_rotl32(rand32, 13); - *src = rand32; - return RAND_rotl32(rand32, 27); -#undef RAND_rotl32 -} - -#define DISTSIZE (8192) - -/* Write `size` bytes into `ptr`, all of which are less than or equal to `maxSymb` */ -static void RAND_bufferMaxSymb(U32* seed, void* ptr, size_t size, int maxSymb) -{ - size_t i; - BYTE* op = ptr; - - for (i = 0; i < size; i++) { - op[i] = (BYTE) (RAND(seed) % (maxSymb + 1)); - } -} - -/* Write `size` random bytes into `ptr` */ -static void RAND_buffer(U32* seed, void* ptr, size_t size) -{ - size_t i; - BYTE* op = ptr; - - for (i = 0; i + 4 <= size; i += 4) { - MEM_writeLE32(op + i, RAND(seed)); - } - for (; i < size; i++) { - op[i] = RAND(seed) & 0xff; - } -} - -/* Write `size` bytes into `ptr` following the distribution `dist` */ -static void RAND_bufferDist(U32* seed, BYTE* dist, void* ptr, size_t size) -{ - size_t i; - BYTE* op = ptr; - - for (i = 0; i < size; i++) { - op[i] = dist[RAND(seed) % DISTSIZE]; - } -} - -/* Generate a random distribution where the frequency of each symbol follows a - * geometric distribution defined by `weight` - * `dist` should have size at least `DISTSIZE` */ -static void RAND_genDist(U32* seed, BYTE* dist, double weight) -{ - size_t i = 0; - size_t statesLeft = DISTSIZE; - BYTE symb = (BYTE) (RAND(seed) % 256); - BYTE step = (BYTE) ((RAND(seed) % 256) | 1); /* force it to be odd so it's relatively prime to 256 */ - - while (i < DISTSIZE) { - size_t states = ((size_t)(weight * statesLeft)) + 1; - size_t j; - for (j = 0; j < states && i < DISTSIZE; j++, i++) { - dist[i] = symb; - } - - symb += step; - statesLeft -= states; - } -} - -/* Generates a random number in the range [min, max) */ -static inline U32 RAND_range(U32* seed, U32 min, U32 max) -{ - return (RAND(seed) % (max-min)) + min; -} - -#define ROUND(x) ((U32)(x + 0.5)) - -/* Generates a random number in an exponential distribution with mean `mean` */ -static double RAND_exp(U32* seed, double mean) -{ - double const u = RAND(seed) / (double) UINT_MAX; - return log(1-u) * (-mean); -} - -/*-******************************************************* -* Constants and Structs -*********************************************************/ -const char *BLOCK_TYPES[] = {"raw", "rle", "compressed"}; - -#define MAX_DECOMPRESSED_SIZE_LOG 20 -#define MAX_DECOMPRESSED_SIZE (1ULL << MAX_DECOMPRESSED_SIZE_LOG) - -#define MAX_WINDOW_LOG 22 /* Recommended support is 8MB, so limit to 4MB + mantissa */ - -#define MIN_SEQ_LEN (3) -#define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN) - -BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE]; -BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2]; -BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; - -seqDef SEQUENCE_BUFFER[MAX_NB_SEQ]; -BYTE SEQUENCE_LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; /* storeSeq expects a place to copy literals to */ -BYTE SEQUENCE_LLCODE[ZSTD_BLOCKSIZE_MAX]; -BYTE SEQUENCE_MLCODE[ZSTD_BLOCKSIZE_MAX]; -BYTE SEQUENCE_OFCODE[ZSTD_BLOCKSIZE_MAX]; - -unsigned WKSP[1024]; - -typedef struct { - size_t contentSize; /* 0 means unknown (unless contentSize == windowSize == 0) */ - unsigned windowSize; /* contentSize >= windowSize means single segment */ -} frameHeader_t; - -/* For repeat modes */ -typedef struct { - U32 rep[ZSTD_REP_NUM]; - - int hufInit; - /* the distribution used in the previous block for repeat mode */ - BYTE hufDist[DISTSIZE]; - U32 hufTable [256]; /* HUF_CElt is an incomplete type */ - - int fseInit; - FSE_CTable offcodeCTable [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; - FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; - FSE_CTable litlengthCTable [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; - - /* Symbols that were present in the previous distribution, for use with - * set_repeat */ - BYTE litlengthSymbolSet[36]; - BYTE offsetSymbolSet[29]; - BYTE matchlengthSymbolSet[53]; -} cblockStats_t; - -typedef struct { - void* data; - void* dataStart; - void* dataEnd; - - void* src; - void* srcStart; - void* srcEnd; - - frameHeader_t header; - - cblockStats_t stats; - cblockStats_t oldStats; /* so they can be rolled back if uncompressible */ -} frame_t; - -typedef struct { - int useDict; - U32 dictID; - size_t dictContentSize; - BYTE* dictContent; -} dictInfo; - -typedef enum { - gt_frame = 0, /* generate frames */ - gt_block, /* generate compressed blocks without block/frame headers */ -} genType_e; - -/*-******************************************************* -* Global variables (set from command line) -*********************************************************/ -U32 g_maxDecompressedSizeLog = MAX_DECOMPRESSED_SIZE_LOG; /* <= 20 */ -U32 g_maxBlockSize = ZSTD_BLOCKSIZE_MAX; /* <= 128 KB */ - -/*-******************************************************* -* Generator Functions -*********************************************************/ - -struct { - int contentSize; /* force the content size to be present */ -} opts; /* advanced options on generation */ - -/* Generate and write a random frame header */ -static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info) -{ - BYTE* const op = frame->data; - size_t pos = 0; - frameHeader_t fh; - - BYTE windowByte = 0; - - int singleSegment = 0; - int contentSizeFlag = 0; - int fcsCode = 0; - - memset(&fh, 0, sizeof(fh)); - - /* generate window size */ - { - /* Follow window algorithm from specification */ - int const exponent = RAND(seed) % (MAX_WINDOW_LOG - 10); - int const mantissa = RAND(seed) % 8; - windowByte = (BYTE) ((exponent << 3) | mantissa); - fh.windowSize = (1U << (exponent + 10)); - fh.windowSize += fh.windowSize / 8 * mantissa; - } - - { - /* Generate random content size */ - size_t highBit; - if (RAND(seed) & 7 && g_maxDecompressedSizeLog > 7) { - /* do content of at least 128 bytes */ - highBit = 1ULL << RAND_range(seed, 7, g_maxDecompressedSizeLog); - } else if (RAND(seed) & 3) { - /* do small content */ - highBit = 1ULL << RAND_range(seed, 0, MIN(7, 1U << g_maxDecompressedSizeLog)); - } else { - /* 0 size frame */ - highBit = 0; - } - fh.contentSize = highBit ? highBit + (RAND(seed) % highBit) : 0; - - /* provide size sometimes */ - contentSizeFlag = opts.contentSize | (RAND(seed) & 1); - - if (contentSizeFlag && (fh.contentSize == 0 || !(RAND(seed) & 7))) { - /* do single segment sometimes */ - fh.windowSize = (U32) fh.contentSize; - singleSegment = 1; - } - } - - if (contentSizeFlag) { - /* Determine how large fcs field has to be */ - int minFcsCode = (fh.contentSize >= 256) + - (fh.contentSize >= 65536 + 256) + - (fh.contentSize > 0xFFFFFFFFU); - if (!singleSegment && !minFcsCode) { - minFcsCode = 1; - } - fcsCode = minFcsCode + (RAND(seed) % (4 - minFcsCode)); - if (fcsCode == 1 && fh.contentSize < 256) fcsCode++; - } - - /* write out the header */ - MEM_writeLE32(op + pos, ZSTD_MAGICNUMBER); - pos += 4; - - { - /* - * fcsCode: 2-bit flag specifying how many bytes used to represent Frame_Content_Size (bits 7-6) - * singleSegment: 1-bit flag describing if data must be regenerated within a single continuous memory segment. (bit 5) - * contentChecksumFlag: 1-bit flag that is set if frame includes checksum at the end -- set to 1 below (bit 2) - * dictBits: 2-bit flag describing how many bytes Dictionary_ID uses -- set to 3 (bits 1-0) - * For more information: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_header - */ - int const dictBits = info.useDict ? 3 : 0; - BYTE const frameHeaderDescriptor = - (BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2) | dictBits); - op[pos++] = frameHeaderDescriptor; - } - - if (!singleSegment) { - op[pos++] = windowByte; - } - if (info.useDict) { - MEM_writeLE32(op + pos, (U32) info.dictID); - pos += 4; - } - if (contentSizeFlag) { - switch (fcsCode) { - default: /* Impossible */ - case 0: op[pos++] = (BYTE) fh.contentSize; break; - case 1: MEM_writeLE16(op + pos, (U16) (fh.contentSize - 256)); pos += 2; break; - case 2: MEM_writeLE32(op + pos, (U32) fh.contentSize); pos += 4; break; - case 3: MEM_writeLE64(op + pos, (U64) fh.contentSize); pos += 8; break; - } - } - - DISPLAYLEVEL(3, " frame content size:\t%u\n", (U32)fh.contentSize); - DISPLAYLEVEL(3, " frame window size:\t%u\n", fh.windowSize); - DISPLAYLEVEL(3, " content size flag:\t%d\n", contentSizeFlag); - DISPLAYLEVEL(3, " single segment flag:\t%d\n", singleSegment); - - frame->data = op + pos; - frame->header = fh; -} - -/* Write a literal block in either raw or RLE form, return the literals size */ -static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t contentSize) -{ - BYTE* op = (BYTE*)frame->data; - int const type = RAND(seed) % 2; - int const sizeFormatDesc = RAND(seed) % 8; - size_t litSize; - size_t maxLitSize = MIN(contentSize, g_maxBlockSize); - - if (sizeFormatDesc == 0) { - /* Size_FormatDesc = ?0 */ - maxLitSize = MIN(maxLitSize, 31); - } else if (sizeFormatDesc <= 4) { - /* Size_FormatDesc = 01 */ - maxLitSize = MIN(maxLitSize, 4095); - } else { - /* Size_Format = 11 */ - maxLitSize = MIN(maxLitSize, 1048575); - } - - litSize = RAND(seed) % (maxLitSize + 1); - if (frame->src == frame->srcStart && litSize == 0) { - litSize = 1; /* no empty literals if there's nothing preceding this block */ - } - if (litSize + 3 > contentSize) { - litSize = contentSize; /* no matches shorter than 3 are allowed */ - } - /* use smallest size format that fits */ - if (litSize < 32) { - op[0] = (type | (0 << 2) | (litSize << 3)) & 0xff; - op += 1; - } else if (litSize < 4096) { - op[0] = (type | (1 << 2) | (litSize << 4)) & 0xff; - op[1] = (litSize >> 4) & 0xff; - op += 2; - } else { - op[0] = (type | (3 << 2) | (litSize << 4)) & 0xff; - op[1] = (litSize >> 4) & 0xff; - op[2] = (litSize >> 12) & 0xff; - op += 3; - } - - if (type == 0) { - /* Raw literals */ - DISPLAYLEVEL(4, " raw literals\n"); - - RAND_buffer(seed, LITERAL_BUFFER, litSize); - memcpy(op, LITERAL_BUFFER, litSize); - op += litSize; - } else { - /* RLE literals */ - BYTE const symb = (BYTE) (RAND(seed) % 256); - - DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (U32)symb); - - memset(LITERAL_BUFFER, symb, litSize); - op[0] = symb; - op++; - } - - frame->data = op; - - return litSize; -} - -/* Generate a Huffman header for the given source */ -static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t dstSize, - const void* src, size_t srcSize) -{ - BYTE* const ostart = (BYTE*)dst; - BYTE* op = ostart; - - unsigned huffLog = 11; - U32 maxSymbolValue = 255; - - U32 count[HUF_SYMBOLVALUE_MAX+1]; - - /* Scan input and build symbol stats */ - { size_t const largest = FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, WKSP); - if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 0; } /* single symbol, rle */ - if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ - } - - /* Build Huffman Tree */ - /* Max Huffman log is 11, min is highbit(maxSymbolValue)+1 */ - huffLog = RAND_range(seed, ZSTD_highbit32(maxSymbolValue)+1, huffLog+1); - DISPLAYLEVEL(6, " huffman log: %u\n", huffLog); - { size_t const maxBits = HUF_buildCTable_wksp (hufTable, count, maxSymbolValue, huffLog, WKSP, sizeof(WKSP)); - CHECKERR(maxBits); - huffLog = (U32)maxBits; - } - - /* Write table description header */ - { size_t const hSize = HUF_writeCTable (op, dstSize, hufTable, maxSymbolValue, huffLog); - if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */ - op += hSize; - } - - return op - ostart; -} - -/* Write a Huffman coded literals block and return the literals size */ -static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t contentSize) -{ - BYTE* origop = (BYTE*)frame->data; - BYTE* opend = (BYTE*)frame->dataEnd; - BYTE* op; - BYTE* const ostart = origop; - int const sizeFormat = RAND(seed) % 4; - size_t litSize; - size_t hufHeaderSize = 0; - size_t compressedSize = 0; - size_t maxLitSize = MIN(contentSize-3, g_maxBlockSize); - - symbolEncodingType_e hType; - - if (contentSize < 64) { - /* make sure we get reasonably-sized literals for compression */ - return ERROR(GENERIC); - } - - DISPLAYLEVEL(4, " compressed literals\n"); - - switch (sizeFormat) { - case 0: /* fall through, size is the same as case 1 */ - case 1: - maxLitSize = MIN(maxLitSize, 1023); - origop += 3; - break; - case 2: - maxLitSize = MIN(maxLitSize, 16383); - origop += 4; - break; - case 3: - maxLitSize = MIN(maxLitSize, 262143); - origop += 5; - break; - default:; /* impossible */ - } - - do { - op = origop; - do { - litSize = RAND(seed) % (maxLitSize + 1); - } while (litSize < 32); /* avoid small literal sizes */ - if (litSize + 3 > contentSize) { - litSize = contentSize; /* no matches shorter than 3 are allowed */ - } - - /* most of the time generate a new distribution */ - if ((RAND(seed) & 3) || !frame->stats.hufInit) { - do { - if (RAND(seed) & 3) { - /* add 10 to ensure some compressability */ - double const weight = ((RAND(seed) % 90) + 10) / 100.0; - - DISPLAYLEVEL(5, " distribution weight: %d%%\n", - (int)(weight * 100)); - - RAND_genDist(seed, frame->stats.hufDist, weight); - } else { - /* sometimes do restricted range literals to force - * non-huffman headers */ - DISPLAYLEVEL(5, " small range literals\n"); - RAND_bufferMaxSymb(seed, frame->stats.hufDist, DISTSIZE, - 15); - } - RAND_bufferDist(seed, frame->stats.hufDist, LITERAL_BUFFER, - litSize); - - /* generate the header from the distribution instead of the - * actual data to avoid bugs with symbols that were in the - * distribution but never showed up in the output */ - hufHeaderSize = writeHufHeader( - seed, (HUF_CElt*)frame->stats.hufTable, op, opend - op, - frame->stats.hufDist, DISTSIZE); - CHECKERR(hufHeaderSize); - /* repeat until a valid header is written */ - } while (hufHeaderSize == 0); - op += hufHeaderSize; - hType = set_compressed; - - frame->stats.hufInit = 1; - } else { - /* repeat the distribution/table from last time */ - DISPLAYLEVEL(5, " huffman repeat stats\n"); - RAND_bufferDist(seed, frame->stats.hufDist, LITERAL_BUFFER, - litSize); - hufHeaderSize = 0; - hType = set_repeat; - } - - do { - compressedSize = - sizeFormat == 0 - ? HUF_compress1X_usingCTable( - op, opend - op, LITERAL_BUFFER, litSize, - (HUF_CElt*)frame->stats.hufTable) - : HUF_compress4X_usingCTable( - op, opend - op, LITERAL_BUFFER, litSize, - (HUF_CElt*)frame->stats.hufTable); - CHECKERR(compressedSize); - /* this only occurs when it could not compress or similar */ - } while (compressedSize <= 0); - - op += compressedSize; - - compressedSize += hufHeaderSize; - DISPLAYLEVEL(5, " regenerated size: %u\n", (U32)litSize); - DISPLAYLEVEL(5, " compressed size: %u\n", (U32)compressedSize); - if (compressedSize >= litSize) { - DISPLAYLEVEL(5, " trying again\n"); - /* if we have to try again, reset the stats so we don't accidentally - * try to repeat a distribution we just made */ - frame->stats = frame->oldStats; - } else { - break; - } - } while (1); - - /* write header */ - switch (sizeFormat) { - case 0: /* fall through, size is the same as case 1 */ - case 1: { - U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) | - ((U32)compressedSize << 14); - MEM_writeLE24(ostart, header); - break; - } - case 2: { - U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) | - ((U32)compressedSize << 18); - MEM_writeLE32(ostart, header); - break; - } - case 3: { - U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) | - ((U32)compressedSize << 22); - MEM_writeLE32(ostart, header); - ostart[4] = (BYTE)(compressedSize >> 10); - break; - } - default:; /* impossible */ - } - - frame->data = op; - return litSize; -} - -static size_t writeLiteralsBlock(U32* seed, frame_t* frame, size_t contentSize) -{ - /* only do compressed for larger segments to avoid compressibility issues */ - if (RAND(seed) & 7 && contentSize >= 64) { - return writeLiteralsBlockCompressed(seed, frame, contentSize); - } else { - return writeLiteralsBlockSimple(seed, frame, contentSize); - } -} - -static inline void initSeqStore(seqStore_t *seqStore) { - seqStore->sequencesStart = SEQUENCE_BUFFER; - seqStore->litStart = SEQUENCE_LITERAL_BUFFER; - seqStore->llCode = SEQUENCE_LLCODE; - seqStore->mlCode = SEQUENCE_MLCODE; - seqStore->ofCode = SEQUENCE_OFCODE; - - ZSTD_resetSeqStore(seqStore); -} - -/* Randomly generate sequence commands */ -static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, - size_t contentSize, size_t literalsSize, dictInfo info) -{ - /* The total length of all the matches */ - size_t const remainingMatch = contentSize - literalsSize; - size_t excessMatch = 0; - U32 numSequences = 0; - - U32 i; - - - const BYTE* literals = LITERAL_BUFFER; - BYTE* srcPtr = frame->src; - - if (literalsSize != contentSize) { - /* each match must be at least MIN_SEQ_LEN, so this is the maximum - * number of sequences we can have */ - U32 const maxSequences = (U32)remainingMatch / MIN_SEQ_LEN; - numSequences = (RAND(seed) % maxSequences) + 1; - - /* the extra match lengths we have to allocate to each sequence */ - excessMatch = remainingMatch - numSequences * MIN_SEQ_LEN; - } - - DISPLAYLEVEL(5, " total match lengths: %u\n", (U32)remainingMatch); - for (i = 0; i < numSequences; i++) { - /* Generate match and literal lengths by exponential distribution to - * ensure nice numbers */ - U32 matchLen = - MIN_SEQ_LEN + - ROUND(RAND_exp(seed, excessMatch / (double)(numSequences - i))); - U32 literalLen = - (RAND(seed) & 7) - ? ROUND(RAND_exp(seed, - literalsSize / - (double)(numSequences - i))) - : 0; - /* actual offset, code to send, and point to copy up to when shifting - * codes in the repeat offsets history */ - U32 offset, offsetCode, repIndex; - - /* bounds checks */ - matchLen = (U32) MIN(matchLen, excessMatch + MIN_SEQ_LEN); - literalLen = MIN(literalLen, (U32) literalsSize); - if (i == 0 && srcPtr == frame->srcStart && literalLen == 0) literalLen = 1; - if (i + 1 == numSequences) matchLen = MIN_SEQ_LEN + (U32) excessMatch; - - memcpy(srcPtr, literals, literalLen); - srcPtr += literalLen; - do { - if (RAND(seed) & 7) { - /* do a normal offset */ - U32 const dataDecompressed = (U32)((BYTE*)srcPtr-(BYTE*)frame->srcStart); - offset = (RAND(seed) % - MIN(frame->header.windowSize, - (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) + - 1; - if (info.useDict && (RAND(seed) & 1) && i + 1 != numSequences && dataDecompressed < frame->header.windowSize) { - /* need to occasionally generate offsets that go past the start */ - /* including i+1 != numSequences because the last sequences has to adhere to predetermined contentSize */ - U32 lenPastStart = (RAND(seed) % info.dictContentSize) + 1; - offset = (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart)+lenPastStart; - if (offset > frame->header.windowSize) { - if (lenPastStart < MIN_SEQ_LEN) { - /* when offset > windowSize, matchLen bound by end of dictionary (lenPastStart) */ - /* this also means that lenPastStart must be greater than MIN_SEQ_LEN */ - /* make sure lenPastStart does not go past dictionary start though */ - lenPastStart = MIN(lenPastStart+MIN_SEQ_LEN, (U32)info.dictContentSize); - offset = (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) + lenPastStart; - } - { - U32 const matchLenBound = MIN(frame->header.windowSize, lenPastStart); - matchLen = MIN(matchLen, matchLenBound); - } - } - } - offsetCode = offset + ZSTD_REP_MOVE; - repIndex = 2; - } else { - /* do a repeat offset */ - offsetCode = RAND(seed) % 3; - if (literalLen > 0) { - offset = frame->stats.rep[offsetCode]; - repIndex = offsetCode; - } else { - /* special case */ - offset = offsetCode == 2 ? frame->stats.rep[0] - 1 - : frame->stats.rep[offsetCode + 1]; - repIndex = MIN(2, offsetCode + 1); - } - } - } while (((!info.useDict) && (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) || offset == 0); - - { - size_t j; - BYTE* const dictEnd = info.dictContent + info.dictContentSize; - for (j = 0; j < matchLen; j++) { - if ((U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) < offset) { - /* copy from dictionary instead of literals */ - size_t const dictOffset = offset - (srcPtr - (BYTE*)frame->srcStart); - *srcPtr = *(dictEnd - dictOffset); - } - else { - *srcPtr = *(srcPtr-offset); - } - srcPtr++; - } - } - - { int r; - for (r = repIndex; r > 0; r--) { - frame->stats.rep[r] = frame->stats.rep[r - 1]; - } - frame->stats.rep[0] = offset; - } - - DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u", literalLen, offset, matchLen); - DISPLAYLEVEL(7, " srcPos: %8u seqNb: %3u", - (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart), i); - DISPLAYLEVEL(6, "\n"); - if (offsetCode < 3) { - DISPLAYLEVEL(7, " repeat offset: %d\n", repIndex); - } - /* use libzstd sequence handling */ - ZSTD_storeSeq(seqStore, literalLen, literals, offsetCode, - matchLen - MINMATCH); - - literalsSize -= literalLen; - excessMatch -= (matchLen - MIN_SEQ_LEN); - literals += literalLen; - } - - memcpy(srcPtr, literals, literalsSize); - srcPtr += literalsSize; - DISPLAYLEVEL(6, " excess literals: %5u", (U32)literalsSize); - DISPLAYLEVEL(7, " srcPos: %8u", (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart)); - DISPLAYLEVEL(6, "\n"); - - return numSequences; -} - -static void initSymbolSet(const BYTE* symbols, size_t len, BYTE* set, BYTE maxSymbolValue) -{ - size_t i; - - memset(set, 0, (size_t)maxSymbolValue+1); - - for (i = 0; i < len; i++) { - set[symbols[i]] = 1; - } -} - -static int isSymbolSubset(const BYTE* symbols, size_t len, const BYTE* set, BYTE maxSymbolValue) -{ - size_t i; - - for (i = 0; i < len; i++) { - if (symbols[i] > maxSymbolValue || !set[symbols[i]]) { - return 0; - } - } - return 1; -} - -static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, - size_t nbSeq) -{ - /* This code is mostly copied from ZSTD_compressSequences in zstd_compress.c */ - U32 count[MaxSeq+1]; - S16 norm[MaxSeq+1]; - FSE_CTable* CTable_LitLength = frame->stats.litlengthCTable; - FSE_CTable* CTable_OffsetBits = frame->stats.offcodeCTable; - FSE_CTable* CTable_MatchLength = frame->stats.matchlengthCTable; - U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ - const seqDef* const sequences = seqStorePtr->sequencesStart; - const BYTE* const ofCodeTable = seqStorePtr->ofCode; - const BYTE* const llCodeTable = seqStorePtr->llCode; - const BYTE* const mlCodeTable = seqStorePtr->mlCode; - BYTE* const oend = (BYTE*)frame->dataEnd; - BYTE* op = (BYTE*)frame->data; - BYTE* seqHead; - BYTE scratchBuffer[1<>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; - else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; - - /* seqHead : flags for FSE encoding type */ - seqHead = op++; - - if (nbSeq==0) { - frame->data = op; - - return 0; - } - - /* convert length/distances into codes */ - ZSTD_seqToCodes(seqStorePtr); - - /* CTable for Literal Lengths */ - { U32 max = MaxLL; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, WKSP); - if (mostFrequent == nbSeq) { - /* do RLE if we have the chance */ - *op++ = llCodeTable[0]; - FSE_buildCTable_rle(CTable_LitLength, (BYTE)max); - LLtype = set_rle; - } else if (frame->stats.fseInit && !(RAND(seed) & 3) && - isSymbolSubset(llCodeTable, nbSeq, - frame->stats.litlengthSymbolSet, 35)) { - /* maybe do repeat mode if we're allowed to */ - LLtype = set_repeat; - } else if (!(RAND(seed) & 3)) { - /* maybe use the default distribution */ - FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); - LLtype = set_basic; - } else { - /* fall back on a full table */ - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); - if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); - { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) return ERROR(GENERIC); - op += NCountSize; } - FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); - LLtype = set_compressed; - } } - - /* CTable for Offsets */ - /* see Literal Lengths for descriptions of mode choices */ - { U32 max = MaxOff; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, WKSP); - if (mostFrequent == nbSeq) { - *op++ = ofCodeTable[0]; - FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max); - Offtype = set_rle; - } else if (frame->stats.fseInit && !(RAND(seed) & 3) && - isSymbolSubset(ofCodeTable, nbSeq, - frame->stats.offsetSymbolSet, 28)) { - Offtype = set_repeat; - } else if (!(RAND(seed) & 3)) { - FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, DefaultMaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); - Offtype = set_basic; - } else { - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); - if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); - { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) return ERROR(GENERIC); - op += NCountSize; } - FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); - Offtype = set_compressed; - } } - - /* CTable for MatchLengths */ - /* see Literal Lengths for descriptions of mode choices */ - { U32 max = MaxML; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, WKSP); - if (mostFrequent == nbSeq) { - *op++ = *mlCodeTable; - FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max); - MLtype = set_rle; - } else if (frame->stats.fseInit && !(RAND(seed) & 3) && - isSymbolSubset(mlCodeTable, nbSeq, - frame->stats.matchlengthSymbolSet, 52)) { - MLtype = set_repeat; - } else if (!(RAND(seed) & 3)) { - /* sometimes do default distribution */ - FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); - MLtype = set_basic; - } else { - /* fall back on table */ - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); - if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); - { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) return ERROR(GENERIC); - op += NCountSize; } - FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); - MLtype = set_compressed; - } } - frame->stats.fseInit = 1; - initSymbolSet(llCodeTable, nbSeq, frame->stats.litlengthSymbolSet, 35); - initSymbolSet(ofCodeTable, nbSeq, frame->stats.offsetSymbolSet, 28); - initSymbolSet(mlCodeTable, nbSeq, frame->stats.matchlengthSymbolSet, 52); - - DISPLAYLEVEL(5, " LL type: %d OF type: %d ML type: %d\n", LLtype, Offtype, MLtype); - - *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); - - /* Encoding Sequences */ - { BIT_CStream_t blockStream; - FSE_CState_t stateMatchLength; - FSE_CState_t stateOffsetBits; - FSE_CState_t stateLitLength; - - CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */ - - /* first symbols */ - FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); - FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); - FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); - BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); - BIT_flushBits(&blockStream); - - { size_t n; - for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) - BIT_flushBits(&blockStream); /* (7)*/ - BIT_addBits(&blockStream, sequences[n].litLength, llBits); - if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); - if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/ - BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ - BIT_flushBits(&blockStream); /* (7)*/ - } } - - FSE_flushCState(&blockStream, &stateMatchLength); - FSE_flushCState(&blockStream, &stateOffsetBits); - FSE_flushCState(&blockStream, &stateLitLength); - - { size_t const streamSize = BIT_closeCStream(&blockStream); - if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ - op += streamSize; - } } - - frame->data = op; - - return 0; -} - -static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize, - size_t literalsSize, dictInfo info) -{ - seqStore_t seqStore; - size_t numSequences; - - - initSeqStore(&seqStore); - - /* randomly generate sequences */ - numSequences = generateSequences(seed, frame, &seqStore, contentSize, literalsSize, info); - /* write them out to the frame data */ - CHECKERR(writeSequences(seed, frame, &seqStore, numSequences)); - - return numSequences; -} - -static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize, dictInfo info) -{ - BYTE* const blockStart = (BYTE*)frame->data; - size_t literalsSize; - size_t nbSeq; - - DISPLAYLEVEL(4, " compressed block:\n"); - - literalsSize = writeLiteralsBlock(seed, frame, contentSize); - - DISPLAYLEVEL(4, " literals size: %u\n", (U32)literalsSize); - - nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize, info); - - DISPLAYLEVEL(4, " number of sequences: %u\n", (U32)nbSeq); - - return (BYTE*)frame->data - blockStart; -} - -static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, - int lastBlock, dictInfo info) -{ - int const blockTypeDesc = RAND(seed) % 8; - size_t blockSize; - int blockType; - - BYTE *const header = (BYTE*)frame->data; - BYTE *op = header + 3; - - DISPLAYLEVEL(4, " block:\n"); - DISPLAYLEVEL(4, " block content size: %u\n", (U32)contentSize); - DISPLAYLEVEL(4, " last block: %s\n", lastBlock ? "yes" : "no"); - - if (blockTypeDesc == 0) { - /* Raw data frame */ - - RAND_buffer(seed, frame->src, contentSize); - memcpy(op, frame->src, contentSize); - - op += contentSize; - blockType = 0; - blockSize = contentSize; - } else if (blockTypeDesc == 1) { - /* RLE */ - BYTE const symbol = RAND(seed) & 0xff; - - op[0] = symbol; - memset(frame->src, symbol, contentSize); - - op++; - blockType = 1; - blockSize = contentSize; - } else { - /* compressed, most common */ - size_t compressedSize; - blockType = 2; - - frame->oldStats = frame->stats; - - frame->data = op; - compressedSize = writeCompressedBlock(seed, frame, contentSize, info); - if (compressedSize >= contentSize) { /* compressed block must be strictly smaller than uncompressed one */ - blockType = 0; - memcpy(op, frame->src, contentSize); - - op += contentSize; - blockSize = contentSize; /* fall back on raw block if data doesn't - compress */ - - frame->stats = frame->oldStats; /* don't update the stats */ - } else { - op += compressedSize; - blockSize = compressedSize; - } - } - frame->src = (BYTE*)frame->src + contentSize; - - DISPLAYLEVEL(4, " block type: %s\n", BLOCK_TYPES[blockType]); - DISPLAYLEVEL(4, " block size field: %u\n", (U32)blockSize); - - header[0] = (BYTE) ((lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff); - MEM_writeLE16(header + 1, (U16) (blockSize >> 5)); - - frame->data = op; -} - -static void writeBlocks(U32* seed, frame_t* frame, dictInfo info) -{ - size_t contentLeft = frame->header.contentSize; - size_t const maxBlockSize = MIN(g_maxBlockSize, frame->header.windowSize); - while (1) { - /* 1 in 4 chance of ending frame */ - int const lastBlock = contentLeft > maxBlockSize ? 0 : !(RAND(seed) & 3); - size_t blockContentSize; - if (lastBlock) { - blockContentSize = contentLeft; - } else { - if (contentLeft > 0 && (RAND(seed) & 7)) { - /* some variable size block */ - blockContentSize = RAND(seed) % (MIN(maxBlockSize, contentLeft)+1); - } else if (contentLeft > maxBlockSize && (RAND(seed) & 1)) { - /* some full size block */ - blockContentSize = maxBlockSize; - } else { - /* some empty block */ - blockContentSize = 0; - } - } - - writeBlock(seed, frame, blockContentSize, lastBlock, info); - - contentLeft -= blockContentSize; - if (lastBlock) break; - } -} - -static void writeChecksum(frame_t* frame) -{ - /* write checksum so implementations can verify their output */ - U64 digest = XXH64(frame->srcStart, (BYTE*)frame->src-(BYTE*)frame->srcStart, 0); - DISPLAYLEVEL(3, " checksum: %08x\n", (U32)digest); - MEM_writeLE32(frame->data, (U32)digest); - frame->data = (BYTE*)frame->data + 4; -} - -static void outputBuffer(const void* buf, size_t size, const char* const path) -{ - /* write data out to file */ - const BYTE* ip = (const BYTE*)buf; - FILE* out; - if (path) { - out = fopen(path, "wb"); - } else { - out = stdout; - } - if (!out) { - fprintf(stderr, "Failed to open file at %s: ", path); - perror(NULL); - exit(1); - } - - { size_t fsize = size; - size_t written = 0; - while (written < fsize) { - written += fwrite(ip + written, 1, fsize - written, out); - if (ferror(out)) { - fprintf(stderr, "Failed to write to file at %s: ", path); - perror(NULL); - exit(1); - } - } - } - - if (path) { - fclose(out); - } -} - -static void initFrame(frame_t* fr) -{ - memset(fr, 0, sizeof(*fr)); - fr->data = fr->dataStart = FRAME_BUFFER; - fr->dataEnd = FRAME_BUFFER + sizeof(FRAME_BUFFER); - fr->src = fr->srcStart = CONTENT_BUFFER; - fr->srcEnd = CONTENT_BUFFER + sizeof(CONTENT_BUFFER); - - /* init repeat codes */ - fr->stats.rep[0] = 1; - fr->stats.rep[1] = 4; - fr->stats.rep[2] = 8; -} - -/** - * Generated a single zstd compressed block with no block/frame header. - * Returns the final seed. - */ -static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info) -{ - size_t blockContentSize; - int blockWritten = 0; - BYTE* op; - DISPLAYLEVEL(4, "block seed: %u\n", seed); - initFrame(frame); - op = (BYTE*)frame->data; - - while (!blockWritten) { - size_t cSize; - /* generate window size */ - { int const exponent = RAND(&seed) % (MAX_WINDOW_LOG - 10); - int const mantissa = RAND(&seed) % 8; - frame->header.windowSize = (1U << (exponent + 10)); - frame->header.windowSize += (frame->header.windowSize / 8) * mantissa; - } - - /* generate content size */ - { size_t const maxBlockSize = MIN(g_maxBlockSize, frame->header.windowSize); - if (RAND(&seed) & 15) { - /* some full size blocks */ - blockContentSize = maxBlockSize; - } else if (RAND(&seed) & 7 && g_maxBlockSize >= (1U << 7)) { - /* some small blocks <= 128 bytes*/ - blockContentSize = RAND(&seed) % (1U << 7); - } else { - /* some variable size blocks */ - blockContentSize = RAND(&seed) % maxBlockSize; - } - } - - /* try generating a compressed block */ - frame->oldStats = frame->stats; - frame->data = op; - cSize = writeCompressedBlock(&seed, frame, blockContentSize, info); - if (cSize >= blockContentSize) { /* compressed size must be strictly smaller than decompressed size : https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#blocks */ - /* data doesn't compress -- try again */ - frame->stats = frame->oldStats; /* don't update the stats */ - DISPLAYLEVEL(5, " can't compress block : try again \n"); - } else { - blockWritten = 1; - DISPLAYLEVEL(4, " block size: %u \n", (U32)cSize); - frame->src = (BYTE*)frame->src + blockContentSize; - } - } - return seed; -} - -/* Return the final seed */ -static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info) -{ - /* generate a complete frame */ - DISPLAYLEVEL(3, "frame seed: %u\n", seed); - initFrame(fr); - - writeFrameHeader(&seed, fr, info); - writeBlocks(&seed, fr, info); - writeChecksum(fr); - - return seed; -} - -/*_******************************************************* -* Dictionary Helper Functions -*********************************************************/ -/* returns 0 if successful, otherwise returns 1 upon error */ -static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict) -{ - /* allocate space for samples */ - int ret = 0; - unsigned const numSamples = 4; - size_t sampleSizes[4]; - BYTE* const samples = malloc(5000*sizeof(BYTE)); - if (samples == NULL) { - DISPLAY("Error: could not allocate space for samples\n"); - return 1; - } - - /* generate samples */ - { unsigned literalValue = 1; - unsigned samplesPos = 0; - size_t currSize = 1; - while (literalValue <= 4) { - sampleSizes[literalValue - 1] = currSize; - { size_t k; - for (k = 0; k < currSize; k++) { - *(samples + (samplesPos++)) = (BYTE)literalValue; - } } - literalValue++; - currSize *= 16; - } } - - { size_t dictWriteSize = 0; - ZDICT_params_t zdictParams; - size_t const headerSize = MAX(dictSize/4, 256); - size_t const dictContentSize = dictSize - headerSize; - BYTE* const dictContent = fullDict + headerSize; - if (dictContentSize < ZDICT_CONTENTSIZE_MIN || dictSize < ZDICT_DICTSIZE_MIN) { - DISPLAY("Error: dictionary size is too small\n"); - ret = 1; - goto exitGenRandomDict; - } - - /* init dictionary params */ - memset(&zdictParams, 0, sizeof(zdictParams)); - zdictParams.dictID = dictID; - zdictParams.notificationLevel = 1; - - /* fill in dictionary content */ - RAND_buffer(&seed, (void*)dictContent, dictContentSize); - - /* finalize dictionary with random samples */ - dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, - dictContent, dictContentSize, - samples, sampleSizes, numSamples, - zdictParams); - - if (ZDICT_isError(dictWriteSize)) { - DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); - ret = 1; - } - } - -exitGenRandomDict: - free(samples); - return ret; -} - -static dictInfo initDictInfo(int useDict, size_t dictContentSize, BYTE* dictContent, U32 dictID){ - /* allocate space statically */ - dictInfo dictOp; - memset(&dictOp, 0, sizeof(dictOp)); - dictOp.useDict = useDict; - dictOp.dictContentSize = dictContentSize; - dictOp.dictContent = dictContent; - dictOp.dictID = dictID; - return dictOp; -} - -/*-******************************************************* -* Test Mode -*********************************************************/ - -BYTE DECOMPRESSED_BUFFER[MAX_DECOMPRESSED_SIZE]; - -static size_t testDecodeSimple(frame_t* fr) -{ - /* test decoding the generated data with the simple API */ - size_t const ret = ZSTD_decompress(DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, - fr->dataStart, (BYTE*)fr->data - (BYTE*)fr->dataStart); - - if (ZSTD_isError(ret)) return ret; - - if (memcmp(DECOMPRESSED_BUFFER, fr->srcStart, - (BYTE*)fr->src - (BYTE*)fr->srcStart) != 0) { - return ERROR(corruption_detected); - } - - return ret; -} - -static size_t testDecodeStreaming(frame_t* fr) -{ - /* test decoding the generated data with the streaming API */ - ZSTD_DStream* zd = ZSTD_createDStream(); - ZSTD_inBuffer in; - ZSTD_outBuffer out; - size_t ret; - - if (!zd) return ERROR(memory_allocation); - - in.src = fr->dataStart; - in.pos = 0; - in.size = (BYTE*)fr->data - (BYTE*)fr->dataStart; - - out.dst = DECOMPRESSED_BUFFER; - out.pos = 0; - out.size = ZSTD_DStreamOutSize(); - - ZSTD_initDStream(zd); - while (1) { - ret = ZSTD_decompressStream(zd, &out, &in); - if (ZSTD_isError(ret)) goto cleanup; /* error */ - if (ret == 0) break; /* frame is done */ - - /* force decoding to be done in chunks */ - out.size += MIN(ZSTD_DStreamOutSize(), MAX_DECOMPRESSED_SIZE - out.size); - } - - ret = out.pos; - - if (memcmp(out.dst, fr->srcStart, out.pos) != 0) { - return ERROR(corruption_detected); - } - -cleanup: - ZSTD_freeDStream(zd); - return ret; -} - -static size_t testDecodeWithDict(U32 seed, genType_e genType) -{ - /* create variables */ - size_t const dictSize = RAND(&seed) % (10 << 20) + ZDICT_DICTSIZE_MIN + ZDICT_CONTENTSIZE_MIN; - U32 const dictID = RAND(&seed); - size_t errorDetected = 0; - BYTE* const fullDict = malloc(dictSize); - if (fullDict == NULL) { - return ERROR(GENERIC); - } - - /* generate random dictionary */ - if (genRandomDict(dictID, seed, dictSize, fullDict)) { /* return 0 on success */ - errorDetected = ERROR(GENERIC); - goto dictTestCleanup; - } - - - { frame_t fr; - dictInfo info; - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - size_t ret; - - /* get dict info */ - { size_t const headerSize = MAX(dictSize/4, 256); - size_t const dictContentSize = dictSize-headerSize; - BYTE* const dictContent = fullDict+headerSize; - info = initDictInfo(1, dictContentSize, dictContent, dictID); - } - - /* manually decompress and check difference */ - if (genType == gt_frame) { - /* Test frame */ - generateFrame(seed, &fr, info); - ret = ZSTD_decompress_usingDict(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, - fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, - fullDict, dictSize); - } else { - /* Test block */ - generateCompressedBlock(seed, &fr, info); - ret = ZSTD_decompressBegin_usingDict(dctx, fullDict, dictSize); - if (ZSTD_isError(ret)) { - errorDetected = ret; - ZSTD_freeDCtx(dctx); - goto dictTestCleanup; - } - ret = ZSTD_decompressBlock(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, - fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart); - } - ZSTD_freeDCtx(dctx); - - if (ZSTD_isError(ret)) { - errorDetected = ret; - goto dictTestCleanup; - } - - if (memcmp(DECOMPRESSED_BUFFER, fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart) != 0) { - errorDetected = ERROR(corruption_detected); - goto dictTestCleanup; - } - } - -dictTestCleanup: - free(fullDict); - return errorDetected; -} - -static size_t testDecodeRawBlock(frame_t* fr) -{ - ZSTD_DCtx* dctx = ZSTD_createDCtx(); - size_t ret = ZSTD_decompressBegin(dctx); - if (ZSTD_isError(ret)) return ret; - - ret = ZSTD_decompressBlock( - dctx, - DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, - fr->dataStart, (BYTE*)fr->data - (BYTE*)fr->dataStart); - ZSTD_freeDCtx(dctx); - if (ZSTD_isError(ret)) return ret; - - if (memcmp(DECOMPRESSED_BUFFER, fr->srcStart, - (BYTE*)fr->src - (BYTE*)fr->srcStart) != 0) { - return ERROR(corruption_detected); - } - - return ret; -} - -static int runBlockTest(U32* seed) -{ - frame_t fr; - U32 const seedCopy = *seed; - { dictInfo const info = initDictInfo(0, 0, NULL, 0); - *seed = generateCompressedBlock(*seed, &fr, info); - } - - { size_t const r = testDecodeRawBlock(&fr); - if (ZSTD_isError(r)) { - DISPLAY("Error in block mode on test seed %u: %s\n", seedCopy, - ZSTD_getErrorName(r)); - return 1; - } - } - - { size_t const r = testDecodeWithDict(*seed, gt_block); - if (ZSTD_isError(r)) { - DISPLAY("Error in block mode with dictionary on test seed %u: %s\n", - seedCopy, ZSTD_getErrorName(r)); - return 1; - } - } - return 0; -} - -static int runFrameTest(U32* seed) -{ - frame_t fr; - U32 const seedCopy = *seed; - { dictInfo const info = initDictInfo(0, 0, NULL, 0); - *seed = generateFrame(*seed, &fr, info); - } - - { size_t const r = testDecodeSimple(&fr); - if (ZSTD_isError(r)) { - DISPLAY("Error in simple mode on test seed %u: %s\n", - seedCopy, ZSTD_getErrorName(r)); - return 1; - } - } - { size_t const r = testDecodeStreaming(&fr); - if (ZSTD_isError(r)) { - DISPLAY("Error in streaming mode on test seed %u: %s\n", - seedCopy, ZSTD_getErrorName(r)); - return 1; - } - } - { size_t const r = testDecodeWithDict(*seed, gt_frame); /* avoid big dictionaries */ - if (ZSTD_isError(r)) { - DISPLAY("Error in dictionary mode on test seed %u: %s\n", - seedCopy, ZSTD_getErrorName(r)); - return 1; - } - } - return 0; -} - -static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS, - genType_e genType) -{ - unsigned fnum; - - UTIL_time_t const startClock = UTIL_getTime(); - U64 const maxClockSpan = testDurationS * SEC_TO_MICRO; - - if (numFiles == 0 && !testDurationS) numFiles = 1; - - DISPLAY("seed: %u\n", seed); - - for (fnum = 0; fnum < numFiles || UTIL_clockSpanMicro(startClock) < maxClockSpan; fnum++) { - if (fnum < numFiles) - DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - else - DISPLAYUPDATE("\r%u ", fnum); - - { int const ret = (genType == gt_frame) ? - runFrameTest(&seed) : - runBlockTest(&seed); - if (ret) return ret; - } - } - - DISPLAY("\r%u tests completed: ", fnum); - DISPLAY("OK\n"); - - return 0; -} - -/*-******************************************************* -* File I/O -*********************************************************/ - -static int generateFile(U32 seed, const char* const path, - const char* const origPath, genType_e genType) -{ - frame_t fr; - - DISPLAY("seed: %u\n", seed); - - { dictInfo const info = initDictInfo(0, 0, NULL, 0); - if (genType == gt_frame) { - generateFrame(seed, &fr, info); - } else { - generateCompressedBlock(seed, &fr, info); - } - } - outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path); - if (origPath) { - outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, origPath); - } - return 0; -} - -static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, - const char* const origPath, genType_e genType) -{ - char outPath[MAX_PATH]; - unsigned fnum; - - DISPLAY("seed: %u\n", seed); - - for (fnum = 0; fnum < numFiles; fnum++) { - frame_t fr; - - DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - - { dictInfo const info = initDictInfo(0, 0, NULL, 0); - if (genType == gt_frame) { - seed = generateFrame(seed, &fr, info); - } else { - seed = generateCompressedBlock(seed, &fr, info); - } - } - - if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { - DISPLAY("Error: path too long\n"); - return 1; - } - outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); - - if (origPath) { - if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { - DISPLAY("Error: path too long\n"); - return 1; - } - outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); - } - } - - DISPLAY("\r%u/%u \n", fnum, numFiles); - - return 0; -} - -static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const path, - const char* const origPath, const size_t dictSize, - genType_e genType) -{ - char outPath[MAX_PATH]; - BYTE* fullDict; - U32 const dictID = RAND(&seed); - int errorDetected = 0; - - if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { - DISPLAY("Error: path too long\n"); - return 1; - } - - /* allocate space for the dictionary */ - fullDict = malloc(dictSize); - if (fullDict == NULL) { - DISPLAY("Error: could not allocate space for full dictionary.\n"); - return 1; - } - - /* randomly generate the dictionary */ - { int const ret = genRandomDict(dictID, seed, dictSize, fullDict); - if (ret != 0) { - errorDetected = ret; - goto dictCleanup; - } - } - - /* write out dictionary */ - if (numFiles != 0) { - if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { - DISPLAY("Error: dictionary path too long\n"); - errorDetected = 1; - goto dictCleanup; - } - outputBuffer(fullDict, dictSize, outPath); - } - else { - outputBuffer(fullDict, dictSize, "dictionary"); - } - - /* generate random compressed/decompressed files */ - { unsigned fnum; - for (fnum = 0; fnum < MAX(numFiles, 1); fnum++) { - frame_t fr; - DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - { - size_t const headerSize = MAX(dictSize/4, 256); - size_t const dictContentSize = dictSize-headerSize; - BYTE* const dictContent = fullDict+headerSize; - dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); - if (genType == gt_frame) { - seed = generateFrame(seed, &fr, info); - } else { - seed = generateCompressedBlock(seed, &fr, info); - } - } - - if (numFiles != 0) { - if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { - DISPLAY("Error: path too long\n"); - errorDetected = 1; - goto dictCleanup; - } - outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); - - if (origPath) { - if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { - DISPLAY("Error: path too long\n"); - errorDetected = 1; - goto dictCleanup; - } - outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); - } - } - else { - outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path); - if (origPath) { - outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, origPath); - } - } - } - } - -dictCleanup: - free(fullDict); - return errorDetected; -} - - -/*_******************************************************* -* Command line -*********************************************************/ -static U32 makeSeed(void) -{ - U32 t = (U32) time(NULL); - return XXH32(&t, sizeof(t), 0) % 65536; -} - -static unsigned readInt(const char** argument) -{ - unsigned val = 0; - while ((**argument>='0') && (**argument<='9')) { - val *= 10; - val += **argument - '0'; - (*argument)++; - } - return val; -} - -static void usage(const char* programName) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -p : select output path (default:stdout)\n"); - DISPLAY( " in multiple files mode this should be a directory\n"); - DISPLAY( " -o : select path to output original file (default:no output)\n"); - DISPLAY( " in multiple files mode this should be a directory\n"); - DISPLAY( " -s# : select seed (default:random based on time)\n"); - DISPLAY( " -n# : number of files to generate (default:1)\n"); - DISPLAY( " -t : activate test mode (test files against libzstd instead of outputting them)\n"); - DISPLAY( " -T# : length of time to run tests for\n"); - DISPLAY( " -v : increase verbosity level (default:0, max:7)\n"); - DISPLAY( " -h/H : display help/long help and exit\n"); -} - -static void advancedUsage(const char* programName) -{ - usage(programName); - DISPLAY( "\n"); - DISPLAY( "Advanced arguments :\n"); - DISPLAY( " --content-size : always include the content size in the frame header\n"); - DISPLAY( " --use-dict=# : include a dictionary used to decompress the corpus\n"); - DISPLAY( " --gen-blocks : generate raw compressed blocks without block/frame headers\n"); - DISPLAY( " --max-block-size-log=# : max block size log, must be in range [2, 17]\n"); - DISPLAY( " --max-content-size-log=# : max content size log, must be <= 20\n"); - DISPLAY( " (this is ignored with gen-blocks)\n"); -} - -/*! readU32FromChar() : - @return : unsigned integer value read from input in `char` format - allows and interprets K, KB, KiB, M, MB and MiB suffix. - Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : function result can overflow if digit string > MAX_UINT */ -static unsigned readU32FromChar(const char** stringPtr) -{ - unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; - if ((**stringPtr=='K') || (**stringPtr=='M')) { - result <<= 10; - if (**stringPtr=='M') result <<= 10; - (*stringPtr)++ ; - if (**stringPtr=='i') (*stringPtr)++; - if (**stringPtr=='B') (*stringPtr)++; - } - return result; -} - -/** longCommandWArg() : - * check if *stringPtr is the same as longCommand. - * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. - * @return 0 and doesn't modify *stringPtr otherwise. - */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) -{ - size_t const comSize = strlen(longCommand); - int const result = !strncmp(*stringPtr, longCommand, comSize); - if (result) *stringPtr += comSize; - return result; -} - -int main(int argc, char** argv) -{ - U32 seed = 0; - int seedset = 0; - unsigned numFiles = 0; - unsigned testDuration = 0; - int testMode = 0; - const char* path = NULL; - const char* origPath = NULL; - int useDict = 0; - unsigned dictSize = (10 << 10); /* 10 kB default */ - genType_e genType = gt_frame; - - int argNb; - - /* Check command line */ - for (argNb=1; argNbVi8VgP*Qj(lfvmE1c0RaG879=~%<1Ug5Gim?; diff --git a/sys/contrib/zstd/tests/fullbench.c b/sys/contrib/zstd/tests/fullbench.c deleted file mode 100644 index 6abdd4da00f..00000000000 --- a/sys/contrib/zstd/tests/fullbench.c +++ /dev/null @@ -1,659 +0,0 @@ -/* - * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -/*_************************************ -* Includes -**************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize */ -#include /* malloc */ -#include /* fprintf, fopen, ftello64 */ -#include /* assert */ - -#include "mem.h" /* U32 */ -#ifndef ZSTD_DLL_IMPORT - #include "zstd_internal.h" /* ZSTD_blockHeaderSize, blockType_e, KB, MB */ -#else - #define KB *(1 <<10) - #define MB *(1 <<20) - #define GB *(1U<<30) - typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; -#endif -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressBegin, ZSTD_compressContinue, etc. */ -#include "zstd.h" /* ZSTD_versionString */ -#include "util.h" /* time functions */ -#include "datagen.h" - - -/*_************************************ -* Constants -**************************************/ -#define PROGRAM_DESCRIPTION "Zstandard speed analyzer" -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_versionString(), (int)(sizeof(void*)*8), AUTHOR, __DATE__ - -#define NBLOOPS 6 -#define TIMELOOP_S 2 - -#define KNUTH 2654435761U -#define MAX_MEM (1984 MB) - -#define COMPRESSIBILITY_DEFAULT 0.50 -static const size_t g_sampleSize = 10000000; - - -/*_************************************ -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) - - -/*_************************************ -* Benchmark Parameters -**************************************/ -static U32 g_nbIterations = NBLOOPS; -static double g_compressibility = COMPRESSIBILITY_DEFAULT; - -static void BMK_SetNbIterations(U32 nbLoops) -{ - g_nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", g_nbIterations); -} - - -/*_******************************************************* -* Private functions -*********************************************************/ -static size_t BMK_findMaxMem(U64 requiredMem) -{ - size_t const step = 64 MB; - void* testmem = NULL; - - requiredMem = (((requiredMem >> 26) + 1) << 26); - if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; - - requiredMem += step; - do { - testmem = malloc ((size_t)requiredMem); - requiredMem -= step; - } while (!testmem); - - free (testmem); - return (size_t) requiredMem; -} - - -/*_******************************************************* -* Benchmark wrappers -*********************************************************/ -size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) -{ - (void)buff2; - return ZSTD_compress(dst, dstSize, src, srcSize, 1); -} - -static size_t g_cSize = 0; -size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) -{ - (void)src; (void)srcSize; - return ZSTD_decompress(dst, dstSize, buff2, g_cSize); -} - -static ZSTD_DCtx* g_zdc = NULL; - -#ifndef ZSTD_DLL_IMPORT -extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize); -size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) -{ - (void)src; (void)srcSize; (void)dst; (void)dstSize; - return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize); -} - -extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize); -size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) -{ - int nbSeq; - (void)src; (void)srcSize; (void)dst; (void)dstSize; - return ZSTD_decodeSeqHeaders(g_zdc, &nbSeq, buff2, g_cSize); -} -#endif - -static ZSTD_CStream* g_cstream= NULL; -size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) -{ - ZSTD_outBuffer buffOut; - ZSTD_inBuffer buffIn; - (void)buff2; - ZSTD_initCStream(g_cstream, 1); - buffOut.dst = dst; - buffOut.size = dstCapacity; - buffOut.pos = 0; - buffIn.src = src; - buffIn.size = srcSize; - buffIn.pos = 0; - ZSTD_compressStream(g_cstream, &buffOut, &buffIn); - ZSTD_endStream(g_cstream, &buffOut); - return buffOut.pos; -} - -static size_t local_ZSTD_compress_generic_end(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) -{ - ZSTD_outBuffer buffOut; - ZSTD_inBuffer buffIn; - (void)buff2; - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1); - buffOut.dst = dst; - buffOut.size = dstCapacity; - buffOut.pos = 0; - buffIn.src = src; - buffIn.size = srcSize; - buffIn.pos = 0; - ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end); - return buffOut.pos; -} - -static size_t local_ZSTD_compress_generic_continue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) -{ - ZSTD_outBuffer buffOut; - ZSTD_inBuffer buffIn; - (void)buff2; - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1); - buffOut.dst = dst; - buffOut.size = dstCapacity; - buffOut.pos = 0; - buffIn.src = src; - buffIn.size = srcSize; - buffIn.pos = 0; - ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue); - ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end); - return buffOut.pos; -} - -static size_t local_ZSTD_compress_generic_T2_end(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) -{ - ZSTD_outBuffer buffOut; - ZSTD_inBuffer buffIn; - (void)buff2; - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2); - buffOut.dst = dst; - buffOut.size = dstCapacity; - buffOut.pos = 0; - buffIn.src = src; - buffIn.size = srcSize; - buffIn.pos = 0; - while (ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {} - return buffOut.pos; -} - -static size_t local_ZSTD_compress_generic_T2_continue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) -{ - ZSTD_outBuffer buffOut; - ZSTD_inBuffer buffIn; - (void)buff2; - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2); - buffOut.dst = dst; - buffOut.size = dstCapacity; - buffOut.pos = 0; - buffIn.src = src; - buffIn.size = srcSize; - buffIn.pos = 0; - ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue); - while(ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {} - return buffOut.pos; -} - -static ZSTD_DStream* g_dstream= NULL; -static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) -{ - ZSTD_outBuffer buffOut; - ZSTD_inBuffer buffIn; - (void)src; (void)srcSize; - ZSTD_initDStream(g_dstream); - buffOut.dst = dst; - buffOut.size = dstCapacity; - buffOut.pos = 0; - buffIn.src = buff2; - buffIn.size = g_cSize; - buffIn.pos = 0; - ZSTD_decompressStream(g_dstream, &buffOut, &buffIn); - return buffOut.pos; -} - -static ZSTD_CCtx* g_zcc = NULL; - -#ifndef ZSTD_DLL_IMPORT -size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) -{ - (void)buff2; - ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */); - return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize); -} - -#define FIRST_BLOCK_SIZE 8 -size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) -{ - BYTE firstBlockBuf[FIRST_BLOCK_SIZE]; - - (void)buff2; - memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE); - ZSTD_compressBegin(g_zcc, 1); - - { size_t const compressResult = ZSTD_compressContinue(g_zcc, dst, dstCapacity, firstBlockBuf, FIRST_BLOCK_SIZE); - if (ZSTD_isError(compressResult)) { DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", ZSTD_getErrorName(compressResult)); return compressResult; } - dst = (BYTE*)dst + compressResult; - dstCapacity -= compressResult; - } - return ZSTD_compressEnd(g_zcc, dst, dstCapacity, (const BYTE*)src + FIRST_BLOCK_SIZE, srcSize - FIRST_BLOCK_SIZE); -} - -size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) -{ - size_t regeneratedSize = 0; - const BYTE* ip = (const BYTE*)buff2; - const BYTE* const iend = ip + g_cSize; - BYTE* op = (BYTE*)dst; - size_t remainingCapacity = dstCapacity; - - (void)src; (void)srcSize; - ZSTD_decompressBegin(g_zdc); - while (ip < iend) { - size_t const iSize = ZSTD_nextSrcSizeToDecompress(g_zdc); - size_t const decodedSize = ZSTD_decompressContinue(g_zdc, op, remainingCapacity, ip, iSize); - ip += iSize; - regeneratedSize += decodedSize; - op += decodedSize; - remainingCapacity -= decodedSize; - } - - return regeneratedSize; -} -#endif - - -/*_******************************************************* -* Bench functions -*********************************************************/ -static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) -{ - BYTE* dstBuff; - size_t const dstBuffSize = ZSTD_compressBound(srcSize); - void* buff2; - const char* benchName; - size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize); - double bestTime = 100000000.; - - /* Selection */ - switch(benchNb) - { - case 1: - benchFunction = local_ZSTD_compress; benchName = "compress(1)"; - break; - case 2: - benchFunction = local_ZSTD_decompress; benchName = "decompress"; - break; -#ifndef ZSTD_DLL_IMPORT - case 11: - benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue(1)"; - break; - case 12: - benchFunction = local_ZSTD_compressContinue_extDict; benchName = "compressContinue_extDict"; - break; - case 13: - benchFunction = local_ZSTD_decompressContinue; benchName = "decompressContinue"; - break; - case 31: - benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "decodeLiteralsBlock"; - break; - case 32: - benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "decodeSeqHeaders"; - break; -#endif - case 41: - benchFunction = local_ZSTD_compressStream; benchName = "compressStream(1)"; - break; - case 42: - benchFunction = local_ZSTD_decompressStream; benchName = "decompressStream"; - break; - case 51: - benchFunction = local_ZSTD_compress_generic_continue; benchName = "compress_generic, continue"; - break; - case 52: - benchFunction = local_ZSTD_compress_generic_end; benchName = "compress_generic, end"; - break; - case 61: - benchFunction = local_ZSTD_compress_generic_T2_continue; benchName = "compress_generic, -T2, continue"; - break; - case 62: - benchFunction = local_ZSTD_compress_generic_T2_end; benchName = "compress_generic, -T2, end"; - break; - default : - return 0; - } - - /* Allocation */ - dstBuff = (BYTE*)malloc(dstBuffSize); - buff2 = malloc(dstBuffSize); - if ((!dstBuff) || (!buff2)) { - DISPLAY("\nError: not enough memory!\n"); - free(dstBuff); free(buff2); - return 12; - } - if (g_zcc==NULL) g_zcc = ZSTD_createCCtx(); - if (g_zdc==NULL) g_zdc = ZSTD_createDCtx(); - if (g_cstream==NULL) g_cstream = ZSTD_createCStream(); - if (g_dstream==NULL) g_dstream = ZSTD_createDStream(); - - /* Preparation */ - switch(benchNb) - { - case 2: - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); - break; -#ifndef ZSTD_DLL_IMPORT - case 13 : - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); - break; - case 31: /* ZSTD_decodeLiteralsBlock */ - { blockProperties_t bp; - ZSTD_frameHeader zfp; - size_t frameHeaderSize, skippedSize; - g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); - frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min); - if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; - ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */ - if (bp.blockType != bt_compressed) { - DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); - goto _cleanOut; - } - skippedSize = frameHeaderSize + ZSTD_blockHeaderSize; - memcpy(buff2, dstBuff+skippedSize, g_cSize-skippedSize); - srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ - ZSTD_decompressBegin(g_zdc); - break; - } - case 32: /* ZSTD_decodeSeqHeaders */ - { blockProperties_t bp; - ZSTD_frameHeader zfp; - const BYTE* ip = dstBuff; - const BYTE* iend; - size_t frameHeaderSize, cBlockSize; - ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */ - g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); - frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min); - if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; - ip += frameHeaderSize; /* Skip frame Header */ - cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ - if (bp.blockType != bt_compressed) { - DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); - goto _cleanOut; - } - iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */ - ip += ZSTD_blockHeaderSize; /* skip block header */ - ZSTD_decompressBegin(g_zdc); - ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */ - g_cSize = iend-ip; - memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ - srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ - break; - } -#else - case 31: - goto _cleanOut; -#endif - case 42 : - g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); - break; - - /* test functions */ - /* convention: test functions have ID > 100 */ - - default : ; - } - - /* warming up memory */ - { size_t i; for (i=0; i %s !! \n", benchName, ZSTD_getErrorName(benchResult)); - exit(1); - } } - { U64 const clockSpanNano = UTIL_clockSpanNano(clockStart); - double const averageTime = (double)clockSpanNano / TIME_SEC_NANOSEC / nbRounds; - if (clockSpanNano > 0) { - if (averageTime < bestTime) bestTime = averageTime; - assert(bestTime > (1./2000000000)); - nbRounds = (U32)(1. / bestTime); /* aim for 1 sec */ - DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", - loopNb, benchName, - (double)srcSize / (1 MB) / bestTime, - (U32)benchResult); - } else { - assert(nbRounds < 40000000); /* avoid overflow */ - nbRounds *= 100; - } - } } } - DISPLAY("%2u\n", benchNb); - -_cleanOut: - free(dstBuff); - free(buff2); - ZSTD_freeCCtx(g_zcc); g_zcc=NULL; - ZSTD_freeDCtx(g_zdc); g_zdc=NULL; - ZSTD_freeCStream(g_cstream); g_cstream=NULL; - ZSTD_freeDStream(g_dstream); g_dstream=NULL; - return 0; -} - - -static int benchSample(U32 benchNb) -{ - size_t const benchedSize = g_sampleSize; - const char* name = "Sample 10MiB"; - - /* Allocation */ - void* origBuff = malloc(benchedSize); - if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; } - - /* Fill buffer */ - RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY(" %s : \n", name); - if (benchNb) - benchMem(origBuff, benchedSize, benchNb); - else - for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb); - - free(origBuff); - return 0; -} - - -static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb) -{ - /* Loop for each file */ - int fileIdx; - for (fileIdx=0; fileIdx inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %u MB only...\n", inFileName, (U32)(benchedSize>>20)); - - /* Alloc */ - origBuff = malloc(benchedSize); - if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; } - - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { - size_t readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if (readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } } - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY(" %s : \n", inFileName); - if (benchNb) - benchMem(origBuff, benchedSize, benchNb); - else - for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb); - - free(origBuff); - } - - return 0; -} - - -static int usage(const char* exename) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); - DISPLAY( "Arguments :\n"); - DISPLAY( " -H/-h : Help (this text + advanced options)\n"); - return 0; -} - -static int usage_advanced(const char* exename) -{ - usage(exename); - DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -b# : test only function # \n"); - DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); - DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100); - return 0; -} - -static int badusage(const char* exename) -{ - DISPLAY("Wrong parameters\n"); - usage(exename); - return 1; -} - -int main(int argc, const char** argv) -{ - int i, filenamesStart=0, result; - const char* exename = argv[0]; - const char* input_filename = NULL; - U32 benchNb = 0, main_pause = 0; - - DISPLAY(WELCOME_MESSAGE); - if (argc<1) return badusage(exename); - - for(i=1; i= '0') && (argument[1]<= '9')) { - benchNb *= 10; - benchNb += argument[1] - '0'; - argument++; - } - break; - - /* Modify Nb Iterations */ - case 'i': - if ((argument[1] >='0') && (argument[1] <='9')) { - int iters = argument[1] - '0'; - BMK_SetNbIterations(iters); - argument++; - } - break; - - /* Select compressibility of synthetic sample */ - case 'P': - { U32 proba32 = 0; - while ((argument[1]>= '0') && (argument[1]<= '9')) { - proba32 *= 10; - proba32 += argument[1] - '0'; - argument++; - } - g_compressibility = (double)proba32 / 100.; - } - break; - - /* Unknown command */ - default : return badusage(exename); - } - } - continue; - } - - /* first provided filename is input */ - if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } - } - - if (filenamesStart==0) /* no input file */ - result = benchSample(benchNb); - else - result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb); - - if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } - - return result; -} diff --git a/sys/contrib/zstd/tests/fuzz/Makefile b/sys/contrib/zstd/tests/fuzz/Makefile deleted file mode 100644 index d9b00fd2af7..00000000000 --- a/sys/contrib/zstd/tests/fuzz/Makefile +++ /dev/null @@ -1,127 +0,0 @@ -# ################################################################ -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# ################################################################ - -# Optionally user defined flags -CFLAGS ?= -O3 -CXXFLAGS ?= -O3 -CPPFLAGS ?= -LDFLAGS ?= -ARFLAGS ?= -LIB_FUZZING_ENGINE ?= libregression.a -PYTHON ?= python -ifeq ($(shell uname), Darwin) - DOWNLOAD?=curl -L -o -else - DOWNLOAD?=wget -O -endif -CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/ - -ZSTDDIR = ../../lib -PRGDIR = ../../programs - -FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ - -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \ - $(CPPFLAGS) -FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ - -Wstrict-prototypes -Wundef -Wformat-security \ - -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls \ - -g -fno-omit-frame-pointer -FUZZ_CFLAGS := $(FUZZ_EXTRA_FLAGS) $(CFLAGS) -FUZZ_CXXFLAGS := $(FUZZ_EXTRA_FLAGS) -std=c++11 $(CXXFLAGS) -FUZZ_LDFLAGS := $(LDFLAGS) -FUZZ_ARFLAGS := $(ARFLAGS) -FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS) - -FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h -FUZZ_SRC := zstd_helpers.c - -ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c -ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c -ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c -FUZZ_SRC := \ - $(FUZZ_SRC) \ - $(ZSTDDECOMP_SRC) \ - $(ZSTDCOMMON_SRC) \ - $(ZSTDCOMP_SRC) - -FUZZ_OBJ := $(patsubst %.c,%.o, $(wildcard $(FUZZ_SRC))) - - -.PHONY: default all clean cleanall - -default: all - -FUZZ_TARGETS := \ - simple_round_trip \ - stream_round_trip \ - block_round_trip \ - simple_decompress \ - stream_decompress \ - block_decompress - -all: $(FUZZ_TARGETS) - -%.o: %.c - $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $^ -c -o $@ - -simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) simple_round_trip.o - $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ - -stream_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_round_trip.o - $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) stream_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ - -block_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_round_trip.o - $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ - -simple_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) simple_decompress.o - $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) simple_decompress.o $(LIB_FUZZING_ENGINE) -o $@ - -stream_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_decompress.o - $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) stream_decompress.o $(LIB_FUZZING_ENGINE) -o $@ - -block_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_decompress.o - $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@ - -libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o - $(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o - -# Install libfuzzer (not usable for MSAN testing) -# Provided for convienence. To use this library run make libFuzzer and -# set LDFLAGS=-L. -.PHONY: libFuzzer -libFuzzer: - @$(RM) -rf Fuzzer - @git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer - @cd Fuzzer && ./build.sh - -corpora/%_seed_corpus.zip: - @mkdir -p corpora - $(DOWNLOAD) $@ $(CORPORA_URL_PREFIX)$*_seed_corpus.zip - -corpora/%: corpora/%_seed_corpus.zip - unzip -q $^ -d $@ - -.PHONY: corpora -corpora: $(patsubst %,corpora/%,$(FUZZ_TARGETS)) - -regressiontest: corpora - CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all - $(PYTHON) ./fuzz.py regression all - -clean: - @$(MAKE) -C $(ZSTDDIR) clean - @$(RM) *.a *.o - @$(RM) simple_round_trip stream_round_trip simple_decompress \ - stream_decompress block_decompress block_round_trip - -cleanall: - @$(RM) -r Fuzzer - @$(RM) -r corpora diff --git a/sys/contrib/zstd/tests/fuzz/README.md b/sys/contrib/zstd/tests/fuzz/README.md deleted file mode 100644 index f184be646ef..00000000000 --- a/sys/contrib/zstd/tests/fuzz/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Fuzzing - -Each fuzzing target can be built with multiple engines. -Zstd provides a fuzz corpus for each target that can be downloaded with -the command: - -``` -make corpora -``` - -It will download each corpus into `./corpora/TARGET`. - -## fuzz.py - -`fuzz.py` is a helper script for building and running fuzzers. -Run `./fuzz.py -h` for the commands and run `./fuzz.py COMMAND -h` for -command specific help. - -### Generating Data - -`fuzz.py` provides a utility to generate seed data for each fuzzer. - -``` -make -C ../tests decodecorpus -./fuzz.py gen TARGET -``` - -By default it outputs 100 samples, each at most 8KB into `corpora/TARGET-seed`, -but that can be configured with the `--number`, `--max-size-log` and `--seed` -flags. - -### Build -It respects the usual build environment variables `CC`, `CFLAGS`, etc. -The environment variables can be overridden with the corresponding flags -`--cc`, `--cflags`, etc. -The specific fuzzing engine is selected with `LIB_FUZZING_ENGINE` or -`--lib-fuzzing-engine`, the default is `libregression.a`. -It has flags that can easily set up sanitizers `--enable-{a,ub,m}san`, and -coverage instrumentation `--enable-coverage`. -It sets sane defaults which can be overriden with flags `--debug`, -`--enable-ubsan-pointer-overlow`, etc. -Run `./fuzz.py build -h` for help. - -### Running Fuzzers - -`./fuzz.py` can run `libfuzzer`, `afl`, and `regression` tests. -See the help of the relevant command for options. -Flags not parsed by `fuzz.py` are passed to the fuzzing engine. -The command used to run the fuzzer is printed for debugging. - -## LibFuzzer - -``` -# Build libfuzzer if necessary -make libFuzzer -# Build the fuzz targets -./fuzz.py build all --enable-coverage --enable-asan --enable-ubsan --lib-fuzzing-engine Fuzzer/libFuzzer.a --cc clang --cxx clang++ -# OR equivalently -CC=clang CXX=clang++ LIB_FUZZING_ENGINE=Fuzzer/libFuzzer.a ./fuzz.py build all --enable-coverage --enable-asan --enable-ubsan -# Run the fuzzer -./fuzz.py libfuzzer TARGET -max_len=8192 -jobs=4 -``` - -where `TARGET` could be `simple_decompress`, `stream_round_trip`, etc. - -### MSAN - -Fuzzing with `libFuzzer` and `MSAN` will require building a C++ standard library -and libFuzzer with MSAN. -`fuzz.py` respects the environment variables / flags `MSAN_EXTRA_CPPFLAGS`, -`MSAN_EXTRA_CFLAGS`, `MSAN_EXTRA_CXXFLAGS`, `MSAN_EXTRA_LDFLAGS` to easily pass -the extra parameters only for MSAN. - -## AFL - -The default `LIB_FUZZING_ENGINE` is `libregression.a`, which produces a binary -that AFL can use. - -``` -# Build the fuzz targets -CC=afl-clang CXX=afl-clang++ ./fuzz.py build all --enable-asan --enable-ubsan -# Run the fuzzer without a memory limit because of ASAN -./fuzz.py afl TARGET -m none -``` - -## Regression Testing - -The regression rest supports the `all` target to run all the fuzzers in one -command. - -``` -CC=clang CXX=clang++ ./fuzz.py build all --enable-asan --enable-ubsan -./fuzz.py regression all -CC=clang CXX=clang++ ./fuzz.py build all --enable-msan -./fuzz.py regression all -``` diff --git a/sys/contrib/zstd/tests/fuzz/block_decompress.c b/sys/contrib/zstd/tests/fuzz/block_decompress.c deleted file mode 100644 index 3cccc32f4e5..00000000000 --- a/sys/contrib/zstd/tests/fuzz/block_decompress.c +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * This fuzz target attempts to decompress the fuzzed data with the simple - * decompression function to ensure the decompressor never crashes. - */ - -#define ZSTD_STATIC_LINKING_ONLY - -#include -#include -#include -#include "fuzz_helpers.h" -#include "zstd.h" - -static ZSTD_DCtx *dctx = NULL; -static void* rBuf = NULL; -static size_t bufSize = 0; - -int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) -{ - size_t const neededBufSize = ZSTD_BLOCKSIZE_MAX; - - FUZZ_seed(&src, &size); - - /* Allocate all buffers and contexts if not already allocated */ - if (neededBufSize > bufSize) { - free(rBuf); - rBuf = malloc(neededBufSize); - bufSize = neededBufSize; - FUZZ_ASSERT(rBuf); - } - if (!dctx) { - dctx = ZSTD_createDCtx(); - FUZZ_ASSERT(dctx); - } - ZSTD_decompressBegin(dctx); - ZSTD_decompressBlock(dctx, rBuf, neededBufSize, src, size); - -#ifndef STATEFUL_FUZZING - ZSTD_freeDCtx(dctx); dctx = NULL; -#endif - return 0; -} diff --git a/sys/contrib/zstd/tests/fuzz/block_round_trip.c b/sys/contrib/zstd/tests/fuzz/block_round_trip.c deleted file mode 100644 index 64ca5fc40f9..00000000000 --- a/sys/contrib/zstd/tests/fuzz/block_round_trip.c +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * This fuzz target performs a zstd round-trip test (compress & decompress), - * compares the result with the original, and calls abort() on corruption. - */ - -#define ZSTD_STATIC_LINKING_ONLY - -#include -#include -#include -#include -#include "fuzz_helpers.h" -#include "zstd.h" - -static const int kMaxClevel = 19; - -static ZSTD_CCtx *cctx = NULL; -static ZSTD_DCtx *dctx = NULL; -static void* cBuf = NULL; -static void* rBuf = NULL; -static size_t bufSize = 0; -static uint32_t seed; - -static size_t roundTripTest(void *result, size_t resultCapacity, - void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) -{ - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; - ZSTD_parameters const params = ZSTD_getParams(cLevel, srcSize, 0); - size_t ret = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, srcSize); - FUZZ_ZASSERT(ret); - - ret = ZSTD_compressBlock(cctx, compressed, compressedCapacity, src, srcSize); - FUZZ_ZASSERT(ret); - if (ret == 0) { - FUZZ_ASSERT(resultCapacity >= srcSize); - memcpy(result, src, srcSize); - return srcSize; - } - ZSTD_decompressBegin(dctx); - return ZSTD_decompressBlock(dctx, result, resultCapacity, compressed, ret); -} - -int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) -{ - size_t neededBufSize; - - seed = FUZZ_seed(&src, &size); - neededBufSize = size; - if (size > ZSTD_BLOCKSIZE_MAX) - return 0; - - /* Allocate all buffers and contexts if not already allocated */ - if (neededBufSize > bufSize || !cBuf || !rBuf) { - free(cBuf); - free(rBuf); - cBuf = malloc(neededBufSize); - rBuf = malloc(neededBufSize); - bufSize = neededBufSize; - FUZZ_ASSERT(cBuf && rBuf); - } - if (!cctx) { - cctx = ZSTD_createCCtx(); - FUZZ_ASSERT(cctx); - } - if (!dctx) { - dctx = ZSTD_createDCtx(); - FUZZ_ASSERT(dctx); - } - - { - size_t const result = - roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size); - FUZZ_ZASSERT(result); - FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); - FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); - } -#ifndef STATEFUL_FUZZING - ZSTD_freeCCtx(cctx); cctx = NULL; - ZSTD_freeDCtx(dctx); dctx = NULL; -#endif - return 0; -} diff --git a/sys/contrib/zstd/tests/fuzz/default.options b/sys/contrib/zstd/tests/fuzz/default.options deleted file mode 100644 index 8ea8588375d..00000000000 --- a/sys/contrib/zstd/tests/fuzz/default.options +++ /dev/null @@ -1,2 +0,0 @@ -[libfuzzer] -max_len = 8192 diff --git a/sys/contrib/zstd/tests/fuzz/fuzz.h b/sys/contrib/zstd/tests/fuzz/fuzz.h deleted file mode 100644 index a64845473c2..00000000000 --- a/sys/contrib/zstd/tests/fuzz/fuzz.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * Fuzz target interface. - * Fuzz targets have some common parameters passed as macros during compilation. - * Check the documentation for each individual fuzzer for more parameters. - * - * @param STATEFUL_FUZZING: - * Define this to reuse state between fuzzer runs. This can be useful to - * test code paths which are only executed when contexts are reused. - * WARNING: Makes reproducing crashes much harder. - * Default: Not defined. - * @param FUZZ_RNG_SEED_SIZE: - * The number of bytes of the source to look at when constructing a seed - * for the deterministic RNG. These bytes are discarded before passing - * the data to zstd functions. Every fuzzer initializes the RNG exactly - * once before doing anything else, even if it is unused. - * Default: 4. - * @param ZSTD_DEBUG: - * This is a parameter for the zstd library. Defining `ZSTD_DEBUG=1` - * enables assert() statements in the zstd library. Higher levels enable - * logging, so aren't recommended. Defining `ZSTD_DEBUG=1` is - * recommended. - * @param MEM_FORCE_MEMORY_ACCESS: - * This flag controls how the zstd library accesses unaligned memory. - * It can be undefined, or 0 through 2. If it is undefined, it selects - * the method to use based on the compiler. If testing with UBSAN set - * MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method. - * @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - * This is the canonical flag to enable deterministic builds for fuzzing. - * Changes to zstd for fuzzing are gated behind this define. - * It is recommended to define this when building zstd for fuzzing. - */ - -#ifndef FUZZ_H -#define FUZZ_H - -#ifndef FUZZ_RNG_SEED_SIZE -# define FUZZ_RNG_SEED_SIZE 4 -#endif - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/sys/contrib/zstd/tests/fuzz/fuzz.py b/sys/contrib/zstd/tests/fuzz/fuzz.py deleted file mode 100755 index b591e4f6734..00000000000 --- a/sys/contrib/zstd/tests/fuzz/fuzz.py +++ /dev/null @@ -1,818 +0,0 @@ -#!/usr/bin/env python - -# ################################################################ -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# ########################################################################## - -import argparse -import contextlib -import os -import re -import shutil -import subprocess -import sys -import tempfile - - -def abs_join(a, *p): - return os.path.abspath(os.path.join(a, *p)) - - -# Constants -FUZZ_DIR = os.path.abspath(os.path.dirname(__file__)) -CORPORA_DIR = abs_join(FUZZ_DIR, 'corpora') -TARGETS = [ - 'simple_round_trip', - 'stream_round_trip', - 'block_round_trip', - 'simple_decompress', - 'stream_decompress', - 'block_decompress', -] -ALL_TARGETS = TARGETS + ['all'] -FUZZ_RNG_SEED_SIZE = 4 - -# Standard environment variables -CC = os.environ.get('CC', 'cc') -CXX = os.environ.get('CXX', 'c++') -CPPFLAGS = os.environ.get('CPPFLAGS', '') -CFLAGS = os.environ.get('CFLAGS', '-O3') -CXXFLAGS = os.environ.get('CXXFLAGS', CFLAGS) -LDFLAGS = os.environ.get('LDFLAGS', '') -MFLAGS = os.environ.get('MFLAGS', '-j') - -# Fuzzing environment variables -LIB_FUZZING_ENGINE = os.environ.get('LIB_FUZZING_ENGINE', 'libregression.a') -AFL_FUZZ = os.environ.get('AFL_FUZZ', 'afl-fuzz') -DECODECORPUS = os.environ.get('DECODECORPUS', - abs_join(FUZZ_DIR, '..', 'decodecorpus')) - -# Sanitizer environment variables -MSAN_EXTRA_CPPFLAGS = os.environ.get('MSAN_EXTRA_CPPFLAGS', '') -MSAN_EXTRA_CFLAGS = os.environ.get('MSAN_EXTRA_CFLAGS', '') -MSAN_EXTRA_CXXFLAGS = os.environ.get('MSAN_EXTRA_CXXFLAGS', '') -MSAN_EXTRA_LDFLAGS = os.environ.get('MSAN_EXTRA_LDFLAGS', '') - - -def create(r): - d = os.path.abspath(r) - if not os.path.isdir(d): - os.mkdir(d) - return d - - -def check(r): - d = os.path.abspath(r) - if not os.path.isdir(d): - return None - return d - - -@contextlib.contextmanager -def tmpdir(): - dirpath = tempfile.mkdtemp() - try: - yield dirpath - finally: - shutil.rmtree(dirpath, ignore_errors=True) - - -def parse_targets(in_targets): - targets = set() - for target in in_targets: - if not target: - continue - if target == 'all': - targets = targets.union(TARGETS) - elif target in TARGETS: - targets.add(target) - else: - raise RuntimeError('{} is not a valid target'.format(target)) - return list(targets) - - -def targets_parser(args, description): - parser = argparse.ArgumentParser(prog=args.pop(0), description=description) - parser.add_argument( - 'TARGET', - nargs='*', - type=str, - help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS))) - args, extra = parser.parse_known_args(args) - args.extra = extra - - args.TARGET = parse_targets(args.TARGET) - - return args - - -def parse_env_flags(args, flags): - """ - Look for flags set by environment variables. - """ - san_flags = ','.join(re.findall('-fsanitize=((?:[a-z]+,?)+)', flags)) - nosan_flags = ','.join(re.findall('-fno-sanitize=((?:[a-z]+,?)+)', flags)) - - def set_sanitizer(sanitizer, default, san, nosan): - if sanitizer in san and sanitizer in nosan: - raise RuntimeError('-fno-sanitize={s} and -fsanitize={s} passed'. - format(s=sanitizer)) - if sanitizer in san: - return True - if sanitizer in nosan: - return False - return default - - san = set(san_flags.split(',')) - nosan = set(nosan_flags.split(',')) - - args.asan = set_sanitizer('address', args.asan, san, nosan) - args.msan = set_sanitizer('memory', args.msan, san, nosan) - args.ubsan = set_sanitizer('undefined', args.ubsan, san, nosan) - - args.sanitize = args.asan or args.msan or args.ubsan - - return args - - -def compiler_version(cc, cxx): - """ - Determines the compiler and version. - Only works for clang and gcc. - """ - cc_version_bytes = subprocess.check_output([cc, "--version"]) - cxx_version_bytes = subprocess.check_output([cxx, "--version"]) - if cc_version_bytes.startswith(b'clang'): - assert(cxx_version_bytes.startswith(b'clang')) - compiler = 'clang' - if cc_version_bytes.startswith(b'gcc'): - assert(cxx_version_bytes.startswith(b'g++')) - compiler = 'gcc' - version_regex = b'([0-9])+\.([0-9])+\.([0-9])+' - version_match = re.search(version_regex, cc_version_bytes) - version = tuple(int(version_match.group(i)) for i in range(1, 4)) - return compiler, version - - -def overflow_ubsan_flags(cc, cxx): - compiler, version = compiler_version(cc, cxx) - if compiler == 'gcc': - return ['-fno-sanitize=signed-integer-overflow'] - if compiler == 'clang' and version >= (5, 0, 0): - return ['-fno-sanitize=pointer-overflow'] - return [] - - -def build_parser(args): - description = """ - Cleans the repository and builds a fuzz target (or all). - Many flags default to environment variables (default says $X='y'). - Options that aren't enabling features default to the correct values for - zstd. - Enable sanitizers with --enable-*san. - For regression testing just build. - For libFuzzer set LIB_FUZZING_ENGINE and pass --enable-coverage. - For AFL set CC and CXX to AFL's compilers and set - LIB_FUZZING_ENGINE='libregression.a'. - """ - parser = argparse.ArgumentParser(prog=args.pop(0), description=description) - parser.add_argument( - '--lib-fuzzing-engine', - dest='lib_fuzzing_engine', - type=str, - default=LIB_FUZZING_ENGINE, - help=('The fuzzing engine to use e.g. /path/to/libFuzzer.a ' - "(default: $LIB_FUZZING_ENGINE='{})".format(LIB_FUZZING_ENGINE))) - parser.add_argument( - '--enable-coverage', - dest='coverage', - action='store_true', - help='Enable coverage instrumentation (-fsanitize-coverage)') - parser.add_argument( - '--enable-asan', dest='asan', action='store_true', help='Enable UBSAN') - parser.add_argument( - '--enable-ubsan', - dest='ubsan', - action='store_true', - help='Enable UBSAN') - parser.add_argument( - '--enable-ubsan-pointer-overflow', - dest='ubsan_pointer_overflow', - action='store_true', - help='Enable UBSAN pointer overflow check (known failure)') - parser.add_argument( - '--enable-msan', dest='msan', action='store_true', help='Enable MSAN') - parser.add_argument( - '--enable-msan-track-origins', dest='msan_track_origins', - action='store_true', help='Enable MSAN origin tracking') - parser.add_argument( - '--msan-extra-cppflags', - dest='msan_extra_cppflags', - type=str, - default=MSAN_EXTRA_CPPFLAGS, - help="Extra CPPFLAGS for MSAN (default: $MSAN_EXTRA_CPPFLAGS='{}')". - format(MSAN_EXTRA_CPPFLAGS)) - parser.add_argument( - '--msan-extra-cflags', - dest='msan_extra_cflags', - type=str, - default=MSAN_EXTRA_CFLAGS, - help="Extra CFLAGS for MSAN (default: $MSAN_EXTRA_CFLAGS='{}')".format( - MSAN_EXTRA_CFLAGS)) - parser.add_argument( - '--msan-extra-cxxflags', - dest='msan_extra_cxxflags', - type=str, - default=MSAN_EXTRA_CXXFLAGS, - help="Extra CXXFLAGS for MSAN (default: $MSAN_EXTRA_CXXFLAGS='{}')". - format(MSAN_EXTRA_CXXFLAGS)) - parser.add_argument( - '--msan-extra-ldflags', - dest='msan_extra_ldflags', - type=str, - default=MSAN_EXTRA_LDFLAGS, - help="Extra LDFLAGS for MSAN (default: $MSAN_EXTRA_LDFLAGS='{}')". - format(MSAN_EXTRA_LDFLAGS)) - parser.add_argument( - '--enable-sanitize-recover', - dest='sanitize_recover', - action='store_true', - help='Non-fatal sanitizer errors where possible') - parser.add_argument( - '--debug', - dest='debug', - type=int, - default=1, - help='Set ZSTD_DEBUG (default: 1)') - parser.add_argument( - '--force-memory-access', - dest='memory_access', - type=int, - default=0, - help='Set MEM_FORCE_MEMORY_ACCESS (default: 0)') - parser.add_argument( - '--fuzz-rng-seed-size', - dest='fuzz_rng_seed_size', - type=int, - default=4, - help='Set FUZZ_RNG_SEED_SIZE (default: 4)') - parser.add_argument( - '--disable-fuzzing-mode', - dest='fuzzing_mode', - action='store_false', - help='Do not define FUZZING_BUILD_MORE_UNSAFE_FOR_PRODUCTION') - parser.add_argument( - '--enable-stateful-fuzzing', - dest='stateful_fuzzing', - action='store_true', - help='Reuse contexts between runs (makes reproduction impossible)') - parser.add_argument( - '--cc', - dest='cc', - type=str, - default=CC, - help="CC (default: $CC='{}')".format(CC)) - parser.add_argument( - '--cxx', - dest='cxx', - type=str, - default=CXX, - help="CXX (default: $CXX='{}')".format(CXX)) - parser.add_argument( - '--cppflags', - dest='cppflags', - type=str, - default=CPPFLAGS, - help="CPPFLAGS (default: $CPPFLAGS='{}')".format(CPPFLAGS)) - parser.add_argument( - '--cflags', - dest='cflags', - type=str, - default=CFLAGS, - help="CFLAGS (default: $CFLAGS='{}')".format(CFLAGS)) - parser.add_argument( - '--cxxflags', - dest='cxxflags', - type=str, - default=CXXFLAGS, - help="CXXFLAGS (default: $CXXFLAGS='{}')".format(CXXFLAGS)) - parser.add_argument( - '--ldflags', - dest='ldflags', - type=str, - default=LDFLAGS, - help="LDFLAGS (default: $LDFLAGS='{}')".format(LDFLAGS)) - parser.add_argument( - '--mflags', - dest='mflags', - type=str, - default=MFLAGS, - help="Extra Make flags (default: $MFLAGS='{}')".format(MFLAGS)) - parser.add_argument( - 'TARGET', - nargs='*', - type=str, - help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS)) - ) - args = parser.parse_args(args) - args = parse_env_flags(args, ' '.join( - [args.cppflags, args.cflags, args.cxxflags, args.ldflags])) - - # Check option sanitiy - if args.msan and (args.asan or args.ubsan): - raise RuntimeError('MSAN may not be used with any other sanitizers') - if args.msan_track_origins and not args.msan: - raise RuntimeError('--enable-msan-track-origins requires MSAN') - if args.ubsan_pointer_overflow and not args.ubsan: - raise RuntimeError('--enable-ubsan-pointer-overlow requires UBSAN') - if args.sanitize_recover and not args.sanitize: - raise RuntimeError('--enable-sanitize-recover but no sanitizers used') - - return args - - -def build(args): - try: - args = build_parser(args) - except Exception as e: - print(e) - return 1 - # The compilation flags we are setting - targets = args.TARGET - cc = args.cc - cxx = args.cxx - cppflags = [args.cppflags] - cflags = [args.cflags] - ldflags = [args.ldflags] - cxxflags = [args.cxxflags] - mflags = [args.mflags] if args.mflags else [] - # Flags to be added to both cflags and cxxflags - common_flags = [] - - cppflags += [ - '-DZSTD_DEBUG={}'.format(args.debug), - '-DMEM_FORCE_MEMORY_ACCESS={}'.format(args.memory_access), - '-DFUZZ_RNG_SEED_SIZE={}'.format(args.fuzz_rng_seed_size), - ] - - mflags += ['LIB_FUZZING_ENGINE={}'.format(args.lib_fuzzing_engine)] - - # Set flags for options - if args.coverage: - common_flags += [ - '-fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp' - ] - - if args.sanitize_recover: - recover_flags = ['-fsanitize-recover=all'] - else: - recover_flags = ['-fno-sanitize-recover=all'] - if args.sanitize: - common_flags += recover_flags - - if args.msan: - msan_flags = ['-fsanitize=memory'] - if args.msan_track_origins: - msan_flags += ['-fsanitize-memory-track-origins'] - common_flags += msan_flags - # Append extra MSAN flags (it might require special setup) - cppflags += [args.msan_extra_cppflags] - cflags += [args.msan_extra_cflags] - cxxflags += [args.msan_extra_cxxflags] - ldflags += [args.msan_extra_ldflags] - - if args.asan: - common_flags += ['-fsanitize=address'] - - if args.ubsan: - ubsan_flags = ['-fsanitize=undefined'] - if not args.ubsan_pointer_overflow: - ubsan_flags += overflow_ubsan_flags(cc, cxx) - common_flags += ubsan_flags - - if args.stateful_fuzzing: - cppflags += ['-DSTATEFUL_FUZZING'] - - if args.fuzzing_mode: - cppflags += ['-DFUZZING_BUILD_MORE_UNSAFE_FOR_PRODUCTION'] - - if args.lib_fuzzing_engine == 'libregression.a': - targets = ['libregression.a'] + targets - - # Append the common flags - cflags += common_flags - cxxflags += common_flags - - # Prepare the flags for Make - cc_str = "CC={}".format(cc) - cxx_str = "CXX={}".format(cxx) - cppflags_str = "CPPFLAGS={}".format(' '.join(cppflags)) - cflags_str = "CFLAGS={}".format(' '.join(cflags)) - cxxflags_str = "CXXFLAGS={}".format(' '.join(cxxflags)) - ldflags_str = "LDFLAGS={}".format(' '.join(ldflags)) - - # Print the flags - print('MFLAGS={}'.format(' '.join(mflags))) - print(cc_str) - print(cxx_str) - print(cppflags_str) - print(cflags_str) - print(cxxflags_str) - print(ldflags_str) - - # Clean and build - clean_cmd = ['make', 'clean'] + mflags - print(' '.join(clean_cmd)) - subprocess.check_call(clean_cmd) - build_cmd = [ - 'make', - cc_str, - cxx_str, - cppflags_str, - cflags_str, - cxxflags_str, - ldflags_str, - ] + mflags + targets - print(' '.join(build_cmd)) - subprocess.check_call(build_cmd) - return 0 - - -def libfuzzer_parser(args): - description = """ - Runs a libfuzzer binary. - Passes all extra arguments to libfuzzer. - The fuzzer should have been build with LIB_FUZZING_ENGINE pointing to - libFuzzer.a. - Generates output in the CORPORA directory, puts crashes in the ARTIFACT - directory, and takes extra input from the SEED directory. - To merge AFL's output pass the SEED as AFL's output directory and pass - '-merge=1'. - """ - parser = argparse.ArgumentParser(prog=args.pop(0), description=description) - parser.add_argument( - '--corpora', - type=str, - help='Override the default corpora dir (default: {})'.format( - abs_join(CORPORA_DIR, 'TARGET'))) - parser.add_argument( - '--artifact', - type=str, - help='Override the default artifact dir (default: {})'.format( - abs_join(CORPORA_DIR, 'TARGET-crash'))) - parser.add_argument( - '--seed', - type=str, - help='Override the default seed dir (default: {})'.format( - abs_join(CORPORA_DIR, 'TARGET-seed'))) - parser.add_argument( - 'TARGET', - type=str, - help='Fuzz target(s) to build {{{}}}'.format(', '.join(TARGETS))) - args, extra = parser.parse_known_args(args) - args.extra = extra - - if args.TARGET and args.TARGET not in TARGETS: - raise RuntimeError('{} is not a valid target'.format(args.TARGET)) - - return args - - -def libfuzzer(target, corpora=None, artifact=None, seed=None, extra_args=None): - if corpora is None: - corpora = abs_join(CORPORA_DIR, target) - if artifact is None: - artifact = abs_join(CORPORA_DIR, '{}-crash'.format(target)) - if seed is None: - seed = abs_join(CORPORA_DIR, '{}-seed'.format(target)) - if extra_args is None: - extra_args = [] - - target = abs_join(FUZZ_DIR, target) - - corpora = [create(corpora)] - artifact = create(artifact) - seed = check(seed) - - corpora += [artifact] - if seed is not None: - corpora += [seed] - - cmd = [target, '-artifact_prefix={}/'.format(artifact)] - cmd += corpora + extra_args - print(' '.join(cmd)) - subprocess.check_call(cmd) - - -def libfuzzer_cmd(args): - try: - args = libfuzzer_parser(args) - except Exception as e: - print(e) - return 1 - libfuzzer(args.TARGET, args.corpora, args.artifact, args.seed, args.extra) - return 0 - - -def afl_parser(args): - description = """ - Runs an afl-fuzz job. - Passes all extra arguments to afl-fuzz. - The fuzzer should have been built with CC/CXX set to the AFL compilers, - and with LIB_FUZZING_ENGINE='libregression.a'. - Takes input from CORPORA and writes output to OUTPUT. - Uses AFL_FUZZ as the binary (set from flag or environment variable). - """ - parser = argparse.ArgumentParser(prog=args.pop(0), description=description) - parser.add_argument( - '--corpora', - type=str, - help='Override the default corpora dir (default: {})'.format( - abs_join(CORPORA_DIR, 'TARGET'))) - parser.add_argument( - '--output', - type=str, - help='Override the default AFL output dir (default: {})'.format( - abs_join(CORPORA_DIR, 'TARGET-afl'))) - parser.add_argument( - '--afl-fuzz', - type=str, - default=AFL_FUZZ, - help='AFL_FUZZ (default: $AFL_FUZZ={})'.format(AFL_FUZZ)) - parser.add_argument( - 'TARGET', - type=str, - help='Fuzz target(s) to build {{{}}}'.format(', '.join(TARGETS))) - args, extra = parser.parse_known_args(args) - args.extra = extra - - if args.TARGET and args.TARGET not in TARGETS: - raise RuntimeError('{} is not a valid target'.format(args.TARGET)) - - if not args.corpora: - args.corpora = abs_join(CORPORA_DIR, args.TARGET) - if not args.output: - args.output = abs_join(CORPORA_DIR, '{}-afl'.format(args.TARGET)) - - return args - - -def afl(args): - try: - args = afl_parser(args) - except Exception as e: - print(e) - return 1 - target = abs_join(FUZZ_DIR, args.TARGET) - - corpora = create(args.corpora) - output = create(args.output) - - cmd = [args.afl_fuzz, '-i', corpora, '-o', output] + args.extra - cmd += [target, '@@'] - print(' '.join(cmd)) - subprocess.call(cmd) - return 0 - - -def regression(args): - try: - description = """ - Runs one or more regression tests. - The fuzzer should have been built with with - LIB_FUZZING_ENGINE='libregression.a'. - Takes input from CORPORA. - """ - args = targets_parser(args, description) - except Exception as e: - print(e) - return 1 - for target in args.TARGET: - corpora = create(abs_join(CORPORA_DIR, target)) - target = abs_join(FUZZ_DIR, target) - cmd = [target, corpora] - print(' '.join(cmd)) - subprocess.check_call(cmd) - return 0 - - -def gen_parser(args): - description = """ - Generate a seed corpus appropiate for TARGET with data generated with - decodecorpus. - The fuzz inputs are prepended with a seed before the zstd data, so the - output of decodecorpus shouldn't be used directly. - Generates NUMBER samples prepended with FUZZ_RNG_SEED_SIZE random bytes and - puts the output in SEED. - DECODECORPUS is the decodecorpus binary, and must already be built. - """ - parser = argparse.ArgumentParser(prog=args.pop(0), description=description) - parser.add_argument( - '--number', - '-n', - type=int, - default=100, - help='Number of samples to generate') - parser.add_argument( - '--max-size-log', - type=int, - default=13, - help='Maximum sample size to generate') - parser.add_argument( - '--seed', - type=str, - help='Override the default seed dir (default: {})'.format( - abs_join(CORPORA_DIR, 'TARGET-seed'))) - parser.add_argument( - '--decodecorpus', - type=str, - default=DECODECORPUS, - help="decodecorpus binary (default: $DECODECORPUS='{}')".format( - DECODECORPUS)) - parser.add_argument( - '--fuzz-rng-seed-size', - type=int, - default=4, - help="FUZZ_RNG_SEED_SIZE used for generate the samples (must match)" - ) - parser.add_argument( - 'TARGET', - type=str, - help='Fuzz target(s) to build {{{}}}'.format(', '.join(TARGETS))) - args, extra = parser.parse_known_args(args) - args.extra = extra - - if args.TARGET and args.TARGET not in TARGETS: - raise RuntimeError('{} is not a valid target'.format(args.TARGET)) - - if not args.seed: - args.seed = abs_join(CORPORA_DIR, '{}-seed'.format(args.TARGET)) - - if not os.path.isfile(args.decodecorpus): - raise RuntimeError("{} is not a file run 'make -C {} decodecorpus'". - format(args.decodecorpus, abs_join(FUZZ_DIR, '..'))) - - return args - - -def gen(args): - try: - args = gen_parser(args) - except Exception as e: - print(e) - return 1 - - seed = create(args.seed) - with tmpdir() as compressed: - with tmpdir() as decompressed: - cmd = [ - args.decodecorpus, - '-n{}'.format(args.number), - '-p{}/'.format(compressed), - '-o{}'.format(decompressed), - ] - - if 'block_' in args.TARGET: - cmd += [ - '--gen-blocks', - '--max-block-size-log={}'.format(args.max_size_log) - ] - else: - cmd += ['--max-content-size-log={}'.format(args.max_size_log)] - - print(' '.join(cmd)) - subprocess.check_call(cmd) - - if '_round_trip' in args.TARGET: - print('using decompressed data in {}'.format(decompressed)) - samples = decompressed - elif '_decompress' in args.TARGET: - print('using compressed data in {}'.format(compressed)) - samples = compressed - - # Copy the samples over and prepend the RNG seeds - for name in os.listdir(samples): - samplename = abs_join(samples, name) - outname = abs_join(seed, name) - rng_seed = os.urandom(args.fuzz_rng_seed_size) - with open(samplename, 'rb') as sample: - with open(outname, 'wb') as out: - out.write(rng_seed) - CHUNK_SIZE = 131072 - chunk = sample.read(CHUNK_SIZE) - while len(chunk) > 0: - out.write(chunk) - chunk = sample.read(CHUNK_SIZE) - return 0 - - -def minimize(args): - try: - description = """ - Runs a libfuzzer fuzzer with -merge=1 to build a minimal corpus in - TARGET_seed_corpus. All extra args are passed to libfuzzer. - """ - args = targets_parser(args, description) - except Exception as e: - print(e) - return 1 - - for target in args.TARGET: - # Merge the corpus + anything else into the seed_corpus - corpus = abs_join(CORPORA_DIR, target) - seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target)) - extra_args = [corpus, "-merge=1"] + args.extra - libfuzzer(target, corpora=seed_corpus, extra_args=extra_args) - seeds = set(os.listdir(seed_corpus)) - # Copy all crashes directly into the seed_corpus if not already present - crashes = abs_join(CORPORA_DIR, '{}-crash'.format(target)) - for crash in os.listdir(crashes): - if crash not in seeds: - shutil.copy(abs_join(crashes, crash), seed_corpus) - seeds.add(crash) - - -def zip_cmd(args): - try: - description = """ - Zips up the seed corpus. - """ - args = targets_parser(args, description) - except Exception as e: - print(e) - return 1 - - for target in args.TARGET: - # Zip the seed_corpus - seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target)) - seeds = [abs_join(seed_corpus, f) for f in os.listdir(seed_corpus)] - zip_file = "{}.zip".format(seed_corpus) - cmd = ["zip", "-q", "-j", "-9", zip_file] - print(' '.join(cmd + [abs_join(seed_corpus, '*')])) - subprocess.check_call(cmd + seeds) - - -def list_cmd(args): - print("\n".join(TARGETS)) - - -def short_help(args): - name = args[0] - print("Usage: {} [OPTIONS] COMMAND [ARGS]...\n".format(name)) - - -def help(args): - short_help(args) - print("\tfuzzing helpers (select a command and pass -h for help)\n") - print("Options:") - print("\t-h, --help\tPrint this message") - print("") - print("Commands:") - print("\tbuild\t\tBuild a fuzzer") - print("\tlibfuzzer\tRun a libFuzzer fuzzer") - print("\tafl\t\tRun an AFL fuzzer") - print("\tregression\tRun a regression test") - print("\tgen\t\tGenerate a seed corpus for a fuzzer") - print("\tminimize\tMinimize the test corpora") - print("\tzip\t\tZip the minimized corpora up") - print("\tlist\t\tList the available targets") - - -def main(): - args = sys.argv - if len(args) < 2: - help(args) - return 1 - if args[1] == '-h' or args[1] == '--help' or args[1] == '-H': - help(args) - return 1 - command = args.pop(1) - args[0] = "{} {}".format(args[0], command) - if command == "build": - return build(args) - if command == "libfuzzer": - return libfuzzer_cmd(args) - if command == "regression": - return regression(args) - if command == "afl": - return afl(args) - if command == "gen": - return gen(args) - if command == "minimize": - return minimize(args) - if command == "zip": - return zip_cmd(args) - if command == "list": - return list_cmd(args) - short_help(args) - print("Error: No such command {} (pass -h for help)".format(command)) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/sys/contrib/zstd/tests/fuzz/fuzz_helpers.h b/sys/contrib/zstd/tests/fuzz/fuzz_helpers.h deleted file mode 100644 index 468c39fb42d..00000000000 --- a/sys/contrib/zstd/tests/fuzz/fuzz_helpers.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * Helper functions for fuzzing. - */ - -#ifndef FUZZ_HELPERS_H -#define FUZZ_HELPERS_H - -#include "fuzz.h" -#include "xxhash.h" -#include "zstd.h" -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define MAX(a, b) ((a) > (b) ? (a) : (b)) - -#define FUZZ_QUOTE_IMPL(str) #str -#define FUZZ_QUOTE(str) FUZZ_QUOTE_IMPL(str) - -/** - * Asserts for fuzzing that are always enabled. - */ -#define FUZZ_ASSERT_MSG(cond, msg) \ - ((cond) ? (void)0 \ - : (fprintf(stderr, "%s: %u: Assertion: `%s' failed. %s\n", __FILE__, \ - __LINE__, FUZZ_QUOTE(cond), (msg)), \ - abort())) -#define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), ""); -#define FUZZ_ZASSERT(code) \ - FUZZ_ASSERT_MSG(!ZSTD_isError(code), ZSTD_getErrorName(code)) - -#if defined(__GNUC__) -#define FUZZ_STATIC static __inline __attribute__((unused)) -#elif defined(__cplusplus) || \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#define FUZZ_STATIC static inline -#elif defined(_MSC_VER) -#define FUZZ_STATIC static __inline -#else -#define FUZZ_STATIC static -#endif - -/** - * Determininistically constructs a seed based on the fuzz input. - * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input. - */ -FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) { - uint8_t const *data = *src; - size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size); - *size -= toHash; - *src += toHash; - return XXH32(data, toHash, 0); -} - -#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) - -FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) { - static const uint32_t prime1 = 2654435761U; - static const uint32_t prime2 = 2246822519U; - uint32_t rand32 = *state; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZZ_rotl32(rand32, 13); - *state = rand32; - return rand32 >> 5; -} - -/* Returns a random numer in the range [min, max]. */ -FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) { - uint32_t random = FUZZ_rand(state); - return min + (random % (max - min + 1)); -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/sys/contrib/zstd/tests/fuzz/regression_driver.c b/sys/contrib/zstd/tests/fuzz/regression_driver.c deleted file mode 100644 index 2b714d29e9d..00000000000 --- a/sys/contrib/zstd/tests/fuzz/regression_driver.c +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -#include "fuzz.h" -#include "fuzz_helpers.h" -#include "util.h" -#include -#include -#include -#include - -int main(int argc, char const **argv) { - size_t const kMaxFileSize = (size_t)1 << 20; - int const kFollowLinks = 1; - char *fileNamesBuf = NULL; - char const **files = argv + 1; - unsigned numFiles = argc - 1; - uint8_t *buffer = NULL; - size_t bufferSize = 0; - unsigned i; - int ret; - -#ifdef UTIL_HAS_CREATEFILELIST - files = UTIL_createFileList(files, numFiles, &fileNamesBuf, &numFiles, - kFollowLinks); - if (!files) - numFiles = 0; -#endif - if (numFiles == 0) - fprintf(stderr, "WARNING: No files passed to %s\n", argv[0]); - for (i = 0; i < numFiles; ++i) { - char const *fileName = files[i]; - size_t const fileSize = UTIL_getFileSize(fileName); - size_t readSize; - FILE *file; - - /* Check that it is a regular file, and that the fileSize is valid */ - FUZZ_ASSERT_MSG(UTIL_isRegularFile(fileName), fileName); - FUZZ_ASSERT_MSG(fileSize <= kMaxFileSize, fileName); - /* Ensure we have a large enough buffer allocated */ - if (fileSize > bufferSize) { - free(buffer); - buffer = (uint8_t *)malloc(fileSize); - FUZZ_ASSERT_MSG(buffer, fileName); - bufferSize = fileSize; - } - /* Open the file */ - file = fopen(fileName, "rb"); - FUZZ_ASSERT_MSG(file, fileName); - /* Read the file */ - readSize = fread(buffer, 1, fileSize, file); - FUZZ_ASSERT_MSG(readSize == fileSize, fileName); - /* Close the file */ - fclose(file); - /* Run the fuzz target */ - LLVMFuzzerTestOneInput(buffer, fileSize); - } - - ret = 0; - free(buffer); -#ifdef UTIL_HAS_CREATEFILELIST - UTIL_freeFileList(files, fileNamesBuf); -#endif - return ret; -} diff --git a/sys/contrib/zstd/tests/fuzz/simple_decompress.c b/sys/contrib/zstd/tests/fuzz/simple_decompress.c deleted file mode 100644 index bba272c6225..00000000000 --- a/sys/contrib/zstd/tests/fuzz/simple_decompress.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * This fuzz target attempts to decompress the fuzzed data with the simple - * decompression function to ensure the decompressor never crashes. - */ - -#include -#include -#include -#include "fuzz_helpers.h" -#include "zstd.h" - -static ZSTD_DCtx *dctx = NULL; -static void* rBuf = NULL; -static size_t bufSize = 0; - -int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) -{ - size_t neededBufSize; - - FUZZ_seed(&src, &size); - neededBufSize = MAX(20 * size, (size_t)256 << 10); - - /* Allocate all buffers and contexts if not already allocated */ - if (neededBufSize > bufSize) { - free(rBuf); - rBuf = malloc(neededBufSize); - bufSize = neededBufSize; - FUZZ_ASSERT(rBuf); - } - if (!dctx) { - dctx = ZSTD_createDCtx(); - FUZZ_ASSERT(dctx); - } - ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, src, size); - -#ifndef STATEFUL_FUZZING - ZSTD_freeDCtx(dctx); dctx = NULL; -#endif - return 0; -} diff --git a/sys/contrib/zstd/tests/fuzz/simple_round_trip.c b/sys/contrib/zstd/tests/fuzz/simple_round_trip.c deleted file mode 100644 index 0921106dea8..00000000000 --- a/sys/contrib/zstd/tests/fuzz/simple_round_trip.c +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * This fuzz target performs a zstd round-trip test (compress & decompress), - * compares the result with the original, and calls abort() on corruption. - */ - -#define ZSTD_STATIC_LINKING_ONLY - -#include -#include -#include -#include -#include "fuzz_helpers.h" -#include "zstd_helpers.h" - -static const int kMaxClevel = 19; - -static ZSTD_CCtx *cctx = NULL; -static ZSTD_DCtx *dctx = NULL; -static void* cBuf = NULL; -static void* rBuf = NULL; -static size_t bufSize = 0; -static uint32_t seed; - -static size_t roundTripTest(void *result, size_t resultCapacity, - void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) -{ - size_t cSize; - if (FUZZ_rand(&seed) & 1) { - ZSTD_inBuffer in = {src, srcSize, 0}; - ZSTD_outBuffer out = {compressed, compressedCapacity, 0}; - size_t err; - - ZSTD_CCtx_reset(cctx); - FUZZ_setRandomParameters(cctx, srcSize, &seed); - err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); - FUZZ_ZASSERT(err); - FUZZ_ASSERT(err == 0); - cSize = out.pos; - } else { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; - cSize = ZSTD_compressCCtx( - cctx, compressed, compressedCapacity, src, srcSize, cLevel); - } - FUZZ_ZASSERT(cSize); - return ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize); -} - -int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) -{ - size_t neededBufSize; - - seed = FUZZ_seed(&src, &size); - neededBufSize = ZSTD_compressBound(size); - - /* Allocate all buffers and contexts if not already allocated */ - if (neededBufSize > bufSize) { - free(cBuf); - free(rBuf); - cBuf = malloc(neededBufSize); - rBuf = malloc(neededBufSize); - bufSize = neededBufSize; - FUZZ_ASSERT(cBuf && rBuf); - } - if (!cctx) { - cctx = ZSTD_createCCtx(); - FUZZ_ASSERT(cctx); - } - if (!dctx) { - dctx = ZSTD_createDCtx(); - FUZZ_ASSERT(dctx); - } - - { - size_t const result = - roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size); - FUZZ_ZASSERT(result); - FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); - FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); - } -#ifndef STATEFUL_FUZZING - ZSTD_freeCCtx(cctx); cctx = NULL; - ZSTD_freeDCtx(dctx); dctx = NULL; -#endif - return 0; -} diff --git a/sys/contrib/zstd/tests/fuzz/stream_decompress.c b/sys/contrib/zstd/tests/fuzz/stream_decompress.c deleted file mode 100644 index 7ad571221df..00000000000 --- a/sys/contrib/zstd/tests/fuzz/stream_decompress.c +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * This fuzz target attempts to decompress the fuzzed data with the simple - * decompression function to ensure the decompressor never crashes. - */ - -#define ZSTD_STATIC_LINKING_ONLY - -#include -#include -#include -#include "fuzz_helpers.h" -#include "zstd.h" - -static size_t const kBufSize = ZSTD_BLOCKSIZE_MAX; - -static ZSTD_DStream *dstream = NULL; -static void* buf = NULL; -uint32_t seed; - -static ZSTD_outBuffer makeOutBuffer(void) -{ - ZSTD_outBuffer buffer = { buf, 0, 0 }; - - buffer.size = (FUZZ_rand(&seed) % kBufSize) + 1; - FUZZ_ASSERT(buffer.size <= kBufSize); - - return buffer; -} - -static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) -{ - ZSTD_inBuffer buffer = { *src, 0, 0 }; - - FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_rand(&seed) % *size) + 1; - FUZZ_ASSERT(buffer.size <= *size); - *src += buffer.size; - *size -= buffer.size; - - return buffer; -} - -int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) -{ - seed = FUZZ_seed(&src, &size); - - /* Allocate all buffers and contexts if not already allocated */ - if (!buf) { - buf = malloc(kBufSize); - FUZZ_ASSERT(buf); - } - - if (!dstream) { - dstream = ZSTD_createDStream(); - FUZZ_ASSERT(dstream); - FUZZ_ASSERT(!ZSTD_isError(ZSTD_initDStream(dstream))); - } else { - FUZZ_ASSERT(!ZSTD_isError(ZSTD_resetDStream(dstream))); - } - - while (size > 0) { - ZSTD_inBuffer in = makeInBuffer(&src, &size); - while (in.pos != in.size) { - ZSTD_outBuffer out = makeOutBuffer(); - size_t const rc = ZSTD_decompressStream(dstream, &out, &in); - if (ZSTD_isError(rc)) goto error; - if (rc == 0) FUZZ_ASSERT(!ZSTD_isError(ZSTD_resetDStream(dstream))); - } - } - -error: -#ifndef STATEFUL_FUZZING - ZSTD_freeDStream(dstream); dstream = NULL; -#endif - return 0; -} diff --git a/sys/contrib/zstd/tests/fuzz/stream_round_trip.c b/sys/contrib/zstd/tests/fuzz/stream_round_trip.c deleted file mode 100644 index 72d70495f83..00000000000 --- a/sys/contrib/zstd/tests/fuzz/stream_round_trip.c +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * This fuzz target performs a zstd round-trip test (compress & decompress), - * compares the result with the original, and calls abort() on corruption. - */ - -#define ZSTD_STATIC_LINKING_ONLY - -#include -#include -#include -#include -#include "fuzz_helpers.h" -#include "zstd_helpers.h" - -ZSTD_CCtx *cctx = NULL; -static ZSTD_DCtx *dctx = NULL; -static uint8_t* cBuf = NULL; -static uint8_t* rBuf = NULL; -static size_t bufSize = 0; -static uint32_t seed; - -static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity) -{ - ZSTD_outBuffer buffer = { dst, 0, 0 }; - - FUZZ_ASSERT(capacity > 0); - buffer.size = (FUZZ_rand(&seed) % capacity) + 1; - FUZZ_ASSERT(buffer.size <= capacity); - - return buffer; -} - -static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) -{ - ZSTD_inBuffer buffer = { *src, 0, 0 }; - - FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_rand(&seed) % *size) + 1; - FUZZ_ASSERT(buffer.size <= *size); - *src += buffer.size; - *size -= buffer.size; - - return buffer; -} - -static size_t compress(uint8_t *dst, size_t capacity, - const uint8_t *src, size_t srcSize) -{ - size_t dstSize = 0; - ZSTD_CCtx_reset(cctx); - FUZZ_setRandomParameters(cctx, srcSize, &seed); - - while (srcSize > 0) { - ZSTD_inBuffer in = makeInBuffer(&src, &srcSize); - /* Mode controls the action. If mode == -1 we pick a new mode */ - int mode = -1; - while (in.pos < in.size) { - ZSTD_outBuffer out = makeOutBuffer(dst, capacity); - /* Previous action finished, pick a new mode. */ - if (mode == -1) mode = FUZZ_rand(&seed) % 10; - switch (mode) { - case 0: /* fall-though */ - case 1: /* fall-though */ - case 2: { - size_t const ret = - ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_flush); - FUZZ_ZASSERT(ret); - if (ret == 0) - mode = -1; - break; - } - case 3: { - size_t ret = - ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); - FUZZ_ZASSERT(ret); - /* Reset the compressor when the frame is finished */ - if (ret == 0) { - ZSTD_CCtx_reset(cctx); - if ((FUZZ_rand(&seed) & 7) == 0) { - size_t const remaining = in.size - in.pos; - FUZZ_setRandomParameters(cctx, remaining, &seed); - } - mode = -1; - } - break; - } - default: { - size_t const ret = - ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue); - FUZZ_ZASSERT(ret); - mode = -1; - } - } - dst += out.pos; - dstSize += out.pos; - capacity -= out.pos; - } - } - for (;;) { - ZSTD_inBuffer in = {NULL, 0, 0}; - ZSTD_outBuffer out = makeOutBuffer(dst, capacity); - size_t const ret = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); - FUZZ_ZASSERT(ret); - - dst += out.pos; - dstSize += out.pos; - capacity -= out.pos; - if (ret == 0) - break; - } - return dstSize; -} - -int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) -{ - size_t neededBufSize; - - seed = FUZZ_seed(&src, &size); - neededBufSize = ZSTD_compressBound(size) * 2; - - /* Allocate all buffers and contexts if not already allocated */ - if (neededBufSize > bufSize) { - free(cBuf); - free(rBuf); - cBuf = (uint8_t*)malloc(neededBufSize); - rBuf = (uint8_t*)malloc(neededBufSize); - bufSize = neededBufSize; - FUZZ_ASSERT(cBuf && rBuf); - } - if (!cctx) { - cctx = ZSTD_createCCtx(); - FUZZ_ASSERT(cctx); - } - if (!dctx) { - dctx = ZSTD_createDCtx(); - FUZZ_ASSERT(dctx); - } - - { - size_t const cSize = compress(cBuf, neededBufSize, src, size); - size_t const rSize = - ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize); - FUZZ_ZASSERT(rSize); - FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size"); - FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); - } - -#ifndef STATEFUL_FUZZING - ZSTD_freeCCtx(cctx); cctx = NULL; - ZSTD_freeDCtx(dctx); dctx = NULL; -#endif - return 0; -} diff --git a/sys/contrib/zstd/tests/fuzz/zstd_helpers.c b/sys/contrib/zstd/tests/fuzz/zstd_helpers.c deleted file mode 100644 index 6fc38361b7a..00000000000 --- a/sys/contrib/zstd/tests/fuzz/zstd_helpers.c +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -#define ZSTD_STATIC_LINKING_ONLY - -#include "zstd_helpers.h" -#include "fuzz_helpers.h" -#include "zstd.h" - -static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned value) -{ - FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value)); -} - -static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min, - unsigned max, uint32_t *state) { - unsigned const value = FUZZ_rand32(state, min, max); - set(cctx, param, value); -} - -ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state) -{ - /* Select compression parameters */ - ZSTD_compressionParameters cParams; - cParams.windowLog = FUZZ_rand32(state, ZSTD_WINDOWLOG_MIN, 15); - cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15); - cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16); - cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9); - cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN, - ZSTD_SEARCHLENGTH_MAX); - cParams.targetLength = FUZZ_rand32(state, ZSTD_TARGETLENGTH_MIN, - 512); - cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra); - return ZSTD_adjustCParams(cParams, srcSize, 0); -} - -ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state) -{ - /* Select frame parameters */ - ZSTD_frameParameters fParams; - fParams.contentSizeFlag = FUZZ_rand32(state, 0, 1); - fParams.checksumFlag = FUZZ_rand32(state, 0, 1); - fParams.noDictIDFlag = FUZZ_rand32(state, 0, 1); - return fParams; -} - -ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state) -{ - ZSTD_parameters params; - params.cParams = FUZZ_randomCParams(srcSize, state); - params.fParams = FUZZ_randomFParams(state); - return params; -} - -void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) -{ - ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state); - set(cctx, ZSTD_p_windowLog, cParams.windowLog); - set(cctx, ZSTD_p_hashLog, cParams.hashLog); - set(cctx, ZSTD_p_chainLog, cParams.chainLog); - set(cctx, ZSTD_p_searchLog, cParams.searchLog); - set(cctx, ZSTD_p_minMatch, cParams.searchLength); - set(cctx, ZSTD_p_targetLength, cParams.targetLength); - set(cctx, ZSTD_p_compressionStrategy, cParams.strategy); - /* Select frame parameters */ - setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state); - setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state); - setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state); - /* Select long distance matchig parameters */ - setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state); - setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state); - setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, - ZSTD_LDM_MINMATCH_MAX, state); - setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX, - state); - setRand(cctx, ZSTD_p_ldmHashEveryLog, 0, - ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, state); -} diff --git a/sys/contrib/zstd/tests/fuzz/zstd_helpers.h b/sys/contrib/zstd/tests/fuzz/zstd_helpers.h deleted file mode 100644 index 3856bebecf7..00000000000 --- a/sys/contrib/zstd/tests/fuzz/zstd_helpers.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/** - * Helper functions for fuzzing. - */ - -#ifndef ZSTD_HELPERS_H -#define ZSTD_HELPERS_H - -#include "zstd.h" -#include - -#ifdef __cplusplus -extern "C" { -#endif - -void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state); - -ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state); -ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state); -ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state); - - -#ifdef __cplusplus -} -#endif - -#endif /* ZSTD_HELPERS_H */ diff --git a/sys/contrib/zstd/tests/fuzzer.c b/sys/contrib/zstd/tests/fuzzer.c deleted file mode 100644 index e97b841e853..00000000000 --- a/sys/contrib/zstd/tests/fuzzer.c +++ /dev/null @@ -1,1824 +0,0 @@ -/* - * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -/*-************************************ -* Compiler specific -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ -#endif - - -/*-************************************ -* Includes -**************************************/ -#include /* free */ -#include /* fgets, sscanf */ -#include /* strcmp */ -#include -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ -#include "zstd.h" /* ZSTD_VERSION_STRING */ -#include "zstd_errors.h" /* ZSTD_getErrorCode */ -#include "zstdmt_compress.h" -#define ZDICT_STATIC_LINKING_ONLY -#include "zdict.h" /* ZDICT_trainFromBuffer */ -#include "datagen.h" /* RDG_genBuffer */ -#include "mem.h" -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH64 */ -#include "util.h" - - -/*-************************************ -* Constants -**************************************/ -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -static const U32 FUZ_compressibility_default = 50; -static const U32 nbTestsDefault = 30000; - - -/*-************************************ -* Display Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 2; - -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ - { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); } } - - -#undef MIN -#undef MAX -void FUZ_bug976(void) -{ /* these constants shall not depend on MIN() macro */ - assert(ZSTD_HASHLOG_MAX < 31); - assert(ZSTD_CHAINLOG_MAX < 31); -} - -/*-******************************************************* -* Internal functions -*********************************************************/ -#define MIN(a,b) ((a)<(b)?(a):(b)) -#define MAX(a,b) ((a)>(b)?(a):(b)) - -#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -static unsigned FUZ_rand(unsigned* src) -{ - static const U32 prime1 = 2654435761U; - static const U32 prime2 = 2246822519U; - U32 rand32 = *src; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZ_rotl32(rand32, 13); - *src = rand32; - return rand32 >> 5; -} - -static unsigned FUZ_highbit32(U32 v32) -{ - unsigned nbBits = 0; - if (v32==0) return 0; - while (v32) v32 >>= 1, nbBits++; - return nbBits; -} - - -/*============================================= -* Test macros -=============================================*/ -#define CHECK_Z(f) { \ - size_t const err = f; \ - if (ZSTD_isError(err)) { \ - DISPLAY("Error => %s : %s ", \ - #f, ZSTD_getErrorName(err)); \ - exit(1); \ -} } - -#define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error -#define CHECK(fn) { CHECK_V(err, fn); } -#define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; } - - -/*============================================= -* Memory Tests -=============================================*/ -#if defined(__APPLE__) && defined(__MACH__) - -#include /* malloc_size */ - -typedef struct { - unsigned long long totalMalloc; - size_t currentMalloc; - size_t peakMalloc; - unsigned nbMalloc; - unsigned nbFree; -} mallocCounter_t; - -static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 }; - -static void* FUZ_mallocDebug(void* counter, size_t size) -{ - mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; - void* const ptr = malloc(size); - if (ptr==NULL) return NULL; - DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n", - (U32)(size >> 10), (U32)(malloc_size(ptr) >> 10)); /* OS-X specific */ - mcPtr->totalMalloc += size; - mcPtr->currentMalloc += size; - if (mcPtr->currentMalloc > mcPtr->peakMalloc) - mcPtr->peakMalloc = mcPtr->currentMalloc; - mcPtr->nbMalloc += 1; - return ptr; -} - -static void FUZ_freeDebug(void* counter, void* address) -{ - mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; - DISPLAYLEVEL(4, "freeing %u KB \n", (U32)(malloc_size(address) >> 10)); - mcPtr->nbFree += 1; - mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */ - free(address); -} - -static void FUZ_displayMallocStats(mallocCounter_t count) -{ - DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n", - (U32)(count.peakMalloc >> 10), - count.nbMalloc, - (U32)(count.totalMalloc >> 10)); -} - -static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) -{ - size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */ - size_t const outSize = ZSTD_compressBound(inSize); - void* const inBuffer = malloc(inSize); - void* const outBuffer = malloc(outSize); - - /* test only played in verbose mode, as they are long */ - if (g_displayLevel<3) return 0; - - /* Create compressible noise */ - if (!inBuffer || !outBuffer) { - DISPLAY("Not enough memory, aborting\n"); - exit(1); - } - RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed); - - /* simple compression tests */ - if (part <= 1) - { int compressionLevel; - for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { - mallocCounter_t malcount = INIT_MALLOC_COUNTER; - ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; - ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); - CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) ); - ZSTD_freeCCtx(cctx); - DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel); - FUZ_displayMallocStats(malcount); - } } - - /* streaming compression tests */ - if (part <= 2) - { int compressionLevel; - for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { - mallocCounter_t malcount = INIT_MALLOC_COUNTER; - ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; - ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem); - ZSTD_outBuffer out = { outBuffer, outSize, 0 }; - ZSTD_inBuffer in = { inBuffer, inSize, 0 }; - CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) ); - CHECK_Z( ZSTD_compressStream(cstream, &out, &in) ); - CHECK_Z( ZSTD_endStream(cstream, &out) ); - ZSTD_freeCStream(cstream); - DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel); - FUZ_displayMallocStats(malcount); - } } - - /* advanced MT API test */ - if (part <= 3) - { U32 nbThreads; - for (nbThreads=1; nbThreads<=4; nbThreads++) { - int compressionLevel; - for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { - mallocCounter_t malcount = INIT_MALLOC_COUNTER; - ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; - ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); - ZSTD_outBuffer out = { outBuffer, outSize, 0 }; - ZSTD_inBuffer in = { inBuffer, inSize, 0 }; - CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel) ); - CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbWorkers, nbThreads) ); - while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {} - ZSTD_freeCCtx(cctx); - DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ", - nbThreads, compressionLevel); - FUZ_displayMallocStats(malcount); - } } } - - /* advanced MT streaming API test */ - if (part <= 4) - { U32 nbThreads; - for (nbThreads=1; nbThreads<=4; nbThreads++) { - int compressionLevel; - for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { - mallocCounter_t malcount = INIT_MALLOC_COUNTER; - ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; - ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); - ZSTD_outBuffer out = { outBuffer, outSize, 0 }; - ZSTD_inBuffer in = { inBuffer, inSize, 0 }; - CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel) ); - CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbWorkers, nbThreads) ); - CHECK_Z( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue) ); - while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {} - ZSTD_freeCCtx(cctx); - DISPLAYLEVEL(3, "compress_generic,-T%u,continue level %i : ", - nbThreads, compressionLevel); - FUZ_displayMallocStats(malcount); - } } } - - return 0; -} - -#else - -static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) -{ - (void)seed; (void)compressibility; (void)part; - return 0; -} - -#endif - -/*============================================= -* Unit tests -=============================================*/ - -static int basicUnitTests(U32 seed, double compressibility) -{ - size_t const CNBuffSize = 5 MB; - void* const CNBuffer = malloc(CNBuffSize); - size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); - void* const compressedBuffer = malloc(compressedBufferSize); - void* const decodedBuffer = malloc(CNBuffSize); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); - int testResult = 0; - U32 testNb=0; - size_t cSize; - - /* Create compressible noise */ - if (!CNBuffer || !compressedBuffer || !decodedBuffer) { - DISPLAY("Not enough memory, aborting\n"); - testResult = 1; - goto _end; - } - RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); - - /* Basic tests */ - DISPLAYLEVEL(3, "test%3i : ZSTD_getErrorName : ", testNb++); - { const char* errorString = ZSTD_getErrorName(0); - DISPLAYLEVEL(3, "OK : %s \n", errorString); - } - - DISPLAYLEVEL(3, "test%3i : ZSTD_getErrorName with wrong value : ", testNb++); - { const char* errorString = ZSTD_getErrorName(499); - DISPLAYLEVEL(3, "OK : %s \n", errorString); - } - - - DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize); - { ZSTD_CCtx* cctx = ZSTD_createCCtx(); - if (cctx==NULL) goto _output_error; - CHECKPLUS(r, ZSTD_compressCCtx(cctx, - compressedBuffer, compressedBufferSize, - CNBuffer, CNBuffSize, 1), - cSize=r ); - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); - - DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++); - { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); - DISPLAYLEVEL(3, "%u bytes \n", (U32)cctxSize); - } - ZSTD_freeCCtx(cctx); - } - - - DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); - { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); - if (rSize != CNBuffSize) goto _output_error; - } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++); - { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize); - if (rSize != CNBuffSize) goto _output_error; - } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize); - { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); - if (r != CNBuffSize) goto _output_error; } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); - { size_t u; - for (u=0; u (1U << 20)) goto _output_error; - ZSTD_freeCCtx(cctx); - } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++); - { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); - assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */ - CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) ); - CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */ - - CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */ - { size_t const inSize = 2* 128 KB; - size_t const outSize = ZSTD_compressBound(inSize); - CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) ); - /* will fail if blockSize is not resized */ - } - ZSTD_freeCCtx(cctx); - } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3d : large window log smaller data : ", testNb++); - { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_parameters params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); - size_t const nbCompressions = (1U << 31) / CNBuffSize + 1; - size_t i; - params.fParams.contentSizeFlag = 0; - params.cParams.windowLog = ZSTD_WINDOWLOG_MAX; - for (i = 0; i < nbCompressions; ++i) { - CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */ - CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) ); - } - ZSTD_freeCCtx(cctx); - } - DISPLAYLEVEL(3, "OK \n"); - - /* Static CCtx tests */ -#define STATIC_CCTX_LEVEL 3 - DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL); - { size_t const staticCCtxSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL); - void* const staticCCtxBuffer = malloc(staticCCtxSize); - size_t const staticDCtxSize = ZSTD_estimateDCtxSize(); - void* const staticDCtxBuffer = malloc(staticDCtxSize); - if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) { - free(staticCCtxBuffer); - free(staticDCtxBuffer); - DISPLAY("Not enough memory, aborting\n"); - testResult = 1; - goto _end; - } - { ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); - ZSTD_DCtx* staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize); - if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error; - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); - { size_t const r = ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL); - if (ZSTD_isError(r)) goto _output_error; } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : simple compression test with static CCtx : ", testNb++); - CHECKPLUS(r, ZSTD_compressCCtx(staticCCtx, - compressedBuffer, compressedBufferSize, - CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL), - cSize=r ); - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", - (U32)cSize, (double)cSize/CNBuffSize*100); - - DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++); - { size_t const r = ZSTD_decompressDCtx(staticDCtx, - decodedBuffer, CNBuffSize, - compressedBuffer, cSize); - if (r != CNBuffSize) goto _output_error; } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); - { size_t u; - for (u=0; u same size */ - } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); - - DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); - CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, - decodedBuffer, CNBuffSize, - compressedBuffer, cSize, - CNBuffer, dictSize), - if (r != CNBuffSize - dictSize) goto _output_error); - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++); - { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize); - size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); - if (r != CNBuffSize - dictSize) goto _output_error; - DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict)); - ZSTD_freeDDict(ddict); - } - - DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++); - { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); - void* ddictBuffer = malloc(ddictBufferSize); - if (ddictBuffer == NULL) goto _output_error; - { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); - size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); - if (r != CNBuffSize - dictSize) goto _output_error; - } - free(ddictBuffer); - DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (U32)ddictBufferSize); - } - - DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); - { size_t const testSize = CNBuffSize / 3; - { ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize); - p.fParams.contentSizeFlag = 1; - CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); - } - CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) ); - - CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), - (const char*)CNBuffer + dictSize, testSize), - cSize = r); - { ZSTD_frameHeader zfh; - if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error; - if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error; - } } - DISPLAYLEVEL(3, "OK \n"); - - ZSTD_freeCCtx(ctxOrig); - ZSTD_freeCCtx(ctxDuplicated); - } - - /* Dictionary and dictBuilder tests */ - { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - size_t const dictBufferCapacity = 16 KB; - void* dictBuffer = malloc(dictBufferCapacity); - size_t const totalSampleSize = 1 MB; - size_t const sampleUnitSize = 8 KB; - U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); - size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); - size_t dictSize; - U32 dictID; - - if (dictBuffer==NULL || samplesSizes==NULL) { - free(dictBuffer); - free(samplesSizes); - goto _output_error; - } - - DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++); - assert(compressedBufferSize >= totalSampleSize); - { U32 u; for (u=0; u= dictLimit) goto _output_error; - MEM_writeLE32(dictPtr + 0, 10); - MEM_writeLE32(dictPtr + 4, 10); - MEM_writeLE32(dictPtr + 8, 10); - /* Set the last 8 bytes to 'x' */ - memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8); - } - /* The optimal parser checks all the repcodes. - * Make sure at least one is a match >= targetLength so that it is - * immediately chosen. This will make sure that the compressor and - * decompressor agree on at least one of the repcodes. - */ - { size_t dSize; - BYTE data[1024]; - ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize); - ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, - ZSTD_dlm_byRef, ZSTD_dct_auto, - cParams, ZSTD_defaultCMem); - memset(data, 'x', sizeof(data)); - cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, - data, sizeof(data), cdict); - ZSTD_freeCDict(cdict); - if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; } - dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize); - if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; } - if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; } - } - DISPLAYLEVEL(3, "OK \n"); - - ZSTD_freeCCtx(cctx); - free(dictBuffer); - free(samplesSizes); - } - - /* COVER dictionary builder tests */ - { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - size_t dictSize = 16 KB; - size_t optDictSize = dictSize; - void* dictBuffer = malloc(dictSize); - size_t const totalSampleSize = 1 MB; - size_t const sampleUnitSize = 8 KB; - U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); - size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); - ZDICT_cover_params_t params; - U32 dictID; - - if (dictBuffer==NULL || samplesSizes==NULL) { - free(dictBuffer); - free(samplesSizes); - goto _output_error; - } - - DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++); - { U32 u; for (u=0; u %u bytes)\n", (U32)inputSize, (U32)cSize); - ZSTD_freeCCtx(cctx); - } - - { ZSTD_CCtx* cctx = ZSTD_createCCtx(); - DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) ); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) ); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) ); - { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; - ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; - size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); - if (result != 0) goto _output_error; - if (in.pos != in.size) goto _output_error; - if (out.pos != cSize) goto _output_error; /* must result in same compressed result, hence same size */ - if (XXH64(out.dst, out.pos, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ - DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)out.pos); - } - ZSTD_freeCCtx(cctx); - } - } - - /* custom formats tests */ - { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */ - - /* basic block compression */ - DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_format, ZSTD_f_zstd1_magicless) ); - { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; - ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; - size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); - if (result != 0) goto _output_error; - if (in.pos != in.size) goto _output_error; - cSize = out.pos; - } - DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize); - - DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++); - { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); - if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error; - DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult)); - } - - DISPLAYLEVEL(3, "test%3i : decompress with magic-less instruction : ", testNb++); - ZSTD_DCtx_reset(dctx); - CHECK( ZSTD_DCtx_setFormat(dctx, ZSTD_f_zstd1_magicless) ); - { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; - ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; - size_t const result = ZSTD_decompress_generic(dctx, &out, &in); - if (result != 0) goto _output_error; - if (in.pos != in.size) goto _output_error; - if (out.pos != inputSize) goto _output_error; - DISPLAYLEVEL(3, "OK : regenerated %u bytes \n", (U32)out.pos); - } - - ZSTD_freeCCtx(cctx); - } - - /* block API tests */ - { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - static const size_t dictSize = 65 KB; - static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ - size_t cSize2; - - /* basic block compression */ - DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); - CHECK( ZSTD_compressBegin(cctx, 5) ); - CHECK( ZSTD_getBlockSize(cctx) >= blockSize); - cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); - if (ZSTD_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++); - CHECK( ZSTD_decompressBegin(dctx) ); - { CHECK_V(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); - if (r != blockSize) goto _output_error; } - DISPLAYLEVEL(3, "OK \n"); - - /* dictionary block compression */ - DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++); - CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); - cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize); - if (ZSTD_isError(cSize)) goto _output_error; - cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize); - if (ZSTD_isError(cSize2)) goto _output_error; - memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* fake non-compressed block */ - cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize), - (char*)CNBuffer+dictSize+2*blockSize, blockSize); - if (ZSTD_isError(cSize2)) goto _output_error; - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++); - CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); - { CHECK_V( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); - if (r != blockSize) goto _output_error; } - ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */ - { CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, CNBuffSize, (char*)compressedBuffer+cSize+blockSize, cSize2) ); - if (r != blockSize) goto _output_error; } - DISPLAYLEVEL(3, "OK \n"); - - ZSTD_freeCCtx(cctx); - } - ZSTD_freeDCtx(dctx); - - /* long rle test */ - { size_t sampleSize = 0; - DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); - RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1); - memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); - sampleSize += 256 KB - 1; - RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2); - sampleSize += 96 KB; - cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); - if (ZSTD_isError(cSize)) goto _output_error; - { CHECK_V(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); - if (regenSize!=sampleSize) goto _output_error; } - DISPLAYLEVEL(3, "OK \n"); - } - - /* All zeroes test (test bug #137) */ - #define ZEROESLENGTH 100 - DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); - memset(CNBuffer, 0, ZEROESLENGTH); - { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); - cSize = r; } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100); - - DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); - { CHECK_V(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); - if (r != ZEROESLENGTH) goto _output_error; } - DISPLAYLEVEL(3, "OK \n"); - - /* nbSeq limit test */ - #define _3BYTESTESTLENGTH 131000 - #define NB3BYTESSEQLOG 9 - #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG) - #define NB3BYTESSEQMASK (NB3BYTESSEQ-1) - /* creates a buffer full of 3-bytes sequences */ - { BYTE _3BytesSeqs[NB3BYTESSEQ][3]; - U32 rSeed = 1; - - /* create batch of 3-bytes sequences */ - { int i; - for (i=0; i < NB3BYTESSEQ; i++) { - _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); - _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); - _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); - } } - - /* randomly fills CNBuffer with prepared 3-bytes sequences */ - { int i; - for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ - U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; - ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; - ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; - ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; - } } } - DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++); - { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), - CNBuffer, _3BYTESTESTLENGTH, 19) ); - cSize = r; } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100); - - DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++); - { CHECK_V(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); - if (r != _3BYTESTESTLENGTH) goto _output_error; } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++); - RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed); - { /* Train a dictionary on low characters */ - size_t dictSize = 16 KB; - void* const dictBuffer = malloc(dictSize); - size_t const totalSampleSize = 1 MB; - size_t const sampleUnitSize = 8 KB; - U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); - size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); - if (!dictBuffer || !samplesSizes) goto _output_error; - { U32 u; for (u=0; u "); \ - DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \ - goto _output_error; \ -} } - -#undef CHECK_Z -#define CHECK_Z(f) { \ - size_t const err = f; \ - if (ZSTD_isError(err)) { \ - DISPLAY("Error => %s : %s ", \ - #f, ZSTD_getErrorName(err)); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \ - goto _output_error; \ -} } - - -static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests) -{ - static const U32 maxSrcLog = 23; - static const U32 maxSampleLog = 22; - size_t const srcBufferSize = (size_t)1<= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } - else { DISPLAYUPDATE(2, "\r%6u ", testNb); } - - FUZ_rand(&coreSeed); - { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; } - - /* srcBuffer selection [0-4] */ - { U32 buffNb = FUZ_rand(&lseed) & 0x7F; - if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ - else { - buffNb >>= 3; - if (buffNb & 7) { - const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ - buffNb = tnb[buffNb >> 3]; - } else { - const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ - buffNb = tnb[buffNb >> 3]; - } } - srcBuffer = cNoiseBuffer[buffNb]; - } - - /* select src segment */ - sampleSize = FUZ_randomLength(&lseed, maxSampleLog); - - /* create sample buffer (to catch read error with valgrind & sanitizers) */ - sampleBuffer = (BYTE*)malloc(sampleSize); - CHECK(sampleBuffer==NULL, "not enough memory for sample buffer"); - { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); - memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); } - crcOrig = XXH64(sampleBuffer, sampleSize, 0); - - /* compression tests */ - { int const cLevelPositive = - ( FUZ_rand(&lseed) % - (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / cLevelLimiter)) ) - + 1; - int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ? - - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */ - cLevelPositive; - DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel); - cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); - CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize)); - - /* compression failure test : too small dest buffer */ - if (cSize > 3) { - const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ - const size_t tooSmallSize = cSize - missing; - const U32 endMark = 0x4DC2B1A9; - memcpy(dstBuffer+tooSmallSize, &endMark, 4); - { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); - CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); } - { U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4); - CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } - } } - - /* frame header decompression test */ - { ZSTD_frameHeader zfh; - CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) ); - CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect"); - } - - /* Decompressed size test */ - { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize); - CHECK(rSize != sampleSize, "decompressed size incorrect"); - } - - /* successful decompression test */ - DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb); - { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; - size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); - CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize); - { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0); - CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize); - } } - - free(sampleBuffer); /* no longer useful after this point */ - - /* truncated src decompression test */ - DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb); - { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ - size_t const tooSmallSize = cSize - missing; - void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */ - CHECK(cBufferTooSmall == NULL, "not enough memory !"); - memcpy(cBufferTooSmall, cBuffer, tooSmallSize); - { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); - CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); } - free(cBufferTooSmall); - } - - /* too small dst decompression test */ - DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb); - if (sampleSize > 3) { - size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ - size_t const tooSmallSize = sampleSize - missing; - static const BYTE token = 0xA9; - dstBuffer[tooSmallSize] = token; - { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); - CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize); } - CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); - } - - /* noisy src decompression test */ - if (cSize > 6) { - /* insert noise into src */ - { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4)); - size_t pos = 4; /* preserve magic number (too easy to detect) */ - for (;;) { - /* keep some original src */ - { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits; - size_t const mask = (1<= cSize) break; - /* add noise */ - { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits; - U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; - size_t const mask = (1<sampleSize), - "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)decompressResult, (U32)sampleSize); - } - { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4); - CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); - } } } /* noisy src decompression test */ - - /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/ - DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb); - { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - int const cLevel = (FUZ_rand(&lseed) % - (ZSTD_maxCLevel() - - (MAX(testLog, dictLog) / cLevelLimiter))) + - 1; - maxTestSize = FUZ_rLogLength(&lseed, testLog); - if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; - - dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */ - dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); - - DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n", - testNb, (U32)maxTestSize, cLevel, (U32)dictSize); - - if (FUZ_rand(&lseed) & 0xF) { - CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) ); - } else { - ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); - ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, - !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, - 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ - ZSTD_parameters const p = FUZ_makeParams(cPar, fPar); - CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) ); - } - CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) ); - } - - { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; - U32 n; - XXH64_state_t xxhState; - XXH64_reset(&xxhState, 0); - for (totalTestSize=0, cSize=0, n=0 ; n maxTestSize) break; - - { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize); - CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); - cSize += compressResult; - } - XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); - memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); - totalTestSize += segmentSize; - } - - { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); - CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); - cSize += flushResult; - } - crcOrig = XXH64_digest(&xxhState); - } - - /* streaming decompression test */ - DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb); - /* ensure memory requirement is good enough (should always be true) */ - { ZSTD_frameHeader zfh; - CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_frameHeaderSize_max), - "ZSTD_getFrameHeader(): error retrieving frame information"); - { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize); - CHECK_Z(roundBuffSize); - CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN), - "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)", - (U32)roundBuffSize, (U32)totalTestSize ); - } } - if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ - CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) ); - totalCSize = 0; - totalGenSize = 0; - while (totalCSize < cSize) { - size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx); - size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); - CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize)); - totalGenSize += genSize; - totalCSize += inSize; - } - CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); - CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size") - CHECK (totalCSize != cSize, "compressed data should be fully read") - { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); - if (crcDest!=crcOrig) { - size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize); - CHECK (1, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)", - (U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]); - } } - } /* for ( ; (testNb <= nbTests) */ - DISPLAY("\r%u fuzzer tests completed \n", testNb-1); - -_cleanup: - ZSTD_freeCCtx(refCtx); - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - free(cNoiseBuffer[0]); - free(cNoiseBuffer[1]); - free(cNoiseBuffer[2]); - free(cNoiseBuffer[3]); - free(cNoiseBuffer[4]); - free(cBuffer); - free(dstBuffer); - free(mirrorBuffer); - return result; - -_output_error: - result = 1; - goto _cleanup; -} - - -/*_******************************************************* -* Command line -*********************************************************/ -static int FUZ_usage(const char* programName) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); - DISPLAY( " -s# : Select seed (default:prompt user)\n"); - DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -P# : Select compressibility in %% (default:%u%%)\n", FUZ_compressibility_default); - DISPLAY( " -v : verbose\n"); - DISPLAY( " -p : pause at the end\n"); - DISPLAY( " -h : display help and exit\n"); - return 0; -} - -/*! readU32FromChar() : - @return : unsigned integer value read from input in `char` format - allows and interprets K, KB, KiB, M, MB and MiB suffix. - Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : function result can overflow if digit string > MAX_UINT */ -static unsigned readU32FromChar(const char** stringPtr) -{ - unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; - if ((**stringPtr=='K') || (**stringPtr=='M')) { - result <<= 10; - if (**stringPtr=='M') result <<= 10; - (*stringPtr)++ ; - if (**stringPtr=='i') (*stringPtr)++; - if (**stringPtr=='B') (*stringPtr)++; - } - return result; -} - -/** longCommandWArg() : - * check if *stringPtr is the same as longCommand. - * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. - * @return 0 and doesn't modify *stringPtr otherwise. - */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) -{ - size_t const comSize = strlen(longCommand); - int const result = !strncmp(*stringPtr, longCommand, comSize); - if (result) *stringPtr += comSize; - return result; -} - -int main(int argc, const char** argv) -{ - U32 seed = 0; - int seedset = 0; - int argNb; - int nbTests = nbTestsDefault; - int testNb = 0; - U32 proba = FUZ_compressibility_default; - int result = 0; - U32 mainPause = 0; - U32 maxDuration = 0; - int bigTests = 1; - U32 memTestsOnly = 0; - const char* const programName = argv[0]; - - /* Check command line */ - for (argNb=1; argNb100) proba = 100; - break; - - default: - return (FUZ_usage(programName), 1); - } } } } /* for (argNb=1; argNb $(VOID) - @$(RM) *.trs *.log - @echo Cleaning completed - - -#------------------------------------------------------------------------------ -# validated only for Linux, OSX, Hurd and some BSD targets -#------------------------------------------------------------------------------ -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) - -test-%: zstd - @./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh - # || echo ignoring error - -endif diff --git a/sys/contrib/zstd/tests/gzip/gzip-env.sh b/sys/contrib/zstd/tests/gzip/gzip-env.sh deleted file mode 100755 index 120e52d78d4..00000000000 --- a/sys/contrib/zstd/tests/gzip/gzip-env.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# Test the obsolescent GZIP environment variable. - -# Copyright 2015-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -#echo PATH=$PATH -#gzip --version - -echo a >exp || framework_failure_ -gzip in || framework_failure_ - -fail=0 -GZIP=-qv gzip -d out 2>err || fail=1 -compare exp out || fail=1 - -for badopt in -- -c --stdout -d --decompress -f --force -h --help -k --keep \ - -l --list -L --license -r --recursive -Sxxx --suffix=xxx '--suffix xxx' \ - -t --test -V --version -do - GZIP=$badopt gzip -d out 2>err && fail=1 -done - -for goodopt in -n --no-name -N --name -q --quiet -v --verbose \ - -1 --fast -2 -3 -4 -5 -6 -7 -8 -9 --best -do - GZIP=$goodopt gzip -d out 2>err || fail=1 - compare exp out || fail=1 -done - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/helin-segv.sh b/sys/contrib/zstd/tests/gzip/helin-segv.sh deleted file mode 100755 index f182c8066f3..00000000000 --- a/sys/contrib/zstd/tests/gzip/helin-segv.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -# Before gzip-1.4, gzip -d would segfault on some inputs. - -# Copyright (C) 2010-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -# This test case was provided by Aki Helin. -printf '\037\235\220\0\0\0\304' > helin.gz || framework_failure_ -printf '\0\0' > exp || framework_failure_ - -fail=0 - -gzip -dc helin.gz > out || fail=1 -compare exp out || fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/help-version.sh b/sys/contrib/zstd/tests/gzip/help-version.sh deleted file mode 100755 index ee0c19f7d1f..00000000000 --- a/sys/contrib/zstd/tests/gzip/help-version.sh +++ /dev/null @@ -1,270 +0,0 @@ -#! /bin/sh -# Make sure all these programs work properly -# when invoked with --help or --version. - -# Copyright (C) 2000-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Ensure that $SHELL is set to *some* value and exported. -# This is required for dircolors, which would fail e.g., when -# invoked via debuild (which removes SHELL from the environment). -test "x$SHELL" = x && SHELL=/bin/sh -export SHELL - -. "${srcdir=.}/init.sh"; path_prepend_ . - -expected_failure_status_chroot=125 -expected_failure_status_env=125 -expected_failure_status_nice=125 -expected_failure_status_nohup=125 -expected_failure_status_stdbuf=125 -expected_failure_status_su=125 -expected_failure_status_timeout=125 -expected_failure_status_printenv=2 -expected_failure_status_tty=3 -expected_failure_status_sort=2 -expected_failure_status_expr=3 -expected_failure_status_lbracket=2 -expected_failure_status_dir=2 -expected_failure_status_ls=2 -expected_failure_status_vdir=2 - -expected_failure_status_cmp=2 -expected_failure_status_zcmp=2 -expected_failure_status_sdiff=2 -expected_failure_status_diff3=2 -expected_failure_status_diff=2 -expected_failure_status_zdiff=2 -expected_failure_status_zgrep=2 -expected_failure_status_zegrep=2 -expected_failure_status_zfgrep=2 - -expected_failure_status_grep=2 -expected_failure_status_egrep=2 -expected_failure_status_fgrep=2 - -test "$built_programs" \ - || fail_ "built_programs not specified!?!" - -test "$VERSION" \ - || fail_ "set envvar VERSION; it is required for a PATH sanity-check" - -# Extract version from --version output of the first program -for i in $built_programs; do - v=$(env $i --version | sed -n '1s/.* //p;q') - break -done - -# Ensure that it matches $VERSION. -test "x$v" = "x$VERSION" \ - || fail_ "--version-\$VERSION mismatch" - -for lang in C fr da; do - for i in $built_programs; do - - # Skip `test'; it doesn't accept --help or --version. - test $i = test && continue; - - # false fails even when invoked with --help or --version. - if test $i = false; then - env LC_MESSAGES=$lang $i --help >/dev/null && fail=1 - env LC_MESSAGES=$lang $i --version >/dev/null && fail=1 - continue - fi - - args= - - # The just-built install executable is always named `ginstall'. - test $i = install && i=ginstall - - # Make sure they exit successfully, under normal conditions. - eval "env \$i $args --help > h-\$i " || fail=1 - eval "env \$i $args --version >/dev/null" || fail=1 - - # Make sure they mention the bug-reporting address in --help output. - grep "$PACKAGE_BUGREPORT" h-$i > /dev/null || fail=1 - rm -f h-$i - - # Make sure they fail upon `disk full' error. - if test -w /dev/full && test -c /dev/full; then - eval "env \$i $args --help >/dev/full 2>/dev/null" && fail=1 - eval "env \$i $args --version >/dev/full 2>/dev/null" && fail=1 - status=$? - test $i = [ && prog=lbracket || prog=$i - eval "expected=\$expected_failure_status_$prog" - test x$expected = x && expected=1 - if test $status = $expected; then - : # ok - else - fail=1 - echo "*** $i: bad exit status \`$status' (expected $expected)," 1>&2 - echo " with --help or --version output redirected to /dev/full" 1>&2 - fi - fi - done -done - -bigZ_in=bigZ-in.Z -zin=zin.gz -zin2=zin2.gz - -tmp=tmp-$$ -tmp_in=in-$$ -tmp_in2=in2-$$ -tmp_dir=dir-$$ -tmp_out=out-$$ -mkdir $tmp || fail=1 -cd $tmp || fail=1 - -comm_setup () { args="$tmp_in $tmp_in"; } -csplit_setup () { args="$tmp_in //"; } -cut_setup () { args='-f 1'; } -join_setup () { args="$tmp_in $tmp_in"; } -tr_setup () { args='a a'; } - -chmod_setup () { args="a+x $tmp_in"; } -# Punt on these. -chgrp_setup () { args=--version; } -chown_setup () { args=--version; } -mkfifo_setup () { args=--version; } -mknod_setup () { args=--version; } -# Punt on uptime, since it fails (e.g., failing to get boot time) -# on some systems, and we shouldn't let that stop `make check'. -uptime_setup () { args=--version; } - -# Create a file in the current directory, not in $TMPDIR. -mktemp_setup () { args=mktemp.XXXX; } - -cmp_setup () { args="$tmp_in $tmp_in2"; } - -# Tell dd not to print the line with transfer rate and total. -# The transfer rate would vary between runs. -dd_setup () { args=status=noxfer; } - -zdiff_setup () { args="$args $zin $zin2"; } -zcmp_setup () { zdiff_setup; } -zcat_setup () { args="$args $zin"; } -gunzip_setup () { zcat_setup; } -zmore_setup () { zcat_setup; } -zless_setup () { zcat_setup; } -znew_setup () { args="$args $bigZ_in"; } -zforce_setup () { zcat_setup; } -zgrep_setup () { args="$args z $zin"; } -zegrep_setup () { zgrep_setup; } -zfgrep_setup () { zgrep_setup; } -gzexe_setup () { args="$args $tmp_in"; } - -# We know that $tmp_in contains a "0" -grep_setup () { args="0 $tmp_in"; } -egrep_setup () { args="0 $tmp_in"; } -fgrep_setup () { args="0 $tmp_in"; } - -diff_setup () { args="$tmp_in $tmp_in2"; } -sdiff_setup () { args="$tmp_in $tmp_in2"; } -diff3_setup () { args="$tmp_in $tmp_in2 $tmp_in2"; } -cp_setup () { args="$tmp_in $tmp_in2"; } -ln_setup () { args="$tmp_in ln-target"; } -ginstall_setup () { args="$tmp_in $tmp_in2"; } -mv_setup () { args="$tmp_in $tmp_in2"; } -mkdir_setup () { args=$tmp_dir/subdir; } -rmdir_setup () { args=$tmp_dir; } -rm_setup () { args=$tmp_in; } -shred_setup () { args=$tmp_in; } -touch_setup () { args=$tmp_in2; } -truncate_setup () { args="--reference=$tmp_in $tmp_in2"; } - -basename_setup () { args=$tmp_in; } -dirname_setup () { args=$tmp_in; } -expr_setup () { args=foo; } - -# Punt, in case GNU `id' hasn't been installed yet. -groups_setup () { args=--version; } - -pathchk_setup () { args=$tmp_in; } -yes_setup () { args=--version; } -logname_setup () { args=--version; } -nohup_setup () { args=--version; } -printf_setup () { args=foo; } -seq_setup () { args=10; } -sleep_setup () { args=0; } -su_setup () { args=--version; } -stdbuf_setup () { args="-oL true"; } -timeout_setup () { args=--version; } - -# I'd rather not run sync, since it spins up disks that I've -# deliberately caused to spin down (but not unmounted). -sync_setup () { args=--version; } - -test_setup () { args=foo; } - -# This is necessary in the unusual event that there is -# no valid entry in /etc/mtab. -df_setup () { args=/; } - -# This is necessary in the unusual event that getpwuid (getuid ()) fails. -id_setup () { args=-u; } - -# Use env to avoid invoking built-in sleep of Solaris 11's /bin/sh. -kill_setup () { - env sleep 10m & - args=$! -} - -link_setup () { args="$tmp_in link-target"; } -unlink_setup () { args=$tmp_in; } - -readlink_setup () { - ln -s . slink - args=slink; -} - -stat_setup () { args=$tmp_in; } -unlink_setup () { args=$tmp_in; } -lbracket_setup () { args=": ]"; } - -# Ensure that each program "works" (exits successfully) when doing -# something more than --help or --version. -for i in $built_programs; do - # Skip these. - case $i in chroot|stty|tty|false|chcon|runcon) continue;; esac - - rm -rf $tmp_in $tmp_in2 $tmp_dir $tmp_out $bigZ_in $zin $zin2 - echo z |gzip > $zin - cp $zin $zin2 - cp $zin $bigZ_in - - # This is sort of kludgey: use numbers so this is valid input for factor, - # and two tokens so it's valid input for tsort. - echo 2147483647 0 > $tmp_in - # Make $tmp_in2 identical. Then, using $tmp_in and $tmp_in2 as arguments - # to the likes of cmp and diff makes them exit successfully. - cp $tmp_in $tmp_in2 - mkdir $tmp_dir - # echo ================== $i - test $i = [ && prog=lbracket || prog=$i - args= - if type ${prog}_setup > /dev/null 2>&1; then - ${prog}_setup - fi - if eval "env \$i $args < \$tmp_in > \$tmp_out"; then - : # ok - else - echo FAIL: $i - fail=1 - fi - rm -rf $tmp_in $tmp_in2 $tmp_out $tmp_dir -done - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/hufts.sh b/sys/contrib/zstd/tests/gzip/hufts.sh deleted file mode 100755 index 9b9576ce34e..00000000000 --- a/sys/contrib/zstd/tests/gzip/hufts.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# Exercise a bug whereby an invalid input could make gzip -d misbehave. - -# Copyright (C) 2009-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -printf '\n...: invalid compressed data--format violated\n' > exp \ - || framework_failure_ - -fail=0 -gzip -dc "$abs_srcdir/hufts-segv.gz" > out 2> err -test $? = 1 || fail=1 - -compare /dev/null out || fail=1 - -sed 's/.*hufts-segv.gz: /...: /' err > k; mv k err || fail=1 -compare exp err || fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/init.cfg b/sys/contrib/zstd/tests/gzip/init.cfg deleted file mode 100644 index 901209ceae9..00000000000 --- a/sys/contrib/zstd/tests/gzip/init.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# This file is sourced by init.sh, *before* its initialization. - -# This goes hand in hand with the "exec 9>&2;" in Makefile.am's -# TESTS_ENVIRONMENT definition. -stderr_fileno_=9 diff --git a/sys/contrib/zstd/tests/gzip/init.sh b/sys/contrib/zstd/tests/gzip/init.sh deleted file mode 100755 index 97e4e4ba5e6..00000000000 --- a/sys/contrib/zstd/tests/gzip/init.sh +++ /dev/null @@ -1,616 +0,0 @@ -# source this file; set up for tests - -# Copyright (C) 2009-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Using this file in a test -# ========================= -# -# The typical skeleton of a test looks like this: -# -# #!/bin/sh -# . "${srcdir=.}/init.sh"; path_prepend_ . -# Execute some commands. -# Note that these commands are executed in a subdirectory, therefore you -# need to prepend "../" to relative filenames in the build directory. -# Note that the "path_prepend_ ." is useful only if the body of your -# test invokes programs residing in the initial directory. -# For example, if the programs you want to test are in src/, and this test -# script is named tests/test-1, then you would use "path_prepend_ ../src", -# or perhaps export PATH='$(abs_top_builddir)/src$(PATH_SEPARATOR)'"$$PATH" -# to all tests via automake's TESTS_ENVIRONMENT. -# Set the exit code 0 for success, 77 for skipped, or 1 or other for failure. -# Use the skip_ and fail_ functions to print a diagnostic and then exit -# with the corresponding exit code. -# Exit $? - -# Executing a test that uses this file -# ==================================== -# -# Running a single test: -# $ make check TESTS=test-foo.sh -# -# Running a single test, with verbose output: -# $ make check TESTS=test-foo.sh VERBOSE=yes -# -# Running a single test, with single-stepping: -# 1. Go into a sub-shell: -# $ bash -# 2. Set relevant environment variables from TESTS_ENVIRONMENT in the -# Makefile: -# $ export srcdir=../../tests # this is an example -# 3. Execute the commands from the test, copy&pasting them one by one: -# $ . "$srcdir/init.sh"; path_prepend_ . -# ... -# 4. Finally -# $ exit - -ME_=`expr "./$0" : '.*/\(.*\)$'` - -# We use a trap below for cleanup. This requires us to go through -# hoops to get the right exit status transported through the handler. -# So use 'Exit STATUS' instead of 'exit STATUS' inside of the tests. -# Turn off errexit here so that we don't trip the bug with OSF1/Tru64 -# sh inside this function. -Exit () { set +e; (exit $1); exit $1; } - -# Print warnings (e.g., about skipped and failed tests) to this file number. -# Override by defining to say, 9, in init.cfg, and putting say, -# export ...ENVVAR_SETTINGS...; $(SHELL) 9>&2 -# in the definition of TESTS_ENVIRONMENT in your tests/Makefile.am file. -# This is useful when using automake's parallel tests mode, to print -# the reason for skip/failure to console, rather than to the .log files. -: ${stderr_fileno_=2} - -# Note that correct expansion of "$*" depends on IFS starting with ' '. -# Always write the full diagnostic to stderr. -# When stderr_fileno_ is not 2, also emit the first line of the -# diagnostic to that file descriptor. -warn_ () -{ - # If IFS does not start with ' ', set it and emit the warning in a subshell. - case $IFS in - ' '*) printf '%s\n' "$*" >&2 - test $stderr_fileno_ = 2 \ - || { printf '%s\n' "$*" | sed 1q >&$stderr_fileno_ ; } ;; - *) (IFS=' '; warn_ "$@");; - esac -} -fail_ () { warn_ "$ME_: failed test: $@"; Exit 1; } -skip_ () { warn_ "$ME_: skipped test: $@"; Exit 77; } -fatal_ () { warn_ "$ME_: hard error: $@"; Exit 99; } -framework_failure_ () { warn_ "$ME_: set-up failure: $@"; Exit 99; } - -# This is used to simplify checking of the return value -# which is useful when ensuring a command fails as desired. -# I.e., just doing `command ... &&fail=1` will not catch -# a segfault in command for example. With this helper you -# instead check an explicit exit code like -# returns_ 1 command ... || fail -returns_ () { - # Disable tracing so it doesn't interfere with stderr of the wrapped command - { set +x; } 2>/dev/null - - local exp_exit="$1" - shift - "$@" - test $? -eq $exp_exit && ret_=0 || ret_=1 - - if test "$VERBOSE" = yes && test "$gl_set_x_corrupts_stderr_" = false; then - set -x - fi - { return $ret_; } 2>/dev/null -} - -# Sanitize this shell to POSIX mode, if possible. -DUALCASE=1; export DUALCASE -if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then - emulate sh - NULLCMD=: - alias -g '${1+"$@"}'='"$@"' - setopt NO_GLOB_SUBST -else - case `(set -o) 2>/dev/null` in - *posix*) set -o posix ;; - esac -fi - -# We require $(...) support unconditionally. -# We require a few additional shell features only when $EXEEXT is nonempty, -# in order to support automatic $EXEEXT emulation: -# - hyphen-containing alias names -# - we prefer to use ${var#...} substitution, rather than having -# to work around lack of support for that feature. -# The following code attempts to find a shell with support for these features. -# If the current shell passes the test, we're done. Otherwise, test other -# shells until we find one that passes. If one is found, re-exec it. -# If no acceptable shell is found, skip the current test. -# -# The "...set -x; P=1 true 2>err..." test is to disqualify any shell that -# emits "P=1" into err, as /bin/sh from SunOS 5.11 and OpenBSD 4.7 do. -# -# Use "9" to indicate success (rather than 0), in case some shell acts -# like Solaris 10's /bin/sh but exits successfully instead of with status 2. - -# Eval this code in a subshell to determine a shell's suitability. -# 10 - passes all tests; ok to use -# 9 - ok, but enabling "set -x" corrupts app stderr; prefer higher score -# ? - not ok -gl_shell_test_script_=' -test $(echo y) = y || exit 1 -f_local_() { local v=1; }; f_local_ || exit 1 -score_=10 -if test "$VERBOSE" = yes; then - test -n "$( (exec 3>&1; set -x; P=1 true 2>&3) 2> /dev/null)" && score_=9 -fi -test -z "$EXEEXT" && exit $score_ -shopt -s expand_aliases -alias a-b="echo zoo" -v=abx - test ${v%x} = ab \ - && test ${v#a} = bx \ - && test $(a-b) = zoo \ - && exit $score_ -' - -if test "x$1" = "x--no-reexec"; then - shift -else - # Assume a working shell. Export to subshells (setup_ needs this). - gl_set_x_corrupts_stderr_=false - export gl_set_x_corrupts_stderr_ - - # Record the first marginally acceptable shell. - marginal_= - - # Search for a shell that meets our requirements. - for re_shell_ in __current__ "${CONFIG_SHELL:-no_shell}" \ - /bin/sh bash dash zsh pdksh fail - do - test "$re_shell_" = no_shell && continue - - # If we've made it all the way to the sentinel, "fail" without - # finding even a marginal shell, skip this test. - if test "$re_shell_" = fail; then - test -z "$marginal_" && skip_ failed to find an adequate shell - re_shell_=$marginal_ - break - fi - - # When testing the current shell, simply "eval" the test code. - # Otherwise, run it via $re_shell_ -c ... - if test "$re_shell_" = __current__; then - # 'eval'ing this code makes Solaris 10's /bin/sh exit with - # $? set to 2. It does not evaluate any of the code after the - # "unexpected" first '('. Thus, we must run it in a subshell. - ( eval "$gl_shell_test_script_" ) > /dev/null 2>&1 - else - "$re_shell_" -c "$gl_shell_test_script_" 2>/dev/null - fi - - st_=$? - - # $re_shell_ works just fine. Use it. - if test $st_ = 10; then - gl_set_x_corrupts_stderr_=false - break - fi - - # If this is our first marginally acceptable shell, remember it. - if test "$st_:$marginal_" = 9: ; then - marginal_="$re_shell_" - gl_set_x_corrupts_stderr_=true - fi - done - - if test "$re_shell_" != __current__; then - # Found a usable shell. Preserve -v and -x. - case $- in - *v*x* | *x*v*) opts_=-vx ;; - *v*) opts_=-v ;; - *x*) opts_=-x ;; - *) opts_= ;; - esac - re_shell=$re_shell_ - export re_shell - exec "$re_shell_" $opts_ "$0" --no-reexec "$@" - echo "$ME_: exec failed" 1>&2 - exit 127 - fi -fi - -# If this is bash, turn off all aliases. -test -n "$BASH_VERSION" && unalias -a - -# Note that when supporting $EXEEXT (transparently mapping from PROG_NAME to -# PROG_NAME.exe), we want to support hyphen-containing names like test-acos. -# That is part of the shell-selection test above. Why use aliases rather -# than functions? Because support for hyphen-containing aliases is more -# widespread than that for hyphen-containing function names. -test -n "$EXEEXT" && shopt -s expand_aliases - -# Enable glibc's malloc-perturbing option. -# This is useful for exposing code that depends on the fact that -# malloc-related functions often return memory that is mostly zeroed. -# If you have the time and cycles, use valgrind to do an even better job. -: ${MALLOC_PERTURB_=87} -export MALLOC_PERTURB_ - -# This is a stub function that is run upon trap (upon regular exit and -# interrupt). Override it with a per-test function, e.g., to unmount -# a partition, or to undo any other global state changes. -cleanup_ () { :; } - -# Emit a header similar to that from diff -u; Print the simulated "diff" -# command so that the order of arguments is clear. Don't bother with @@ lines. -emit_diff_u_header_ () -{ - printf '%s\n' "diff -u $*" \ - "--- $1 1970-01-01" \ - "+++ $2 1970-01-01" -} - -# Arrange not to let diff or cmp operate on /dev/null, -# since on some systems (at least OSF/1 5.1), that doesn't work. -# When there are not two arguments, or no argument is /dev/null, return 2. -# When one argument is /dev/null and the other is not empty, -# cat the nonempty file to stderr and return 1. -# Otherwise, return 0. -compare_dev_null_ () -{ - test $# = 2 || return 2 - - if test "x$1" = x/dev/null; then - test -s "$2" || return 0 - emit_diff_u_header_ "$@"; sed 's/^/+/' "$2" - return 1 - fi - - if test "x$2" = x/dev/null; then - test -s "$1" || return 0 - emit_diff_u_header_ "$@"; sed 's/^/-/' "$1" - return 1 - fi - - return 2 -} - -if diff_out_=`exec 2>/dev/null; diff -u "$0" "$0" < /dev/null` \ - && diff -u Makefile "$0" 2>/dev/null | grep '^[+]#!' >/dev/null; then - # diff accepts the -u option and does not (like AIX 7 'diff') produce an - # extra space on column 1 of every content line. - if test -z "$diff_out_"; then - compare_ () { diff -u "$@"; } - else - compare_ () - { - if diff -u "$@" > diff.out; then - # No differences were found, but Solaris 'diff' produces output - # "No differences encountered". Hide this output. - rm -f diff.out - true - else - cat diff.out - rm -f diff.out - false - fi - } - fi -elif - for diff_opt_ in -U3 -c '' no; do - test "$diff_opt_" = no && break - diff_out_=`exec 2>/dev/null; diff $diff_opt_ "$0" "$0" diff.out; then - # No differences were found, but AIX and HP-UX 'diff' produce output - # "No differences encountered" or "There are no differences between the - # files.". Hide this output. - rm -f diff.out - true - else - cat diff.out - rm -f diff.out - false - fi - } - fi -elif cmp -s /dev/null /dev/null 2>/dev/null; then - compare_ () { cmp -s "$@"; } -else - compare_ () { cmp "$@"; } -fi - -# Usage: compare EXPECTED ACTUAL -# -# Given compare_dev_null_'s preprocessing, defer to compare_ if 2 or more. -# Otherwise, propagate $? to caller: any diffs have already been printed. -compare () -{ - # This looks like it can be factored to use a simple "case $?" - # after unchecked compare_dev_null_ invocation, but that would - # fail in a "set -e" environment. - if compare_dev_null_ "$@"; then - return 0 - else - case $? in - 1) return 1;; - *) compare_ "$@";; - esac - fi -} - -# An arbitrary prefix to help distinguish test directories. -testdir_prefix_ () { printf gt; } - -# Run the user-overridable cleanup_ function, remove the temporary -# directory and exit with the incoming value of $?. -remove_tmp_ () -{ - __st=$? - cleanup_ - # cd out of the directory we're about to remove - cd "$initial_cwd_" || cd / || cd /tmp - chmod -R u+rwx "$test_dir_" - # If removal fails and exit status was to be 0, then change it to 1. - rm -rf "$test_dir_" || { test $__st = 0 && __st=1; } - exit $__st -} - -# Given a directory name, DIR, if every entry in it that matches *.exe -# contains only the specified bytes (see the case stmt below), then print -# a space-separated list of those names and return 0. Otherwise, don't -# print anything and return 1. Naming constraints apply also to DIR. -find_exe_basenames_ () -{ - feb_dir_=$1 - feb_fail_=0 - feb_result_= - feb_sp_= - for feb_file_ in $feb_dir_/*.exe; do - # If there was no *.exe file, or there existed a file named "*.exe" that - # was deleted between the above glob expansion and the existence test - # below, just skip it. - test "x$feb_file_" = "x$feb_dir_/*.exe" && test ! -f "$feb_file_" \ - && continue - # Exempt [.exe, since we can't create a function by that name, yet - # we can't invoke [ by PATH search anyways due to shell builtins. - test "x$feb_file_" = "x$feb_dir_/[.exe" && continue - case $feb_file_ in - *[!-a-zA-Z/0-9_.+]*) feb_fail_=1; break;; - *) # Remove leading file name components as well as the .exe suffix. - feb_file_=${feb_file_##*/} - feb_file_=${feb_file_%.exe} - feb_result_="$feb_result_$feb_sp_$feb_file_";; - esac - feb_sp_=' ' - done - test $feb_fail_ = 0 && printf %s "$feb_result_" - return $feb_fail_ -} - -# Consider the files in directory, $1. -# For each file name of the form PROG.exe, create an alias named -# PROG that simply invokes PROG.exe, then return 0. If any selected -# file name or the directory name, $1, contains an unexpected character, -# define no alias and return 1. -create_exe_shims_ () -{ - case $EXEEXT in - '') return 0 ;; - .exe) ;; - *) echo "$0: unexpected \$EXEEXT value: $EXEEXT" 1>&2; return 1 ;; - esac - - base_names_=`find_exe_basenames_ $1` \ - || { echo "$0 (exe_shim): skipping directory: $1" 1>&2; return 0; } - - if test -n "$base_names_"; then - for base_ in $base_names_; do - alias "$base_"="$base_$EXEEXT" - done - fi - - return 0 -} - -# Use this function to prepend to PATH an absolute name for each -# specified, possibly-$initial_cwd_-relative, directory. -path_prepend_ () -{ - while test $# != 0; do - path_dir_=$1 - case $path_dir_ in - '') fail_ "invalid path dir: '$1'";; - /*) abs_path_dir_=$path_dir_;; - *) abs_path_dir_=$initial_cwd_/$path_dir_;; - esac - case $abs_path_dir_ in - *:*) fail_ "invalid path dir: '$abs_path_dir_'";; - esac - PATH="$abs_path_dir_:$PATH" - - # Create an alias, FOO, for each FOO.exe in this directory. - create_exe_shims_ "$abs_path_dir_" \ - || fail_ "something failed (above): $abs_path_dir_" - shift - done - export PATH -} - -setup_ () -{ - if test "$VERBOSE" = yes; then - # Test whether set -x may cause the selected shell to corrupt an - # application's stderr. Many do, including zsh-4.3.10 and the /bin/sh - # from SunOS 5.11, OpenBSD 4.7 and Irix 5.x and 6.5. - # If enabling verbose output this way would cause trouble, simply - # issue a warning and refrain. - if $gl_set_x_corrupts_stderr_; then - warn_ "using SHELL=$SHELL with 'set -x' corrupts stderr" - else - set -x - fi - fi - - initial_cwd_=$PWD - - pfx_=`testdir_prefix_` - test_dir_=`mktempd_ "$initial_cwd_" "$pfx_-$ME_.XXXX"` \ - || fail_ "failed to create temporary directory in $initial_cwd_" - cd "$test_dir_" || fail_ "failed to cd to temporary directory" - - # As autoconf-generated configure scripts do, ensure that IFS - # is defined initially, so that saving and restoring $IFS works. - gl_init_sh_nl_=' -' - IFS=" "" $gl_init_sh_nl_" - - # This trap statement, along with a trap on 0 below, ensure that the - # temporary directory, $test_dir_, is removed upon exit as well as - # upon receipt of any of the listed signals. - for sig_ in 1 2 3 13 15; do - eval "trap 'Exit $(expr $sig_ + 128)' $sig_" - done -} - -# Create a temporary directory, much like mktemp -d does. -# Written by Jim Meyering. -# -# Usage: mktempd_ /tmp phoey.XXXXXXXXXX -# -# First, try to use the mktemp program. -# Failing that, we'll roll our own mktemp-like function: -# - try to get random bytes from /dev/urandom -# - failing that, generate output from a combination of quickly-varying -# sources and gzip. Ignore non-varying gzip header, and extract -# "random" bits from there. -# - given those bits, map to file-name bytes using tr, and try to create -# the desired directory. -# - make only $MAX_TRIES_ attempts - -# Helper function. Print $N pseudo-random bytes from a-zA-Z0-9. -rand_bytes_ () -{ - n_=$1 - - # Maybe try openssl rand -base64 $n_prime_|tr '+/=\012' abcd first? - # But if they have openssl, they probably have mktemp, too. - - chars_=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 - dev_rand_=/dev/urandom - if test -r "$dev_rand_"; then - # Note: 256-length($chars_) == 194; 3 copies of $chars_ is 186 + 8 = 194. - dd ibs=$n_ count=1 if=$dev_rand_ 2>/dev/null \ - | LC_ALL=C tr -c $chars_ 01234567$chars_$chars_$chars_ - return - fi - - n_plus_50_=`expr $n_ + 50` - cmds_='date; date +%N; free; who -a; w; ps auxww; ps ef; netstat -n' - data_=` (eval "$cmds_") 2>&1 | gzip ` - - # Ensure that $data_ has length at least 50+$n_ - while :; do - len_=`echo "$data_"|wc -c` - test $n_plus_50_ -le $len_ && break; - data_=` (echo "$data_"; eval "$cmds_") 2>&1 | gzip ` - done - - echo "$data_" \ - | dd bs=1 skip=50 count=$n_ 2>/dev/null \ - | LC_ALL=C tr -c $chars_ 01234567$chars_$chars_$chars_ -} - -mktempd_ () -{ - case $# in - 2);; - *) fail_ "Usage: mktempd_ DIR TEMPLATE";; - esac - - destdir_=$1 - template_=$2 - - MAX_TRIES_=4 - - # Disallow any trailing slash on specified destdir: - # it would subvert the post-mktemp "case"-based destdir test. - case $destdir_ in - / | //) destdir_slash_=$destdir;; - */) fail_ "invalid destination dir: remove trailing slash(es)";; - *) destdir_slash_=$destdir_/;; - esac - - case $template_ in - *XXXX) ;; - *) fail_ \ - "invalid template: $template_ (must have a suffix of at least 4 X's)";; - esac - - # First, try to use mktemp. - d=`unset TMPDIR; { mktemp -d -t -p "$destdir_" "$template_"; } 2>/dev/null` && - - # The resulting name must be in the specified directory. - case $d in "$destdir_slash_"*) :;; *) false;; esac && - - # It must have created the directory. - test -d "$d" && - - # It must have 0700 permissions. Handle sticky "S" bits. - perms=`ls -dgo "$d" 2>/dev/null` && - case $perms in drwx--[-S]---*) :;; *) false;; esac && { - echo "$d" - return - } - - # If we reach this point, we'll have to create a directory manually. - - # Get a copy of the template without its suffix of X's. - base_template_=`echo "$template_"|sed 's/XX*$//'` - - # Calculate how many X's we've just removed. - template_length_=`echo "$template_" | wc -c` - nx_=`echo "$base_template_" | wc -c` - nx_=`expr $template_length_ - $nx_` - - err_= - i_=1 - while :; do - X_=`rand_bytes_ $nx_` - candidate_dir_="$destdir_slash_$base_template_$X_" - err_=`mkdir -m 0700 "$candidate_dir_" 2>&1` \ - && { echo "$candidate_dir_"; return; } - test $MAX_TRIES_ -le $i_ && break; - i_=`expr $i_ + 1` - done - fail_ "$err_" -} - -# If you want to override the testdir_prefix_ function, -# or to add more utility functions, use this file. -test -f "$srcdir/init.cfg" \ - && . "$srcdir/init.cfg" - -setup_ "$@" -# This trap is here, rather than in the setup_ function, because some -# shells run the exit trap at shell function exit, rather than script exit. -trap remove_tmp_ 0 diff --git a/sys/contrib/zstd/tests/gzip/keep.sh b/sys/contrib/zstd/tests/gzip/keep.sh deleted file mode 100755 index ab9a21811d3..00000000000 --- a/sys/contrib/zstd/tests/gzip/keep.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -# Exercise the --keep option. - -# Copyright (C) 2013-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -echo fooooooooo > in || framework_failure_ -cp in orig || framework_failure_ - -fail=0 - -# Compress and decompress both with and without --keep. -for k in --keep ''; do - # With --keep, the source must be retained, otherwise, it must be removed. - case $k in --keep) op='||' ;; *) op='&&' ;; esac - - gzip $k in || fail=1 - eval "test -f in $op fail=1" - test -f in.gz || fail=1 - rm -f in || fail=1 - - gzip -d $k in.gz || fail=1 - eval "test -f in.gz $op fail=1" - test -f in || fail=1 - compare in orig || fail=1 - rm -f in.gz || fail=1 -done - -cp orig in || framework_failure_ -log=$(gzip -kv in 2>&1) || fail=1 -case $log in - *'created in.gz'*) ;; - *) fail=1;; -esac - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/list.sh b/sys/contrib/zstd/tests/gzip/list.sh deleted file mode 100755 index 75912e1e26d..00000000000 --- a/sys/contrib/zstd/tests/gzip/list.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -# Exercise the --list option. - -# Copyright 2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -echo zoology zucchini > in || framework_failure_ -cp in orig || framework_failure_ - -gzip -l in && fail=1 -gzip -9 in || fail=1 -gzip -l in.gz >out1 || fail=1 -gzip -l in.gz | cat >out2 || fail=1 -compare out1 out2 || fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/memcpy-abuse.sh b/sys/contrib/zstd/tests/gzip/memcpy-abuse.sh deleted file mode 100755 index 7d5c056debc..00000000000 --- a/sys/contrib/zstd/tests/gzip/memcpy-abuse.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# Before gzip-1.4, this the use of memcpy in inflate_codes could -# mistakenly operate on overlapping regions. Exercise that code. - -# Copyright (C) 2010-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -# The input must be larger than 32KiB and slightly -# less uniform than e.g., all zeros. -printf wxy%032767d 0 | tee in | gzip > in.gz || framework_failure_ - -fail=0 - -# Before the fix, this would call memcpy with overlapping regions. -gzip -dc in.gz > out || fail=1 - -compare in out || fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/mixed.sh b/sys/contrib/zstd/tests/gzip/mixed.sh deleted file mode 100755 index 383a54f5e46..00000000000 --- a/sys/contrib/zstd/tests/gzip/mixed.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh -# Ensure that gzip -cdf handles mixed compressed/not-compressed data -# Before gzip-1.5, it would produce invalid output. - -# Copyright (C) 2010-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -printf 'xxx\nyyy\n' > exp2 || framework_failure_ -printf 'aaa\nbbb\nccc\n' > exp3 || framework_failure_ - -fail=0 - -(echo xxx; echo yyy) > in || fail=1 -gzip -cdf < in > out || fail=1 -compare exp2 out || fail=1 - -# Uncompressed input, followed by compressed data. -# Currently fails, so skip it. -# (echo xxx; echo yyy|gzip) > in || fail=1 -# gzip -cdf < in > out || fail=1 -# compare exp2 out || fail=1 - -# Compressed input, followed by regular (not-compressed) data. -(echo xxx|gzip; echo yyy) > in || fail=1 -gzip -cdf < in > out || fail=1 -compare exp2 out || fail=1 - -(echo xxx|gzip; echo yyy|gzip) > in || fail=1 -gzip -cdf < in > out || fail=1 -compare exp2 out || fail=1 - -in_str=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-+=% -for i in 0 1 2 3 4 5 6 7 8 9 a; do in_str="$in_str$in_str" ;done - -# Start with some small sizes. $(seq 64) -sizes=$(i=0; while :; do echo $i; test $i = 64 && break; i=$(expr $i + 1); done) - -# gzip's internal buffer size is 32KiB + 64 bytes: -sizes="$sizes 32831 32832 32833" - -# 128KiB, +/- 1 -sizes="$sizes 131071 131072 131073" - -# Ensure that "gzip -cdf" acts like cat, for a range of small input files. -i=0 -for i in $sizes; do - echo $i - printf %$i.${i}s $in_str > in - gzip -cdf < in > out - compare in out || fail=1 -done - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/null-suffix-clobber.sh b/sys/contrib/zstd/tests/gzip/null-suffix-clobber.sh deleted file mode 100755 index 0efd0e34490..00000000000 --- a/sys/contrib/zstd/tests/gzip/null-suffix-clobber.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# Before gzip-1.5, gzip -d -S '' k.gz would delete F.gz and not create "F" - -# Copyright (C) 2010-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -printf anything | gzip > F.gz || framework_failure_ -echo y > yes || framework_failure_ -echo "gzip: invalid suffix ''" > expected-err || framework_failure_ - -fail=0 - -gzip ---presume-input-tty -d -S '' F.gz < yes > out 2>err && fail=1 - -compare /dev/null out || fail=1 -compare expected-err err || fail=1 - -test -f F.gz || fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/stdin.sh b/sys/contrib/zstd/tests/gzip/stdin.sh deleted file mode 100755 index eef4cd8b107..00000000000 --- a/sys/contrib/zstd/tests/gzip/stdin.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -# Ensure that gzip interprets "-" as stdin. - -# Copyright (C) 2009-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -printf a | gzip > in || framework_failure_ -printf aaa > exp || framework_failure_ - -fail=0 -gzip -dc in - in < in > out 2>err || fail=1 - -compare exp out || fail=1 -compare /dev/null err || fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/test-driver.sh b/sys/contrib/zstd/tests/gzip/test-driver.sh deleted file mode 100755 index 649c084e4b0..00000000000 --- a/sys/contrib/zstd/tests/gzip/test-driver.sh +++ /dev/null @@ -1,150 +0,0 @@ -#! /bin/sh -# test-driver - basic testsuite driver script. - -scriptversion=2016-01-11.22; # UTC - -# Copyright (C) 2011-2015 Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - -# This file is maintained in Automake, please report -# bugs to or send patches to -# . - -# Make unconditional expansion of undefined variables an error. This -# helps a lot in preventing typo-related bugs. -set -u - -usage_error () -{ - echo "$0: $*" >&2 - print_usage >&2 - exit 2 -} - -print_usage () -{ - cat <$log_file 2>&1 -estatus=$? - -if test $enable_hard_errors = no && test $estatus -eq 99; then - tweaked_estatus=1 -else - tweaked_estatus=$estatus -fi - -case $tweaked_estatus:$expect_failure in - 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;; - 0:*) col=$grn res=PASS recheck=no gcopy=no;; - 77:*) col=$blu res=SKIP recheck=no gcopy=yes;; - 99:*) col=$mgn res=ERROR recheck=yes gcopy=yes;; - *:yes) col=$lgn res=XFAIL recheck=no gcopy=yes;; - *:*) col=$red res=FAIL recheck=yes gcopy=yes;; -esac - -# Report the test outcome and exit status in the logs, so that one can -# know whether the test passed or failed simply by looking at the '.log' -# file, without the need of also peaking into the corresponding '.trs' -# file (automake bug#11814). -echo "$res $test_name (exit status: $estatus)" >>$log_file - -# Report outcome to console. -echo "${col}${res}${std}: $test_name" - -# Register the test result, and other relevant metadata. -echo ":test-result: $res" > $trs_file -echo ":global-test-result: $res" >> $trs_file -echo ":recheck: $recheck" >> $trs_file -echo ":copy-in-global-log: $gcopy" >> $trs_file - -# Local Variables: -# mode: shell-script -# sh-indentation: 2 -# eval: (add-hook 'write-file-hooks 'time-stamp) -# time-stamp-start: "scriptversion=" -# time-stamp-format: "%:y-%02m-%02d.%02H" -# time-stamp-time-zone: "UTC0" -# time-stamp-end: "; # UTC" -# End: - -exit $tweaked_estatus diff --git a/sys/contrib/zstd/tests/gzip/trailing-nul.sh b/sys/contrib/zstd/tests/gzip/trailing-nul.sh deleted file mode 100755 index 7b15d5e5578..00000000000 --- a/sys/contrib/zstd/tests/gzip/trailing-nul.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# gzip accepts trailing NUL bytes; don't fail if there is exactly one. -# Before gzip-1.4, this would fail. - -# Copyright (C) 2009-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -(echo 0 | gzip; printf '\0') > 0.gz || framework_failure_ -(echo 00 | gzip; printf '\0\0') > 00.gz || framework_failure_ -(echo 1 | gzip; printf '\1') > 1.gz || framework_failure_ - -fail=0 - -for i in 0 00 1; do - gzip -d $i.gz; ret=$? - test $ret -eq $i || fail=1 - test $ret = 1 && continue - echo $i > exp || fail=1 - compare exp $i || fail=1 -done - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/unpack-invalid.sh b/sys/contrib/zstd/tests/gzip/unpack-invalid.sh deleted file mode 100755 index fe8384d73cd..00000000000 --- a/sys/contrib/zstd/tests/gzip/unpack-invalid.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# gzip should report invalid 'unpack' input when uncompressing. -# With gzip-1.5, it would output invalid data instead. - -# Copyright (C) 2012-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -for input in \ - '\037\036\000\000\037\213\010\000\000\000\000\000\002\003\036\000\000\000\002\003\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\037\000\302\240\037\213\010\000\000\000\000\000\002\003\355\301' \ - '\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\076\366\017\370\036\016\030\000\000\000\000\000\000\000\000\000\034\010\105\140\104\025\020\047\000\000\037\036\016\030\000\000\000'; do - - printf "$input" >in || framework_failure_ - - if gzip -d out 2>err; then - fail=1 - else - fail=0 - fi -done - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/z-suffix.sh b/sys/contrib/zstd/tests/gzip/z-suffix.sh deleted file mode 100755 index a870a5408de..00000000000 --- a/sys/contrib/zstd/tests/gzip/z-suffix.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# Check that -Sz works. - -# Copyright 2014-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -printf anything > F && cp F G || framework_failure_ -gzip -Sz F || fail=1 -test ! -f F || fail=1 -test -f Fz || fail=1 -gzip -dSz F || fail=1 -test ! -f Fz || fail=1 -compare F G || fail\1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/zdiff.sh b/sys/contrib/zstd/tests/gzip/zdiff.sh deleted file mode 100755 index d62a84606ba..00000000000 --- a/sys/contrib/zstd/tests/gzip/zdiff.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh -# Exercise zdiff with two compressed inputs. -# Before gzip-1.4, this would fail. - -# Copyright (C) 2009-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -echo a > a || framework_failure_ -echo b > b || framework_failure_ -gzip a b || framework_failure_ - -cat < exp -1c1 -< a ---- -> b -EOF - -fail=0 -zdiff a.gz b.gz > out 2>&1 -test $? = 1 || fail=1 - -compare exp out || fail=1 - -rm -f out -# expect success, for equal files -zdiff a.gz a.gz > out 2> err || fail=1 -# expect no output -test -s out && fail=1 -# expect no stderr -test -s err && fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/zgrep-context.sh b/sys/contrib/zstd/tests/gzip/zgrep-context.sh deleted file mode 100755 index c8648b7e4f5..00000000000 --- a/sys/contrib/zstd/tests/gzip/zgrep-context.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -# Ensure that zgrep -15 works. Before gzip-1.5, it would fail. - -# Copyright (C) 2012-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -# A limited replacement for seq: handle 1 or 2 args; increment must be 1 -seq() -{ - case $# in - 1) start=1 final=$1;; - 2) start=$1 final=$2;; - *) echo you lose 1>&2; exit 1;; - esac - awk 'BEGIN{for(i='$start';i<='$final';i++) print i}' < /dev/null -} - -seq 40 > in || framework_failure_ -gzip < in > in.gz || framework_failure_ -seq 2 32 > exp || framework_failure_ - -: ${GREP=grep} -$GREP -15 17 - < in > out && compare exp out || { - echo >&2 "$0: $GREP does not support context options; skipping this test" - exit 77 -} - -fail=0 -zgrep -15 17 - < in.gz > out || fail=1 -compare exp out || fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/zgrep-f.sh b/sys/contrib/zstd/tests/gzip/zgrep-f.sh deleted file mode 100755 index d0cf27f7e23..00000000000 --- a/sys/contrib/zstd/tests/gzip/zgrep-f.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# Ensure that zgrep -f - works like grep -f - -# Before gzip-1.4, it would fail. - -# Copyright (C) 2009-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -printf 'needle\nn2\n' > n || framework_failure_ -cp n haystack || framework_failure_ -gzip haystack || framework_failure_ - -fail=0 -zgrep -f - haystack.gz < n > out 2>&1 || fail=1 - -compare out n || fail=1 - -if ${BASH_VERSION+:} false; then - set +o posix - # This failed with gzip 1.6. - cat n n >nn || framework_failure_ - eval 'zgrep -h -f <(cat n) haystack.gz haystack.gz' >out || fail=1 - compare out nn || fail=1 -fi - -# This failed with gzip 1.4. -echo a-b | zgrep -e - > /dev/null || fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/zgrep-signal.sh b/sys/contrib/zstd/tests/gzip/zgrep-signal.sh deleted file mode 100755 index a8c53881add..00000000000 --- a/sys/contrib/zstd/tests/gzip/zgrep-signal.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh -# Check that zgrep is terminated gracefully by signal when -# its grep/sed pipeline is terminated by a signal. - -# Copyright (C) 2010-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -echo a | gzip -c > f.gz || framework_failure_ - -test "x$PERL" = x && PERL=perl -("$PERL" -e 'use POSIX qw(dup2)') >/dev/null 2>&1 || - skip_ "no suitable perl found" - -# Run the arguments as a command, in a process where stdout is a -# dangling pipe and SIGPIPE has the default signal-handling action. -# This can't be done portably in the shell, because if SIGPIPE is -# ignored when the shell is entered, the shell might refuse to trap -# it. Fall back on Perl+POSIX, if available. Take care to close the -# pipe's read end before running the program; the equivalent of the -# shell's "command | :" has a race condition in that COMMAND could -# write before ":" exits. -write_to_dangling_pipe () { - program=${1?} - shift - args= - for arg; do - args="$args, '$arg'" - done - "$PERL" -e ' - use POSIX qw(dup2); - $SIG{PIPE} = "DEFAULT"; - pipe my ($read_end, $write_end) or die "pipe: $!\n"; - dup2 fileno $write_end, 1 or die "dup2: $!\n"; - close $read_end or die "close: $!\n"; - exec '"'$program'$args"'; - ' -} - -write_to_dangling_pipe cat f.gz f.gz -signal_status=$? -test 128 -lt $signal_status || - framework_failure_ 'signal handling busted on this host' - -fail=0 - -write_to_dangling_pipe zgrep a f.gz f.gz -test $? -eq $signal_status || fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/gzip/znew-k.sh b/sys/contrib/zstd/tests/gzip/znew-k.sh deleted file mode 100755 index 6c239e28ea8..00000000000 --- a/sys/contrib/zstd/tests/gzip/znew-k.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -# Check that znew -K works without compress(1). - -# Copyright (C) 2010-2016 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# limit so don't run it by default. - -. "${srcdir=.}/init.sh"; path_prepend_ . - -cat <<'EOF' >compress || framework_failure_ -#!/bin/sh -echo >&2 'compress has been invoked' -exit 1 -EOF -chmod +x compress || framework_failure_ - -# Note that the basename must have a length of 6 or greater. -# Otherwise, "test -f $name" below would fail. -name=123456.Z - -printf '%1012977s' ' ' | gzip -c > $name || framework_failure_ - -fail=0 - -znew -K $name || fail=1 -test -f $name || fail=1 - -Exit $fail diff --git a/sys/contrib/zstd/tests/invalidDictionaries.c b/sys/contrib/zstd/tests/invalidDictionaries.c deleted file mode 100644 index b23db036f48..00000000000 --- a/sys/contrib/zstd/tests/invalidDictionaries.c +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#include -#include "zstd.h" - -static const char invalidRepCode[] = { - 0x37, 0xa4, 0x30, 0xec, 0x2a, 0x00, 0x00, 0x00, 0x39, 0x10, 0xc0, 0xc2, - 0xa6, 0x00, 0x0c, 0x30, 0xc0, 0x00, 0x03, 0x0c, 0x30, 0x20, 0x72, 0xf8, - 0xb4, 0x6d, 0x4b, 0x9f, 0xfc, 0x97, 0x29, 0x49, 0xb2, 0xdf, 0x4b, 0x29, - 0x7d, 0x4a, 0xfc, 0x83, 0x18, 0x22, 0x75, 0x23, 0x24, 0x44, 0x4d, 0x02, - 0xb7, 0x97, 0x96, 0xf6, 0xcb, 0xd1, 0xcf, 0xe8, 0x22, 0xea, 0x27, 0x36, - 0xb7, 0x2c, 0x40, 0x46, 0x01, 0x08, 0x23, 0x01, 0x00, 0x00, 0x06, 0x1e, - 0x3c, 0x83, 0x81, 0xd6, 0x18, 0xd4, 0x12, 0x3a, 0x04, 0x00, 0x80, 0x03, - 0x08, 0x0e, 0x12, 0x1c, 0x12, 0x11, 0x0d, 0x0e, 0x0a, 0x0b, 0x0a, 0x09, - 0x10, 0x0c, 0x09, 0x05, 0x04, 0x03, 0x06, 0x06, 0x06, 0x02, 0x00, 0x03, - 0x00, 0x00, 0x02, 0x02, 0x00, 0x04, 0x06, 0x03, 0x06, 0x08, 0x24, 0x6b, - 0x0d, 0x01, 0x10, 0x04, 0x81, 0x07, 0x00, 0x00, 0x04, 0xb9, 0x58, 0x18, - 0x06, 0x59, 0x92, 0x43, 0xce, 0x28, 0xa5, 0x08, 0x88, 0xc0, 0x80, 0x88, - 0x8c, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x08, 0x00, 0x00, 0x00 -}; - -typedef struct dictionary_s { - const char *data; - size_t size; -} dictionary; - -static const dictionary dictionaries[] = { - {invalidRepCode, sizeof(invalidRepCode)}, - {NULL, 0}, -}; - -int main(int argc, const char** argv) { - const dictionary *dict; - for (dict = dictionaries; dict->data != NULL; ++dict) { - ZSTD_CDict *cdict; - ZSTD_DDict *ddict; - cdict = ZSTD_createCDict(dict->data, dict->size, 1); - if (cdict) { - ZSTD_freeCDict(cdict); - return 1; - } - ddict = ZSTD_createDDict(dict->data, dict->size); - if (ddict) { - ZSTD_freeDDict(ddict); - return 2; - } - } - - (void)argc; - (void)argv; - return 0; -} diff --git a/sys/contrib/zstd/tests/legacy.c b/sys/contrib/zstd/tests/legacy.c deleted file mode 100644 index 847e1d25e96..00000000000 --- a/sys/contrib/zstd/tests/legacy.c +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -/* - This program uses hard-coded data compressed with Zstd legacy versions - and tests that the API decompresses them correctly -*/ - -/*=========================================== -* Dependencies -*==========================================*/ -#include /* size_t */ -#include /* malloc, free */ -#include /* fprintf */ -#include /* strlen */ -#include "zstd.h" -#include "zstd_errors.h" - -/*=========================================== -* Macros -*==========================================*/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) - -/*=========================================== -* Precompressed frames -*==========================================*/ -const char* const COMPRESSED; /* content is at end of file */ -size_t const COMPRESSED_SIZE = 917; -const char* const EXPECTED; /* content is at end of file */ - - -int testSimpleAPI(void) -{ - size_t const size = strlen(EXPECTED); - char* const output = malloc(size); - - if (!output) { - DISPLAY("ERROR: Not enough memory!\n"); - return 1; - } - - { - size_t const ret = ZSTD_decompress(output, size, COMPRESSED, COMPRESSED_SIZE); - if (ZSTD_isError(ret)) { - if (ret == ZSTD_error_prefix_unknown) { - DISPLAY("ERROR: Invalid frame magic number, was this compiled " - "without legacy support?\n"); - } else { - DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret)); - } - return 1; - } - if (ret != size) { - DISPLAY("ERROR: Wrong decoded size\n"); - } - } - if (memcmp(EXPECTED, output, size) != 0) { - DISPLAY("ERROR: Wrong decoded output produced\n"); - return 1; - } - - free(output); - DISPLAY("Simple API OK\n"); - return 0; -} - -int testStreamingAPI(void) -{ - size_t const outBuffSize = ZSTD_DStreamOutSize(); - char* const outBuff = malloc(outBuffSize); - ZSTD_DStream* const stream = ZSTD_createDStream(); - ZSTD_inBuffer input = { COMPRESSED, COMPRESSED_SIZE, 0 }; - size_t outputPos = 0; - int needsInit = 1; - - if (outBuff == NULL) { - DISPLAY("ERROR: Could not allocate memory\n"); - return 1; - } - if (stream == NULL) { - DISPLAY("ERROR: Could not create dstream\n"); - return 1; - } - - while (1) { - ZSTD_outBuffer output = {outBuff, outBuffSize, 0}; - if (needsInit) { - size_t const ret = ZSTD_initDStream(stream); - if (ZSTD_isError(ret)) { - DISPLAY("ERROR: ZSTD_initDStream: %s\n", ZSTD_getErrorName(ret)); - return 1; - } } - - { size_t const ret = ZSTD_decompressStream(stream, &output, &input); - if (ZSTD_isError(ret)) { - DISPLAY("ERROR: ZSTD_decompressStream: %s\n", ZSTD_getErrorName(ret)); - return 1; - } - - if (ret == 0) { - needsInit = 1; - } } - - if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) { - DISPLAY("ERROR: Wrong decoded output produced\n"); - return 1; - } - outputPos += output.pos; - if (input.pos == input.size && output.pos < output.size) { - break; - } - } - - free(outBuff); - ZSTD_freeDStream(stream); - DISPLAY("Streaming API OK\n"); - return 0; -} - -int main(void) -{ - { int const ret = testSimpleAPI(); - if (ret) return ret; } - { int const ret = testStreamingAPI(); - if (ret) return ret; } - - DISPLAY("OK\n"); - return 0; -} - -/* Consists of the "EXPECTED" string compressed with default settings on - - v0.4.3 - - v0.5.0 - - v0.6.0 - - v0.7.0 - - v0.8.0 -*/ -const char* const COMPRESSED = - "\x24\xB5\x2F\xFD\x00\x00\x00\xBB\xB0\x02\xC0\x10\x00\x1E\xB0\x01" - "\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF\xE9\xF3\xEF\x53" - "\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89\x17\x00\x18\x00" - "\x18\x00\x3F\xE6\xE2\xE3\x74\xD6\xEC\xC9\x4A\xE0\x71\x71\x42\x3E" - "\x64\x4F\x6A\x45\x4E\x78\xEC\x49\x03\x3F\xC6\x80\xAB\x8F\x75\x5E" - "\x6F\x2E\x3E\x7E\xC6\xDC\x45\x69\x6C\xC5\xFD\xC7\x40\xB8\x84\x8A" - "\x01\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B\x67" - "\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\x19\x03\x01\x50\x67\x56\xF5\x9F" - "\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF" - "\xC6\xBA\x01\x0E\x00\x54\x00\x00\x19\x00\x00\x54\x14\x00\x24\x24" - "\x04\xFE\x04\x84\x4E\x41\x00\x27\xE2\x02\xC4\xB1\x00\xD2\x51\x00" - "\x79\x58\x41\x28\x00\xE0\x0C\x01\x68\x65\x00\x04\x13\x0C\xDA\x0C" - "\x80\x22\x06\xC0\x00\x00\x25\xB5\x2F\xFD\x00\x00\x00\xAD\x12\xB0" - "\x7D\x1E\xB0\x01\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF" - "\xE9\xF3\xEF\x53\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89" - "\x03\x01\x50\x67\x56\xF5\x9F\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC" - "\xAB\xAB\xE0\xE2\x81\xFA\xCF\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C" - "\x64\xF8\xEB\x53\xE6\x18\x0B\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A" - "\xF9\x63\x0C\xB8\xFA\x58\xE7\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6" - "\x56\xDC\x7F\x0C\x84\x4B\xA8\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC" - "\x04\x1E\x17\x27\xE4\x43\xF6\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00" - "\x00\x32\x40\x80\xA8\x00\x01\x49\x81\xE0\x3C\x01\x29\x1D\x00\x87" - "\xCE\x80\x75\x08\x80\x72\x24\x00\x7B\x52\x00\x94\x00\x20\xCC\x01" - "\x86\xD2\x00\x81\x09\x83\xC1\x34\xA0\x88\x01\xC0\x00\x00\x26\xB5" - "\x2F\xFD\x42\xEF\x00\x00\xA6\x12\xB0\x7D\x1E\xB0\x01\x02\x00\x00" - "\x54\xA0\xBA\x24\x8D\xC4\x25\xF2\x77\xFA\xFC\xFB\x94\x7A\xC7\xC9" - "\x13\x03\x11\x24\x43\x63\x3C\x6D\x22\x03\x01\x50\x67\x56\xF5\x9F" - "\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF" - "\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B" - "\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\xF9\x63\x0C\xB8\xFA\x58\xE7" - "\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6\x56\xDC\x7F\x0C\x84\x4B\xA8" - "\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC\x04\x1E\x17\x27\xE4\x43\xF6" - "\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00\x35\x0B\x71\xB5\xC0\x2A\x5C" - "\x26\x94\x22\x20\x8B\x4C\x8D\x13\x47\x58\x67\x15\x6C\xF1\x1C\x4B" - "\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0\x00\x00" - "\x27\xB5\x2F\xFD\x20\xEF\x00\x00\xA6\x12\xE4\x84\x1F\xB0\x01\x10" - "\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF\x9F\xEF" - "\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21\x80\x32" - "\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20\x5F\x45" - "\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB\x44\x14" - "\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB\x89\x51" - "\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F\x59\xDB" - "\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E\xDF\x78" - "\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55\x4A\xD4" - "\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39\x43\xF1" - "\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0" - "\x00\x00\x28\xB5\x2F\xFD\x24\xEF\x35\x05\x00\x92\x0B\x21\x1F\xB0" - "\x01\x10\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF" - "\x9F\xEF\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21" - "\x80\x32\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20" - "\x5F\x45\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB" - "\x44\x14\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB" - "\x89\x51\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F" - "\x59\xDB\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E" - "\xDF\x78\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55" - "\x4A\xD4\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39" - "\x43\xF1\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D" - "\x01\xD2\x2F\x21\x80"; - -const char* const EXPECTED = - "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" - "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" - "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" - - "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" - "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" - "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" - - "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" - "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" - "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" - - "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" - "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" - "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" - - "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" - "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" - "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"; diff --git a/sys/contrib/zstd/tests/longmatch.c b/sys/contrib/zstd/tests/longmatch.c deleted file mode 100644 index ed3861571d9..00000000000 --- a/sys/contrib/zstd/tests/longmatch.c +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2017-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -#include -#include -#include -#include -#include "mem.h" -#define ZSTD_STATIC_LINKING_ONLY -#include "zstd.h" - -int compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size) { - ZSTD_inBuffer in = { data, size, 0 }; - while (in.pos < in.size) { - ZSTD_outBuffer tmp = out; - const size_t rc = ZSTD_compressStream(ctx, &tmp, &in); - if (ZSTD_isError(rc)) { - return 1; - } - } - { - ZSTD_outBuffer tmp = out; - const size_t rc = ZSTD_flushStream(ctx, &tmp); - if (rc != 0) { return 1; } - } - return 0; -} - -int main(int argc, const char** argv) { - ZSTD_CStream *ctx; - ZSTD_parameters params; - size_t rc; - unsigned windowLog; - (void)argc; - (void)argv; - /* Create stream */ - ctx = ZSTD_createCStream(); - if (!ctx) { return 1; } - /* Set parameters */ - memset(¶ms, 0, sizeof(params)); - params.cParams.windowLog = 18; - params.cParams.chainLog = 13; - params.cParams.hashLog = 14; - params.cParams.searchLog = 1; - params.cParams.searchLength = 7; - params.cParams.targetLength = 16; - params.cParams.strategy = ZSTD_fast; - windowLog = params.cParams.windowLog; - /* Initialize stream */ - rc = ZSTD_initCStream_advanced(ctx, NULL, 0, params, 0); - if (ZSTD_isError(rc)) { return 2; } - { - U64 compressed = 0; - const U64 toCompress = ((U64)1) << 33; - const size_t size = 1 << windowLog; - size_t pos = 0; - char *srcBuffer = (char*) malloc(1 << windowLog); - char *dstBuffer = (char*) malloc(ZSTD_compressBound(1 << windowLog)); - ZSTD_outBuffer out = { dstBuffer, ZSTD_compressBound(1 << windowLog), 0 }; - const char match[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; - const size_t randomData = (1 << windowLog) - 2*sizeof(match); - size_t i; - printf("\n === Long Match Test === \n"); - printf("Creating random data to produce long matches \n"); - for (i = 0; i < sizeof(match); ++i) { - srcBuffer[i] = match[i]; - } - for (i = 0; i < randomData; ++i) { - srcBuffer[sizeof(match) + i] = (char)(rand() & 0xFF); - } - for (i = 0; i < sizeof(match); ++i) { - srcBuffer[sizeof(match) + randomData + i] = match[i]; - } - printf("Compressing, trying to generate a segfault \n"); - if (compress(ctx, out, srcBuffer, size)) { - return 1; - } - compressed += size; - while (compressed < toCompress) { - const size_t block = rand() % (size - pos + 1); - if (pos == size) { pos = 0; } - if (compress(ctx, out, srcBuffer + pos, block)) { - return 1; - } - pos += block; - compressed += block; - } - printf("Compression completed successfully (no error triggered)\n"); - free(srcBuffer); - free(dstBuffer); - } - return 0; -} diff --git a/sys/contrib/zstd/tests/namespaceTest.c b/sys/contrib/zstd/tests/namespaceTest.c deleted file mode 100644 index 5b7095f67c8..00000000000 --- a/sys/contrib/zstd/tests/namespaceTest.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - - -#include /* size_t */ -#include /* strlen */ - -/* symbol definition */ -extern unsigned XXH32(const void* src, size_t srcSize, unsigned seed); - -int main(int argc, const char** argv) -{ - const char* exename = argv[0]; - unsigned result = XXH32(exename, strlen(exename), argc); - return !result; -} diff --git a/sys/contrib/zstd/tests/paramgrill.c b/sys/contrib/zstd/tests/paramgrill.c deleted file mode 100644 index 13b102b2d04..00000000000 --- a/sys/contrib/zstd/tests/paramgrill.c +++ /dev/null @@ -1,1051 +0,0 @@ -/* - * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -/*-************************************ -* Dependencies -**************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize */ -#include /* malloc */ -#include /* fprintf, fopen, ftello64 */ -#include /* strcmp */ -#include /* log */ -#include - -#include "mem.h" -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */ -#include "zstd.h" -#include "datagen.h" -#include "xxhash.h" -#include "util.h" - - -/*-************************************ -* Constants -**************************************/ -#define PROGRAM_DESCRIPTION "ZSTD parameters tester" -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR, __DATE__ - - -#define KB *(1<<10) -#define MB *(1<<20) -#define GB *(1ULL<<30) - -#define NBLOOPS 2 -#define TIMELOOP (2 * SEC_TO_MICRO) - -#define NB_LEVELS_TRACKED 30 - -static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); - -#define COMPRESSIBILITY_DEFAULT 0.50 -static const size_t sampleSize = 10000000; - -static const double g_grillDuration_s = 90000; /* about 24 hours */ -static const U64 g_maxParamTime = 15 * SEC_TO_MICRO; -static const U64 g_maxVariationTime = 60 * SEC_TO_MICRO; -static const int g_maxNbVariations = 64; - - -/*-************************************ -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) - -#undef MIN -#undef MAX -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) -#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) - - -/*-************************************ -* Benchmark Parameters -**************************************/ -static U32 g_nbIterations = NBLOOPS; -static double g_compressibility = COMPRESSIBILITY_DEFAULT; -static U32 g_blockSize = 0; -static U32 g_rand = 1; -static U32 g_singleRun = 0; -static U32 g_target = 0; -static U32 g_noSeed = 0; -static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; - -void BMK_SetNbIterations(int nbLoops) -{ - g_nbIterations = nbLoops; - DISPLAY("- %u iterations -\n", g_nbIterations); -} - - -/*-******************************************************* -* Private functions -*********************************************************/ - -/* accuracy in seconds only, span can be multiple years */ -static double BMK_timeSpan(time_t tStart) { return difftime(time(NULL), tStart); } - -static size_t BMK_findMaxMem(U64 requiredMem) -{ - size_t const step = 64 MB; - void* testmem = NULL; - - requiredMem = (((requiredMem >> 26) + 1) << 26); - if (requiredMem > maxMemory) requiredMem = maxMemory; - - requiredMem += 2*step; - while (!testmem) { - requiredMem -= step; - testmem = malloc ((size_t)requiredMem); - } - - free (testmem); - return (size_t) (requiredMem - step); -} - - -static U32 FUZ_rotl32(U32 x, U32 r) -{ - return ((x << r) | (x >> (32 - r))); -} - -U32 FUZ_rand(U32* src) -{ - const U32 prime1 = 2654435761U; - const U32 prime2 = 2246822519U; - U32 rand32 = *src; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZ_rotl32(rand32, 13); - *src = rand32; - return rand32 >> 5; -} - - -/*-******************************************************* -* Bench functions -*********************************************************/ -typedef struct { - size_t cSize; - double cSpeed; /* bytes / sec */ - double dSpeed; -} BMK_result_t; - -typedef struct -{ - const char* srcPtr; - size_t srcSize; - char* cPtr; - size_t cRoom; - size_t cSize; - char* resPtr; - size_t resSize; -} blockParam_t; - - -static size_t BMK_benchParam(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, - const ZSTD_compressionParameters cParams) -{ - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize); - blockParam_t* const blockTable = (blockParam_t*) malloc(nbBlocks * sizeof(blockParam_t)); - const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize); - void* const compressedBuffer = malloc(maxCompressedSize); - void* const resultBuffer = malloc(srcSize); - ZSTD_parameters params; - U32 Wlog = cParams.windowLog; - U32 Clog = cParams.chainLog; - U32 Hlog = cParams.hashLog; - U32 Slog = cParams.searchLog; - U32 Slength = cParams.searchLength; - U32 Tlength = cParams.targetLength; - ZSTD_strategy strat = cParams.strategy; - char name[30] = { 0 }; - U64 crcOrig; - - /* init result for early exit */ - resultPtr->cSize = srcSize; - resultPtr->cSpeed = 0.; - resultPtr->dSpeed = 0.; - - /* Memory allocation & restrictions */ - snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%03uS%1u", Wlog, Clog, Hlog, Slog, Slength, Tlength, strat); - if (!compressedBuffer || !resultBuffer || !blockTable) { - DISPLAY("\nError: not enough memory!\n"); - free(compressedBuffer); - free(resultBuffer); - free(blockTable); - return 12; - } - - /* Calculating input Checksum */ - crcOrig = XXH64(srcBuffer, srcSize, 0); - - /* Init blockTable data */ - { - U32 i; - size_t remaining = srcSize; - const char* srcPtr = (const char*)srcBuffer; - char* cPtr = (char*)compressedBuffer; - char* resPtr = (char*)resultBuffer; - for (i=0; i g_maxParamTime) break; } - - /* Compression */ - DISPLAY("\r%1u-%s : %9u ->", loopNb, name, (U32)srcSize); - memset(compressedBuffer, 0xE5, maxCompressedSize); - - nbLoops = 0; - UTIL_waitForNextTick(); - roundStart = UTIL_getTime(); - while (UTIL_clockSpanMicro(roundStart) < TIMELOOP) { - for (blockNb=0; blockNb", loopNb, name, (U32)srcSize); - DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.); - resultPtr->cSize = cSize; - resultPtr->cSpeed = (double)srcSize / fastestC; - -#if 1 - /* Decompression */ - memset(resultBuffer, 0xD6, srcSize); - - nbLoops = 0; - UTIL_waitForNextTick(); - roundStart = UTIL_getTime(); - for ( ; UTIL_clockSpanMicro(roundStart) < TIMELOOP; nbLoops++) { - for (blockNb=0; blockNb ", loopNb, name, (U32)srcSize); - DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.); - DISPLAY("%7.1f MB/s", (double)srcSize / fastestD / 1000000.); - resultPtr->dSpeed = (double)srcSize / fastestD; - - /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); - if (crcOrig!=crcCheck) { - unsigned u; - unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize)); - DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck); - for (u=0; u O_ratio) - DISPLAY ("Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", - W_ratio, (double)(W_DMemUsed) / 1024 / 1024, - O_ratio, (double)(O_DMemUsed) / 1024 / 1024, cLevel); - continue; - } - if (W_CMemUsed_note < O_CMemUsed_note) { - /* uses too much memory for compression for too little benefit */ - if (W_ratio > O_ratio) - DISPLAY ("Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", - W_ratio, (double)(W_CMemUsed) / 1024 / 1024, - O_ratio, (double)(O_CMemUsed) / 1024 / 1024, cLevel); - continue; - } - if (W_CSpeed_note < O_CSpeed_note ) { - /* too large compression speed difference for the compression benefit */ - if (W_ratio > O_ratio) - DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, testResult.cSpeed / 1000000, - O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel); - continue; - } - if (W_DSpeed_note < O_DSpeed_note ) { - /* too large decompression speed difference for the compression benefit */ - if (W_ratio > O_ratio) - DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, testResult.dSpeed / 1000000., - O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel); - continue; - } - - if (W_ratio < O_ratio) - DISPLAY("Solution %4.3f selected over %4.3f at level %i, due to better secondary statistics \n", W_ratio, O_ratio, cLevel); - - winners[cLevel].result = testResult; - winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, srcSize); - - better = 1; - } } - - return better; -} - - -/* nullified useless params, to ensure count stats */ -static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params) -{ - g_params = params; - if (params.strategy == ZSTD_fast) - g_params.chainLog = 0, g_params.searchLog = 0; - if (params.strategy == ZSTD_dfast) - g_params.searchLog = 0; - if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra) - g_params.targetLength = 0; - return &g_params; -} - - -static void paramVariation(ZSTD_compressionParameters* ptr) -{ - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; - p = *ptr; - for ( ; nbChanges ; nbChanges--) { - const U32 changeID = FUZ_rand(&g_rand) % 14; - switch(changeID) - { - case 0: - p.chainLog++; break; - case 1: - p.chainLog--; break; - case 2: - p.hashLog++; break; - case 3: - p.hashLog--; break; - case 4: - p.searchLog++; break; - case 5: - p.searchLog--; break; - case 6: - p.windowLog++; break; - case 7: - p.windowLog--; break; - case 8: - p.searchLength++; break; - case 9: - p.searchLength--; break; - case 10: - p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break; - case 11: - p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break; - case 12: - p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; - case 13: - p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; - } - } - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - *ptr = p; -} - - -#define PARAMTABLELOG 25 -#define PARAMTABLESIZE (1<> 3) & PARAMTABLEMASK] - - -static void playAround(FILE* f, winnerInfo_t* winners, - ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx) -{ - int nbVariations = 0; - UTIL_time_t const clockStart = UTIL_getTime(); - - while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { - ZSTD_compressionParameters p = params; - - if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p); - - /* exclude faster if already played params */ - if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) - continue; - - /* test */ - NB_TESTS_PLAYED(p)++; - if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx)) continue; - - /* improvement found => search more */ - BMK_printWinners(f, winners, srcSize); - playAround(f, winners, p, srcBuffer, srcSize, ctx); - } - -} - - -static ZSTD_compressionParameters randomParams(void) -{ - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; - p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; - p.targetLength=(FUZ_rand(&g_rand) % (512)) + ZSTD_TARGETLENGTH_MIN; - p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - return p; -} - -static void BMK_selectRandomStart( - FILE* f, winnerInfo_t* winners, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx) -{ - U32 const id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1)); - if ((id==0) || (winners[id].params.windowLog==0)) { - /* totally random entry */ - ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); - playAround(f, winners, p, srcBuffer, srcSize, ctx); - } - else - playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx); -} - - -static void BMK_benchMem(void* srcBuffer, size_t srcSize) -{ - ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_compressionParameters params; - winnerInfo_t winners[NB_LEVELS_TRACKED]; - const char* const rfName = "grillResults.txt"; - FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - - /* init */ - if (ctx==NULL) { DISPLAY("ZSTD_createCCtx() failed \n"); exit(1); } - memset(winners, 0, sizeof(winners)); - if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } - - if (g_singleRun) { - BMK_result_t testResult; - g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params); - DISPLAY("\n"); - return; - } - - if (g_target) - g_cSpeedTarget[1] = g_target * 1000000; - else { - /* baseline config for level 1 */ - BMK_result_t testResult; - params = ZSTD_getCParams(1, blockSize, 0); - BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); - g_cSpeedTarget[1] = (testResult.cSpeed * 31) / 32; - } - - /* establish speed objectives (relative to level 1) */ - { int i; - for (i=2; i<=ZSTD_maxCLevel(); i++) - g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) / 32; - } - - /* populate initial solution */ - { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - int i; - for (i=0; i<=maxSeeds; i++) { - params = ZSTD_getCParams(i, blockSize, 0); - BMK_seed(winners, params, srcBuffer, srcSize, ctx); - } } - BMK_printWinners(f, winners, srcSize); - - /* start tests */ - { const time_t grillStart = time(NULL); - do { - BMK_selectRandomStart(f, winners, srcBuffer, srcSize, ctx); - } while (BMK_timeSpan(grillStart) < g_grillDuration_s); - } - - /* end summary */ - BMK_printWinners(f, winners, srcSize); - DISPLAY("grillParams operations completed \n"); - - /* clean up*/ - fclose(f); - ZSTD_freeCCtx(ctx); -} - - -static int benchSample(void) -{ - void* origBuff; - size_t const benchedSize = sampleSize; - const char* const name = "Sample 10MiB"; - - /* Allocation */ - origBuff = malloc(benchedSize); - if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; } - - /* Fill buffer */ - RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); - BMK_benchMem(origBuff, benchedSize); - - free(origBuff); - return 0; -} - - -int benchFiles(const char** fileNamesTable, int nbFiles) -{ - int fileIdx=0; - - /* Loop for each file */ - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - origBuff = malloc(benchedSize); - if (origBuff==NULL) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } - - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } } - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("using %s : \n", inFileName); - BMK_benchMem(origBuff, benchedSize); - - /* clean */ - free(origBuff); - } - - return 0; -} - - -static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) -{ - DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); -} - -/* optimizeForSize(): - * targetSpeed : expressed in MB/s */ -int optimizeForSize(const char* inFileName, U32 targetSpeed) -{ - FILE* const inFile = fopen( inFileName, "rb" ); - U64 const inFileSize = UTIL_getFileSize(inFileName); - size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; - void* origBuff; - - /* Init */ - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - if (inFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Pb evaluatin size of %s \n", inFileName); - fclose(inFile); - return 11; - } - - /* Memory allocation & restrictions */ - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) { - DISPLAY("Not enough memory for '%s' \n", inFileName); - fclose(inFile); - return 11; - } - - /* Alloc */ - origBuff = malloc(benchedSize); - if(!origBuff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } - - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } } - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed); - targetSpeed *= 1000000; - - { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - winnerInfo_t winner; - BMK_result_t candidate; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - - /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} - memset(&winner, 0, sizeof(winner)); - winner.result.cSize = (size_t)(-1); - - /* find best solution from default params */ - { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - int i; - for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, CParams); - if (candidate.cSpeed < targetSpeed) - break; - if ( (candidate.cSize < winner.result.cSize) - | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) - { - winner.params = CParams; - winner.result = candidate; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } } - } - BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - - /* start tests */ - { time_t const grillStart = time(NULL); - do { - ZSTD_compressionParameters params = winner.params; - paramVariation(¶ms); - if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ - params = ZSTD_adjustCParams(params, blockSize, 0); - - /* exclude faster if already played set of params */ - if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; - - /* test */ - NB_TESTS_PLAYED(params)++; - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params); - - /* improvement found => new winner */ - if ( (candidate.cSpeed > targetSpeed) - & ( (candidate.cSize < winner.result.cSize) - | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) ) - { - winner.params = params; - winner.result = candidate; - BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - } - } while (BMK_timeSpan(grillStart) < g_grillDuration_s); - } - - /* end summary */ - BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); - DISPLAY("grillParams size - optimizer completed \n"); - - /* clean up*/ - ZSTD_freeCCtx(ctx); - } - - free(origBuff); - return 0; -} - - -static int usage(const char* exename) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] file\n", exename); - DISPLAY( "Arguments :\n"); - DISPLAY( " file : path to the file used as reference (if none, generates a compressible sample)\n"); - DISPLAY( " -H/-h : Help (this text + advanced options)\n"); - return 0; -} - -static int usage_advanced(void) -{ - DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -T# : set level 1 speed objective \n"); - DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); - DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS); - DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n"); - DISPLAY( " -S : Single run \n"); - DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); - return 0; -} - -static int badusage(const char* exename) -{ - DISPLAY("Wrong parameters\n"); - usage(exename); - return 1; -} - -int main(int argc, const char** argv) -{ - int i, - filenamesStart=0, - result; - const char* exename=argv[0]; - const char* input_filename=0; - U32 optimizer = 0; - U32 main_pause = 0; - U32 targetSpeed = 0; - - /* checks */ - if (NB_LEVELS_TRACKED <= ZSTD_maxCLevel()) { - DISPLAY("Error : NB_LEVELS_TRACKED <= ZSTD_maxCLevel() \n"); - exit(1); - } - - /* Welcome message */ - DISPLAY(WELCOME_MESSAGE); - - if (argc<1) { badusage(exename); return 1; } - - for(i=1; i='0') & (argument[0] <='9')) - g_nbIterations = *argument++ - '0'; - break; - - /* Sample compressibility (when no file provided) */ - case 'P': - argument++; - { U32 proba32 = 0; - while ((argument[0]>= '0') & (argument[0]<= '9')) - proba32 = (proba32*10) + (*argument++ - '0'); - g_compressibility = (double)proba32 / 100.; - } - break; - - case 'O': - argument++; - optimizer=1; - targetSpeed = 0; - while ((*argument >= '0') & (*argument <= '9')) - targetSpeed = (targetSpeed*10) + (*argument++ - '0'); - break; - - /* Run Single conf */ - case 'S': - g_singleRun = 1; - argument++; - g_params = ZSTD_getCParams(2, g_blockSize, 0); - for ( ; ; ) { - switch(*argument) - { - case 'w': - g_params.windowLog = 0; - argument++; - while ((*argument>= '0') && (*argument<='9')) - g_params.windowLog *= 10, g_params.windowLog += *argument++ - '0'; - continue; - case 'c': - g_params.chainLog = 0; - argument++; - while ((*argument>= '0') && (*argument<='9')) - g_params.chainLog *= 10, g_params.chainLog += *argument++ - '0'; - continue; - case 'h': - g_params.hashLog = 0; - argument++; - while ((*argument>= '0') && (*argument<='9')) - g_params.hashLog *= 10, g_params.hashLog += *argument++ - '0'; - continue; - case 's': - g_params.searchLog = 0; - argument++; - while ((*argument>= '0') && (*argument<='9')) - g_params.searchLog *= 10, g_params.searchLog += *argument++ - '0'; - continue; - case 'l': /* search length */ - g_params.searchLength = 0; - argument++; - while ((*argument>= '0') && (*argument<='9')) - g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0'; - continue; - case 't': /* target length */ - g_params.targetLength = 0; - argument++; - while ((*argument>= '0') && (*argument<='9')) - g_params.targetLength *= 10, g_params.targetLength += *argument++ - '0'; - continue; - case 'S': /* strategy */ - argument++; - while ((*argument>= '0') && (*argument<='9')) - g_params.strategy = (ZSTD_strategy)(*argument++ - '0'); - continue; - case 'L': - { int cLevel = 0; - argument++; - while ((*argument>= '0') && (*argument<='9')) - cLevel *= 10, cLevel += *argument++ - '0'; - g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); - continue; - } - default : ; - } - break; - } - break; - - /* target level1 speed objective, in MB/s */ - case 'T': - argument++; - g_target = 0; - while ((*argument >= '0') && (*argument <= '9')) - g_target = (g_target*10) + (*argument++ - '0'); - break; - - /* cut input into blocks */ - case 'B': - g_blockSize = 0; - argument++; - while ((*argument >='0') & (*argument <='9')) - g_blockSize = (g_blockSize*10) + (*argument++ - '0'); - if (*argument=='K') g_blockSize<<=10, argument++; /* allows using KB notation */ - if (*argument=='M') g_blockSize<<=20, argument++; - if (*argument=='B') argument++; - DISPLAY("using %u KB block size \n", g_blockSize>>10); - break; - - /* Unknown command */ - default : return badusage(exename); - } - } - continue; - } /* if (argument[0]=='-') */ - - /* first provided filename is input */ - if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } - } - - if (filenamesStart==0) - result = benchSample(); - else { - if (optimizer) - result = optimizeForSize(input_filename, targetSpeed); - else - result = benchFiles(argv+filenamesStart, argc-filenamesStart); - } - - if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } - - return result; -} diff --git a/sys/contrib/zstd/tests/playTests.sh b/sys/contrib/zstd/tests/playTests.sh deleted file mode 100755 index 41d8263b6a6..00000000000 --- a/sys/contrib/zstd/tests/playTests.sh +++ /dev/null @@ -1,771 +0,0 @@ -#!/bin/sh -e - -die() { - $ECHO "$@" 1>&2 - exit 1 -} - -roundTripTest() { - if [ -n "$3" ]; then - cLevel="$3" - proba="$2" - else - cLevel="$2" - proba="" - fi - if [ -n "$4" ]; then - dLevel="$4" - else - dLevel="$cLevel" - fi - - rm -f tmp1 tmp2 - $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel" - ./datagen $1 $proba | $MD5SUM > tmp1 - ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel | $MD5SUM > tmp2 - $DIFF -q tmp1 tmp2 -} - -fileRoundTripTest() { - if [ -n "$3" ]; then - local_c="$3" - local_p="$2" - else - local_c="$2" - local_p="" - fi - if [ -n "$4" ]; then - local_d="$4" - else - local_d="$local_c" - fi - - rm -f tmp.zstd tmp.md5.1 tmp.md5.2 - $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d" - ./datagen $1 $local_p > tmp - < tmp $MD5SUM > tmp.md5.1 - $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2 - $DIFF -q tmp.md5.1 tmp.md5.2 -} - -isTerminal=false -if [ -t 0 ] && [ -t 1 ] -then - isTerminal=true -fi - -isWindows=false -INTOVOID="/dev/null" -DEVDEVICE="/dev/zero" -case "$OS" in - Windows*) - isWindows=true - INTOVOID="NUL" - DEVDEVICE="NUL" - ;; -esac - -UNAME=$(uname) -case "$UNAME" in - Darwin) MD5SUM="md5 -r" ;; - FreeBSD) MD5SUM="gmd5sum" ;; - *) MD5SUM="md5sum" ;; -esac - -DIFF="diff" -case "$UNAME" in - SunOS) DIFF="gdiff" ;; -esac - -ECHO="echo -e" -case "$UNAME" in - Darwin) ECHO="echo" ;; -esac - -$ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'" - -[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" - -if [ -n "$(echo hello | $ZSTD -v -T2 2>&1 > $INTOVOID | grep 'multi-threading is disabled')" ] -then - hasMT="" -else - hasMT="true" -fi - - -$ECHO "\n===> simple tests " - -./datagen > tmp -$ECHO "test : basic compression " -$ZSTD -f tmp # trivial compression case, creates tmp.zst -$ECHO "test : basic decompression" -$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) -$ECHO "test : too large compression level => auto-fix" -$ZSTD -99 -f tmp # too large compression level, automatic sized down -$ECHO "test : --fast aka negative compression levels" -$ZSTD --fast -f tmp # == -1 -$ZSTD --fast=3 -f tmp # == -3 -$ZSTD --fast=200000 -f tmp # == no compression -$ECHO "test : compress to stdout" -$ZSTD tmp -c > tmpCompressed -$ZSTD tmp --stdout > tmpCompressed # long command format -$ECHO "test : compress to named file" -rm tmpCompressed -$ZSTD tmp -o tmpCompressed -test -f tmpCompressed # file must be created -$ECHO "test : -o must be followed by filename (must fail)" -$ZSTD tmp -of tmpCompressed && die "-o must be followed by filename " -$ECHO "test : force write, correct order" -$ZSTD tmp -fo tmpCompressed -$ECHO "test : forgotten argument" -cp tmp tmp2 -$ZSTD tmp2 -fo && die "-o must be followed by filename " -$ECHO "test : implied stdout when input is stdin" -$ECHO bob | $ZSTD | $ZSTD -d -if [ "$isTerminal" = true ]; then -$ECHO "test : compressed data to terminal" -$ECHO bob | $ZSTD && die "should have refused : compressed data to terminal" -$ECHO "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)" -$ZSTD -d > $INTOVOID && die "should have refused : compressed data from terminal" -fi -$ECHO "test : null-length file roundtrip" -$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout -$ECHO "test : ensure small file doesn't add 3-bytes null block" -./datagen -g1 > tmp1 -$ZSTD tmp1 -c | wc -c | grep "14" -$ZSTD < tmp1 | wc -c | grep "14" -$ECHO "test : decompress file with wrong suffix (must fail)" -$ZSTD -d tmpCompressed && die "wrong suffix error not detected!" -$ZSTD -df tmp && die "should have refused : wrong extension" -$ECHO "test : decompress into stdout" -$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout -$ZSTD --decompress tmpCompressed -c > tmpResult -$ZSTD --decompress tmpCompressed --stdout > tmpResult -$ECHO "test : decompress from stdin into stdout" -$ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout -$ZSTD -dc - < tmp.zst > $INTOVOID -$ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used -$ZSTD -d - < tmp.zst > $INTOVOID -$ECHO "test : impose memory limitation (must fail)" -$ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" -$ZSTD -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command -$ZSTD -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command -$ZSTD -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command -$ECHO "test : overwrite protection" -$ZSTD -q tmp && die "overwrite check failed!" -$ECHO "test : force overwrite" -$ZSTD -q -f tmp -$ZSTD -q --force tmp -$ECHO "test : overwrite readonly file" -rm -f tmpro tmpro.zst -$ECHO foo > tmpro.zst -$ECHO foo > tmpro -chmod 400 tmpro.zst -$ZSTD -q tmpro && die "should have refused to overwrite read-only file" -$ZSTD -q -f tmpro -rm -f tmpro tmpro.zst -$ECHO "test : file removal" -$ZSTD -f --rm tmp -test ! -f tmp # tmp should no longer be present -$ZSTD -f -d --rm tmp.zst -test ! -f tmp.zst # tmp.zst should no longer be present -$ECHO "test : should quietly not remove non-regular file" -$ECHO hello > tmp -$ZSTD tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" -grep -v "Refusing to remove non-regular file" tmplog -rm -f tmplog -$ZSTD tmp -f -o "$INTONULL" 2>&1 | grep -v "Refusing to remove non-regular file" -$ECHO "test : --rm on stdin" -$ECHO a | $ZSTD --rm > $INTOVOID # --rm should remain silent -rm tmp -$ZSTD -f tmp && die "tmp not present : should have failed" -test ! -f tmp.zst # tmp.zst should not be created - -$ECHO "test : compress multiple files" -$ECHO hello > tmp1 -$ECHO world > tmp2 -$ZSTD tmp1 tmp2 -o "$INTOVOID" -$ZSTD tmp1 tmp2 -c | $ZSTD -t -$ZSTD tmp1 tmp2 -o tmp.zst -test ! -f tmp1.zst -test ! -f tmp2.zst -$ZSTD tmp1 tmp2 -$ZSTD -t tmp1.zst tmp2.zst -$ZSTD -dc tmp1.zst tmp2.zst -$ZSTD tmp1.zst tmp2.zst -o "$INTOVOID" -$ZSTD -d tmp1.zst tmp2.zst -o tmp -touch tmpexists -$ZSTD tmp1 tmp2 -f -o tmpexists -$ZSTD tmp1 tmp2 -o tmpexists && die "should have refused to overwrite" -# Bug: PR #972 -if [ "$?" -eq 139 ]; then - die "should not have segfaulted" -fi -rm tmp* - - -$ECHO "\n===> Advanced compression parameters " -$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" -$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" -$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" -test ! -f tmp.zst # tmp.zst should not be created -roundTripTest -g512K -roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6" -roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" -roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6" -roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmSearchLength=64,ldmBucketSizeLog=1,ldmHashEveryLog=7" -roundTripTest -g512K " --single-thread --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7" -roundTripTest -g512K 19 - - -$ECHO "\n===> Pass-Through mode " -$ECHO "Hello world 1!" | $ZSTD -df -$ECHO "Hello world 2!" | $ZSTD -dcf -$ECHO "Hello world 3!" > tmp1 -$ZSTD -dcf tmp1 - - -$ECHO "\n===> frame concatenation " - -$ECHO "hello " > hello.tmp -$ECHO "world!" > world.tmp -cat hello.tmp world.tmp > helloworld.tmp -$ZSTD -c hello.tmp > hello.zstd -$ZSTD -c world.tmp > world.zstd -cat hello.zstd world.zstd > helloworld.zstd -$ZSTD -dc helloworld.zstd > result.tmp -cat result.tmp -$DIFF helloworld.tmp result.tmp -$ECHO "frame concatenation without checksum" -$ZSTD -c hello.tmp > hello.zstd --no-check -$ZSTD -c world.tmp > world.zstd --no-check -cat hello.zstd world.zstd > helloworld.zstd -$ZSTD -dc helloworld.zstd > result.tmp -$DIFF helloworld.tmp result.tmp -$ECHO "testing zstdcat symlink" -ln -sf $ZSTD zstdcat -./zstdcat helloworld.zstd > result.tmp -$DIFF helloworld.tmp result.tmp -rm zstdcat -rm result.tmp -$ECHO "testing zcat symlink" -ln -sf $ZSTD zcat -./zcat helloworld.zstd > result.tmp -$DIFF helloworld.tmp result.tmp -rm zcat -rm ./*.tmp ./*.zstd -$ECHO "frame concatenation tests completed" - - -if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then -$ECHO "\n**** flush write error test **** " - -$ECHO "$ECHO foo | $ZSTD > /dev/full" -$ECHO foo | $ZSTD > /dev/full && die "write error not detected!" -$ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full" -$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" - - -$ECHO "\n===> symbolic link test " - -rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst -$ECHO "hello world" > hello.tmp -ln -s hello.tmp world.tmp -$ZSTD world.tmp hello.tmp -test -f hello.tmp.zst # regular file should have been compressed! -test ! -f world.tmp.zst # symbolic link should not have been compressed! -$ZSTD world.tmp hello.tmp -f -test -f world.tmp.zst # symbolic link should have been compressed with --force -rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst - -fi - - -$ECHO "\n===> test sparse file support " - -./datagen -g5M -P100 > tmpSparse -$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen -$DIFF -s tmpSparse tmpSparseRegen -$ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse -$DIFF -s tmpSparse tmpOutSparse -$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse -$DIFF -s tmpSparse tmpOutNoSparse -ls -ls tmpSparse* # look at file size and block size on disk -./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) -./datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd -ls -ls tmpSparseOdd # look at file size and block size on disk -$ECHO "\n Sparse Compatibility with Console :" -$ECHO "Hello World 1 !" | $ZSTD | $ZSTD -d -c -$ECHO "Hello World 2 !" | $ZSTD | $ZSTD -d | cat -$ECHO "\n Sparse Compatibility with Append :" -./datagen -P100 -g1M > tmpSparse1M -cat tmpSparse1M tmpSparse1M > tmpSparse2M -$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed -$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated -$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated -ls -ls tmpSparse* # look at file size and block size on disk -$DIFF tmpSparse2M tmpSparseRegenerated -rm tmpSparse* - - -$ECHO "\n===> multiple files tests " - -./datagen -s1 > tmp1 2> $INTOVOID -./datagen -s2 -g100K > tmp2 2> $INTOVOID -./datagen -s3 -g1M > tmp3 2> $INTOVOID -$ECHO "compress tmp* : " -$ZSTD -f tmp* -ls -ls tmp* -rm tmp1 tmp2 tmp3 -$ECHO "decompress tmp* : " -$ZSTD -df *.zst -ls -ls tmp* -$ECHO "compress tmp* into stdout > tmpall : " -$ZSTD -c tmp1 tmp2 tmp3 > tmpall -ls -ls tmp* # check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) -$ECHO "decompress tmpall* into stdout > tmpdec : " -cp tmpall tmpall2 -$ZSTD -dc tmpall* > tmpdec -ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) -$ECHO "compress multiple files including a missing one (notHere) : " -$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" - - -$ECHO "\n===> dictionary tests " - -$ECHO "- test with raw dict (content only) " -./datagen > tmpDict -./datagen -g1M | $MD5SUM > tmp1 -./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2 -$DIFF -q tmp1 tmp2 -$ECHO "- Create first dictionary " -TESTFILE=../programs/zstdcli.c -$ZSTD --train *.c ../programs/*.c -o tmpDict -cp $TESTFILE tmp -$ECHO "- Dictionary compression roundtrip" -$ZSTD -f tmp -D tmpDict -$ZSTD -d tmp.zst -D tmpDict -fo result -$DIFF $TESTFILE result -$ECHO "- Dictionary compression with btlazy2 strategy" -$ZSTD -f tmp -D tmpDict --zstd=strategy=6 -$ZSTD -d tmp.zst -D tmpDict -fo result -$DIFF $TESTFILE result -if [ -n "$hasMT" ] -then - $ECHO "- Test dictionary compression with multithreading " - ./datagen -g5M | $ZSTD -T2 -D tmpDict | $ZSTD -t -D tmpDict # fails with v1.3.2 -fi -$ECHO "- Create second (different) dictionary " -$ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC -$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" -$ECHO "- Create dictionary with short dictID" -$ZSTD --train *.c ../programs/*.c --dictID=1 -o tmpDict1 -cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" -$ECHO "- Create dictionary with wrong dictID parameter order (must fail)" -$ZSTD --train *.c ../programs/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " -$ECHO "- Create dictionary with size limit" -$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict=4K -v -$ECHO "- Create dictionary with small size limit" -$ZSTD --train *.c ../programs/*.c -o tmpDict3 --maxdict=1K -v -$ECHO "- Create dictionary with wrong parameter order (must fail)" -$ZSTD --train *.c ../programs/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " -$ECHO "- Compress without dictID" -$ZSTD -f tmp -D tmpDict1 --no-dictID -$ZSTD -d tmp.zst -D tmpDict -fo result -$DIFF $TESTFILE result -$ECHO "- Compress with wrong argument order (must fail)" -$ZSTD tmp -Df tmpDict1 -c > $INTOVOID && die "-D must be followed by dictionary name " -$ECHO "- Compress multiple files with dictionary" -rm -rf dirTestDict -mkdir dirTestDict -cp *.c dirTestDict -cp ../programs/*.c dirTestDict -cp ../programs/*.h dirTestDict -$MD5SUM dirTestDict/* > tmph1 -$ZSTD -f --rm dirTestDict/* -D tmpDictC -$ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default -case "$UNAME" in - Darwin) $ECHO "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 - *) $MD5SUM -c tmph1 ;; -esac -rm -rf dirTestDict -$ECHO "- dictionary builder on bogus input" -$ECHO "Hello World" > tmp -$ZSTD --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" -./datagen -P0 -g10M > tmp -$ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" -rm tmp* - - -$ECHO "\n===> cover dictionary builder : advanced options " - -TESTFILE=../programs/zstdcli.c -./datagen > tmpDict -$ECHO "- Create first dictionary" -$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c -o tmpDict -cp $TESTFILE tmp -$ZSTD -f tmp -D tmpDict -$ZSTD -d tmp.zst -D tmpDict -fo result -$DIFF $TESTFILE result -$ECHO "- Create second (different) dictionary" -$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC -$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" -$ECHO "- Create dictionary with short dictID" -$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c --dictID=1 -o tmpDict1 -cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" -$ECHO "- Create dictionary with size limit" -$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K -rm tmp* - -$ECHO "\n===> legacy dictionary builder " - -TESTFILE=../programs/zstdcli.c -./datagen > tmpDict -$ECHO "- Create first dictionary" -$ZSTD --train-legacy=selectivity=8 *.c ../programs/*.c -o tmpDict -cp $TESTFILE tmp -$ZSTD -f tmp -D tmpDict -$ZSTD -d tmp.zst -D tmpDict -fo result -$DIFF $TESTFILE result -$ECHO "- Create second (different) dictionary" -$ZSTD --train-legacy=s=5 *.c ../programs/*.c ../programs/*.h -o tmpDictC -$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" -$ECHO "- Create dictionary with short dictID" -$ZSTD --train-legacy -s5 *.c ../programs/*.c --dictID=1 -o tmpDict1 -cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" -$ECHO "- Create dictionary with size limit" -$ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K -rm tmp* - - -$ECHO "\n===> integrity tests " - -$ECHO "test one file (tmp1.zst) " -./datagen > tmp1 -$ZSTD tmp1 -$ZSTD -t tmp1.zst -$ZSTD --test tmp1.zst -$ECHO "test multiple files (*.zst) " -$ZSTD -t *.zst -$ECHO "test bad files (*) " -$ZSTD -t * && die "bad files not detected !" -$ZSTD -t tmp1 && die "bad file not detected !" -cp tmp1 tmp2.zst -$ZSTD -t tmp2.zst && die "bad file not detected !" -./datagen -g0 > tmp3 -$ZSTD -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad -$ECHO "test --rm and --test combined " -$ZSTD -t --rm tmp1.zst -test -f tmp1.zst # check file is still present -split -b16384 tmp1.zst tmpSplit. -$ZSTD -t tmpSplit.* && die "bad file not detected !" -./datagen | $ZSTD -c | $ZSTD -t - - - -$ECHO "\n===> golden files tests " - -$ZSTD -t -r files -$ZSTD -c -r files | $ZSTD -t - - -$ECHO "\n===> benchmark mode tests " - -$ECHO "bench one file" -./datagen > tmp1 -$ZSTD -bi0 tmp1 -$ECHO "bench multiple levels" -$ZSTD -i0b0e3 tmp1 -$ECHO "bench negative level" -$ZSTD -bi0 --fast tmp1 -$ECHO "with recursive and quiet modes" -$ZSTD -rqi1b1e2 tmp1 - - -$ECHO "\n===> gzip compatibility tests " - -GZIPMODE=1 -$ZSTD --format=gzip -V || GZIPMODE=0 -if [ $GZIPMODE -eq 1 ]; then - $ECHO "gzip support detected" - GZIPEXE=1 - gzip -V || GZIPEXE=0 - if [ $GZIPEXE -eq 1 ]; then - ./datagen > tmp - $ZSTD --format=gzip -f tmp - gzip -t -v tmp.gz - gzip -f tmp - $ZSTD -d -f -v tmp.gz - rm tmp* - else - $ECHO "gzip binary not detected" - fi -else - $ECHO "gzip mode not supported" -fi - - -$ECHO "\n===> gzip frame tests " - -if [ $GZIPMODE -eq 1 ]; then - ./datagen > tmp - $ZSTD -f --format=gzip tmp - $ZSTD -f tmp - cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp - head -c -1 tmp.gz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" - rm tmp* -else - $ECHO "gzip mode not supported" -fi - - -$ECHO "\n===> xz compatibility tests " - -LZMAMODE=1 -$ZSTD --format=xz -V || LZMAMODE=0 -if [ $LZMAMODE -eq 1 ]; then - $ECHO "xz support detected" - XZEXE=1 - xz -V && lzma -V || XZEXE=0 - if [ $XZEXE -eq 1 ]; then - $ECHO "Testing zstd xz and lzma support" - ./datagen > tmp - $ZSTD --format=lzma -f tmp - $ZSTD --format=xz -f tmp - xz -t -v tmp.xz - xz -t -v tmp.lzma - xz -f -k tmp - lzma -f -k --lzma1 tmp - $ZSTD -d -f -v tmp.xz - $ZSTD -d -f -v tmp.lzma - rm tmp* - $ECHO "Creating symlinks" - ln -s $ZSTD ./xz - ln -s $ZSTD ./unxz - ln -s $ZSTD ./lzma - ln -s $ZSTD ./unlzma - $ECHO "Testing xz and lzma symlinks" - ./datagen > tmp - ./xz tmp - xz -d tmp.xz - ./lzma tmp - lzma -d tmp.lzma - $ECHO "Testing unxz and unlzma symlinks" - xz tmp - ./xz -d tmp.xz - lzma tmp - ./lzma -d tmp.lzma - rm xz unxz lzma unlzma - rm tmp* - else - $ECHO "xz binary not detected" - fi -else - $ECHO "xz mode not supported" -fi - - -$ECHO "\n===> xz frame tests " - -if [ $LZMAMODE -eq 1 ]; then - ./datagen > tmp - $ZSTD -f --format=xz tmp - $ZSTD -f --format=lzma tmp - $ZSTD -f tmp - cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp - head -c -1 tmp.xz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" - head -c -1 tmp.lzma | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" - rm tmp* -else - $ECHO "xz mode not supported" -fi - -$ECHO "\n===> lz4 compatibility tests " - -LZ4MODE=1 -$ZSTD --format=lz4 -V || LZ4MODE=0 -if [ $LZ4MODE -eq 1 ]; then - $ECHO "lz4 support detected" - LZ4EXE=1 - lz4 -V || LZ4EXE=0 - if [ $LZ4EXE -eq 1 ]; then - ./datagen > tmp - $ZSTD --format=lz4 -f tmp - lz4 -t -v tmp.lz4 - lz4 -f tmp - $ZSTD -d -f -v tmp.lz4 - rm tmp* - else - $ECHO "lz4 binary not detected" - fi -else - $ECHO "lz4 mode not supported" -fi - - -$ECHO "\n===> lz4 frame tests " - -if [ $LZ4MODE -eq 1 ]; then - ./datagen > tmp - $ZSTD -f --format=lz4 tmp - $ZSTD -f tmp - cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | $ZSTD -d -f -o tmp - head -c -1 tmp.lz4 | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" - rm tmp* -else - $ECHO "lz4 mode not supported" -fi - -$ECHO "\n===> zstd round-trip tests " - -roundTripTest -roundTripTest -g15K # TableID==3 -roundTripTest -g127K # TableID==2 -roundTripTest -g255K # TableID==1 -roundTripTest -g522K # TableID==0 -roundTripTest -g519K 6 # greedy, hash chain -roundTripTest -g517K 16 # btlazy2 -roundTripTest -g516K 19 # btopt - -fileRoundTripTest -g500K - -$ECHO "\n===> zstd long distance matching round-trip tests " -roundTripTest -g0 "2 --single-thread --long" -roundTripTest -g1000K "1 --single-thread --long" -roundTripTest -g517K "6 --single-thread --long" -roundTripTest -g516K "16 --single-thread --long" -roundTripTest -g518K "19 --single-thread --long" -fileRoundTripTest -g5M "3 --single-thread --long" - - -roundTripTest -g96K "5 --single-thread" -if [ -n "$hasMT" ] -then - $ECHO "\n===> zstdmt round-trip tests " - roundTripTest -g4M "1 -T0" - roundTripTest -g8M "3 -T2" - roundTripTest -g8000K "2 --threads=2" - fileRoundTripTest -g4M "19 -T2 -B1M" - - $ECHO "\n===> zstdmt long distance matching round-trip tests " - roundTripTest -g8M "3 --long=24 -T2" -else - $ECHO "\n===> no multithreading, skipping zstdmt tests " -fi - -rm tmp* - -$ECHO "\n===> zstd --list/-l single frame tests " -./datagen > tmp1 -./datagen > tmp2 -./datagen > tmp3 -$ZSTD tmp* -$ZSTD -l *.zst -$ZSTD -lv *.zst | grep "Decompressed Size:" # check that decompressed size is present in header -$ZSTD --list *.zst -$ZSTD --list -v *.zst - -$ECHO "\n===> zstd --list/-l multiple frame tests " -cat tmp1.zst tmp2.zst > tmp12.zst -cat tmp12.zst tmp3.zst > tmp123.zst -$ZSTD -l *.zst -$ZSTD -lv *.zst - -$ECHO "\n===> zstd --list/-l error detection tests " -! $ZSTD -l tmp1 tmp1.zst -! $ZSTD --list tmp* -! $ZSTD -lv tmp1* -! $ZSTD --list -v tmp2 tmp12.zst - -$ECHO "\n===> zstd --list/-l test with null files " -./datagen -g0 > tmp5 -$ZSTD tmp5 -$ZSTD -l tmp5.zst -! $ZSTD -l tmp5* -$ZSTD -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header -! $ZSTD -lv tmp5* - -$ECHO "\n===> zstd --list/-l test with no content size field " -./datagen -g513K | $ZSTD > tmp6.zst -$ZSTD -l tmp6.zst -! $ZSTD -lv tmp6.zst | grep "Decompressed Size:" # must NOT be present in header - -$ECHO "\n===> zstd --list/-l test with no checksum " -$ZSTD -f --no-check tmp1 -$ZSTD -l tmp1.zst -$ZSTD -lv tmp1.zst - -rm tmp* - - -$ECHO "\n===> zstd long distance matching tests " -roundTripTest -g0 " --single-thread --long" -roundTripTest -g9M "2 --single-thread --long" -# Test parameter parsing -roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB" -roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB" -roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB" -roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" - - -if [ "$1" != "--test-large-data" ]; then - $ECHO "Skipping large data tests" - exit 0 -fi - -$ECHO "\n===> large files tests " - -roundTripTest -g270000000 1 -roundTripTest -g250000000 2 -roundTripTest -g230000000 3 - -roundTripTest -g140000000 -P60 4 -roundTripTest -g130000000 -P62 5 -roundTripTest -g120000000 -P65 6 - -roundTripTest -g70000000 -P70 7 -roundTripTest -g60000000 -P71 8 -roundTripTest -g50000000 -P73 9 - -roundTripTest -g35000000 -P75 10 -roundTripTest -g30000000 -P76 11 -roundTripTest -g25000000 -P78 12 - -roundTripTest -g18000013 -P80 13 -roundTripTest -g18000014 -P80 14 -roundTripTest -g18000015 -P81 15 -roundTripTest -g18000016 -P84 16 -roundTripTest -g18000017 -P88 17 -roundTripTest -g18000018 -P94 18 -roundTripTest -g18000019 -P96 19 - -roundTripTest -g5000000000 -P99 1 -roundTripTest -g1700000000 -P0 "1 --zstd=strategy=6" # ensure btlazy2 can survive an overflow rescale - -fileRoundTripTest -g4193M -P99 1 - - -$ECHO "\n===> zstd long, long distance matching round-trip tests " -roundTripTest -g270000000 "1 --single-thread --long" -roundTripTest -g130000000 -P60 "5 --single-thread --long" -roundTripTest -g35000000 -P70 "8 --single-thread --long" -roundTripTest -g18000001 -P80 "18 --single-thread --long" -# Test large window logs -roundTripTest -g700M -P50 "1 --single-thread --long=29" -roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" - - -if [ -n "$hasMT" ] -then - $ECHO "\n===> zstdmt long round-trip tests " - roundTripTest -g80000000 -P99 "19 -T2" " " - roundTripTest -g5000000000 -P99 "1 -T2" " " - roundTripTest -g500000000 -P97 "1 -T999" " " - fileRoundTripTest -g4103M -P98 " -T0" " " - roundTripTest -g400000000 -P97 "1 --long=24 -T2" " " -else - $ECHO "\n**** no multithreading, skipping zstdmt tests **** " -fi - -rm tmp* diff --git a/sys/contrib/zstd/tests/poolTests.c b/sys/contrib/zstd/tests/poolTests.c deleted file mode 100644 index 00ee830154c..00000000000 --- a/sys/contrib/zstd/tests/poolTests.c +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -#include "pool.h" -#include "threading.h" -#include "util.h" -#include -#include - -#define ASSERT_TRUE(p) \ - do { \ - if (!(p)) { \ - return 1; \ - } \ - } while (0) -#define ASSERT_FALSE(p) ASSERT_TRUE(!(p)) -#define ASSERT_EQ(lhs, rhs) ASSERT_TRUE((lhs) == (rhs)) - -struct data { - pthread_mutex_t mutex; - unsigned data[16]; - size_t i; -}; - -void fn(void *opaque) { - struct data *data = (struct data *)opaque; - pthread_mutex_lock(&data->mutex); - data->data[data->i] = data->i; - ++data->i; - pthread_mutex_unlock(&data->mutex); -} - -int testOrder(size_t numThreads, size_t queueSize) { - struct data data; - POOL_ctx *ctx = POOL_create(numThreads, queueSize); - ASSERT_TRUE(ctx); - data.i = 0; - pthread_mutex_init(&data.mutex, NULL); - { - size_t i; - for (i = 0; i < 16; ++i) { - POOL_add(ctx, &fn, &data); - } - } - POOL_free(ctx); - ASSERT_EQ(16, data.i); - { - size_t i; - for (i = 0; i < data.i; ++i) { - ASSERT_EQ(i, data.data[i]); - } - } - pthread_mutex_destroy(&data.mutex); - return 0; -} - -void waitFn(void *opaque) { - (void)opaque; - UTIL_sleepMilli(1); -} - -/* Tests for deadlock */ -int testWait(size_t numThreads, size_t queueSize) { - struct data data; - POOL_ctx *ctx = POOL_create(numThreads, queueSize); - ASSERT_TRUE(ctx); - { - size_t i; - for (i = 0; i < 16; ++i) { - POOL_add(ctx, &waitFn, &data); - } - } - POOL_free(ctx); - return 0; -} - -int main(int argc, const char **argv) { - size_t numThreads; - for (numThreads = 1; numThreads <= 4; ++numThreads) { - size_t queueSize; - for (queueSize = 0; queueSize <= 2; ++queueSize) { - if (testOrder(numThreads, queueSize)) { - printf("FAIL: testOrder\n"); - return 1; - } - if (testWait(numThreads, queueSize)) { - printf("FAIL: testWait\n"); - return 1; - } - } - } - printf("PASS: testOrder\n"); - (void)argc; - (void)argv; - return (POOL_create(0, 1)) ? printf("FAIL: testInvalid\n"), 1 - : printf("PASS: testInvalid\n"), 0; - return 0; -} diff --git a/sys/contrib/zstd/tests/roundTripCrash.c b/sys/contrib/zstd/tests/roundTripCrash.c deleted file mode 100644 index 7d937fceebc..00000000000 --- a/sys/contrib/zstd/tests/roundTripCrash.c +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -/* - This program takes a file in input, - performs a zstd round-trip test (compression - decompress) - compares the result with original - and generates a crash (double free) on corruption detection. -*/ - -/*=========================================== -* Dependencies -*==========================================*/ -#include /* size_t */ -#include /* malloc, free, exit */ -#include /* fprintf */ -#include /* strcmp */ -#include /* stat */ -#include /* stat */ -#include "xxhash.h" - -#define ZSTD_STATIC_LINKING_ONLY -#include "zstd.h" - -/*=========================================== -* Macros -*==========================================*/ -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - -static void crash(int errorCode){ - /* abort if AFL/libfuzzer, exit otherwise */ - #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */ - abort(); - #else - exit(errorCode); - #endif -} - -#define CHECK_Z(f) { \ - size_t const err = f; \ - if (ZSTD_isError(err)) { \ - fprintf(stderr, \ - "Error=> %s: %s", \ - #f, ZSTD_getErrorName(err)); \ - crash(1); \ -} } - -/** roundTripTest() : -* Compresses `srcBuff` into `compressedBuff`, -* then decompresses `compressedBuff` into `resultBuff`. -* Compression level used is derived from first content byte. -* @return : result of decompression, which should be == `srcSize` -* or an error code if either compression or decompression fails. -* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)` -* for compression to be guaranteed to work */ -static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity, - void* compressedBuff, size_t compressedBuffCapacity, - const void* srcBuff, size_t srcBuffSize) -{ - static const int maxClevel = 19; - size_t const hashLength = MIN(128, srcBuffSize); - unsigned const h32 = XXH32(srcBuff, hashLength, 0); - int const cLevel = h32 % maxClevel; - size_t const cSize = ZSTD_compress(compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, cLevel); - if (ZSTD_isError(cSize)) { - fprintf(stderr, "Compression error : %s \n", ZSTD_getErrorName(cSize)); - return cSize; - } - return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, cSize); -} - -/** cctxParamRoundTripTest() : - * Same as roundTripTest() except allows experimenting with ZSTD_CCtx_params. */ -static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity, - void* compressedBuff, size_t compressedBuffCapacity, - const void* srcBuff, size_t srcBuffSize) -{ - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams(); - ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 }; - ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 }; - - static const int maxClevel = 19; - size_t const hashLength = MIN(128, srcBuffSize); - unsigned const h32 = XXH32(srcBuff, hashLength, 0); - int const cLevel = h32 % maxClevel; - - /* Set parameters */ - CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, cLevel) ); - CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_nbWorkers, 2) ); - CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_overlapSizeLog, 5) ); - - - /* Apply parameters */ - CHECK_Z( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) ); - - CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) ); - - ZSTD_freeCCtxParams(cctxParams); - ZSTD_freeCCtx(cctx); - - return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, outBuffer.pos); -} - -static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) -{ - const char* ip1 = (const char*)buff1; - const char* ip2 = (const char*)buff2; - size_t pos; - - for (pos=0; pos= `fileSize` */ -static void loadFile(void* buffer, const char* fileName, size_t fileSize) -{ - FILE* const f = fopen(fileName, "rb"); - if (isDirectory(fileName)) { - fprintf(stderr, "Ignoring %s directory \n", fileName); - exit(2); - } - if (f==NULL) { - fprintf(stderr, "Impossible to open %s \n", fileName); - exit(3); - } - { size_t const readSize = fread(buffer, 1, fileSize, f); - if (readSize != fileSize) { - fprintf(stderr, "Error reading %s \n", fileName); - exit(5); - } } - fclose(f); -} - - -static void fileCheck(const char* fileName, int testCCtxParams) -{ - size_t const fileSize = getFileSize(fileName); - void* buffer = malloc(fileSize); - if (!buffer) { - fprintf(stderr, "not enough memory \n"); - exit(4); - } - loadFile(buffer, fileName, fileSize); - roundTripCheck(buffer, fileSize, testCCtxParams); - free (buffer); -} - -int main(int argCount, const char** argv) { - int argNb = 1; - int testCCtxParams = 0; - if (argCount < 2) { - fprintf(stderr, "Error : no argument : need input file \n"); - exit(9); - } - - if (!strcmp(argv[argNb], "--cctxParams")) { - testCCtxParams = 1; - argNb++; - } - - fileCheck(argv[argNb], testCCtxParams); - fprintf(stderr, "no pb detected\n"); - return 0; -} diff --git a/sys/contrib/zstd/tests/seqgen.c b/sys/contrib/zstd/tests/seqgen.c deleted file mode 100644 index 8233ecee1cf..00000000000 --- a/sys/contrib/zstd/tests/seqgen.c +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright (c) 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#include "seqgen.h" -#include "mem.h" -#include - -#define MIN(a, b) ((a) < (b) ? (a) : (b)) - -static const size_t kMatchBytes = 128; - -#define SEQ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -static BYTE SEQ_randByte(U32* src) -{ - static const U32 prime1 = 2654435761U; - static const U32 prime2 = 2246822519U; - U32 rand32 = *src; - rand32 *= prime1; - rand32 ^= prime2; - rand32 = SEQ_rotl32(rand32, 13); - *src = rand32; - return (BYTE)(rand32 >> 5); -} - -SEQ_stream SEQ_initStream(unsigned seed) -{ - SEQ_stream stream; - stream.state = 0; - XXH64_reset(&stream.xxh, 0); - stream.seed = seed; - return stream; -} - -/* Generates a single guard byte, then match length + 1 of a different byte, - * then another guard byte. - */ -static size_t SEQ_gen_matchLength(SEQ_stream* stream, unsigned value, - SEQ_outBuffer* out) -{ - typedef enum { - ml_first_byte = 0, - ml_match_bytes, - ml_last_byte, - } ml_state; - BYTE* const ostart = (BYTE*)out->dst; - BYTE* const oend = ostart + out->size; - BYTE* op = ostart + out->pos; - - switch ((ml_state)stream->state) { - case ml_first_byte: - /* Generate a single byte and pick a different byte for the match */ - if (op >= oend) { - stream->bytesLeft = 1; - break; - } - *op = SEQ_randByte(&stream->seed) & 0xFF; - do { - stream->saved = SEQ_randByte(&stream->seed) & 0xFF; - } while (*op == stream->saved); - ++op; - /* State transition */ - stream->state = ml_match_bytes; - stream->bytesLeft = value + 1; - /* fall-through */ - case ml_match_bytes: { - /* Copy matchLength + 1 bytes to the output buffer */ - size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op)); - if (setLength > 0) { - memset(op, stream->saved, setLength); - op += setLength; - stream->bytesLeft -= setLength; - } - if (stream->bytesLeft > 0) - break; - /* State transition */ - stream->state = ml_last_byte; - } - /* fall-through */ - case ml_last_byte: - /* Generate a single byte and pick a different byte for the match */ - if (op >= oend) { - stream->bytesLeft = 1; - break; - } - do { - *op = SEQ_randByte(&stream->seed) & 0xFF; - } while (*op == stream->saved); - ++op; - /* State transition */ - /* fall-through */ - default: - stream->state = 0; - stream->bytesLeft = 0; - break; - } - XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos); - out->pos = op - ostart; - return stream->bytesLeft; -} - -/* Saves the current seed then generates kMatchBytes random bytes >= 128. - * Generates literal length - kMatchBytes random bytes < 128. - * Generates another kMatchBytes using the saved seed to generate a match. - * This way the match is easy to find for the compressors. - */ -static size_t SEQ_gen_litLength(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out) -{ - typedef enum { - ll_start = 0, - ll_run_bytes, - ll_literals, - ll_run_match, - } ll_state; - BYTE* const ostart = (BYTE*)out->dst; - BYTE* const oend = ostart + out->size; - BYTE* op = ostart + out->pos; - - switch ((ll_state)stream->state) { - case ll_start: - stream->state = ll_run_bytes; - stream->saved = stream->seed; - stream->bytesLeft = MIN(kMatchBytes, value); - /* fall-through */ - case ll_run_bytes: - while (stream->bytesLeft > 0 && op < oend) { - *op++ = SEQ_randByte(&stream->seed) | 0x80; - --stream->bytesLeft; - } - if (stream->bytesLeft > 0) - break; - /* State transition */ - stream->state = ll_literals; - stream->bytesLeft = value - MIN(kMatchBytes, value); - /* fall-through */ - case ll_literals: - while (stream->bytesLeft > 0 && op < oend) { - *op++ = SEQ_randByte(&stream->seed) & 0x7F; - --stream->bytesLeft; - } - if (stream->bytesLeft > 0) - break; - /* State transition */ - stream->state = ll_run_match; - stream->bytesLeft = MIN(kMatchBytes, value); - /* fall-through */ - case ll_run_match: { - while (stream->bytesLeft > 0 && op < oend) { - *op++ = SEQ_randByte(&stream->saved) | 0x80; - --stream->bytesLeft; - } - if (stream->bytesLeft > 0) - break; - } - /* fall-through */ - default: - stream->state = 0; - stream->bytesLeft = 0; - break; - } - XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos); - out->pos = op - ostart; - return stream->bytesLeft; -} - -/* Saves the current seed then generates kMatchBytes random bytes >= 128. - * Generates offset - kMatchBytes of zeros to get a large offset without - * polluting the hash tables. - * Generates another kMatchBytes using the saved seed to generate a with the - * required offset. - */ -static size_t SEQ_gen_offset(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out) -{ - typedef enum { - of_start = 0, - of_run_bytes, - of_offset, - of_run_match, - } of_state; - BYTE* const ostart = (BYTE*)out->dst; - BYTE* const oend = ostart + out->size; - BYTE* op = ostart + out->pos; - - switch ((of_state)stream->state) { - case of_start: - stream->state = of_run_bytes; - stream->saved = stream->seed; - stream->bytesLeft = MIN(value, kMatchBytes); - /* fall-through */ - case of_run_bytes: { - while (stream->bytesLeft > 0 && op < oend) { - *op++ = SEQ_randByte(&stream->seed) | 0x80; - --stream->bytesLeft; - } - if (stream->bytesLeft > 0) - break; - /* State transition */ - stream->state = of_offset; - stream->bytesLeft = value - MIN(value, kMatchBytes); - } - /* fall-through */ - case of_offset: { - /* Copy matchLength + 1 bytes to the output buffer */ - size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op)); - if (setLength > 0) { - memset(op, 0, setLength); - op += setLength; - stream->bytesLeft -= setLength; - } - if (stream->bytesLeft > 0) - break; - /* State transition */ - stream->state = of_run_match; - stream->bytesLeft = MIN(value, kMatchBytes); - } - /* fall-through */ - case of_run_match: { - while (stream->bytesLeft > 0 && op < oend) { - *op++ = SEQ_randByte(&stream->saved) | 0x80; - --stream->bytesLeft; - } - if (stream->bytesLeft > 0) - break; - } - /* fall-through */ - default: - stream->state = 0; - stream->bytesLeft = 0; - break; - } - XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos); - out->pos = op - ostart; - return stream->bytesLeft; -} - -/* Returns the number of bytes left to generate. - * Must pass the same type/value until it returns 0. - */ -size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value, SEQ_outBuffer* out) -{ - switch (type) { - case SEQ_gen_ml: return SEQ_gen_matchLength(stream, value, out); - case SEQ_gen_ll: return SEQ_gen_litLength(stream, value, out); - case SEQ_gen_of: return SEQ_gen_offset(stream, value, out); - case SEQ_gen_max: /* fall-through */ - default: return 0; - } -} - -/* Returns the xxhash of the data produced so far */ -XXH64_hash_t SEQ_digest(SEQ_stream const* stream) -{ - return XXH64_digest(&stream->xxh); -} diff --git a/sys/contrib/zstd/tests/seqgen.h b/sys/contrib/zstd/tests/seqgen.h deleted file mode 100644 index 72d798846aa..00000000000 --- a/sys/contrib/zstd/tests/seqgen.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef SEQGEN_H -#define SEQGEN_H - -#define XXH_STATIC_LINKING_ONLY - -#include "xxhash.h" -#include /* size_t */ - -typedef enum { - SEQ_gen_ml = 0, - SEQ_gen_ll, - SEQ_gen_of, - SEQ_gen_max /* Must be the last value */ -} SEQ_gen_type; - -/* Internal state, do not use */ -typedef struct { - XXH64_state_t xxh; /* xxh state for all the data produced so far (seed=0) */ - unsigned seed; - int state; /* enum to control state machine (clean=0) */ - unsigned saved; - size_t bytesLeft; -} SEQ_stream; - -SEQ_stream SEQ_initStream(unsigned seed); - -typedef struct { - void* dst; - size_t size; - size_t pos; -} SEQ_outBuffer; - -/* Returns non-zero until the current type/value has been generated. - * Must pass the same type/value until it returns 0. - * - * Recommended to pick a value in the middle of the range you want, since there - * may be some noise that causes actual results to be slightly different. - * We try to be more accurate for smaller values. - * - * NOTE: Very small values don't work well (< 6). - */ -size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value, - SEQ_outBuffer* out); - -/* Returns the xxhash of the data produced so far */ -XXH64_hash_t SEQ_digest(SEQ_stream const* stream); - -#endif /* SEQGEN_H */ diff --git a/sys/contrib/zstd/tests/symbols.c b/sys/contrib/zstd/tests/symbols.c deleted file mode 100644 index c0bed2e5d96..00000000000 --- a/sys/contrib/zstd/tests/symbols.c +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -#include -#include "zstd_errors.h" -#define ZSTD_STATIC_LINKING_ONLY -#include "zstd.h" -#define ZBUFF_DISABLE_DEPRECATE_WARNINGS -#define ZBUFF_STATIC_LINKING_ONLY -#include "zbuff.h" -#define ZDICT_DISABLE_DEPRECATE_WARNINGS -#define ZDICT_STATIC_LINKING_ONLY -#include "zdict.h" - -static const void *symbols[] = { -/* zstd.h */ - &ZSTD_versionNumber, - &ZSTD_compress, - &ZSTD_decompress, - &ZSTD_getDecompressedSize, - &ZSTD_findDecompressedSize, - &ZSTD_findFrameCompressedSize, - &ZSTD_getFrameContentSize, - &ZSTD_maxCLevel, - &ZSTD_compressBound, - &ZSTD_isError, - &ZSTD_getErrorName, - &ZSTD_createCCtx, - &ZSTD_freeCCtx, - &ZSTD_compressCCtx, - &ZSTD_createDCtx, - &ZSTD_freeDCtx, - &ZSTD_decompressDCtx, - &ZSTD_compress_usingDict, - &ZSTD_decompress_usingDict, - &ZSTD_createCDict, - &ZSTD_freeCDict, - &ZSTD_compress_usingCDict, - &ZSTD_createDDict, - &ZSTD_freeDDict, - &ZSTD_decompress_usingDDict, - &ZSTD_createCStream, - &ZSTD_freeCStream, - &ZSTD_initCStream, - &ZSTD_compressStream, - &ZSTD_flushStream, - &ZSTD_endStream, - &ZSTD_CStreamInSize, - &ZSTD_CStreamOutSize, - &ZSTD_createDStream, - &ZSTD_freeDStream, - &ZSTD_initDStream, - &ZSTD_decompressStream, - &ZSTD_DStreamInSize, - &ZSTD_DStreamOutSize, -/* zstd.h: advanced functions */ - &ZSTD_estimateCCtxSize, - &ZSTD_createCCtx_advanced, - &ZSTD_sizeof_CCtx, - &ZSTD_createCDict_advanced, - &ZSTD_sizeof_CDict, - &ZSTD_getCParams, - &ZSTD_getParams, - &ZSTD_checkCParams, - &ZSTD_adjustCParams, - &ZSTD_compress_advanced, - &ZSTD_isFrame, - &ZSTD_estimateDCtxSize, - &ZSTD_createDCtx_advanced, - &ZSTD_sizeof_DCtx, - &ZSTD_sizeof_DDict, - &ZSTD_getDictID_fromDict, - &ZSTD_getDictID_fromDDict, - &ZSTD_getDictID_fromFrame, - &ZSTD_createCStream_advanced, - &ZSTD_initCStream_srcSize, - &ZSTD_initCStream_usingDict, - &ZSTD_initCStream_advanced, - &ZSTD_initCStream_usingCDict, - &ZSTD_resetCStream, - &ZSTD_sizeof_CStream, - &ZSTD_createDStream_advanced, - &ZSTD_initDStream_usingDict, - &ZSTD_setDStreamParameter, - &ZSTD_initDStream_usingDDict, - &ZSTD_resetDStream, - &ZSTD_sizeof_DStream, - &ZSTD_compressBegin, - &ZSTD_compressBegin_usingDict, - &ZSTD_compressBegin_advanced, - &ZSTD_copyCCtx, - &ZSTD_compressContinue, - &ZSTD_compressEnd, - &ZSTD_getFrameHeader, - &ZSTD_decompressBegin, - &ZSTD_decompressBegin_usingDict, - &ZSTD_copyDCtx, - &ZSTD_nextSrcSizeToDecompress, - &ZSTD_decompressContinue, - &ZSTD_nextInputType, - &ZSTD_getBlockSize, - &ZSTD_compressBlock, - &ZSTD_decompressBlock, - &ZSTD_insertBlock, -/* zstd_errors.h */ - &ZSTD_getErrorCode, - &ZSTD_getErrorString, -/* zbuff.h */ - &ZBUFF_createCCtx, - &ZBUFF_freeCCtx, - &ZBUFF_compressInit, - &ZBUFF_compressInitDictionary, - &ZBUFF_compressContinue, - &ZBUFF_compressFlush, - &ZBUFF_compressEnd, - &ZBUFF_createDCtx, - &ZBUFF_freeDCtx, - &ZBUFF_decompressInit, - &ZBUFF_decompressInitDictionary, - &ZBUFF_decompressContinue, - &ZBUFF_isError, - &ZBUFF_getErrorName, - &ZBUFF_recommendedCInSize, - &ZBUFF_recommendedCOutSize, - &ZBUFF_recommendedDInSize, - &ZBUFF_recommendedDOutSize, -/* zbuff.h: advanced functions */ - &ZBUFF_createCCtx_advanced, - &ZBUFF_createDCtx_advanced, - &ZBUFF_compressInit_advanced, -/* zdict.h */ - &ZDICT_trainFromBuffer, - &ZDICT_getDictID, - &ZDICT_isError, - &ZDICT_getErrorName, -/* zdict.h: advanced functions */ - &ZDICT_trainFromBuffer_cover, - &ZDICT_optimizeTrainFromBuffer_cover, - &ZDICT_finalizeDictionary, - &ZDICT_trainFromBuffer_legacy, - &ZDICT_addEntropyTablesFromBuffer, - NULL, -}; - -int main(int argc, const char** argv) { - const void **symbol; - (void)argc; - (void)argv; - - for (symbol = symbols; *symbol != NULL; ++symbol) { - printf("%p\n", *symbol); - } - return 0; -} diff --git a/sys/contrib/zstd/tests/test-zstd-speed.py b/sys/contrib/zstd/tests/test-zstd-speed.py deleted file mode 100755 index 1096d5e4e22..00000000000 --- a/sys/contrib/zstd/tests/test-zstd-speed.py +++ /dev/null @@ -1,376 +0,0 @@ -#! /usr/bin/env python3 - -# ################################################################ -# Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# ########################################################################## - -# Limitations: -# - doesn't support filenames with spaces -# - dir1/zstd and dir2/zstd will be merged in a single results file - -import argparse -import os # getloadavg -import string -import subprocess -import time # strftime -import traceback -import hashlib -import platform # system - -script_version = 'v1.1.2 (2017-03-26)' -default_repo_url = 'https://github.com/facebook/zstd.git' -working_dir_name = 'speedTest' -working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest -clone_path = working_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd -email_header = 'ZSTD_speedTest' -pid = str(os.getpid()) -verbose = False -clang_version = "unknown" -gcc_version = "unknown" -args = None - - -def hashfile(hasher, fname, blocksize=65536): - with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(blocksize), b""): - hasher.update(chunk) - return hasher.hexdigest() - - -def log(text): - print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text) - - -def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True): - if print_command: - log("> " + command) - popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd) - stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout) - stderr_lines = stderr_lines.decode("utf-8") - stdout_lines = stdout_lines.decode("utf-8") - if print_output: - if stdout_lines: - print(stdout_lines) - if stderr_lines: - print(stderr_lines) - if popen.returncode is not None and popen.returncode != 0: - if stderr_lines and not print_output and print_error: - print(stderr_lines) - raise RuntimeError(stdout_lines + stderr_lines) - return (stdout_lines + stderr_lines).splitlines() -execute.cwd = None - - -def does_command_exist(command): - try: - execute(command, verbose, False, False) - except Exception: - return False - return True - - -def send_email(emails, topic, text, have_mutt, have_mail): - logFileName = working_path + '/' + 'tmpEmailContent' - with open(logFileName, "w") as myfile: - myfile.writelines(text) - myfile.close() - if have_mutt: - execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose) - elif have_mail: - execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose) - else: - log("e-mail cannot be sent (mail or mutt not found)") - - -def send_email_with_attachments(branch, commit, last_commit, args, text, results_files, - logFileName, have_mutt, have_mail): - with open(logFileName, "w") as myfile: - myfile.writelines(text) - myfile.close() - email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \ - % (email_header, pid, branch, commit, last_commit, - args.lowerLimit, args.ratioLimit) - if have_mutt: - execute('mutt -s "' + email_topic + '" ' + args.emails + ' -a ' + results_files - + ' < ' + logFileName) - elif have_mail: - execute('mail -s "' + email_topic + '" ' + args.emails + ' < ' + logFileName) - else: - log("e-mail cannot be sent (mail or mutt not found)") - - -def git_get_branches(): - execute('git fetch -p', verbose) - branches = execute('git branch -rl', verbose) - output = [] - for line in branches: - if ("HEAD" not in line) and ("coverity_scan" not in line) and ("gh-pages" not in line): - output.append(line.strip()) - return output - - -def git_get_changes(branch, commit, last_commit): - fmt = '--format="%h: (%an) %s, %ar"' - if last_commit is None: - commits = execute('git log -n 10 %s %s' % (fmt, commit)) - else: - commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit)) - return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits) - - -def get_last_results(resultsFileName): - if not os.path.isfile(resultsFileName): - return None, None, None, None - commit = None - csize = [] - cspeed = [] - dspeed = [] - with open(resultsFileName, 'r') as f: - for line in f: - words = line.split() - if len(words) <= 4: # branch + commit + compilerVer + md5 - commit = words[1] - csize = [] - cspeed = [] - dspeed = [] - if (len(words) == 8) or (len(words) == 9): # results: "filename" or "XX files" - csize.append(int(words[1])) - cspeed.append(float(words[3])) - dspeed.append(float(words[5])) - return commit, csize, cspeed, dspeed - - -def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, - testFilePath, fileName, last_csize, last_cspeed, last_dspeed): - sleepTime = 30 - while os.getloadavg()[0] > args.maxLoadAvg: - log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" - % (os.getloadavg()[0], args.maxLoadAvg, sleepTime)) - time.sleep(sleepTime) - start_load = str(os.getloadavg()) - osType = platform.system() - if osType == 'Linux': - cpuSelector = "taskset --cpu-list 0" - else: - cpuSelector = "" - if args.dictionary: - result = execute('%s programs/%s -rqi5b1e%s -D %s %s' % (cpuSelector, executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True) - else: - result = execute('%s programs/%s -rqi5b1e%s %s' % (cpuSelector, executableName, args.lastCLevel, testFilePath), print_output=True) - end_load = str(os.getloadavg()) - linesExpected = args.lastCLevel + 1 - if len(result) != linesExpected: - raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result))) - with open(resultsFileName, "a") as myfile: - myfile.write('%s %s %s md5=%s\n' % (branch, commit, compilerVersion, md5sum)) - myfile.write('\n'.join(result) + '\n') - myfile.close() - if (last_cspeed == None): - log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName)) - return "" - commit, csize, cspeed, dspeed = get_last_results(resultsFileName) - text = "" - for i in range(0, min(len(cspeed), len(last_cspeed))): - print("%s:%s -%d cSpeed=%6.2f cLast=%6.2f cDiff=%1.4f dSpeed=%6.2f dLast=%6.2f dDiff=%1.4f ratioDiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], float(last_csize[i])/csize[i], fileName)) - if (cspeed[i]/last_cspeed[i] < args.lowerLimit): - text += "WARNING: %s -%d cSpeed=%.2f cLast=%.2f cDiff=%.4f %s\n" % (executableName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName) - if (dspeed[i]/last_dspeed[i] < args.lowerLimit): - text += "WARNING: %s -%d dSpeed=%.2f dLast=%.2f dDiff=%.4f %s\n" % (executableName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName) - if (float(last_csize[i])/csize[i] < args.ratioLimit): - text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName) - if text: - text = args.message + ("\nmaxLoadAvg=%s load average at start=%s end=%s\n%s last_commit=%s md5=%s\n" % (args.maxLoadAvg, start_load, end_load, compilerVersion, last_commit, md5sum)) + text - return text - - -def update_config_file(branch, commit): - last_commit = None - commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt" - if os.path.isfile(commitFileName): - with open(commitFileName, 'r') as infile: - last_commit = infile.read() - with open(commitFileName, 'w') as outfile: - outfile.write(commit) - return last_commit - - -def double_check(branch, commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName): - last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName) - if not args.dry_run: - text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed) - if text: - log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit)) - text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed) - return text - - -def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail): - local_branch = branch.split('/')[1] - version = local_branch.rpartition('-')[2] + '_' + commit - if not args.dry_run: - execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version + - 'mv programs/zstd programs/zstd_clang && ' + - 'make -C programs clean zstd zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version) - md5_zstd = hashfile(hashlib.md5(), clone_path + '/programs/zstd') - md5_zstd32 = hashfile(hashlib.md5(), clone_path + '/programs/zstd32') - md5_zstd_clang = hashfile(hashlib.md5(), clone_path + '/programs/zstd_clang') - print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang)) - print("gcc_version=%s clang_version=%s" % (gcc_version, clang_version)) - - logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt" - text_to_send = [] - results_files = "" - if args.dictionary: - dictName = args.dictionary.rpartition('/')[2] - else: - dictName = None - - for filePath in testFilePaths: - fileName = filePath.rpartition('/')[2] - if dictName: - resultsFileName = working_path + "/" + dictName.replace(".", "_") + "_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - else: - resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - text = double_check(branch, commit, args, 'zstd', md5_zstd, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName) - if text: - text_to_send.append(text) - results_files += resultsFileName + " " - resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - text = double_check(branch, commit, args, 'zstd32', md5_zstd32, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName) - if text: - text_to_send.append(text) - results_files += resultsFileName + " " - resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - text = double_check(branch, commit, args, 'zstd_clang', md5_zstd_clang, 'clang_version='+clang_version, resultsFileName, filePath, fileName) - if text: - text_to_send.append(text) - results_files += resultsFileName + " " - if text_to_send: - send_email_with_attachments(branch, commit, last_commit, args, text_to_send, results_files, logFileName, have_mutt, have_mail) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('testFileNames', help='file or directory names list for speed benchmark') - parser.add_argument('emails', help='list of e-mail addresses to send warnings') - parser.add_argument('--dictionary', '-D', help='path to the dictionary') - parser.add_argument('--message', '-m', help='attach an additional message to e-mail', default="") - parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url) - parser.add_argument('--lowerLimit', '-l', type=float, help='send email if speed is lower than given limit', default=0.98) - parser.add_argument('--ratioLimit', '-r', type=float, help='send email if ratio is lower than given limit', default=0.999) - parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75) - parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5) - parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300) - parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=1800) - parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False) - parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False) - args = parser.parse_args() - verbose = args.verbose - - # check if test files are accessible - testFileNames = args.testFileNames.split() - testFilePaths = [] - for fileName in testFileNames: - fileName = os.path.expanduser(fileName) - if os.path.isfile(fileName) or os.path.isdir(fileName): - testFilePaths.append(os.path.abspath(fileName)) - else: - log("ERROR: File/directory not found: " + fileName) - exit(1) - - # check if dictionary is accessible - if args.dictionary: - args.dictionary = os.path.abspath(os.path.expanduser(args.dictionary)) - if not os.path.isfile(args.dictionary): - log("ERROR: Dictionary not found: " + args.dictionary) - exit(1) - - # check availability of e-mail senders - have_mutt = does_command_exist("mutt -h") - have_mail = does_command_exist("mail -V") - if not have_mutt and not have_mail: - log("ERROR: e-mail senders 'mail' or 'mutt' not found") - exit(1) - - clang_version = execute("clang -v 2>&1 | grep ' version ' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0]; - gcc_version = execute("gcc -dumpversion", verbose)[0]; - - if verbose: - print("PARAMETERS:\nrepoURL=%s" % args.repoURL) - print("working_path=%s" % working_path) - print("clone_path=%s" % clone_path) - print("testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)) - print("message=%s" % args.message) - print("emails=%s" % args.emails) - print("dictionary=%s" % args.dictionary) - print("maxLoadAvg=%s" % args.maxLoadAvg) - print("lowerLimit=%s" % args.lowerLimit) - print("ratioLimit=%s" % args.ratioLimit) - print("lastCLevel=%s" % args.lastCLevel) - print("sleepTime=%s" % args.sleepTime) - print("timeout=%s" % args.timeout) - print("dry_run=%s" % args.dry_run) - print("verbose=%s" % args.verbose) - print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail)) - - # clone ZSTD repo if needed - if not os.path.isdir(working_path): - os.mkdir(working_path) - if not os.path.isdir(clone_path): - execute.cwd = working_path - execute('git clone ' + args.repoURL) - if not os.path.isdir(clone_path): - log("ERROR: ZSTD clone not found: " + clone_path) - exit(1) - execute.cwd = clone_path - - # check if speedTest.pid already exists - pidfile = "./speedTest.pid" - if os.path.isfile(pidfile): - log("ERROR: %s already exists, exiting" % pidfile) - exit(1) - - send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail) - with open(pidfile, 'w') as the_file: - the_file.write(pid) - - branch = "" - commit = "" - first_time = True - while True: - try: - if first_time: - first_time = False - else: - time.sleep(args.sleepTime) - loadavg = os.getloadavg()[0] - if (loadavg <= args.maxLoadAvg): - branches = git_get_branches() - for branch in branches: - commit = execute('git show -s --format=%h ' + branch, verbose)[0] - last_commit = update_config_file(branch, commit) - if commit == last_commit: - log("skipping branch %s: head %s already processed" % (branch, commit)) - else: - log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit)) - execute('git checkout -- . && git checkout ' + branch) - print(git_get_changes(branch, commit, last_commit)) - test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail) - else: - log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg)) - if verbose: - log("sleep for %s seconds" % args.sleepTime) - except Exception as e: - stack = traceback.format_exc() - email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit) - send_email(args.emails, email_topic, stack, have_mutt, have_mail) - print(stack) - except KeyboardInterrupt: - os.unlink(pidfile) - send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail) - exit(0) diff --git a/sys/contrib/zstd/tests/test-zstd-versions.py b/sys/contrib/zstd/tests/test-zstd-versions.py deleted file mode 100755 index f2deac1f28d..00000000000 --- a/sys/contrib/zstd/tests/test-zstd-versions.py +++ /dev/null @@ -1,276 +0,0 @@ -#!/usr/bin/env python3 -"""Test zstd interoperability between versions""" - -# ################################################################ -# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# ################################################################ - -import filecmp -import glob -import hashlib -import os -import shutil -import sys -import subprocess -from subprocess import Popen, PIPE - -repo_url = 'https://github.com/facebook/zstd.git' -tmp_dir_name = 'tests/versionsTest' -make_cmd = 'make' -git_cmd = 'git' -test_dat_src = 'README.md' -test_dat = 'test_dat' -head = 'vdevel' -dict_source = 'dict_source' -dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c ' -dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h' - - -def execute(command, print_output=False, print_error=True, param_shell=False): - popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=param_shell) - stdout_lines, stderr_lines = popen.communicate() - stderr_lines = stderr_lines.decode("utf-8") - stdout_lines = stdout_lines.decode("utf-8") - if print_output: - print(stdout_lines) - print(stderr_lines) - if popen.returncode is not None and popen.returncode != 0: - if not print_output and print_error: - print(stderr_lines) - return popen.returncode - - -def proc(cmd_args, pipe=True, dummy=False): - if dummy: - return - if pipe: - subproc = Popen(cmd_args, stdout=PIPE, stderr=PIPE) - else: - subproc = Popen(cmd_args) - return subproc.communicate() - - -def make(args, pipe=True): - return proc([make_cmd] + args, pipe) - - -def git(args, pipe=True): - return proc([git_cmd] + args, pipe) - - -def get_git_tags(): - stdout, stderr = git(['tag', '-l', 'v[0-9].[0-9].[0-9]']) - tags = stdout.decode('utf-8').split() - return tags - - -def create_dict(tag, dict_source_path): - dict_name = 'dict.' + tag - if not os.path.isfile(dict_name): - cFiles = glob.glob(dict_source_path + "/*.c") - hFiles = glob.glob(dict_source_path + "/*.h") - if tag == 'v0.5.0': - result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) - else: - result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) - if result == 0: - print(dict_name + ' created') - else: - print('ERROR: creating of ' + dict_name + ' failed') - else: - print(dict_name + ' already exists') - - -def dict_compress_sample(tag, sample): - dict_name = 'dict.' + tag - DEVNULL = open(os.devnull, 'wb') - if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst') - if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst') - if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst') - if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst') - if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst') - # zstdFiles = glob.glob("*.zst*") - # print(zstdFiles) - print(tag + " : dict compression completed") - - -def compress_sample(tag, sample): - DEVNULL = open(os.devnull, 'wb') - if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst') - if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst') - if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst') - if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst') - if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst') - # zstdFiles = glob.glob("*.zst*") - # print(zstdFiles) - print(tag + " : compression completed") - - -# http://stackoverflow.com/a/19711609/2132223 -def sha1_of_file(filepath): - with open(filepath, 'rb') as f: - return hashlib.sha1(f.read()).hexdigest() - - -def remove_duplicates(): - list_of_zst = sorted(glob.glob('*.zst')) - for i, ref_zst in enumerate(list_of_zst): - if not os.path.isfile(ref_zst): - continue - for j in range(i + 1, len(list_of_zst)): - compared_zst = list_of_zst[j] - if not os.path.isfile(compared_zst): - continue - if filecmp.cmp(ref_zst, compared_zst): - os.remove(compared_zst) - print('duplicated : {} == {}'.format(ref_zst, compared_zst)) - - -def decompress_zst(tag): - dec_error = 0 - list_zst = sorted(glob.glob('*_nodict.zst')) - for file_zst in list_zst: - print(file_zst, end=' ') - print(tag, end=' ') - file_dec = file_zst + '_d64_' + tag + '.dec' - if tag <= 'v0.5.0': - params = ['./zstd.' + tag, '-df', file_zst, file_dec] - else: - params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec] - if execute(params) == 0: - if not filecmp.cmp(file_dec, test_dat): - print('ERR !! ') - dec_error = 1 - else: - print('OK ') - else: - print('command does not work') - dec_error = 1 - return dec_error - - -def decompress_dict(tag): - dec_error = 0 - list_zst = sorted(glob.glob('*_dictio.zst')) - for file_zst in list_zst: - dict_tag = file_zst[0:len(file_zst)-11] # remove "_dictio.zst" - if head in dict_tag: # find vdevel - dict_tag = head - else: - dict_tag = dict_tag[dict_tag.rfind('v'):] - if tag == 'v0.6.0' and dict_tag < 'v0.6.0': - continue - dict_name = 'dict.' + dict_tag - print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ') - file_dec = file_zst + '_d64_' + tag + '.dec' - if tag <= 'v0.5.0': - params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec] - else: - params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec] - if execute(params) == 0: - if not filecmp.cmp(file_dec, test_dat): - print('ERR !! ') - dec_error = 1 - else: - print('OK ') - else: - print('command does not work') - dec_error = 1 - return dec_error - - -if __name__ == '__main__': - error_code = 0 - base_dir = os.getcwd() + '/..' # /path/to/zstd - tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest - clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd - dict_source_path = tmp_dir + '/' + dict_source # /path/to/zstd/tests/versionsTest/dict_source - programs_dir = base_dir + '/programs' # /path/to/zstd/programs - os.makedirs(tmp_dir, exist_ok=True) - - # since Travis clones limited depth, we should clone full repository - if not os.path.isdir(clone_dir): - git(['clone', repo_url, clone_dir]) - - shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat) - - # Retrieve all release tags - print('Retrieve all release tags :') - os.chdir(clone_dir) - alltags = get_git_tags() + [head] - tags = [t for t in alltags if t >= 'v0.4.0'] - print(tags) - - # Build all release zstd - for tag in tags: - os.chdir(base_dir) - dst_zstd = '{}/zstd.{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/zstd. - if not os.path.isfile(dst_zstd) or tag == head: - if tag != head: - r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/ - os.makedirs(r_dir, exist_ok=True) - os.chdir(clone_dir) - git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) - if tag == 'v0.5.0': - os.chdir(r_dir + '/dictBuilder') # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder - make(['clean', 'dictBuilder'], False) - shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag)) - os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest//programs - make(['clean', 'zstd'], False) - else: - os.chdir(programs_dir) - make(['zstd'], False) - shutil.copy2('zstd', dst_zstd) - - # remove any remaining *.zst and *.dec from previous test - os.chdir(tmp_dir) - for compressed in glob.glob("*.zst"): - os.remove(compressed) - for dec in glob.glob("*.dec"): - os.remove(dec) - - # copy *.c and *.h to a temporary directory ("dict_source") - if not os.path.isdir(dict_source_path): - os.mkdir(dict_source_path) - print('cp ' + dict_files + ' ' + dict_source_path) - execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True) - - print('Compress test.dat by all released zstd') - - error_code = 0 - for tag in tags: - print(tag) - if tag >= 'v0.5.0': - create_dict(tag, dict_source_path) - dict_compress_sample(tag, test_dat) - remove_duplicates() - error_code += decompress_dict(tag) - compress_sample(tag, test_dat) - remove_duplicates() - error_code += decompress_zst(tag) - - print('') - print('Enumerate different compressed files') - zstds = sorted(glob.glob('*.zst')) - for zstd in zstds: - print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd)) - - if error_code != 0: - print('====== ERROR !!! =======') - - sys.exit(error_code) diff --git a/sys/contrib/zstd/tests/zbufftest.c b/sys/contrib/zstd/tests/zbufftest.c deleted file mode 100644 index 9b6f7bad6dc..00000000000 --- a/sys/contrib/zstd/tests/zbufftest.c +++ /dev/null @@ -1,618 +0,0 @@ -/* - * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -/*-************************************ -* Compiler specific -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ -#endif - - -/*-************************************ -* Includes -**************************************/ -#include /* free */ -#include /* fgets, sscanf */ -#include /* strcmp */ -#include "mem.h" -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ -#include "zstd.h" /* ZSTD_compressBound */ -#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */ -#include "zbuff.h" /* ZBUFF_isError */ -#include "datagen.h" /* RDG_genBuffer */ -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" /* XXH64_* */ -#include "util.h" - - -/*-************************************ -* Constants -**************************************/ -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -static const U32 nbTestsDefault = 10000; -#define COMPRESSIBLE_NOISE_LENGTH (10 MB) -#define FUZ_COMPRESSIBILITY_DEFAULT 50 -static const U32 prime1 = 2654435761U; -static const U32 prime2 = 2246822519U; - - - -/*-************************************ -* Display Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 2; - -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ - { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); } } - -static U64 g_clockTime = 0; - - -/*-******************************************************* -* Fuzzer functions -*********************************************************/ -#undef MIN -#undef MAX -#define MIN(a,b) ((a)<(b)?(a):(b)) -#define MAX(a,b) ((a)>(b)?(a):(b)) -/*! FUZ_rand() : - @return : a 27 bits random value, from a 32-bits `seed`. - `seed` is also modified */ -# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -unsigned int FUZ_rand(unsigned int* seedPtr) -{ - U32 rand32 = *seedPtr; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZ_rotl32(rand32, 13); - *seedPtr = rand32; - return rand32 >> 5; -} - - -/* -static unsigned FUZ_highbit32(U32 v32) -{ - unsigned nbBits = 0; - if (v32==0) return 0; - for ( ; v32 ; v32>>=1) nbBits++; - return nbBits; -} -*/ - -static void* ZBUFF_allocFunction(void* opaque, size_t size) -{ - void* address = malloc(size); - (void)opaque; - /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ - return address; -} - -static void ZBUFF_freeFunction(void* opaque, void* address) -{ - (void)opaque; - /* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */ - free(address); -} - -static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) -{ - int testResult = 0; - size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; - void* CNBuffer = malloc(CNBufferSize); - size_t const skippableFrameSize = 11; - size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); - void* compressedBuffer = malloc(compressedBufferSize); - size_t const decodedBufferSize = CNBufferSize; - void* decodedBuffer = malloc(decodedBufferSize); - size_t cSize, readSize, readSkipSize, genSize; - U32 testNb=0; - ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem); - ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem); - - /* Create compressible test buffer */ - if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { - DISPLAY("Not enough memory, aborting\n"); - goto _output_error; - } - RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); - - /* generate skippable frame */ - MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); - MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); - cSize = skippableFrameSize + 8; - - /* Basic compression test */ - DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); - readSize = CNBufferSize; - genSize = compressedBufferSize; - { size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize); - if (ZBUFF_isError(r)) goto _output_error; } - if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ - cSize += genSize; - genSize = compressedBufferSize - cSize; - { size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); - if (r != 0) goto _output_error; } /* error, or some data not flushed */ - cSize += genSize; - DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); - - /* skippable frame test */ - DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); - ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); - readSkipSize = cSize; - genSize = CNBufferSize; - { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize); - if (r != 0) goto _output_error; } - if (genSize != 0) goto _output_error; /* skippable frame len is 0 */ - DISPLAYLEVEL(4, "OK \n"); - - /* Basic decompression test */ - DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); - readSize = cSize - readSkipSize; - genSize = CNBufferSize; - { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize); - if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ - if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ - if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */ - DISPLAYLEVEL(4, "OK \n"); - - /* check regenerated data is byte exact */ - DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); - { size_t i; - for (i=0; i "); DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } - -static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) -{ - static const U32 maxSrcLog = 24; - static const U32 maxSampleLog = 19; - BYTE* cNoiseBuffer[5]; - size_t const srcBufferSize = (size_t)1<= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests); - FUZ_rand(&coreSeed); - lseed = coreSeed ^ prime1; - - /* states full reset (unsynchronized) */ - /* some issues only happen when reusing states in a specific sequence of parameters */ - if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); } - if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); } - - /* srcBuffer selection [0-4] */ - { U32 buffNb = FUZ_rand(&lseed) & 0x7F; - if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ - else { - buffNb >>= 3; - if (buffNb & 7) { - const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ - buffNb = tnb[buffNb >> 3]; - } else { - const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ - buffNb = tnb[buffNb >> 3]; - } } - srcBuffer = cNoiseBuffer[buffNb]; - } - - /* compression init */ - { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; - maxTestSize = FUZ_rLogLength(&lseed, testLog); - dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; - /* random dictionary selection */ - { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); - dict = srcBuffer + dictStart; - } - { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize); - params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; - params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; - { size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, ZSTD_CONTENTSIZE_UNKNOWN); - CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError)); - } } } - - /* multi-segments compression test */ - XXH64_reset(&xxhState, 0); - nbChunks = (FUZ_rand(&lseed) & 127) + 2; - for (n=0, cSize=0, totalTestSize=0 ; (n= remainingToFlush; - remainingToFlush = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize); - CHECK (ZBUFF_isError(remainingToFlush), "flush error : %s", ZBUFF_getErrorName(remainingToFlush)); - CHECK (enoughDstSize && remainingToFlush, "ZBUFF_compressEnd() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush); - cSize += dstBuffSize; - } } - crcOrig = XXH64_digest(&xxhState); - - /* multi - fragments decompression test */ - ZBUFF_decompressInitDictionary(zd, dict, dictSize); - errorCode = 1; - for (totalCSize = 0, totalGenSize = 0 ; errorCode ; ) { - size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); - errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize); - CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode)); - totalGenSize += dstBuffSize; - totalCSize += readCSrcSize; - } - CHECK (errorCode != 0, "frame not fully decoded"); - CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") - CHECK (totalCSize != cSize, "compressed data should be fully read") - { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); - if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); - CHECK (crcDest!=crcOrig, "decompressed data corrupted"); } - - /*===== noisy/erroneous src decompression test =====*/ - - /* add some noise */ - { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; - U32 nn; for (nn=0; nn='0') && (*argument<='9')) { - nbTests *= 10; - nbTests += *argument - '0'; - argument++; - } - break; - - case 'T': - argument++; - nbTests=0; g_clockTime=0; - while ((*argument>='0') && (*argument<='9')) { - g_clockTime *= 10; - g_clockTime += *argument - '0'; - argument++; - } - if (*argument=='m') g_clockTime *=60, argument++; - if (*argument=='n') argument++; - g_clockTime *= SEC_TO_MICRO; - break; - - case 's': - argument++; - seed=0; - seedset=1; - while ((*argument>='0') && (*argument<='9')) { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - - case 't': - argument++; - testNb=0; - while ((*argument>='0') && (*argument<='9')) { - testNb *= 10; - testNb += *argument - '0'; - argument++; - } - break; - - case 'P': /* compressibility % */ - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - - default: - return FUZ_usage(programName); - } - } } } /* for(argNb=1; argNb /* free */ -#include /* fgets, sscanf */ -#include /* strcmp */ -#include /* assert */ -#include "mem.h" -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */ -#include "zstd.h" /* ZSTD_compressBound */ -#include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */ -#include "zstdmt_compress.h" -#include "zdict.h" /* ZDICT_trainFromBuffer */ -#include "datagen.h" /* RDG_genBuffer */ -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH64_* */ -#include "seqgen.h" -#include "util.h" - - -/*-************************************ -* Constants -**************************************/ -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -static const U32 nbTestsDefault = 10000; -static const U32 g_cLevelMax_smallTests = 10; -#define COMPRESSIBLE_NOISE_LENGTH (10 MB) -#define FUZ_COMPRESSIBILITY_DEFAULT 50 -static const U32 prime32 = 2654435761U; - - -/*-************************************ -* Display Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \ - DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); } -static U32 g_displayLevel = 2; - -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ - { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); } } - -static U64 g_clockTime = 0; - - -/*-******************************************************* -* Fuzzer functions -*********************************************************/ -#undef MIN -#undef MAX -#define MIN(a,b) ((a)<(b)?(a):(b)) -#define MAX(a,b) ((a)>(b)?(a):(b)) -/*! FUZ_rand() : - @return : a 27 bits random value, from a 32-bits `seed`. - `seed` is also modified */ -#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -unsigned int FUZ_rand(unsigned int* seedPtr) -{ - static const U32 prime2 = 2246822519U; - U32 rand32 = *seedPtr; - rand32 *= prime32; - rand32 += prime2; - rand32 = FUZ_rotl32(rand32, 13); - *seedPtr = rand32; - return rand32 >> 5; -} - -#define CHECK(cond, ...) { \ - if (cond) { \ - DISPLAY("Error => "); \ - DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u, line %u) \n", \ - seed, testNb, __LINE__); \ - goto _output_error; \ -} } - -#define CHECK_Z(f) { \ - size_t const err = f; \ - CHECK(ZSTD_isError(err), "%s : %s ", \ - #f, ZSTD_getErrorName(err)); \ -} - - -/*====================================================== -* Basic Unit tests -======================================================*/ - -typedef struct { - void* start; - size_t size; - size_t filled; -} buffer_t; - -static const buffer_t g_nullBuffer = { NULL, 0 , 0 }; - -static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize) -{ - buffer_t dict = { NULL, 0, 0 }; - size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize; - size_t* const blockSizes = (size_t*) malloc(nbBlocks * sizeof(size_t)); - if (!blockSizes) return dict; - dict.start = malloc(requestedDictSize); - if (!dict.start) { free(blockSizes); return dict; } - { size_t nb; - for (nb=0; nb cSize); - outBuff.size = compressedBufferSize - cSize; - outBuff.pos = 0; - inBuff.src = CNBuffer; - inBuff.size = CNBufferSize; - inBuff.pos = 0; - CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - { size_t const r = ZSTD_endStream(zc, &outBuff); - if (r != 0) goto _output_error; } /* error, or some data not flushed */ - cSize += outBuff.pos; - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); - - /* context size functions */ - DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++); - { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize); - size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams); - size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */ - if (ZSTD_isError(cstreamSize)) goto _output_error; - if (ZSTD_isError(cdictSize)) goto _output_error; - DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)(cstreamSize + cdictSize)); - } - - DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++); - { size_t const s = ZSTD_sizeof_CStream(zc); - if (ZSTD_isError(s)) goto _output_error; - DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s); - } - - /* Attempt bad compression parameters */ - DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++); - { size_t r; - ZSTD_parameters params = ZSTD_getParams(1, 0, 0); - params.cParams.searchLength = 2; - r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); - if (!ZSTD_isError(r)) goto _output_error; - DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r)); - } - - /* skippable frame test */ - DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++); - CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) ); - inBuff.src = compressedBuffer; - inBuff.size = cSize; - inBuff.pos = 0; - outBuff.dst = decodedBuffer; - outBuff.size = CNBufferSize; - outBuff.pos = 0; - { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); - DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (U32)r); - if (r != 0) goto _output_error; - } - if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */ - DISPLAYLEVEL(3, "OK \n"); - - /* Basic decompression test */ - inBuff2 = inBuff; - DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize); - CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000) ); /* large limit */ - { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff); - if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ - if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ - if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ - DISPLAYLEVEL(3, "OK \n"); - - /* Re-use without init */ - DISPLAYLEVEL(3, "test%3i : decompress again without init (re-use previous settings): ", testNb++); - outBuff.pos = 0; - { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2); - if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ - if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ - if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ - DISPLAYLEVEL(3, "OK \n"); - - /* check regenerated data is byte exact */ - DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); - { size_t i; - for (i=0; i 100 bytes */ - DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); } - ZSTD_DCtx_reset(zd); /* leave zd in good shape for next tests */ - - DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++); - { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - int const maxLevel = 16; /* first level with zstd_opt */ - int level; - assert(maxLevel < ZSTD_maxCLevel()); - CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) ); - for (level = 1; level <= maxLevel; ++level) { - ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level); - size_t const maxSize = MIN(1 MB, CNBufferSize); - size_t size; - for (size = 512; size <= maxSize; size <<= 1) { - U64 const crcOrig = XXH64(CNBuffer, size, 0); - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - outBuff.dst = compressedBuffer; - outBuff.size = compressedBufferSize; - outBuff.pos = 0; - inBuff.src = CNBuffer; - inBuff.size = size; - inBuff.pos = 0; - CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); - CHECK_Z(ZSTD_compress_generic(cctx, &outBuff, &inBuff, ZSTD_e_end)); - if (inBuff.pos != inBuff.size) goto _output_error; - { ZSTD_outBuffer decOut = {decodedBuffer, size, 0}; - ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0}; - CHECK_Z( ZSTD_decompress_generic(dctx, &decOut, &decIn) ); - if (decIn.pos != decIn.size) goto _output_error; - if (decOut.pos != size) goto _output_error; - { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0); - if (crcDec != crcOrig) goto _output_error; - } } - ZSTD_freeCCtx(cctx); - } - ZSTD_freeCDict(cdict); - } - ZSTD_freeDCtx(dctx); - } - DISPLAYLEVEL(3, "OK\n"); - - DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++); - { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled); - ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */}; - ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); - size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize); - if (ZSTD_isError(initError)) goto _output_error; - cSize = 0; - outBuff.dst = compressedBuffer; - outBuff.size = compressedBufferSize; - outBuff.pos = 0; - inBuff.src = CNBuffer; - inBuff.size = CNBufferSize; - inBuff.pos = 0; - CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - { size_t const r = ZSTD_endStream(zc, &outBuff); - if (r != 0) goto _output_error; } /* error, or some data not flushed */ - cSize = outBuff.pos; - ZSTD_freeCDict(cdict); - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); - } - - DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++); - { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); - if (did != 0) goto _output_error; - } - DISPLAYLEVEL(3, "OK (not detected) \n"); - - DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++); - { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize); - if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */ - DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); - } - - DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++); - CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) ); - outBuff.dst = compressedBuffer; - outBuff.size = compressedBufferSize; - outBuff.pos = 0; - inBuff.src = CNBuffer; - inBuff.size = CNBufferSize; - inBuff.pos = 0; - CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) ); - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - cSize = outBuff.pos; - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); - - DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++); - CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) ); - outBuff.dst = decodedBuffer; - outBuff.size = CNBufferSize; - outBuff.pos = 0; - inBuff.src = compressedBuffer; - inBuff.size = cSize; - inBuff.pos = 0; - CHECK_Z( ZSTD_decompress_generic(zd, &outBuff, &inBuff) ); - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */ - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++); - { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize); - if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */ - DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); - } - - DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compress_generic : ", testNb++); - outBuff.dst = compressedBuffer; - outBuff.size = compressedBufferSize; - outBuff.pos = 0; - inBuff.src = CNBuffer; - inBuff.size = CNBufferSize; - inBuff.pos = 0; - CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) ); - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - cSize = outBuff.pos; - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); - - DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++); - CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) ); - DISPLAYLEVEL(3, "OK \n"); - - /* Empty srcSize */ - DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++); - { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); - params.fParams.contentSizeFlag = 1; - CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) ); - } /* cstream advanced shall write content size = 0 */ - outBuff.dst = compressedBuffer; - outBuff.size = compressedBufferSize; - outBuff.pos = 0; - inBuff.src = CNBuffer; - inBuff.size = 0; - inBuff.pos = 0; - CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); - if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; - cSize = outBuff.pos; - if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++); - { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); - params.fParams.contentSizeFlag = 1; - CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) ); - } /* cstream advanced shall write content size = 0 */ - inBuff.src = CNBuffer; - inBuff.size = 0; - inBuff.pos = 0; - outBuff.dst = compressedBuffer; - outBuff.size = compressedBufferSize; - outBuff.pos = 0; - CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); - if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; - cSize = outBuff.pos; - if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; - - ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */ - outBuff.dst = compressedBuffer; - outBuff.size = compressedBufferSize; - outBuff.pos = 0; - inBuff.src = CNBuffer; - inBuff.size = 0; - inBuff.pos = 0; - CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); - if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; - cSize = outBuff.pos; - if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; - DISPLAYLEVEL(3, "OK \n"); - - /* Basic multithreading compression test */ - DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - { ZSTD_parameters const params = ZSTD_getParams(1, 0, 0); - CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) ); - } - outBuff.dst = compressedBuffer; - outBuff.size = compressedBufferSize; - outBuff.pos = 0; - inBuff.src = CNBuffer; - inBuff.size = CNBufferSize; - inBuff.pos = 0; - { size_t const compressResult = ZSTDMT_compressStream_generic(mtctx, &outBuff, &inBuff, ZSTD_e_end); - if (compressResult != 0) goto _output_error; /* compression must be completed in a single round */ - } - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - { size_t const compressedSize = ZSTD_findFrameCompressedSize(compressedBuffer, outBuff.pos); - if (compressedSize != outBuff.pos) goto _output_error; /* must be a full valid frame */ - } - DISPLAYLEVEL(3, "OK \n"); - - /* Complex multithreading + dictionary test */ - { U32 const nbWorkers = 2; - size_t const jobSize = 4 * 1 MB; - size_t const srcSize = jobSize * nbWorkers; /* we want each job to have predictable size */ - size_t const segLength = 2 KB; - size_t const offset = 600 KB; /* must be larger than window defined in cdict */ - size_t const start = jobSize + (offset-1); - const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start; - BYTE* const dst = (BYTE*)CNBuffer + start - offset; - DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (U32)srcSize); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, 3) ); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbWorkers, nbWorkers) ); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, jobSize) ); - assert(start > offset); - assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH); - memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */ - outBuff.dst = compressedBuffer; - outBuff.size = compressedBufferSize; - outBuff.pos = 0; - inBuff.src = CNBuffer; - inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH); - inBuff.pos = 0; - } - { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */ - ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); - DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog); - CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) ); - CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) ); - CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */ - ZSTD_freeCDict(cdict); - } - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - cSize = outBuff.pos; - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++); - { ZSTD_DStream* const dstream = ZSTD_createDCtx(); - ZSTD_frameHeader zfh; - ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize); - DISPLAYLEVEL(5, "frame windowsize = %u : ", (U32)zfh.windowSize); - outBuff.dst = decodedBuffer; - outBuff.size = CNBufferSize; - outBuff.pos = 0; - inBuff.src = compressedBuffer; - inBuff.pos = 0; - CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) ); - inBuff.size = 1; /* avoid shortcut to single-pass mode */ - CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) ); - inBuff.size = cSize; - CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) ); - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - ZSTD_freeDStream(dstream); - } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++); - { unsigned const kMaxWindowLog = 24; - unsigned value; - ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024); - ZSTD_CDict* cdict; - ZSTD_DDict* ddict; - SEQ_stream seq = SEQ_initStream(0x87654321); - SEQ_gen_type type; - XXH64_state_t xxh; - - XXH64_reset(&xxh, 0); - cParams.windowLog = kMaxWindowLog; - cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); - ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); - - if (!cdict || !ddict) goto _output_error; - - ZSTD_CCtx_reset(zc); - ZSTD_resetDStream(zd); - CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict)); - CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict)); - CHECK_Z(ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1U << kMaxWindowLog)); - /* Test all values < 300 */ - for (value = 0; value < 300; ++value) { - for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) { - CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value)); - } - } - /* Test values 2^8 to 2^17 */ - for (value = (1 << 8); value < (1 << 17); value <<= 1) { - for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) { - CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value)); - CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2))); - } - } - /* Test offset values up to the max window log */ - for (value = 8; value <= kMaxWindowLog; ++value) { - CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1)); - } - - CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end)); - CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match"); - - ZSTD_freeCDict(cdict); - ZSTD_freeDDict(ddict); - } - DISPLAYLEVEL(3, "OK \n"); - - /* Overlen overwriting window data bug */ - DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++); - { /* This test has a window size of 1024 bytes and consists of 3 blocks: - 1. 'a' repeated 517 times - 2. 'b' repeated 516 times - 3. a compressed block with no literals and 3 sequence commands: - litlength = 0, offset = 24, match length = 24 - litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3) - litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */ - - const char* testCase = - "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A" - "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00" - "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0"; - ZSTD_DStream* const zds = ZSTD_createDStream(); - if (zds==NULL) goto _output_error; - - CHECK_Z( ZSTD_initDStream(zds) ); - inBuff.src = testCase; - inBuff.size = 47; - inBuff.pos = 0; - outBuff.dst = decodedBuffer; - outBuff.size = CNBufferSize; - outBuff.pos = 0; - - while (inBuff.pos < inBuff.size) { - CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) ); - } - - ZSTD_freeDStream(zds); - } - DISPLAYLEVEL(3, "OK \n"); - -_end: - FUZ_freeDictionary(dictionary); - ZSTD_freeCStream(zc); - ZSTD_freeDStream(zd); - ZSTDMT_freeCCtx(mtctx); - free(CNBuffer); - free(compressedBuffer); - free(decodedBuffer); - return testResult; - -_output_error: - testResult = 1; - DISPLAY("Error detected in Unit tests ! \n"); - goto _end; -} - - -/* ====== Fuzzer tests ====== */ - -static size_t findDiff(const void* buf1, const void* buf2, size_t max) -{ - const BYTE* b1 = (const BYTE*)buf1; - const BYTE* b2 = (const BYTE*)buf2; - size_t u; - for (u=0; u No difference detected within %u bytes \n", (U32)max); - return u; - } - DISPLAY("Error at position %u / %u \n", (U32)u, (U32)max); - if (u>=3) - DISPLAY(" %02X %02X %02X ", - b1[u-3], b1[u-2], b1[u-1]); - DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n", - b1[u], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]); - if (u>=3) - DISPLAY(" %02X %02X %02X ", - b2[u-3], b2[u-2], b2[u-1]); - DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n", - b2[u], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]); - return u; -} - -static size_t FUZ_rLogLength(U32* seed, U32 logLength) -{ - size_t const lengthMask = ((size_t)1 << logLength) - 1; - return (lengthMask+1) + (FUZ_rand(seed) & lengthMask); -} - -static size_t FUZ_randomLength(U32* seed, U32 maxLog) -{ - U32 const logLength = FUZ_rand(seed) % maxLog; - return FUZ_rLogLength(seed, logLength); -} - -/* Return value in range minVal <= v <= maxVal */ -static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal) -{ - U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal; - return (U32)((FUZ_rand(seed) % mod) + minVal); -} - -static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests) -{ - U32 const maxSrcLog = bigTests ? 24 : 22; - static const U32 maxSampleLog = 19; - size_t const srcBufferSize = (size_t)1<= testNb) { - DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); - } else { - DISPLAYUPDATE(2, "\r%6u ", testNb); - } - - /* states full reset (deliberately not synchronized) */ - /* some issues can only happen when reusing states */ - if ((FUZ_rand(&lseed) & 0xFF) == 131) { - ZSTD_freeCStream(zc); - zc = ZSTD_createCStream(); - CHECK(zc==NULL, "ZSTD_createCStream : allocation error"); - resetAllowed=0; - } - if ((FUZ_rand(&lseed) & 0xFF) == 132) { - ZSTD_freeDStream(zd); - zd = ZSTD_createDStream(); - CHECK(zd==NULL, "ZSTD_createDStream : allocation error"); - CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */ - } - - /* srcBuffer selection [0-4] */ - { U32 buffNb = FUZ_rand(&lseed) & 0x7F; - if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ - else { - buffNb >>= 3; - if (buffNb & 7) { - const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ - buffNb = tnb[buffNb >> 3]; - } else { - const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ - buffNb = tnb[buffNb >> 3]; - } } - srcBuffer = cNoiseBuffer[buffNb]; - } - - /* compression init */ - if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ - && oldTestLog /* at least one test happened */ && resetAllowed) { - maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); - maxTestSize = MIN(maxTestSize, srcBufferSize-16); - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; - CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) ); - } - } else { - U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const cLevelCandidate = ( FUZ_rand(&lseed) % - (ZSTD_maxCLevel() - - (MAX(testLog, dictLog) / 3))) - + 1; - U32 const cLevel = MIN(cLevelCandidate, cLevelMax); - maxTestSize = FUZ_rLogLength(&lseed, testLog); - oldTestLog = testLog; - /* random dictionary selection */ - dictSize = ((FUZ_rand(&lseed)&7)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; - { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); - dict = srcBuffer + dictStart; - } - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; - ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); - params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; - params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; - params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; - CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); - } } - - /* multi-segments compression test */ - XXH64_reset(&xxhState, 0); - { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; - U32 n; - for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) { - /* compress random chunks into randomly sized dst buffers */ - { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize); - size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); - ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; - outBuff.size = outBuff.pos + dstBuffSize; - - CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); - - XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); - memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); - totalTestSize += inBuff.pos; - } - - /* random flush operation, to mess around */ - if ((FUZ_rand(&lseed) & 15) == 0) { - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); - outBuff.size = outBuff.pos + adjustedDstSize; - CHECK_Z( ZSTD_flushStream(zc, &outBuff) ); - } } - - /* final frame epilogue */ - { size_t remainingToFlush = (size_t)(-1); - while (remainingToFlush) { - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); - outBuff.size = outBuff.pos + adjustedDstSize; - remainingToFlush = ZSTD_endStream(zc, &outBuff); - CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush)); - } } - crcOrig = XXH64_digest(&xxhState); - cSize = outBuff.pos; - } - - /* multi - fragments decompression test */ - if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { - CHECK_Z ( ZSTD_resetDStream(zd) ); - } else { - CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) ); - } - { size_t decompressionResult = 1; - ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; - ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; - for (totalGenSize = 0 ; decompressionResult ; ) { - size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); - inBuff.size = inBuff.pos + readCSrcSize; - outBuff.size = outBuff.pos + dstBuffSize; - decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); - if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) { - DISPLAY("checksum error : \n"); - findDiff(copyBuffer, dstBuffer, totalTestSize); - } - CHECK( ZSTD_isError(decompressionResult), "decompression error : %s", - ZSTD_getErrorName(decompressionResult) ); - } - CHECK (decompressionResult != 0, "frame not fully decoded"); - CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", - (U32)outBuff.pos, (U32)totalTestSize); - CHECK (inBuff.pos != cSize, "compressed data should be fully read") - { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); - if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); - CHECK (crcDest!=crcOrig, "decompressed data corrupted"); - } } - - /*===== noisy/erroneous src decompression test =====*/ - - /* add some noise */ - { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; - U32 nn; for (nn=0; nn= testNb) { - DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); - } else { - DISPLAYUPDATE(2, "\r%6u ", testNb); - } - lseed = coreSeed ^ prime32; - - /* states full reset (deliberately not synchronized) */ - /* some issues can only happen when reusing states */ - if ((FUZ_rand(&lseed) & 0xFF) == 131) { - nbThreads = (FUZ_rand(&lseed) % nbThreadsMax) + 1; - DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads); - ZSTDMT_freeCCtx(zc); - zc = ZSTDMT_createCCtx(nbThreads); - CHECK(zc==NULL, "ZSTDMT_createCCtx allocation error") - } - if ((FUZ_rand(&lseed) & 0xFF) == 132) { - ZSTD_freeDStream(zd); - zd = ZSTD_createDStream(); - CHECK(zd==NULL, "ZSTDMT_createCCtx allocation error") - ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ - } - - /* srcBuffer selection [0-4] */ - { U32 buffNb = FUZ_rand(&lseed) & 0x7F; - if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ - else { - buffNb >>= 3; - if (buffNb & 7) { - const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ - buffNb = tnb[buffNb >> 3]; - } else { - const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ - buffNb = tnb[buffNb >> 3]; - } } - srcBuffer = cNoiseBuffer[buffNb]; - } - - /* compression init */ - { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - int const cLevelCandidate = ( FUZ_rand(&lseed) - % (ZSTD_maxCLevel() - (MAX(testLog, dictLog) / 2)) ) - + 1; - int const cLevelThreadAdjusted = cLevelCandidate - (nbThreads * 2) + 2; /* reduce cLevel when multiple threads to reduce memory consumption */ - int const cLevelMin = MAX(cLevelThreadAdjusted, 1); /* no negative cLevel yet */ - int const cLevel = MIN(cLevelMin, cLevelMax); - maxTestSize = FUZ_rLogLength(&lseed, testLog); - - if (FUZ_rand(&lseed)&1) { /* simple init */ - int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1; - DISPLAYLEVEL(5, "Init with compression level = %i \n", compressionLevel); - CHECK_Z( ZSTDMT_initCStream(zc, compressionLevel) ); - } else { /* advanced init */ - /* random dictionary selection */ - dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; - { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); - dict = srcBuffer + dictStart; - } - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; - ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); - DISPLAYLEVEL(5, "Init with windowLog = %u, pledgedSrcSize = %u, dictSize = %u \n", - params.cParams.windowLog, (U32)pledgedSrcSize, (U32)dictSize); - params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; - params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; - params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; - DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag); - CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) ); - CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custome job size */ - CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); - } } } - - /* multi-segments compression test */ - XXH64_reset(&xxhState, 0); - { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; - U32 n; - for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) { - /* compress random chunks into randomly sized dst buffers */ - { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize); - size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); - ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; - outBuff.size = outBuff.pos + dstBuffSize; - - DISPLAYLEVEL(6, "Sending %u bytes to compress \n", (U32)srcSize); - CHECK_Z( ZSTDMT_compressStream(zc, &outBuff, &inBuff) ); - DISPLAYLEVEL(6, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos); - - XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); - memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); - totalTestSize += inBuff.pos; - } - - /* random flush operation, to mess around */ - if ((FUZ_rand(&lseed) & 15) == 0) { - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); - size_t const previousPos = outBuff.pos; - outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (U32)adjustedDstSize); - CHECK_Z( ZSTDMT_flushStream(zc, &outBuff) ); - assert(outBuff.pos >= previousPos); - DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_flushStream \n", (U32)(outBuff.pos-previousPos)); - } } - - /* final frame epilogue */ - { size_t remainingToFlush = (size_t)(-1); - while (remainingToFlush) { - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); - size_t const previousPos = outBuff.pos; - outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(5, "Ending into dst buffer of size %u \n", (U32)adjustedDstSize); - remainingToFlush = ZSTDMT_endStream(zc, &outBuff); - CHECK (ZSTD_isError(remainingToFlush), "ZSTDMT_endStream error : %s", ZSTD_getErrorName(remainingToFlush)); - assert(outBuff.pos >= previousPos); - DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_endStream \n", (U32)(outBuff.pos-previousPos)); - DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (U32)remainingToFlush); - } } - crcOrig = XXH64_digest(&xxhState); - cSize = outBuff.pos; - DISPLAYLEVEL(5, "Frame completed : %u bytes compressed into %u bytes \n", - (U32)totalTestSize, (U32)cSize); - } - - /* multi - fragments decompression test */ - assert(totalTestSize < dstBufferSize); - memset(dstBuffer, 170, totalTestSize); /* init dest area */ - if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { - CHECK_Z( ZSTD_resetDStream(zd) ); - } else { - CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) ); - } - { size_t decompressionResult = 1; - ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; - ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; - for (totalGenSize = 0 ; decompressionResult ; ) { - size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); - inBuff.size = inBuff.pos + readCSrcSize; - outBuff.size = outBuff.pos + dstBuffSize; - DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes into outBuff %u bytes \n", - (U32)readCSrcSize, (U32)dstBuffSize); - decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); - if (ZSTD_isError(decompressionResult)) { - DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult)); - findDiff(copyBuffer, dstBuffer, totalTestSize); - } - CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); - DISPLAYLEVEL(6, "total ingested (inBuff.pos) = %u and produced (outBuff.pos) = %u \n", - (U32)inBuff.pos, (U32)outBuff.pos); - } - CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize); - CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize); - { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); - if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); - CHECK (crcDest!=crcOrig, "decompressed data corrupted"); - } } - - /*===== noisy/erroneous src decompression test =====*/ - - /* add some noise */ - { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; - U32 nn; for (nn=0; nn= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } - else { DISPLAYUPDATE(2, "\r%6u ", testNb); } - FUZ_rand(&coreSeed); - lseed = coreSeed ^ prime32; - DISPLAYLEVEL(5, " *** Test %u *** \n", testNb); - - /* states full reset (deliberately not synchronized) */ - /* some issues can only happen when reusing states */ - if ((FUZ_rand(&lseed) & 0xFF) == 131) { - DISPLAYLEVEL(5, "Creating new context \n"); - ZSTD_freeCCtx(zc); - zc = ZSTD_createCCtx(); - CHECK(zc==NULL, "ZSTD_createCCtx allocation error"); - resetAllowed=0; - } - if ((FUZ_rand(&lseed) & 0xFF) == 132) { - ZSTD_freeDStream(zd); - zd = ZSTD_createDStream(); - CHECK(zd==NULL, "ZSTD_createDStream allocation error"); - ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ - } - - /* srcBuffer selection [0-4] */ - { U32 buffNb = FUZ_rand(&lseed) & 0x7F; - if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ - else { - buffNb >>= 3; - if (buffNb & 7) { - const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ - buffNb = tnb[buffNb >> 3]; - } else { - const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ - buffNb = tnb[buffNb >> 3]; - } } - srcBuffer = cNoiseBuffer[buffNb]; - } - - /* compression init */ - CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/ - if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ - && oldTestLog /* at least one test happened */ && resetAllowed) { - maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); - if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1; - { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1; - CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_compressionLevel, compressionLevel, useOpaqueAPI) ); - } - } else { - U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const cLevelCandidate = (FUZ_rand(&lseed) % - (ZSTD_maxCLevel() - - (MAX(testLog, dictLog) / 2))) + - 1; - U32 const cLevel = MIN(cLevelCandidate, cLevelMax); - DISPLAYLEVEL(5, "t%u: base cLevel : %u \n", testNb, cLevel); - maxTestSize = FUZ_rLogLength(&lseed, testLog); - DISPLAYLEVEL(5, "t%u: maxTestSize : %u \n", testNb, (U32)maxTestSize); - oldTestLog = testLog; - /* random dictionary selection */ - dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; - { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); - dict = srcBuffer + dictStart; - if (!dictSize) dict=NULL; - } - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; - ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize); - static const U32 windowLogMax = 24; - - /* mess with compression parameters */ - cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1; - cParams.windowLog = MIN(windowLogMax, cParams.windowLog); - cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1; - cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1; - cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1; - cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1; - cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128))); - cParams = ZSTD_adjustCParams(cParams, 0, 0); - - if (FUZ_rand(&lseed) & 1) { - DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) ); - assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */ - windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5; - } - if (FUZ_rand(&lseed) & 1) { - DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) ); - } - if (FUZ_rand(&lseed) & 1) { - DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) ); - } - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) ); - - /* mess with long distance matching parameters */ - if (bigTests) { - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) ); - } - - /* mess with frame parameters */ - if (FUZ_rand(&lseed) & 1) { - U32 const checksumFlag = FUZ_rand(&lseed) & 1; - DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, checksumFlag, useOpaqueAPI) ); - } - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); - if (FUZ_rand(&lseed) & 1) { - DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize); - CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); - } - - /* multi-threading parameters */ - { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; - U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1; - U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax); - DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbWorkers, nbThreads, useOpaqueAPI) ); - if (nbThreads > 1) { - U32 const jobLog = FUZ_rand(&lseed) % (testLog+1); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) ); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), useOpaqueAPI) ); - } - } - - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); - - /* Apply parameters */ - if (useOpaqueAPI) { - DISPLAYLEVEL(6," t%u: applying CCtxParams \n", testNb); - CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) ); - } - - if (FUZ_rand(&lseed) & 1) { - if (FUZ_rand(&lseed) & 1) { - CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); - } else { - CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) ); - } - if (dict && dictSize) { - /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */ - if (useOpaqueAPI) { - size_t const setError = ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams); - CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParametersUsingCCtxParams should have failed"); - } else { - size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1); - CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed"); - } - } - } else { - CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); - } - } } - - /* multi-segments compression test */ - XXH64_reset(&xxhState, 0); - { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; - for (cSize=0, totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) { - /* compress random chunks into randomly sized dst buffers */ - size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize); - size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); - size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); - ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush; - ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; - outBuff.size = outBuff.pos + dstBuffSize; - - CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) ); - DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n", - testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos), (U32)flush, (U32)outBuff.pos); - - XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); - memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); - totalTestSize += inBuff.pos; - } - - /* final frame epilogue */ - { size_t remainingToFlush = 1; - while (remainingToFlush) { - ZSTD_inBuffer inBuff = { NULL, 0, 0 }; - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); - size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); - outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (U32)adjustedDstSize); - remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end); - DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (U32)outBuff.pos); - CHECK( ZSTD_isError(remainingToFlush), - "ZSTD_compress_generic w/ ZSTD_e_end error : %s", - ZSTD_getErrorName(remainingToFlush) ); - } } - crcOrig = XXH64_digest(&xxhState); - cSize = outBuff.pos; - DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize); - } - - /* multi - fragments decompression test */ - if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { - DISPLAYLEVEL(5, "resetting DCtx (dict:%08X) \n", (U32)(size_t)dict); - CHECK_Z( ZSTD_resetDStream(zd) ); - } else { - DISPLAYLEVEL(5, "using dict of size %u \n", (U32)dictSize); - CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) ); - } - { size_t decompressionResult = 1; - ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; - ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; - for (totalGenSize = 0 ; decompressionResult ; ) { - size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); - inBuff.size = inBuff.pos + readCSrcSize; - outBuff.size = outBuff.pos + dstBuffSize; - DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n", - (U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize); - decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); - DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n", - (U32)inBuff.pos, (U32)outBuff.pos); - if (ZSTD_isError(decompressionResult)) { - DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult)); - findDiff(copyBuffer, dstBuffer, totalTestSize); - } - CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); - CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (U32)inBuff.pos, (U32)cSize); - } - CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize); - CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize); - { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); - if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); - CHECK (crcDest!=crcOrig, "decompressed data corrupted"); - } } - - /*===== noisy/erroneous src decompression test =====*/ - - /* add some noise */ - { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; - U32 nn; for (nn=0; nn='0') && (*argument<='9')) { - nbTests *= 10; - nbTests += *argument - '0'; - argument++; - } - break; - - case 'T': /* limit tests by time */ - argument++; - nbTests=0; g_clockTime=0; - while ((*argument>='0') && (*argument<='9')) { - g_clockTime *= 10; - g_clockTime += *argument - '0'; - argument++; - } - if (*argument=='m') { /* -T1m == -T60 */ - g_clockTime *=60, argument++; - if (*argument=='n') argument++; /* -T1mn == -T60 */ - } else if (*argument=='s') argument++; /* -T10s == -T10 */ - g_clockTime *= SEC_TO_MICRO; - break; - - case 's': /* manually select seed */ - argument++; - seedset=1; - seed=0; - while ((*argument>='0') && (*argument<='9')) { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - - case 't': /* select starting test number */ - argument++; - testNb=0; - while ((*argument>='0') && (*argument<='9')) { - testNb *= 10; - testNb += *argument - '0'; - argument++; - } - break; - - case 'P': /* compressibility % */ - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - - default: - return FUZ_usage(programName); - } - } } } /* for(argNb=1; argNb