[miniz] Bump miniz version to 3.0.2 and add patch for zip64 (pytorch#140041)

Summary:
Bump miniz version from 2.1.0 to 3.0.2 and apply these patches:

* pytorch#79636 patches internal BUCK and bazel build
* pytorch#138959 adds `bool compute_crc32` argument
* miniz PR: richgel999/miniz#324 to support
  zip64

Anyone bumping miniz version again, please apply these patches as well.

Test Plan:
Rely on unit test

Imported from OSS

Differential Revision: D65586230

Pull Request resolved: pytorch#140041
Approved by: https://github.com/mikaylagawarecki

Author: larryliu0820

BUILD.bazel

@@ -552,7 +552,7 @@ cc_library(
         ":caffe2_headers",
         ":caffe2_perfkernels_avx",
         ":caffe2_perfkernels_avx2",
-        "//third_party/miniz-2.1.0:miniz",
+        "//third_party/miniz-3.0.2:miniz",
         "@com_google_protobuf//:protobuf",
         "@eigen",
         "@fbgemm//:fbgemm_src_headers",

buckbuild.bzl

@@ -2040,7 +2040,7 @@ def define_buck_targets(
                 ("", "torch/csrc/utils/*.h"),
                 ("", "aten/src/ATen/quantized/*.h"),
             ] + ([
-                ("third_party/miniz-2.1.0", "*.h"),
+                ("third_party/miniz-3.0.2", "*.h"),
             ] if NOT_OSS else []),
             exclude = [
                 "torch/csrc/jit/serialization/mobile_bytecode_generated.h",

build.bzl

@@ -47,7 +47,7 @@ def define_targets(rules):
         deps = [
             ":caffe2_headers",
             "//c10",
-            "//third_party/miniz-2.1.0:miniz",
+            "//third_party/miniz-3.0.2:miniz",
             "@com_github_glog//:glog",
         ],
     )

caffe2/CMakeLists.txt

@@ -1213,7 +1213,7 @@ target_include_directories(torch_cpu PRIVATE
   ${TORCH_SRC_DIR}/csrc)
 
 target_include_directories(torch_cpu PRIVATE
-  ${TORCH_ROOT}/third_party/miniz-2.1.0)
+  ${TORCH_ROOT}/third_party/miniz-3.0.2)
 
 target_include_directories(torch_cpu PRIVATE
   ${TORCH_ROOT}/third_party/kineto/libkineto/include)

caffe2/serialize/CMakeLists.txt

@@ -2,13 +2,13 @@ file(GLOB tmp *_test.cc)
 
 set(Caffe2_CPU_TEST_SRCS ${Caffe2_CPU_TEST_SRCS} ${tmp})
 list(APPEND Caffe2_CPU_SRCS
-  ${PROJECT_SOURCE_DIR}/third_party/miniz-2.1.0/miniz.c
+  ${PROJECT_SOURCE_DIR}/third_party/miniz-3.0.2/miniz.c
   ${CMAKE_CURRENT_SOURCE_DIR}/inline_container.cc
   ${CMAKE_CURRENT_SOURCE_DIR}/istream_adapter.cc
   ${CMAKE_CURRENT_SOURCE_DIR}/file_adapter.cc
   ${CMAKE_CURRENT_SOURCE_DIR}/crc.cc
   ${CMAKE_CURRENT_SOURCE_DIR}/read_adapter_interface.cc)
-list(APPEND Caffe2_CPU_INCLUDE ${PROJECT_SOURCE_DIR}/third_party/miniz-2.1.0)
+list(APPEND Caffe2_CPU_INCLUDE ${PROJECT_SOURCE_DIR}/third_party/miniz-3.0.2)
 
 set(Caffe2_CPU_TEST_SRCS ${Caffe2_CPU_TEST_SRCS} PARENT_SCOPE)
 set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS} PARENT_SCOPE)

third_party/BUCK.oss

@@ -191,9 +191,11 @@ cxx_library(
 
 cxx_library(
     name = "miniz",
-    srcs = ["miniz-2.1.0/miniz.c"],
+    srcs = [
+        "miniz-3.0.2/miniz.c",
+    ],
     header_namespace = "",
-    exported_headers = {"miniz.h": "miniz-2.1.0/miniz.h"},
+    exported_headers = {"miniz.h": "miniz-3.0.2/miniz.h"},
     exported_preprocessor_flags = [
         "-DMINIZ_DISABLE_ZIP_READER_CRC32_CHECKS",
     ],

third_party/miniz-2.1.0/BUILD.bazel third_party/miniz-3.0.2/BUILD.bazel

@@ -1,5 +1,68 @@
 ## Changelog
 
+### 3.0.2
+
+ - Fix buffer overrun in mz_utf8z_to_widechar on Windows
+
+### 3.0.1
+
+ - Fix compilation error with MINIZ_USE_UNALIGNED_LOADS_AND_STORES=1
+
+### 3.0.0
+
+ - Reduce memory usage for inflate. This changes `struct tinfl_decompressor_tag` and therefore requires a major version bump (breaks ABI compatibility)
+ - Add padding to structures so it continues to work if features differ. This also changes some structures
+ - Use _ftelli64, _fseeki64 and stat with MinGW32 and OpenWatcom
+ - Fix varios warnings with OpenWatcom compiler
+ - Avoid using unaligned memory access in UBSan builds
+ - Set MINIZ_LITTLE_ENDIAN only if not set
+ - Add MINIZ_NO_DEFLATE_APIS and MINIZ_NO_INFLATE_APIS
+ - Fix use of uninitialized memory in tinfl_decompress_mem_to_callback()
+ - Use wfopen on windows
+ - Use _wstat64 instead _stat64 on windows
+ - Use level_and_flags after MZ_DEFAULT_COMPRESSION has been handled
+ - Improve endianess detection
+ - Don't use unaligned stores and loads per default
+ - Fix function declaration if MINIZ_NO_STDIO is used
+ - Fix MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_UTF8 not being set
+ - Remove total files check (its 32-bit uint)
+ - tinfl_decompress: avoid NULL ptr arithmetic UB
+ - miniz_zip: fix mz_zip_reader_extract_to_heap to read correct sizes
+ - Eliminate 64-bit operations on 32-bit machines
+ - Disable treating warnings as error with MSVC
+ - Disable building shared lib via CMake by default
+ - Fixed alignment problems on MacOS
+ - Fixed get error string for MZ_ZIP_TOTAL_ERRORS
+ - Write correct FLEVEL 2-bit value in zlib header
+ - miniz.pc.in: fix include path not containing the "miniz" suffix
+ - Fix compatibility with FreeBSD
+ - pkg-config tweaks
+ - Fix integer overflow in header corruption check
+ - Fix some warnings
+ - tdefl_compress_normal: Avoid NULL ptr arithmetic UB
+ - replace use of stdint.h types with mz_ variants
+
+
+### 2.2.0
+
+ - Fix examples with amalgamation
+ - Modified cmake script to support shared library mode and find_package
+ - Fix for misleading doc comment on `mz_zip_reader_init_cfile` function
+ - Add include location tolerance and stop forcing `_GNU_SOURCE`
+ - Fix: mz_zip_reader_locate_file_v2 returns an mz_bool
+ - Fix large file system checks
+ - Add #elif to enable an external mz_crc32() to be linked in
+ - Write with dynamic size (size of file/data to be added not known before adding)
+ - Added uncompress2 for zlib compatibility
+ - Add support for building as a Meson subproject
+ - Added OSSFuzz support; Integrate with CIFuzz
+ - Add pkg-config file
+ - Fixed use-of-uninitialized value msan error when copying dist bytes with no output bytes written.
+ - mz_zip_validate_file(): fix memory leak on errors
+ - Fixed MSAN use-of-uninitialized in tinfl_decompress when invalid dist is decoded. In this instance dist was 31 which s_dist_base translates as 0
+ - Add flag to set (compressed) size in local file header
+ - avoid use of uninitialized value in tdefl_record_literal
+
 ### 2.1.0
 
  - More instances of memcpy instead of cast and use memcpy per default
@@ -82,7 +145,7 @@ The inflator now has a new failure status TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRE
 - The inflator coroutine func. is subtle and complex so I'm being cautious about this release. I would greatly appreciate any help with testing or any feedback.
          I feel good about these changes, and they've been through several hours of automated testing, but they will probably not fix anything for the majority of prev. users so I'm
          going to mark this release as beta for a few weeks and continue testing it at work/home on various things.
-- The inflator in raw (non-zlib) mode is now usable on gzip or similiar data streams that have a bunch of bytes following the raw deflate data (problem discovered by rustyzip author williamw520).
+- The inflator in raw (non-zlib) mode is now usable on gzip or similar data streams that have a bunch of bytes following the raw deflate data (problem discovered by rustyzip author williamw520).
          This version should *never* read beyond the last byte of the raw deflate data independent of how many bytes you pass into the input buffer. This issue was caused by the various Huffman bitbuffer lookahead optimizations, and
          would not be an issue if the caller knew and enforced the precise size of the raw compressed data *or* if the compressed data was in zlib format (i.e. always followed by the byte aligned zlib adler32).
          So in other words, you can now call the inflator on deflate streams that are followed by arbitrary amounts of data and it's guaranteed that decompression will stop exactly on the last byte.
@@ -103,7 +166,7 @@ Merged over a few very minor bug fixes that I fixed in the zip64 branch. This is
 Interim bugfix release while I work on the next major release with zip64 and streaming compression/decompression support. Fixed the MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY bug (thanks [email protected]), which could cause the locate files func to not find files when this flag was specified. Also fixed a bug in mz_zip_reader_extract_to_mem_no_alloc() with user provided read buffers (thanks kymoon). I also merged lots of compiler fixes from various github repo branches and Google Code issue reports. I finally added cmake support (only tested under for Linux so far), compiled and tested with clang v3.3 and gcc 4.6 (under Linux), added defl_write_image_to_png_file_in_memory_ex() (supports Y flipping for OpenGL use, real-time compression), added a new PNG example (example6.c - Mandelbrot), and I added 64-bit file I/O support (stat64(), etc.) for glibc.
 
 - Critical fix for the MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY bug (thanks [email protected]) which could cause locate files to not find files. This bug
-        would only have occured in earlier versions if you explicitly used this flag, OR if you used mz_zip_extract_archive_file_to_heap() or mz_zip_add_mem_to_archive_file_in_place()
+        would only have occurred in earlier versions if you explicitly used this flag, OR if you used mz_zip_extract_archive_file_to_heap() or mz_zip_add_mem_to_archive_file_in_place()
         (which used this flag). If you can't switch to v1.15 but want to fix this bug, just remove the uses of this flag from both helper funcs (and of course don't use the flag).
 - Bugfix in mz_zip_reader_extract_to_mem_no_alloc() from kymoon when pUser_read_buf is not NULL and compressed size is > uncompressed size
 - Fixing mz_zip_reader_extract_*() funcs so they don't try to extract compressed data from directory entries, to account for weird zipfiles which contain zero-size compressed data on dir entries.
@@ -172,5 +235,3 @@ Added statement from unlicense.org
 ### v1.09 - 5/15/11
 
 Initial stable release.
-
-

third_party/miniz-2.1.0/ChangeLog.md third_party/miniz-3.0.2/ChangeLog.md

@@ -13,7 +13,7 @@
 #endif
 
 #include <stdio.h>
-#include "miniz_zip.h"
+#include "miniz.h"
 
 typedef unsigned char uint8;
 typedef unsigned short uint16;

third_party/miniz-2.1.0/LICENSE third_party/miniz-3.0.2/LICENSE

@@ -100,7 +100,7 @@ int main(int argc, char *argv[])
   file_loc = ftell(pInfile);
   fseek(pInfile, 0, SEEK_SET);
 
-  if ((file_loc < 0) || (file_loc > INT_MAX))
+  if ((file_loc < 0) || ((mz_uint64)file_loc > INT_MAX))
   {
      // This is not a limitation of miniz or tinfl, but this example.
      printf("File is too large to be processed by this example.\n");

third_party/miniz-2.1.0/examples/example1.c third_party/miniz-3.0.2/examples/example1.c

@@ -1,6 +1,6 @@
 // example4.c - Uses tinfl.c to decompress a zlib stream in memory to an output file
 // Public domain, May 15 2011, Rich Geldreich, [email protected]. See "unlicense" statement at the end of tinfl.c.
-#include "miniz_tinfl.h"
+#include "miniz.h"
 #include <stdio.h>
 #include <limits.h>
 
@@ -47,7 +47,7 @@ int main(int argc, char *argv[])
   file_loc = ftell(pInfile);
   fseek(pInfile, 0, SEEK_SET);
 
-  if ((file_loc < 0) || (file_loc > INT_MAX))
+  if ((file_loc < 0) || ((mz_uint64)file_loc > INT_MAX))
   {
      // This is not a limitation of miniz or tinfl, but this example.
      printf("File is too large to be processed by this example.\n");

third_party/miniz-2.1.0/examples/example2.c third_party/miniz-3.0.2/examples/example2.c

@@ -132,7 +132,7 @@ int main(int argc, char *argv[])
    file_loc = ftell(pInfile);
    fseek(pInfile, 0, SEEK_SET);
 
-   if ((file_loc < 0) || (file_loc > INT_MAX))
+   if ((file_loc < 0) || ((mz_uint64)file_loc > INT_MAX))
    {
       // This is not a limitation of miniz or tinfl, but this example.
       printf("File is too large to be processed by this example.\n");

third_party/miniz-2.1.0/examples/example3.c third_party/miniz-3.0.2/examples/example3.c

@@ -34,27 +34,29 @@ static void hsv_to_rgb(int hue, int min, int max, rgb_t *p)
   if (!saturation) {
     p->r = p->g = p->b = 255 * (max - hue) / (max - min);
     return;
-  }
-  double h = fmod(color_rotate + 1e-4 + 4.0 * (hue - min) / (max - min), 6);
-  double c = 255.0f * saturation;
-  double X = c * (1 - fabs(fmod(h, 2) - 1));
-
-  p->r = p->g = p->b = 0;
-
-  switch((int)h) {
-  case 0: p->r = c; p->g = X; return;
-  case 1:	p->r = X; p->g = c; return;
-  case 2: p->g = c; p->b = X; return;
-  case 3: p->g = X; p->b = c; return;
-  case 4: p->r = X; p->b = c; return;
-  default:p->r = c; p->b = X;
+  } else {
+    const double h_dbl = fmod(color_rotate + 1e-4 + 4.0 * (hue - min) / (max - min), 6);
+    const double c_dbl = 255 * saturation;
+    const double X_dbl = c_dbl * (1 - fabs(fmod(h_dbl, 2) - 1));
+    const int h = (int)h_dbl;
+    const int c = (int)c_dbl;
+    const int X = (int)X_dbl;
+
+    p->r = p->g = p->b = 0;
+
+    switch(h) {
+    case 0: p->r = c; p->g = X; return;
+    case 1:	p->r = X; p->g = c; return;
+    case 2: p->g = c; p->b = X; return;
+    case 3: p->g = X; p->b = c; return;
+    case 4: p->r = X; p->b = c; return;
+    default:p->r = c; p->b = X;
+    }
   }
 }
 
 int main(int argc, char *argv[])
 {
-  (void)argc, (void)argv;
-
   // Image resolution
   const int iXmax = 4096;
   const int iYmax = 4096;
@@ -89,6 +91,8 @@ int main(int argc, char *argv[])
 
   int MinIter = 9999, MaxIter = 0;
 
+  (void)argc, (void)argv;
+
   for(iY = 0; iY < iYmax; iY++)
   {
     Cy = CyMin + iY * PixelHeight;
@@ -134,7 +138,7 @@ int main(int argc, char *argv[])
 
       uint Iterations = color[0] | (color[1] << 8U);
 
-      hsv_to_rgb(Iterations, MinIter, MaxIter, (rgb_t *)color);
+      hsv_to_rgb((int)Iterations, MinIter, MaxIter, (rgb_t *)color);
     }
   }