ckissane updated this revision to Diff 444400.
ckissane added a comment.
tidy some code
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D128465/new/
https://reviews.llvm.org/D128465
Files:
llvm/CMakeLists.txt
llvm/cmake/config-ix.cmake
llvm/cmake/modules/FindZSTD.cmake
llvm/cmake/modules/LLVMConfig.cmake.in
llvm/include/llvm/Config/llvm-config.h.cmake
llvm/include/llvm/Support/Compression.h
llvm/lib/Support/CMakeLists.txt
llvm/lib/Support/Compression.cpp
llvm/test/lit.site.cfg.py.in
llvm/unittests/Support/CompressionTest.cpp
utils/bazel/llvm_configs/llvm-config.h.cmake
Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===================================================================
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
/* Define if zlib compression is available */
#cmakedefine01 LLVM_ENABLE_ZLIB
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
/* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
#cmakedefine LLVM_HAVE_TF_API
Index: llvm/unittests/Support/CompressionTest.cpp
===================================================================
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -23,11 +23,9 @@
namespace {
#if LLVM_ENABLE_ZLIB
-
-void TestZlibCompression(StringRef Input, int Level) {
+static void testZlibCompression(StringRef Input, int Level) {
SmallString<32> Compressed;
SmallString<32> Uncompressed;
-
zlib::compress(Input, Compressed, Level);
// Check that uncompressed buffer is the same as original.
@@ -43,26 +41,63 @@
}
TEST(CompressionTest, Zlib) {
- TestZlibCompression("", zlib::DefaultCompression);
+ testZlibCompression("", zlib::DefaultCompression);
- TestZlibCompression("hello, world!", zlib::NoCompression);
- TestZlibCompression("hello, world!", zlib::BestSizeCompression);
- TestZlibCompression("hello, world!", zlib::BestSpeedCompression);
- TestZlibCompression("hello, world!", zlib::DefaultCompression);
+ testZlibCompression("hello, world!", zlib::NoCompression);
+ testZlibCompression("hello, world!", zlib::BestSizeCompression);
+ testZlibCompression("hello, world!", zlib::BestSpeedCompression);
+ testZlibCompression("hello, world!", zlib::DefaultCompression);
const size_t kSize = 1024;
char BinaryData[kSize];
- for (size_t i = 0; i < kSize; ++i) {
+ for (size_t i = 0; i < kSize; ++i)
BinaryData[i] = i & 255;
- }
StringRef BinaryDataStr(BinaryData, kSize);
- TestZlibCompression(BinaryDataStr, zlib::NoCompression);
- TestZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
- TestZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
- TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+ testZlibCompression(BinaryDataStr, zlib::NoCompression);
+ testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
+ testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
+ testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+}
+#endif
+
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+ SmallString<32> Compressed;
+ SmallString<32> Uncompressed;
+ zstd::compress(Input, Compressed, Level);
+
+ // Check that uncompressed buffer is the same as original.
+ Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+ consumeError(std::move(E));
+
+ EXPECT_EQ(Input, Uncompressed);
+ if (Input.size() > 0) {
+ // Uncompression fails if expected length is too short.
+ E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+ EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+ }
}
+TEST(CompressionTest, Zstd) {
+ testZstdCompression("", zstd::DefaultCompression);
+
+ testZstdCompression("hello, world!", zstd::NoCompression);
+ testZstdCompression("hello, world!", zstd::BestSizeCompression);
+ testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+ testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+ const size_t kSize = 1024;
+ char BinaryData[kSize];
+ for (size_t i = 0; i < kSize; ++i)
+ BinaryData[i] = i & 255;
+ StringRef BinaryDataStr(BinaryData, kSize);
+
+ testZstdCompression(BinaryDataStr, zstd::NoCompression);
+ testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+ testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+ testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
#endif
}
Index: llvm/test/lit.site.cfg.py.in
===================================================================
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
config.have_libxar = @LLVM_HAVE_LIBXAR@
config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===================================================================
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
#if LLVM_ENABLE_ZLIB
#include <zlib.h>
#endif
+#if LLVM_ENABLE_ZSTD
+#include <zstd.h>
+#endif
using namespace llvm;
using namespace llvm::compression;
@@ -57,7 +60,8 @@
// Tell MemorySanitizer that zlib output buffer is fully initialized.
// This avoids a false report when running LLVM with uninstrumented ZLib.
__msan_unpoison(CompressedBuffer.data(), CompressedSize);
- CompressedBuffer.truncate(CompressedSize);
+ if (CompressedSize < CompressedBuffer.size())
+ CompressedBuffer.truncate(CompressedSize);
}
Error zlib::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
@@ -79,7 +83,8 @@
UncompressedBuffer.resize_for_overwrite(UncompressedSize);
Error E = zlib::uncompress(InputBuffer, UncompressedBuffer.data(),
UncompressedSize);
- UncompressedBuffer.truncate(UncompressedSize);
+ if (UncompressedSize < UncompressedBuffer.size())
+ UncompressedBuffer.truncate(UncompressedSize);
return E;
}
@@ -99,3 +104,65 @@
llvm_unreachable("zlib::uncompress is unavailable");
}
#endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+ SmallVectorImpl<char> &CompressedBuffer, int Level) {
+ unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+ CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+ unsigned long CompressedSize = ::ZSTD_compress(
+ (char *)CompressedBuffer.data(), CompressedBufferSize,
+ (const char *)InputBuffer.data(), InputBuffer.size(), Level);
+ if (ZSTD_isError(CompressedSize))
+ report_bad_alloc_error("Allocation failed");
+ // Tell MemorySanitizer that zstd output buffer is fully initialized.
+ // This avoids a false report when running LLVM with uninstrumented ZLib.
+ __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+ if (CompressedSize < CompressedBuffer.size())
+ CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+ size_t &UncompressedSize) {
+ const size_t Res =
+ ::ZSTD_decompress((char *)UncompressedBuffer, UncompressedSize,
+ (const char *)InputBuffer.data(), InputBuffer.size());
+ UncompressedSize = Res;
+ // Tell MemorySanitizer that zstd output buffer is fully initialized.
+ // This avoids a false report when running LLVM with uninstrumented ZLib.
+ __msan_unpoison(UncompressedBuffer, UncompressedSize);
+ return ZSTD_isError(Res) ? make_error<StringError>(ZSTD_getErrorName(Res),
+ inconvertibleErrorCode())
+ : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+ SmallVectorImpl<char> &UncompressedBuffer,
+ size_t UncompressedSize) {
+ UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+ Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(),
+ UncompressedSize);
+ if (UncompressedSize < UncompressedBuffer.size())
+ UncompressedBuffer.truncate(UncompressedSize);
+ return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+ SmallVectorImpl<char> &CompressedBuffer, int Level) {
+ llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+ size_t &UncompressedSize) {
+ llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer,
+ SmallVectorImpl<char> &UncompressedBuffer,
+ size_t UncompressedSize) {
+ llvm_unreachable("zstd::uncompress is unavailable");
+}
+#endif
Index: llvm/lib/Support/CMakeLists.txt
===================================================================
--- llvm/lib/Support/CMakeLists.txt
+++ llvm/lib/Support/CMakeLists.txt
@@ -25,6 +25,10 @@
set(imported_libs ZLIB::ZLIB)
endif()
+if(LLVM_ENABLE_ZSTD)
+ list(APPEND imported_libs zstd)
+endif()
+
if( MSVC OR MINGW )
# libuuid required for FOLDERID_Profile usage in lib/Support/Windows/Path.inc.
# advapi32 required for CryptAcquireContextW in lib/Support/Windows/Path.inc.
Index: llvm/include/llvm/Support/Compression.h
===================================================================
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -42,6 +42,27 @@
} // End of namespace zlib
+namespace zstd {
+
+constexpr int NoCompression = -5;
+constexpr int BestSpeedCompression = 1;
+constexpr int DefaultCompression = 5;
+constexpr int BestSizeCompression = 12;
+
+bool isAvailable();
+
+void compress(StringRef InputBuffer, SmallVectorImpl<char> &CompressedBuffer,
+ int Level = DefaultCompression);
+
+Error uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+ size_t &UncompressedSize);
+
+Error uncompress(StringRef InputBuffer,
+ SmallVectorImpl<char> &UncompressedBuffer,
+ size_t UncompressedSize);
+
+} // End of namespace zstd
+
} // End of namespace compression
} // End of namespace llvm
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===================================================================
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -95,6 +95,9 @@
/* Define if zlib compression is available */
#cmakedefine01 LLVM_ENABLE_ZLIB
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
/* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
#cmakedefine LLVM_HAVE_TF_API
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===================================================================
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
find_package(ZLIB)
endif()
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+ set(ZSTD_ROOT @ZSTD_ROOT@)
+ find_package(ZSTD)
+endif()
+
set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
if(LLVM_ENABLE_LIBXML2)
find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===================================================================
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,21 @@
+find_path(ZSTD_INCLUDE_DIR
+ NAMES zstd.h
+ HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARY
+ NAMES zstd
+ HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+ ZSTD DEFAULT_MSG
+ ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+if(ZSTD_FOUND)
+ set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+ set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+ ZSTD_INCLUDE_DIR
+ ZSTD_LIBRARY)
Index: llvm/cmake/config-ix.cmake
===================================================================
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -136,6 +136,29 @@
set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
endif()
+if(LLVM_ENABLE_ZSTD)
+ if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+ find_package(ZSTD REQUIRED)
+ elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+ find_package(ZSTD)
+ endif()
+ if(ZSTD_FOUND)
+ # Check if zstd we found is usable; for example, we may have found a 32-bit
+ # library on a 64-bit system which would result in a link-time failure.
+ cmake_push_check_state()
+ list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+ list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+ check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD)
+ cmake_pop_check_state()
+ if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD)
+ message(FATAL_ERROR "Failed to configure zstd")
+ endif()
+ endif()
+ set(LLVM_ENABLE_ZSTD "${HAVE_ZSTD}")
+else()
+ set(LLVM_ENABLE_ZSTD 0)
+endif()
+
if(LLVM_ENABLE_LIBXML2)
if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
find_package(LibXml2 REQUIRED)
Index: llvm/CMakeLists.txt
===================================================================
--- llvm/CMakeLists.txt
+++ llvm/CMakeLists.txt
@@ -438,6 +438,8 @@
set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+set(LLVM_ENABLE_ZSTD "ON" CACHE STRING "Use zstd for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+
set(LLVM_ENABLE_CURL "OFF" CACHE STRING "Use libcurl for the HTTP client if available. Can be ON, OFF, or FORCE_ON")
set(LLVM_ENABLE_HTTPLIB "OFF" CACHE STRING "Use cpp-httplib HTTP server library if available. Can be ON, OFF, or FORCE_ON")
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
