This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 5ea2a8d4d feat(snappy/lz4/zstd): Build snappy/lz4/zstd as a thirdparty
libraries (#1709)
5ea2a8d4d is described below
commit 5ea2a8d4df6e76c43efdb599a7bfff9dcafd90c4
Author: Yingchun Lai <[email protected]>
AuthorDate: Tue Dec 12 14:51:40 2023 +0800
feat(snappy/lz4/zstd): Build snappy/lz4/zstd as a thirdparty libraries
(#1709)
Build snappy/lz4/zstd as thirdparty libraries, and do not dependent on the
library in system. It would be more flexible to upgrade the libraries
version no matter what the library versions in system are.
Another issue this patch resolve is
https://github.com/apache/incubator-pegasus/issues/1632.
---
.github/workflows/lint_and_test_cpp.yaml | 3 -
.licenserc.yaml | 1 +
cmake_modules/BaseFunctions.cmake | 8 +--
docker/pegasus-build-env/centos7/Dockerfile | 4 --
docker/pegasus-build-env/ubuntu1804/Dockerfile | 4 --
docker/pegasus-build-env/ubuntu2004/Dockerfile | 4 --
docker/pegasus-build-env/ubuntu2204/Dockerfile | 4 --
src/aio/CMakeLists.txt | 7 ++-
src/aio/test/CMakeLists.txt | 12 +++-
src/block_service/hdfs/CMakeLists.txt | 7 ++-
src/block_service/local/CMakeLists.txt | 6 +-
src/block_service/test/CMakeLists.txt | 25 ++++----
src/client/test/CMakeLists.txt | 15 +++--
src/common/test/CMakeLists.txt | 5 +-
src/failure_detector/test/CMakeLists.txt | 19 +++---
src/geo/bench/CMakeLists.txt | 8 ++-
src/http/test/CMakeLists.txt | 14 +++--
src/meta/CMakeLists.txt | 31 +++++-----
src/nfs/test/CMakeLists.txt | 12 +++-
src/perf_counter/test/CMakeLists.txt | 10 ++-
src/replica/CMakeLists.txt | 29 +++++----
src/replica/backup/test/CMakeLists.txt | 5 +-
src/replica/bulk_load/test/CMakeLists.txt | 7 ++-
src/replica/duplication/test/CMakeLists.txt | 5 +-
src/replica/storage/simple_kv/CMakeLists.txt | 11 +++-
src/replica/storage/simple_kv/test/CMakeLists.txt | 28 +++++----
src/replica/test/CMakeLists.txt | 34 +++++-----
src/runtime/ranger/CMakeLists.txt | 14 ++---
src/runtime/test/CMakeLists.txt | 15 +++--
src/server/CMakeLists.txt | 32 +++++-----
src/server/test/CMakeLists.txt | 7 ++-
src/shell/CMakeLists.txt | 40 ++++++------
src/test/bench_test/CMakeLists.txt | 8 ++-
src/test/function_test/bulk_load/CMakeLists.txt | 25 ++++----
src/test_util/CMakeLists.txt | 7 ++-
src/utils/CMakeLists.txt | 7 ++-
src/utils/long_adder_bench/CMakeLists.txt | 8 ++-
src/utils/test/CMakeLists.txt | 16 +++--
src/utils/test/nth_element_bench/CMakeLists.txt | 8 ++-
src/zookeeper/test/CMakeLists.txt | 17 ++---
thirdparty/CMakeLists.txt | 75 ++++++++++++++++++++++-
thirdparty/fix_snappy-Wsign-compare-warning.patch | 26 ++++++++
42 files changed, 415 insertions(+), 208 deletions(-)
diff --git a/.github/workflows/lint_and_test_cpp.yaml
b/.github/workflows/lint_and_test_cpp.yaml
index a9b7724eb..85b8eeb09 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -673,9 +673,6 @@ jobs:
run: |
# Preinstalled softwares:
https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md
brew install ccache
- brew install snappy
- brew install lz4
- brew install zstd
brew install [email protected]
- uses: actions/checkout@v3
- name: Setup cache
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 3ddf87be6..1c5804759 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -71,6 +71,7 @@ header:
- 'thirdparty/fix_libevent_for_macos.patch'
- 'thirdparty/fix_prometheus-cpp_limits.patch'
- 'thirdparty/fix_rocksdb-cmake-PORTABLE-option.patch'
+ - 'thirdparty/fix_snappy-Wsign-compare-warning.patch'
- 'thirdparty/fix_s2_build_with_absl_and_gtest.patch'
- 'thirdparty/fix_thrift_for_cpp11.patch'
# TODO(yingchun): shell/* files are import from thirdparties, we can move
them to thirdparty later.
diff --git a/cmake_modules/BaseFunctions.cmake
b/cmake_modules/BaseFunctions.cmake
index 1882353c8..dc0e3b28f 100644
--- a/cmake_modules/BaseFunctions.cmake
+++ b/cmake_modules/BaseFunctions.cmake
@@ -326,15 +326,15 @@ function(dsn_setup_thirdparty_libs)
find_package(fmt REQUIRED)
set(DEFAULT_THIRDPARTY_LIBS ${THRIFT_LIB} fmt::fmt CACHE STRING "default
thirdparty libs" FORCE)
- # rocksdb
+ # rocksdb and dependencies
file(GLOB ROCKSDB_DEPENDS_MODULE_PATH
${THIRDPARTY_ROOT}/build/Source/rocksdb/cmake/modules)
if(NOT ROCKSDB_DEPENDS_MODULE_PATH)
message(WARNING "Cannot find RocksDB depends cmake modules path, might not
find snappy, zstd, lz4")
endif()
list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_DEPENDS_MODULE_PATH}")
- find_package(snappy)
- find_package(zstd)
- find_package(lz4)
+ find_package(Snappy REQUIRED)
+ find_package(zstd REQUIRED)
+ find_package(lz4 REQUIRED)
if(USE_JEMALLOC)
find_package(Jemalloc REQUIRED)
endif()
diff --git a/docker/pegasus-build-env/centos7/Dockerfile
b/docker/pegasus-build-env/centos7/Dockerfile
index 0e89f316b..c8f8da664 100644
--- a/docker/pegasus-build-env/centos7/Dockerfile
+++ b/docker/pegasus-build-env/centos7/Dockerfile
@@ -42,12 +42,8 @@ RUN yum -y install centos-release-scl \
which \
openssl-devel \
libaio-devel \
- snappy-devel \
- bzip2-devel \
zlib \
zlib-devel \
- libzstd-devel \
- lz4-devel \
bison \
flex \
krb5-devel \
diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile
b/docker/pegasus-build-env/ubuntu1804/Dockerfile
index fb4c9a47a..55189b199 100644
--- a/docker/pegasus-build-env/ubuntu1804/Dockerfile
+++ b/docker/pegasus-build-env/ubuntu1804/Dockerfile
@@ -30,10 +30,6 @@ RUN apt-get update -y; \
openjdk-8-jdk \
python3-pip \
libaio-dev \
- libsnappy-dev \
- libbz2-dev \
- libzstd-dev \
- liblz4-dev \
zlib1g \
zlib1g.dev \
patch \
diff --git a/docker/pegasus-build-env/ubuntu2004/Dockerfile
b/docker/pegasus-build-env/ubuntu2004/Dockerfile
index 604b5a0b1..73d58e059 100644
--- a/docker/pegasus-build-env/ubuntu2004/Dockerfile
+++ b/docker/pegasus-build-env/ubuntu2004/Dockerfile
@@ -30,10 +30,6 @@ RUN apt-get update -y; \
openjdk-8-jdk \
python3-pip \
libaio-dev \
- libsnappy-dev \
- libbz2-dev \
- libzstd-dev \
- liblz4-dev \
zlib1g \
zlib1g.dev \
patch \
diff --git a/docker/pegasus-build-env/ubuntu2204/Dockerfile
b/docker/pegasus-build-env/ubuntu2204/Dockerfile
index 0eed16c6c..d7efdb286 100644
--- a/docker/pegasus-build-env/ubuntu2204/Dockerfile
+++ b/docker/pegasus-build-env/ubuntu2204/Dockerfile
@@ -31,10 +31,6 @@ RUN apt-get update -y; \
pkg-config \
python3-pip \
libaio-dev \
- libsnappy-dev \
- libbz2-dev \
- libzstd-dev \
- liblz4-dev \
zlib1g \
zlib1g.dev \
patch \
diff --git a/src/aio/CMakeLists.txt b/src/aio/CMakeLists.txt
index 3754361d0..3a27fbe01 100644
--- a/src/aio/CMakeLists.txt
+++ b/src/aio/CMakeLists.txt
@@ -33,7 +33,12 @@ set(MY_PROJ_SRC "")
#"GLOB" for non - recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS dsn_runtime rocksdb)
+set(MY_PROJ_LIBS
+ dsn_runtime
+ rocksdb
+ lz4
+ zstd
+ snappy)
#Extra files that will be installed
set(MY_BINPLACES "")
diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt
index b8d4ad675..2eb437146 100644
--- a/src/aio/test/CMakeLists.txt
+++ b/src/aio/test/CMakeLists.txt
@@ -33,7 +33,17 @@ set(MY_PROJ_SRC "")
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio test_utils rocksdb)
+set(MY_PROJ_LIBS
+ dsn_meta_server
+ dsn_replication_common
+ gtest
+ dsn_runtime
+ dsn_aio
+ test_utils
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/block_service/hdfs/CMakeLists.txt
b/src/block_service/hdfs/CMakeLists.txt
index 2bba8b96b..ae821f2bf 100644
--- a/src/block_service/hdfs/CMakeLists.txt
+++ b/src/block_service/hdfs/CMakeLists.txt
@@ -26,7 +26,12 @@ set(MY_PROJ_SRC "")
#"GLOB" for non - recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS hdfs rocksdb)
+set(MY_PROJ_LIBS
+ hdfs
+ rocksdb
+ lz4
+ zstd
+ snappy)
#Extra files that will be installed
set(MY_BINPLACES "")
diff --git a/src/block_service/local/CMakeLists.txt
b/src/block_service/local/CMakeLists.txt
index 9d830f782..8fdd0d53a 100644
--- a/src/block_service/local/CMakeLists.txt
+++ b/src/block_service/local/CMakeLists.txt
@@ -26,7 +26,11 @@ set(MY_PROJ_SRC "")
#"GLOB" for non - recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS rocksdb)
+set(MY_PROJ_LIBS
+ rocksdb
+ lz4
+ zstd
+ snappy)
#Extra files that will be installed
set(MY_BINPLACES "")
diff --git a/src/block_service/test/CMakeLists.txt
b/src/block_service/test/CMakeLists.txt
index 6fbfa9ca1..b8aaad652 100644
--- a/src/block_service/test/CMakeLists.txt
+++ b/src/block_service/test/CMakeLists.txt
@@ -22,17 +22,20 @@ set(MY_PROJ_NAME dsn_block_service_test)
set(MY_SRC_SEARCH_MODE "GLOB")
set(MY_PROJ_LIBS
- dsn_replication_common
- dsn.block_service
- dsn.block_service.local
- dsn.block_service.hdfs
- dsn_runtime
- dsn_utils
- gtest
- gtest_main
- hdfs
- test_utils
- rocksdb)
+ dsn_replication_common
+ dsn.block_service
+ dsn.block_service.local
+ dsn.block_service.hdfs
+ dsn_runtime
+ dsn_utils
+ gtest
+ gtest_main
+ hdfs
+ test_utils
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/client/test/CMakeLists.txt b/src/client/test/CMakeLists.txt
index cfe591ce0..60f376276 100644
--- a/src/client/test/CMakeLists.txt
+++ b/src/client/test/CMakeLists.txt
@@ -22,12 +22,15 @@ set(MY_PROJ_SRC "")
set(MY_SRC_SEARCH_MODE "GLOB")
set(MY_PROJ_LIBS
- dsn_client
- dsn_replication_common
- dsn_runtime
- dsn_utils
- gtest
- rocksdb)
+ dsn_client
+ dsn_replication_common
+ dsn_runtime
+ dsn_utils
+ gtest
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/common/test/CMakeLists.txt b/src/common/test/CMakeLists.txt
index 12f53aa52..fab092624 100644
--- a/src/common/test/CMakeLists.txt
+++ b/src/common/test/CMakeLists.txt
@@ -31,7 +31,10 @@ set(MY_PROJ_LIBS
dsn_replication_common
dsn_runtime
gtest
- rocksdb)
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/failure_detector/test/CMakeLists.txt
b/src/failure_detector/test/CMakeLists.txt
index 6bd1e0555..5b5a6832a 100644
--- a/src/failure_detector/test/CMakeLists.txt
+++ b/src/failure_detector/test/CMakeLists.txt
@@ -34,14 +34,17 @@ set(MY_PROJ_SRC "")
set(MY_SRC_SEARCH_MODE "GLOB")
set(MY_PROJ_LIBS
- dsn_runtime
- dsn_meta_server
- dsn_replica_server
- dsn_replication_common
- dsn.failure_detector
- gtest
- hashtable
- rocksdb)
+ dsn_runtime
+ dsn_meta_server
+ dsn_replica_server
+ dsn_replication_common
+ dsn.failure_detector
+ gtest
+ hashtable
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/geo/bench/CMakeLists.txt b/src/geo/bench/CMakeLists.txt
index 6df54a6bc..559747b59 100644
--- a/src/geo/bench/CMakeLists.txt
+++ b/src/geo/bench/CMakeLists.txt
@@ -34,9 +34,11 @@ set(MY_PROJ_LIBS
s2testing
s2
pegasus_client_static
- RocksDB::rocksdb
- dsn_utils
- )
+ rocksdb
+ lz4
+ zstd
+ snappy
+ dsn_utils)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/http/test/CMakeLists.txt b/src/http/test/CMakeLists.txt
index 5ebc142a5..4400c0183 100644
--- a/src/http/test/CMakeLists.txt
+++ b/src/http/test/CMakeLists.txt
@@ -22,12 +22,14 @@ set(MY_PROJ_SRC "")
set(MY_SRC_SEARCH_MODE "GLOB")
set(MY_PROJ_LIBS
- dsn_http
- dsn_runtime
- curl
- gtest
- rocksdb
- )
+ dsn_http
+ dsn_runtime
+ curl
+ gtest
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt
index 477671acb..fcc3a1232 100644
--- a/src/meta/CMakeLists.txt
+++ b/src/meta/CMakeLists.txt
@@ -34,20 +34,23 @@ set(DUPLICATION_SRC
set(MY_PROJ_SRC "${DUPLICATION_SRC}")
set(MY_PROJ_LIBS
- dsn_replication_common
- dsn.block_service
- dsn.block_service.local
- dsn.block_service.hdfs
- dsn.failure_detector
- dsn.replication.zookeeper_provider
- dsn_dist_cmd
- dsn_http
- dsn_runtime
- dsn_aio
- zookeeper
- hashtable
- hdfs
- rocksdb)
+ dsn_replication_common
+ dsn.block_service
+ dsn.block_service.local
+ dsn.block_service.hdfs
+ dsn.failure_detector
+ dsn.replication.zookeeper_provider
+ dsn_dist_cmd
+ dsn_http
+ dsn_runtime
+ dsn_aio
+ zookeeper
+ hashtable
+ hdfs
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/nfs/test/CMakeLists.txt b/src/nfs/test/CMakeLists.txt
index ff591b7e2..4a22ce486 100644
--- a/src/nfs/test/CMakeLists.txt
+++ b/src/nfs/test/CMakeLists.txt
@@ -33,7 +33,17 @@ set(MY_PROJ_SRC "")
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio dsn_http rocksdb test_utils)
+set(MY_PROJ_LIBS
+ dsn_nfs
+ dsn_runtime
+ gtest
+ dsn_aio
+ dsn_http
+ rocksdb
+ lz4
+ zstd
+ snappy
+ test_utils)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/perf_counter/test/CMakeLists.txt
b/src/perf_counter/test/CMakeLists.txt
index 2e02cf868..f924437de 100644
--- a/src/perf_counter/test/CMakeLists.txt
+++ b/src/perf_counter/test/CMakeLists.txt
@@ -33,7 +33,15 @@ set(MY_PROJ_SRC "")
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS gtest dsn_runtime rocksdb)
+set(MY_PROJ_LIBS
+ dsn_meta_server
+ dsn_replication_common
+ gtest
+ dsn_runtime
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt
index fcf789101..5bcd3f57b 100644
--- a/src/replica/CMakeLists.txt
+++ b/src/replica/CMakeLists.txt
@@ -57,19 +57,22 @@ set(MY_PROJ_SRC
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS
- dsn_replication_common
- dsn.failure_detector
- dsn.block_service
- dsn.block_service.local
- dsn.block_service.hdfs
- dsn_nfs
- dsn_dist_cmd
- dsn_http
- dsn_runtime
- dsn_aio
- dsn_meta_server
- rocksdb)
+set(MY_PROJ_LIBS
+ dsn_replication_common
+ dsn.failure_detector
+ dsn.block_service
+ dsn.block_service.local
+ dsn.block_service.hdfs
+ dsn_nfs
+ dsn_dist_cmd
+ dsn_http
+ dsn_runtime
+ dsn_aio
+ dsn_meta_server
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::filesystem)
diff --git a/src/replica/backup/test/CMakeLists.txt
b/src/replica/backup/test/CMakeLists.txt
index 47158ca93..fed60e312 100644
--- a/src/replica/backup/test/CMakeLists.txt
+++ b/src/replica/backup/test/CMakeLists.txt
@@ -29,7 +29,10 @@ set(MY_PROJ_LIBS dsn_meta_server
dsn_utils
hashtable
gtest
- rocksdb)
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/replica/bulk_load/test/CMakeLists.txt
b/src/replica/bulk_load/test/CMakeLists.txt
index 0b0d08385..bda1a8981 100644
--- a/src/replica/bulk_load/test/CMakeLists.txt
+++ b/src/replica/bulk_load/test/CMakeLists.txt
@@ -28,9 +28,12 @@ set(MY_PROJ_LIBS dsn_meta_server
hashtable
gtest
test_utils
- rocksdb)
+ rocksdb
+ lz4
+ zstd
+ snappy)
-set(MY_BOOST_LIBS Boost::system Boost::filesystem rocksdb test_utils)
+set(MY_BOOST_LIBS Boost::system Boost::filesystem)
set(MY_BINPLACES
config-test.ini
diff --git a/src/replica/duplication/test/CMakeLists.txt
b/src/replica/duplication/test/CMakeLists.txt
index 5cc139f91..26e461bd7 100644
--- a/src/replica/duplication/test/CMakeLists.txt
+++ b/src/replica/duplication/test/CMakeLists.txt
@@ -31,7 +31,10 @@ set(MY_PROJ_LIBS dsn_meta_server
hashtable
gtest
test_utils
- rocksdb)
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/replica/storage/simple_kv/CMakeLists.txt
b/src/replica/storage/simple_kv/CMakeLists.txt
index 8584848f1..a2f850677 100644
--- a/src/replica/storage/simple_kv/CMakeLists.txt
+++ b/src/replica/storage/simple_kv/CMakeLists.txt
@@ -37,7 +37,16 @@ set(MY_PROJ_SRC ${SIMPLE_KV_THRIFT_SRCS})
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime
hashtable rocksdb)
+set(MY_PROJ_LIBS
+ dsn_replica_server
+ dsn_meta_server
+ dsn_client
+ dsn_runtime
+ hashtable
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/replica/storage/simple_kv/test/CMakeLists.txt
b/src/replica/storage/simple_kv/test/CMakeLists.txt
index a1e0a69b2..0bce41b71 100644
--- a/src/replica/storage/simple_kv/test/CMakeLists.txt
+++ b/src/replica/storage/simple_kv/test/CMakeLists.txt
@@ -29,18 +29,22 @@ set(MY_PROJ_NAME dsn.rep_tests.simple_kv)
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS dsn_replica_server
- dsn_meta_server
- dsn_replication_common
- dsn_client
- dsn.failure_detector
- dsn.replication.zookeeper_provider
- dsn_runtime
- zookeeper
- hashtable
- gtest
- dsn_utils
- rocksdb)
+set(MY_PROJ_LIBS
+ dsn_replica_server
+ dsn_meta_server
+ dsn_replication_common
+ dsn_client
+ dsn.failure_detector
+ dsn.replication.zookeeper_provider
+ dsn_runtime
+ zookeeper
+ hashtable
+ gtest
+ dsn_utils
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt
index 83af542d1..94b5c8934 100644
--- a/src/replica/test/CMakeLists.txt
+++ b/src/replica/test/CMakeLists.txt
@@ -33,21 +33,25 @@ set(MY_PROJ_SRC "")
#"GLOB" for non - recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS dsn_meta_server
- dsn_replica_server
- dsn.replication.zookeeper_provider
- dsn_replication_common
- dsn.block_service
- dsn.block_service.local
- dsn.block_service.hdfs
- dsn.failure_detector
- dsn_http
- dsn_runtime
- zookeeper
- hashtable
- gtest
- test_utils
- rocksdb)
+set(MY_PROJ_LIBS
+ dsn_meta_server
+ dsn_replica_server
+ dsn.replication.zookeeper_provider
+ dsn_replication_common
+ dsn.block_service
+ dsn.block_service.local
+ dsn.block_service.hdfs
+ dsn.failure_detector
+ dsn_http
+ dsn_runtime
+ zookeeper
+ hashtable
+ gtest
+ test_utils
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/runtime/ranger/CMakeLists.txt
b/src/runtime/ranger/CMakeLists.txt
index 3c3e441a4..fbbceb570 100644
--- a/src/runtime/ranger/CMakeLists.txt
+++ b/src/runtime/ranger/CMakeLists.txt
@@ -17,14 +17,10 @@
set(MY_PROJ_NAME dsn_ranger)
-# Search mode for source files under CURRENT project directory?
-# "GLOB_RECURSE" for recursive search
-# "GLOB" for non-recursive search
+set(MY_PROJ_SRC "")
set(MY_SRC_SEARCH_MODE "GLOB")
+set(MY_PROJ_LIBS
+ dsn_meta_server
+ dsn_replication_common)
-set(MY_PROJ_LIBS "")
-
-# Extra files that will be installed
-set(MY_BINPLACES "")
-
-dsn_add_object()
+dsn_add_static_library()
diff --git a/src/runtime/test/CMakeLists.txt b/src/runtime/test/CMakeLists.txt
index cd5e0daac..38e52a4f5 100644
--- a/src/runtime/test/CMakeLists.txt
+++ b/src/runtime/test/CMakeLists.txt
@@ -29,12 +29,15 @@ set(MY_PROJ_NAME dsn_runtime_tests)
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS gtest
- dsn_runtime
- dsn_aio
- dsn_meta_server
- rocksdb
- )
+set(MY_PROJ_LIBS
+ gtest
+ dsn_runtime
+ dsn_aio
+ dsn_meta_server
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt
index d19408e27..8c8daa4d1 100644
--- a/src/server/CMakeLists.txt
+++ b/src/server/CMakeLists.txt
@@ -27,21 +27,23 @@ set(MY_PROJ_SRC "")
set(MY_SRC_SEARCH_MODE "GLOB")
set(MY_PROJ_LIBS
- dsn_replica_server
- dsn_meta_server
- dsn_replication_common
- dsn_client
- dsn.block_service.local
- dsn.block_service
- dsn.failure_detector
- dsn.replication.zookeeper_provider
- dsn_utils
- RocksDB::rocksdb
- pegasus_base
- pegasus_client_static
- event
- hashtable
- )
+ dsn_replica_server
+ dsn_meta_server
+ dsn_replication_common
+ dsn_client
+ dsn.block_service.local
+ dsn.block_service
+ dsn.failure_detector
+ dsn.replication.zookeeper_provider
+ dsn_utils
+ rocksdb
+ lz4
+ zstd
+ snappy
+ pegasus_base
+ pegasus_client_static
+ event
+ hashtable)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt
index 8a2fbd6f1..2b4c96ac1 100644
--- a/src/server/test/CMakeLists.txt
+++ b/src/server/test/CMakeLists.txt
@@ -45,14 +45,17 @@ set(MY_PROJ_LIBS
dsn.failure_detector
dsn.replication.zookeeper_provider
dsn_utils
- RocksDB::rocksdb
+ rocksdb
+ lz4
+ zstd
+ snappy
pegasus_client_static
event
pegasus_base
gtest
gmock
hashtable
- )
+)
add_definitions(-DPEGASUS_UNIT_TEST)
add_definitions(-DENABLE_FAIL)
diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt
index 909fda9ab..81271006e 100644
--- a/src/shell/CMakeLists.txt
+++ b/src/shell/CMakeLists.txt
@@ -28,25 +28,27 @@ set(MY_PROJ_SRC "linenoise/linenoise.c" "sds/sds.c")
set(MY_SRC_SEARCH_MODE "GLOB_RECURSE")
set(MY_PROJ_LIBS
- pegasus_base
- dsn.replication.tool
- dsn_replica_server
- dsn_meta_server
- dsn_replication_common
- dsn_client
- dsn_utils
- dsn.block_service.local
- dsn.block_service.hdfs
- dsn.block_service
- dsn.failure_detector
- pegasus_client_static
- pegasus_geo_lib
- RocksDB::rocksdb
- absl::flat_hash_set
- absl::strings
- s2
- hdfs
- )
+ pegasus_base
+ dsn.replication.tool
+ dsn_replica_server
+ dsn_meta_server
+ dsn_replication_common
+ dsn_client
+ dsn_utils
+ dsn.block_service.local
+ dsn.block_service.hdfs
+ dsn.block_service
+ dsn.failure_detector
+ pegasus_client_static
+ pegasus_geo_lib
+ rocksdb
+ lz4
+ zstd
+ snappy
+ absl::flat_hash_set
+ absl::strings
+ s2
+ hdfs)
set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/config.ini")
diff --git a/src/test/bench_test/CMakeLists.txt
b/src/test/bench_test/CMakeLists.txt
index 307d543d0..5a52808d8 100644
--- a/src/test/bench_test/CMakeLists.txt
+++ b/src/test/bench_test/CMakeLists.txt
@@ -30,11 +30,13 @@ set(MY_SRC_SEARCH_MODE "GLOB")
set(MY_PROJ_LIBS
pegasus_client_static
dsn_utils
- RocksDB::rocksdb
+ rocksdb
+ lz4
+ zstd
+ snappy
sasl2
gssapi_krb5
- krb5
- )
+ krb5)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/test/function_test/bulk_load/CMakeLists.txt
b/src/test/function_test/bulk_load/CMakeLists.txt
index f9f197649..ce17323ea 100644
--- a/src/test/function_test/bulk_load/CMakeLists.txt
+++ b/src/test/function_test/bulk_load/CMakeLists.txt
@@ -28,17 +28,20 @@ set(MY_PROJ_SRC "")
set(MY_SRC_SEARCH_MODE "GLOB")
set(MY_PROJ_LIBS
- dsn_client
- dsn_replication_common
- dsn_utils
- pegasus_client_static
- gtest
- sasl2
- gssapi_krb5
- krb5
- function_test_utils
- test_utils
- rocksdb)
+ dsn_client
+ dsn_replication_common
+ dsn_utils
+ pegasus_client_static
+ gtest
+ sasl2
+ gssapi_krb5
+ krb5
+ function_test_utils
+ test_utils
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/test_util/CMakeLists.txt b/src/test_util/CMakeLists.txt
index b1e7ac2df..fd4cf1dc1 100644
--- a/src/test_util/CMakeLists.txt
+++ b/src/test_util/CMakeLists.txt
@@ -22,6 +22,11 @@ set(MY_PROJ_NAME test_utils)
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS gtest rocksdb)
+set(MY_PROJ_LIBS
+ gtest
+ rocksdb
+ lz4
+ zstd
+ snappy)
dsn_add_static_library()
diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt
index 8c04e3adc..38390f0ed 100644
--- a/src/utils/CMakeLists.txt
+++ b/src/utils/CMakeLists.txt
@@ -31,7 +31,12 @@ set(MY_SRC_SEARCH_MODE "GLOB")
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
-set(MY_PROJ_LIBS dsn_http rocksdb)
+set(MY_PROJ_LIBS
+ dsn_http
+ rocksdb
+ lz4
+ zstd
+ snappy)
# Extra files that will be installed
set(MY_BINPLACES "")
diff --git a/src/utils/long_adder_bench/CMakeLists.txt
b/src/utils/long_adder_bench/CMakeLists.txt
index d5ace78ca..71568c61a 100644
--- a/src/utils/long_adder_bench/CMakeLists.txt
+++ b/src/utils/long_adder_bench/CMakeLists.txt
@@ -27,7 +27,13 @@ set(MY_PROJ_SRC "")
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb)
+set(MY_PROJ_LIBS
+ dsn_runtime
+ dsn_utils
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt
index 684a5faaf..8502cabb7 100644
--- a/src/utils/test/CMakeLists.txt
+++ b/src/utils/test/CMakeLists.txt
@@ -29,12 +29,16 @@ set(MY_PROJ_NAME dsn_utils_tests)
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS dsn_http
- dsn_runtime
- dsn_utils
- gtest
- test_utils
- rocksdb)
+set(MY_PROJ_LIBS
+ dsn_http
+ dsn_runtime
+ dsn_utils
+ gtest
+ test_utils
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/utils/test/nth_element_bench/CMakeLists.txt
b/src/utils/test/nth_element_bench/CMakeLists.txt
index 7bf0af804..3aae31e4a 100644
--- a/src/utils/test/nth_element_bench/CMakeLists.txt
+++ b/src/utils/test/nth_element_bench/CMakeLists.txt
@@ -27,7 +27,13 @@ set(MY_PROJ_SRC "")
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb)
+set(MY_PROJ_LIBS
+ dsn_runtime
+ dsn_utils
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/src/zookeeper/test/CMakeLists.txt
b/src/zookeeper/test/CMakeLists.txt
index cdb6d4cec..7dabd1faf 100644
--- a/src/zookeeper/test/CMakeLists.txt
+++ b/src/zookeeper/test/CMakeLists.txt
@@ -33,13 +33,16 @@ set(MY_PROJ_SRC "")
# "GLOB" for non-recursive search
set(MY_SRC_SEARCH_MODE "GLOB")
-set(MY_PROJ_LIBS
- dsn.replication.zookeeper_provider
- dsn_runtime
- zookeeper
- hashtable
- gtest
- rocksdb)
+set(MY_PROJ_LIBS
+ dsn.replication.zookeeper_provider
+ dsn_runtime
+ zookeeper
+ hashtable
+ gtest
+ rocksdb
+ lz4
+ zstd
+ snappy)
set(MY_BOOST_LIBS Boost::system Boost::filesystem)
diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt
index e95e24f83..c3a9498f7 100644
--- a/thirdparty/CMakeLists.txt
+++ b/thirdparty/CMakeLists.txt
@@ -361,6 +361,75 @@ ExternalProject_Add(jemalloc
DOWNLOAD_NO_PROGRESS true
)
+set(SNAPPY_OPTIONS
+ -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+ -DCMAKE_BUILD_TYPE=Release
+ -DBUILD_SHARED_LIBS=OFF
+ -DSNAPPY_BUILD_TESTS=OFF
+ -DSNAPPY_BUILD_BENCHMARKS=OFF
+ -DSNAPPY_FUZZING_BUILD=OFF
+ -DSNAPPY_INSTALL=ON)
+execute_process(COMMAND arch OUTPUT_VARIABLE ARCH_NAME
OUTPUT_STRIP_TRAILING_WHITESPACE)
+message(STATUS "ARCH_NAME = ${ARCH_NAME}")
+if (ARCH_NAME EQUAL "x86_64")
+ set(SNAPPY_OPTIONS
+ ${SNAPPY_OPTIONS}
+ -DSNAPPY_REQUIRE_AVX=ON
+ -DSNAPPY_REQUIRE_AVX2=ON)
+endif ()
+ExternalProject_Add(snappy
+ URL ${OSS_URL_PREFIX}/snappy-1.1.10.tar.gz
+ https://github.com/google/snappy/archive/refs/tags/1.1.10.tar.gz
+ URL_MD5 70153395ebe6d72febe2cf2e40026a44
+ PATCH_COMMAND patch -p1 <
${TP_DIR}/fix_snappy-Wsign-compare-warning.patch
+ CMAKE_ARGS ${SNAPPY_OPTIONS}
+ BUILD_COMMAND make -j${PARALLEL}
+ INSTALL_COMMAND make install
+ DOWNLOAD_EXTRACT_TIMESTAMP true
+ DOWNLOAD_NO_PROGRESS true
+)
+
+ExternalProject_Add(zstd
+ URL ${OSS_URL_PREFIX}/zstd-1.5.5.tar.gz
+
https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz
+ URL_MD5 63251602329a106220e0a5ad26ba656f
+ PATCH_COMMAND ""
+ CONFIGURE_COMMAND cmake build/cmake
+ -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+ -DCMAKE_BUILD_TYPE=Release
+ -DZSTD_BUILD_PROGRAMS=OFF
+ -DZSTD_BUILD_TESTS=OFF
+ -DZSTD_BUILD_CONTRIB=OFF
+ -DZSTD_BUILD_SHARED=OFF
+ -DZSTD_BUILD_STATIC=ON
+ BUILD_COMMAND make -j${PARALLEL}
+ INSTALL_COMMAND make install
+ BUILD_IN_SOURCE 1
+ DOWNLOAD_EXTRACT_TIMESTAMP true
+ DOWNLOAD_NO_PROGRESS true
+)
+
+ExternalProject_Add(lz4
+ URL ${OSS_URL_PREFIX}/lz4-1.9.4.tar.gz
+ https://github.com/lz4/lz4/releases/download/v1.9.4/lz4-1.9.4.tar.gz
+ URL_MD5 e9286adb64040071c5e23498bf753261
+ PATCH_COMMAND ""
+ CONFIGURE_COMMAND cmake build/cmake
+ -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
+ -DLZ4_POSITION_INDEPENDENT_LIB=ON
+ -DCMAKE_BUILD_TYPE=Release
+ -DLZ4_BUILD_CLI=OFF
+ -DBUILD_SHARED_LIBS=OFF
+ -DBUILD_STATIC_LIBS=ON
+ BUILD_COMMAND make -j${PARALLEL}
+ INSTALL_COMMAND make install
+ BUILD_IN_SOURCE 1
+ DOWNLOAD_EXTRACT_TIMESTAMP true
+ DOWNLOAD_NO_PROGRESS true
+)
+
option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 =
baseline CPU" 0)
set(ROCKSDB_OPTIONS
-DFAIL_ON_WARNINGS=OFF
@@ -378,6 +447,7 @@ set(ROCKSDB_OPTIONS
-DJEMALLOC_ROOT_DIR=${TP_OUTPUT}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+ -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
-DPORTABLE=${ROCKSDB_PORTABLE})
# Link error on MacOS, disable building encfs plugin.
# See https://github.com/pegasus-kv/encfs/issues/4
@@ -393,9 +463,8 @@ ExternalProject_Add(rocksdb
PATCH_COMMAND patch -p1 <
${TP_DIR}/fix_rocksdb-cmake-PORTABLE-option.patch
COMMAND rm -rf ${TP_DIR}/build/Source/rocksdb/plugin/encfs
COMMAND git clone -b main --depth=1
https://github.com/pegasus-kv/encfs.git
${TP_DIR}/build/Source/rocksdb/plugin/encfs
- DEPENDS googletest jemalloc
- CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
- ${ROCKSDB_OPTIONS}
+ DEPENDS googletest jemalloc lz4 snappy zstd
+ CMAKE_ARGS ${ROCKSDB_OPTIONS}
DOWNLOAD_EXTRACT_TIMESTAMP true
DOWNLOAD_NO_PROGRESS true
)
diff --git a/thirdparty/fix_snappy-Wsign-compare-warning.patch
b/thirdparty/fix_snappy-Wsign-compare-warning.patch
new file mode 100644
index 000000000..0540d0177
--- /dev/null
+++ b/thirdparty/fix_snappy-Wsign-compare-warning.patch
@@ -0,0 +1,26 @@
+From 27f34a580be4a3becf5f8c0cba13433f53c21337 Mon Sep 17 00:00:00 2001
+From: Richard O'Grady <[email protected]>
+Date: Wed, 12 Jul 2023 10:12:01 -0700
+Subject: [PATCH] Fix -Wsign-compare warning
+
+PiperOrigin-RevId: 547529709
+---
+ snappy.cc | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/snappy.cc b/snappy.cc
+index 688065b..6473123 100644
+--- a/snappy.cc
++++ b/snappy.cc
+@@ -1289,7 +1289,7 @@ std::pair<const uint8_t*, ptrdiff_t>
DecompressBranchless(
+ DeferMemCopy(&deferred_src, &deferred_length, from, len);
+ }
+ } while (ip < ip_limit_min_slop &&
+- (op + deferred_length) < op_limit_min_slop);
++ static_cast<ptrdiff_t>(op + deferred_length) <
op_limit_min_slop);
+ exit:
+ ip--;
+ assert(ip <= ip_limit);
+--
+2.42.1
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
