This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new e978e4c78 feat(thirdparty): upgrade rocksdb to 8.5.3 (#1601)
e978e4c78 is described below
commit e978e4c78b1436aa0ae06915365f85c601781ba7
Author: Yingchun Lai <[email protected]>
AuthorDate: Wed Sep 13 22:28:01 2023 +0800
feat(thirdparty): upgrade rocksdb to 8.5.3 (#1601)
https://github.com/apache/incubator-pegasus/issues/1604
This patch upgrade the `rocksdb` to the latest stable version 8.5.3.
Since the 2.5 branch has been created, the new version of rocksdb will
only take effect on version 2.6 and later.
---
.github/workflows/lint_and_test_cpp.yaml | 12 ++++-----
.github/workflows/thirdparty-regular-push.yml | 4 +--
docker/thirdparties-bin/Dockerfile | 2 +-
run.sh | 4 +--
src/server/pegasus_event_listener.cpp | 1 +
src/server/pegasus_server_impl.cpp | 29 ++++++++++++----------
src/server/pegasus_server_impl.h | 2 ++
src/server/pegasus_server_impl_init.cpp | 2 ++
.../test/pegasus_compression_options_test.cpp | 1 +
src/utils/alloc.h | 2 ++
thirdparty/CMakeLists.txt | 10 +++-----
11 files changed, 38 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/lint_and_test_cpp.yaml
b/.github/workflows/lint_and_test_cpp.yaml
index 115f8b124..e405fa3a4 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -91,7 +91,7 @@ jobs:
# Build thirdparties and leave some necessary libraries and source
run: |
mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+ cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
../scripts/download_hadoop.sh hadoop-bin
@@ -151,7 +151,7 @@ jobs:
run: |
rm -f /root/thirdparties-src.zip
mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+ cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -296,7 +296,7 @@ jobs:
run: |
rm -f /root/thirdparties-src.zip
mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+ cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -438,7 +438,7 @@ jobs:
# run: |
# rm -f /root/thirdparties-src.zip
# mkdir build
-# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
+# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
# cmake --build build/ -j $(nproc)
# rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
# find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -576,7 +576,7 @@ jobs:
run: |
rm -f /root/thirdparties-src.zip
mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON
-DUSE_JEMALLOC=ON -B build/
+ cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1
-DUSE_JEMALLOC=ON -B build/
cmake --build build/ -j $(nproc)
rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
@@ -676,7 +676,7 @@ jobs:
run: |
export JAVA_HOME="${JAVA_HOME_8_X64}"
mkdir -p build
- cmake -DCMAKE_BUILD_TYPE=Release -B build/
-DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR}
+ cmake -DCMAKE_BUILD_TYPE=Release -B build/
-DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DROCKSDB_PORTABLE=1
cmake --build build/ -j $(sysctl -n hw.physicalcpu)
- name: Compilation
run: |
diff --git a/.github/workflows/thirdparty-regular-push.yml
b/.github/workflows/thirdparty-regular-push.yml
index 4fb58b59a..48c04e750 100644
--- a/.github/workflows/thirdparty-regular-push.yml
+++ b/.github/workflows/thirdparty-regular-push.yml
@@ -167,7 +167,7 @@ jobs:
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
OS_VERSION=${{ matrix.osversion }}
- ROCKSDB_PORTABLE=ON
+ ROCKSDB_PORTABLE=1
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
- name: Image digest
@@ -203,7 +203,7 @@ jobs:
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
OS_VERSION=${{ matrix.osversion }}
- ROCKSDB_PORTABLE=ON
+ ROCKSDB_PORTABLE=1
USE_JEMALLOC=ON
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
diff --git a/docker/thirdparties-bin/Dockerfile
b/docker/thirdparties-bin/Dockerfile
index ab9171063..27a0d5146 100644
--- a/docker/thirdparties-bin/Dockerfile
+++ b/docker/thirdparties-bin/Dockerfile
@@ -26,7 +26,7 @@ COPY --from=builder /root/thirdparties-src.zip
/root/thirdparties-src.zip
ARG GITHUB_BRANCH=master
ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
-ARG ROCKSDB_PORTABLE=OFF
+ARG ROCKSDB_PORTABLE=0
ARG USE_JEMALLOC=OFF
ARG HADOOP_BIN_PATH
ARG ZOOKEEPER_BIN_PATH
diff --git a/run.sh b/run.sh
index 75b688a8a..5ac6a1964 100755
--- a/run.sh
+++ b/run.sh
@@ -118,7 +118,7 @@ function run_build()
ENABLE_GPERF=ON
SKIP_THIRDPARTY=NO
SANITIZER=""
- ROCKSDB_PORTABLE=OFF
+ ROCKSDB_PORTABLE=0
USE_JEMALLOC=OFF
BUILD_TEST=OFF
IWYU=""
@@ -177,7 +177,7 @@ function run_build()
SKIP_THIRDPARTY=YES
;;
--enable_rocksdb_portable)
- ROCKSDB_PORTABLE=ON
+ ROCKSDB_PORTABLE=1
;;
--use_jemalloc)
ENABLE_GPERF=OFF
diff --git a/src/server/pegasus_event_listener.cpp
b/src/server/pegasus_event_listener.cpp
index e13e2f863..24faf954e 100644
--- a/src/server/pegasus_event_listener.cpp
+++ b/src/server/pegasus_event_listener.cpp
@@ -22,6 +22,7 @@
#include <fmt/core.h>
#include <rocksdb/compaction_job_stats.h>
#include <rocksdb/table_properties.h>
+#include <rocksdb/types.h>
#include <string>
#include "common/gpid.h"
diff --git a/src/server/pegasus_server_impl.cpp
b/src/server/pegasus_server_impl.cpp
index c34895269..8d6ac8c37 100644
--- a/src/server/pegasus_server_impl.cpp
+++ b/src/server/pegasus_server_impl.cpp
@@ -22,7 +22,7 @@
#include <fmt/core.h>
#include <inttypes.h>
#include <limits.h>
-#include <rocksdb/cache.h>
+#include <rocksdb/advanced_cache.h>
#include <rocksdb/convenience.h>
#include <rocksdb/db.h>
#include <rocksdb/env.h>
@@ -1635,12 +1635,12 @@ dsn::error_code pegasus_server_impl::start(int argc,
char **argv)
rocksdb::DBOptions loaded_db_opt;
std::vector<rocksdb::ColumnFamilyDescriptor> loaded_cf_descs;
rocksdb::ColumnFamilyOptions loaded_data_cf_opts;
+ rocksdb::ConfigOptions config_options;
// Set `ignore_unknown_options` true for forward compatibility.
- auto status = rocksdb::LoadLatestOptions(rdb_path,
- rocksdb::Env::Default(),
- &loaded_db_opt,
- &loaded_cf_descs,
-
/*ignore_unknown_options=*/true);
+ config_options.ignore_unknown_options = true;
+ config_options.env = rocksdb::Env::Default();
+ auto status =
+ rocksdb::LoadLatestOptions(config_options, rdb_path,
&loaded_db_opt, &loaded_cf_descs);
if (!status.ok()) {
// Here we ignore an invalid argument error related to
`pegasus_data_version` and
// `pegasus_data` options, which were used in old version rocksdbs
(before 2.1.0).
@@ -1678,11 +1678,14 @@ dsn::error_code pegasus_server_impl::start(int argc,
char **argv)
std::vector<rocksdb::ColumnFamilyDescriptor> column_families(
{{DATA_COLUMN_FAMILY_NAME, _table_data_cf_opts},
{META_COLUMN_FAMILY_NAME, _meta_cf_opts}});
- auto s = rocksdb::CheckOptionsCompatibility(rdb_path,
- rocksdb::Env::Default(),
- _db_opts,
- column_families,
-
/*ignore_unknown_options=*/true);
+ rocksdb::ConfigOptions config_options;
+ config_options.ignore_unknown_options = true;
+ config_options.ignore_unsupported_options = true;
+ config_options.sanity_level =
+ rocksdb::ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible;
+ config_options.env = rocksdb::Env::Default();
+ auto s =
+ rocksdb::CheckOptionsCompatibility(config_options, rdb_path, _db_opts,
column_families);
if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) {
LOG_ERROR_PREFIX("rocksdb::CheckOptionsCompatibility failed, error =
{}", s.ToString());
return dsn::ERR_LOCAL_APP_FAILURE;
@@ -2159,7 +2162,7 @@ private:
{{DATA_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()},
{META_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}});
status = rocksdb::DB::OpenForReadOnly(
- rocksdb::DBOptions(), checkpoint_dir, column_families,
&handles_opened, &snapshot_db);
+ _db_opts, checkpoint_dir, column_families, &handles_opened,
&snapshot_db);
if (!status.ok()) {
LOG_ERROR_PREFIX(
"OpenForReadOnly from {} failed, error = {}", checkpoint_dir,
status.ToString());
@@ -3286,7 +3289,7 @@ bool pegasus_server_impl::set_options(
*missing_meta_cf = true;
*missing_data_cf = true;
std::vector<std::string> column_families;
- auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path,
&column_families);
+ auto s = rocksdb::DB::ListColumnFamilies(_db_opts, path, &column_families);
if (!s.ok()) {
LOG_ERROR_PREFIX("rocksdb::DB::ListColumnFamilies failed, error = {}",
s.ToString());
if (s.IsCorruption() &&
diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h
index c9c5b90ef..02c6421c7 100644
--- a/src/server/pegasus_server_impl.h
+++ b/src/server/pegasus_server_impl.h
@@ -20,6 +20,7 @@
#pragma once
#include <gtest/gtest_prod.h>
+#include <rocksdb/compression_type.h>
#include <rocksdb/options.h>
#include <rocksdb/slice.h>
#include <rocksdb/table.h>
@@ -70,6 +71,7 @@ class WriteBufferManager;
namespace dsn {
class blob;
class message_ex;
+
namespace replication {
class detect_hotkey_request;
class detect_hotkey_response;
diff --git a/src/server/pegasus_server_impl_init.cpp
b/src/server/pegasus_server_impl_init.cpp
index c457ebd9f..528b2bd8e 100644
--- a/src/server/pegasus_server_impl_init.cpp
+++ b/src/server/pegasus_server_impl_init.cpp
@@ -19,6 +19,7 @@
#include <fmt/core.h>
#include <rocksdb/cache.h>
+#include <rocksdb/env.h>
#include <rocksdb/filter_policy.h>
#include <rocksdb/options.h>
#include <rocksdb/rate_limiter.h>
@@ -422,6 +423,7 @@
pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
_rng_rd_opts.rocksdb_iteration_threshold_time_ms =
FLAGS_rocksdb_iteration_threshold_time_ms;
// init rocksdb::DBOptions
+ _db_opts.env = rocksdb::Env::Default();
_db_opts.create_if_missing = true;
// atomic flush data CF and meta CF, aim to keep consistency of 'last
flushed decree' in meta CF
// and data in data CF.
diff --git a/src/server/test/pegasus_compression_options_test.cpp
b/src/server/test/pegasus_compression_options_test.cpp
index 9fa38daf6..b33eeeac1 100644
--- a/src/server/test/pegasus_compression_options_test.cpp
+++ b/src/server/test/pegasus_compression_options_test.cpp
@@ -20,6 +20,7 @@
// IWYU pragma: no_include <gtest/gtest-message.h>
// IWYU pragma: no_include <gtest/gtest-test-part.h>
#include <gtest/gtest.h>
+#include <rocksdb/compression_type.h>
#include <rocksdb/db.h>
#include <rocksdb/options.h>
#include <map>
diff --git a/src/utils/alloc.h b/src/utils/alloc.h
index e224eaff6..542f4efb5 100644
--- a/src/utils/alloc.h
+++ b/src/utils/alloc.h
@@ -17,6 +17,8 @@
#pragma once
+#include <algorithm>
+
#include "utils/ports.h"
// The check for the definition of CACHELINE_SIZE has to be put after
including "utils/ports.h",
diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt
index 069f21d29..7f128c894 100644
--- a/thirdparty/CMakeLists.txt
+++ b/thirdparty/CMakeLists.txt
@@ -358,15 +358,11 @@ ExternalProject_Add(jemalloc
BUILD_IN_SOURCE 1
)
-option(ROCKSDB_PORTABLE "build a portable binary" OFF)
+option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 =
baseline CPU" 0)
-# The patch name '0879c240' means the patch of rocksdb:
-#
https://github.com/facebook/rocksdb/commit/0879c240404b00142ba4718f36cd3f2bd537192d
ExternalProject_Add(rocksdb
- URL ${OSS_URL_PREFIX}/rocksdb-6.6.4.tar.gz
- https://github.com/facebook/rocksdb/archive/refs/tags/v6.6.4.tar.gz
- URL_MD5 7f7fcca3e96b7d83ef332804c90070c8
- PATCH_COMMAND patch -p1 <
${TP_DIR}/rocksdb_fix_atomic_flush_0879c240.patch
+ URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.5.3.tar.gz
+ URL_MD5 f03eac50ec958a21a7cb56183afb8fe4
DEPENDS jemalloc
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
-DFAIL_ON_WARNINGS=OFF
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
