This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 5e9a2a9dc feat(thirdparty): Use facebook/rocksdb and pegasus-kv/encfs
as rocksdb thirdparty (#1678)
5e9a2a9dc is described below
commit 5e9a2a9dcaa8be988e1ea8e745bd7fc8621e2609
Author: Yingchun Lai <[email protected]>
AuthorDate: Thu Nov 9 21:00:14 2023 +0800
feat(thirdparty): Use facebook/rocksdb and pegasus-kv/encfs as rocksdb
thirdparty (#1678)
After the reimplementation of encryption feature on RocksDB as a plugin,
now we can use the facebook official RocksDB plus the pegasus-kv/encfs
plugin. It would be more flexiable rather than modify the core code of
RocksDB.
---
src/utils/env.cpp | 7 ++++---
thirdparty/CMakeLists.txt | 31 +++++++++++++++++++------------
2 files changed, 23 insertions(+), 15 deletions(-)
diff --git a/src/utils/env.cpp b/src/utils/env.cpp
index 7e49d999a..444ac3784 100644
--- a/src/utils/env.cpp
+++ b/src/utils/env.cpp
@@ -40,7 +40,7 @@ DSN_DEFINE_bool(pegasus.server,
DSN_DEFINE_string(pegasus.server,
server_key_for_testing,
- "server_key_for_testing",
+ "0123456789ABCDEF0123456789ABCDEF",
"The encrypted server key to use in the filesystem. NOTE:
only for testing.");
DSN_DEFINE_string(pegasus.server,
@@ -62,8 +62,9 @@ rocksdb::Env *NewEncryptedEnv()
{
// Create an encryption provider.
std::shared_ptr<rocksdb::EncryptionProvider> provider;
- auto provider_id =
- fmt::format("AES:{},{}", FLAGS_server_key_for_testing,
FLAGS_encryption_method);
+ auto provider_id = fmt::format("id=AES;hex_instance_key={};method={}",
+ FLAGS_server_key_for_testing,
+ FLAGS_encryption_method);
auto s = rocksdb::EncryptionProvider::CreateFromString(
rocksdb::ConfigOptions(), provider_id, &provider);
CHECK(s.ok(), "Failed to create encryption provider: {}", s.ToString());
diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt
index 6eaef7f7d..4f00ccbf4 100644
--- a/thirdparty/CMakeLists.txt
+++ b/thirdparty/CMakeLists.txt
@@ -372,15 +372,7 @@ ExternalProject_Add(jemalloc
option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 =
baseline CPU" 0)
-# This is the commit "Update the status badge to pegasus-kv/rocksdb's own
(#18) " on
-# branch v8.5.3-pegasus-encrypt of https://github.com/pegasus-kv/rocksdb.git.
-# The v8.5.3-pegasus-encrypt branch is based on the v8.5.3 tag of
facebook/rocksdb and add
-# the encryption feature.
-ExternalProject_Add(rocksdb
- URL
https://github.com/pegasus-kv/rocksdb/archive/e72106597e6b7924485cadc2433b66029f1aaf17.zip
- URL_MD5 6f6daef703586ce788643bbb8a984fff
- DEPENDS jemalloc
- CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
+set(ROCKSDB_OPTIONS
-DFAIL_ON_WARNINGS=OFF
-DWITH_BENCHMARK_TOOLS=OFF
-DWITH_TOOLS=OFF
@@ -391,14 +383,29 @@ ExternalProject_Add(rocksdb
-DWITH_TESTS=OFF
-DWITH_GFLAGS=OFF
-DUSE_RTTI=ON
- -DWITH_OPENSSL=ON # enable encryption
-DCMAKE_BUILD_TYPE=Release
-DWITH_JEMALLOC=${USE_JEMALLOC}
-DJEMALLOC_ROOT_DIR=${TP_OUTPUT}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
- -DPORTABLE=${ROCKSDB_PORTABLE}
- )
+ -DPORTABLE=${ROCKSDB_PORTABLE})
+# Link error on MacOS, disable building encfs plugin.
+# See https://github.com/pegasus-kv/encfs/issues/4
+if (NOT APPLE)
+ set(ROCKSDB_OPTIONS
+ ${ROCKSDB_OPTIONS}
+ -DROCKSDB_PLUGINS=encfs)
+endif ()
+ExternalProject_Add(rocksdb
+ URL ${OSS_URL_PREFIX}/rocksdb-v8.5.3.tar.gz
+ https://github.com/facebook/rocksdb/archive/refs/tags/v8.5.3.tar.gz
+ URL_MD5 f03eac50ec958a21a7cb56183afb8fe4
+ PATCH_COMMAND ""
+ COMMAND rm -rf ${TP_DIR}/build/Source/rocksdb/plugin/encfs
+ COMMAND git clone -b main --depth=1
https://github.com/pegasus-kv/encfs.git
${TP_DIR}/build/Source/rocksdb/plugin/encfs
+ DEPENDS googletest jemalloc
+ CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
+ ${ROCKSDB_OPTIONS})
ExternalProject_Add(http-parser
URL ${OSS_URL_PREFIX}/http-parser-2.9.4.zip
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
