This is an automated email from the ASF dual-hosted git repository.
hulk pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new eef51ac9 Add speedb as an alternative to rocksdb (#1792)
eef51ac9 is described below
commit eef51ac9a9d48b4e111c7f98a1a40c31b8bc12d7
Author: Twice <[email protected]>
AuthorDate: Fri Oct 13 12:07:40 2023 +0900
Add speedb as an alternative to rocksdb (#1792)
---
.github/workflows/kvrocks.yaml | 6 +++-
CMakeLists.txt | 5 ++++
NOTICE | 3 +-
cmake/modules/FindJeMalloc.cmake | 1 +
cmake/modules/FindZLIB.cmake | 1 +
cmake/modules/Findlz4.cmake | 1 +
cmake/modules/Findzstd.cmake | 1 +
cmake/speedb.cmake | 60 ++++++++++++++++++++++++++++++++++++++++
tests/cppunit/compact_test.cc | 9 ++++++
9 files changed, 85 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml
index f99dc28a..58dcefe4 100644
--- a/.github/workflows/kvrocks.yaml
+++ b/.github/workflows/kvrocks.yaml
@@ -186,6 +186,10 @@ jobs:
os: ubuntu-20.04
compiler: clang
new_encoding: -DENABLE_NEW_ENCODING=TRUE
+ - name: Ubuntu GCC with speedb enabled
+ os: ubuntu-20.04
+ compiler: gcc
+ with_speedb: -DENABLE_SPEEDB=ON
runs-on: ${{ matrix.os }}
steps:
@@ -228,7 +232,7 @@ jobs:
- name: Build Kvrocks
run: |
./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }}
${{ matrix.without_jemalloc }} ${{ matrix.without_luajit }} \
- ${{ matrix.with_ninja }} ${{ matrix.with_sanitizer }} ${{
matrix.with_openssl }} ${{ matrix.new_encoding }} ${{ env.CMAKE_EXTRA_DEFS }}
+ ${{ matrix.with_ninja }} ${{ matrix.with_sanitizer }} ${{
matrix.with_openssl }} ${{ matrix.new_encoding }} ${{ matrix.with_speedb }} ${{
env.CMAKE_EXTRA_DEFS }}
- name: Setup Coredump
if: ${{ startsWith(matrix.os, 'ubuntu') }}
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4aadc0d0..cbbf8472 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,6 +29,7 @@ option(ENABLE_LUAJIT "enable use of luaJIT instead of lua" ON)
option(ENABLE_OPENSSL "enable openssl to support tls connection" OFF)
option(ENABLE_IPO "enable interprocedural optimization" ON)
option(ENABLE_UNWIND "enable libunwind in glog" ON)
+option(ENABLE_SPEEDB "enable speedb instead of rocksdb" OFF)
set(PORTABLE 0 CACHE STRING "build a portable binary (disable arch-specific
optimizations)")
# TODO: set ENABLE_NEW_ENCODING to ON when we are ready
option(ENABLE_NEW_ENCODING "enable new encoding (#1033) for storing 64bit size
and expire time in milliseconds" OFF)
@@ -127,7 +128,11 @@ include(cmake/lz4.cmake)
include(cmake/zlib.cmake)
include(cmake/zstd.cmake)
include(cmake/tbb.cmake)
+if(ENABLE_SPEEDB)
+include(cmake/speedb.cmake)
+else()
include(cmake/rocksdb.cmake)
+endif()
include(cmake/libevent.cmake)
include(cmake/fmt.cmake)
include(cmake/jsoncons.cmake)
diff --git a/NOTICE b/NOTICE
index fbb975e0..a71b5397 100644
--- a/NOTICE
+++ b/NOTICE
@@ -24,6 +24,7 @@ NB: RocksDB is dual-licensed under both the GPLv2 and Apache
2.0 License.
This product uses it under the Apache 2.0 License.
* oneTBB(https://github.com/oneapi-src/oneTBB)
+* speedb(https://github.com/speedb-io/speedb, alternative to rocksdb)
Files src/common/rocksdb_crc32c.h and src/storage/batch_debugger.h are
modified from RocksDB.
Files src/types/bloom_filter.* are modified from Apache Arrow.
@@ -62,8 +63,8 @@ The following components are provided under the MIT License.
See project link fo
The text of each license is also included in licenses/LICENSE-[project].txt
* fmt(https://github.com/fmtlib/fmt)
-* lua(https://github.com/KvrocksLabs/lua)
* LuaJIT(https://github.com/KvrocksLabs/LuaJIT)
+* lua(https://github.com/KvrocksLabs/lua, alternative to LuaJIT)
================================================================
Boost Software License Version 1.0
diff --git a/cmake/modules/FindJeMalloc.cmake b/cmake/modules/FindJeMalloc.cmake
index 23aa812e..4c9cd807 100644
--- a/cmake/modules/FindJeMalloc.cmake
+++ b/cmake/modules/FindJeMalloc.cmake
@@ -22,4 +22,5 @@ if(jemalloc_SOURCE_DIR)
add_library(JeMalloc::JeMalloc ALIAS jemalloc) # rocksdb use it
install(TARGETS jemalloc EXPORT RocksDBTargets) # export for install(...)
+ install(TARGETS jemalloc EXPORT SpeedbTargets) # export for install(...)
endif()
diff --git a/cmake/modules/FindZLIB.cmake b/cmake/modules/FindZLIB.cmake
index 1fa8982f..930267f3 100644
--- a/cmake/modules/FindZLIB.cmake
+++ b/cmake/modules/FindZLIB.cmake
@@ -25,4 +25,5 @@ if(zlib_SOURCE_DIR)
target_link_libraries(zlib_with_headers INTERFACE zlibstatic)
add_library(ZLIB::ZLIB ALIAS zlib_with_headers)
install(TARGETS zlibstatic zlib_with_headers EXPORT RocksDBTargets) # export
for install(...)
+ install(TARGETS zlibstatic zlib_with_headers EXPORT SpeedbTargets) # export
for install(...)
endif()
diff --git a/cmake/modules/Findlz4.cmake b/cmake/modules/Findlz4.cmake
index 16969e4c..6b152bac 100644
--- a/cmake/modules/Findlz4.cmake
+++ b/cmake/modules/Findlz4.cmake
@@ -22,4 +22,5 @@ if(lz4_SOURCE_DIR)
add_library(lz4::lz4 ALIAS lz4) # rocksdb use it
install(TARGETS lz4 EXPORT RocksDBTargets) # export for install(...)
+ install(TARGETS lz4 EXPORT SpeedbTargets) # export for install(...)
endif()
diff --git a/cmake/modules/Findzstd.cmake b/cmake/modules/Findzstd.cmake
index 1f1dbe11..10e98931 100644
--- a/cmake/modules/Findzstd.cmake
+++ b/cmake/modules/Findzstd.cmake
@@ -22,4 +22,5 @@ if(zstd_SOURCE_DIR)
add_library(zstd::zstd ALIAS zstd) # rocksdb use it
install(TARGETS zstd EXPORT RocksDBTargets) # export for install(...)
+ install(TARGETS zstd EXPORT SpeedbTargets) # export for install(...)
endif()
diff --git a/cmake/speedb.cmake b/cmake/speedb.cmake
new file mode 100644
index 00000000..4d172c63
--- /dev/null
+++ b/cmake/speedb.cmake
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+include_guard()
+
+set(COMPILE_WITH_JEMALLOC ON)
+
+if (DISABLE_JEMALLOC)
+ set(COMPILE_WITH_JEMALLOC OFF)
+endif()
+
+include(cmake/utils.cmake)
+
+FetchContent_DeclareGitHubWithMirror(speedb
+ speedb-io/speedb speedb/v2.6.0
+ MD5=caf1bb6f67c79455a8da3c9986ceacc2
+)
+
+FetchContent_GetProperties(jemalloc)
+FetchContent_GetProperties(snappy)
+FetchContent_GetProperties(tbb)
+
+FetchContent_MakeAvailableWithArgs(speedb
+ CMAKE_MODULE_PATH=${PROJECT_SOURCE_DIR}/cmake/modules # to locate
FindJeMalloc.cmake
+ Snappy_DIR=${PROJECT_SOURCE_DIR}/cmake/modules # to locate SnappyConfig.cmake
+ FAIL_ON_WARNINGS=OFF
+ WITH_TESTS=OFF
+ WITH_BENCHMARK_TOOLS=OFF
+ WITH_CORE_TOOLS=OFF
+ WITH_TOOLS=OFF
+ WITH_SNAPPY=ON
+ WITH_LZ4=ON
+ WITH_ZLIB=ON
+ WITH_ZSTD=ON
+ WITH_TOOLS=OFF
+ WITH_GFLAGS=OFF
+ WITH_TBB=ON
+ USE_RTTI=ON
+ ROCKSDB_BUILD_SHARED=OFF
+ WITH_JEMALLOC=${COMPILE_WITH_JEMALLOC}
+ PORTABLE=${PORTABLE}
+)
+
+add_library(rocksdb_with_headers INTERFACE)
+target_include_directories(rocksdb_with_headers INTERFACE
${speedb_SOURCE_DIR}/include)
+target_link_libraries(rocksdb_with_headers INTERFACE speedb)
diff --git a/tests/cppunit/compact_test.cc b/tests/cppunit/compact_test.cc
index 772247b1..42ae3907 100644
--- a/tests/cppunit/compact_test.cc
+++ b/tests/cppunit/compact_test.cc
@@ -48,8 +48,14 @@ TEST(Compact, Filter) {
usleep(10000);
hash->Set(live_hash_key, "f1", "v1", &ret);
hash->Set(live_hash_key, "f2", "v2", &ret);
+
auto status = storage->Compact(nullptr, nullptr);
assert(status.ok());
+ // Compact twice to workaround issue fixed by:
https://github.com/facebook/rocksdb/pull/11468
+ // before rocksdb/speedb 8.1.1. This line can be removed after speedb
upgraded above 8.1.1.
+ status = storage->Compact(nullptr, nullptr);
+ assert(status.ok());
+
rocksdb::DB* db = storage->GetDB();
rocksdb::ReadOptions read_options;
read_options.snapshot = db->GetSnapshot();
@@ -78,6 +84,9 @@ TEST(Compact, Filter) {
st = zset->Expire(expired_zset_key, 1); // expired
usleep(10000);
+ // Same as the above compact, need to compact twice here
+ status = storage->Compact(nullptr, nullptr);
+ assert(status.ok());
status = storage->Compact(nullptr, nullptr);
assert(status.ok());
