This is an automated email from the ASF dual-hosted git repository.
junchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-resilientdb.git
The following commit(s) were added to refs/heads/master by this push:
new d34e9f5a MemLens Database Specific Changes (#170)
d34e9f5a is described below
commit d34e9f5a77e8e39d518849cd7d0b8e9bc8c5ab8a
Author: Harish <[email protected]>
AuthorDate: Thu Mar 20 01:31:35 2025 -0700
MemLens Database Specific Changes (#170)
* global and local build config with profiling flags.
* integrated basic memory hook for RSS calculation
* edited script to generate callgraph using gprof
* enabling level db as default storage engine
* systemd telegraf script to stream prometheus node_exporter data to
InfluxDB.
* added lmdb storage engine interface and partially implemented.
* bootstrapping script to run perf tools
* adding process exporter to bootstrapping script
* Added LRU Cache and Observabiity stats
* lry cache controlled using settings
* removing lmdb implementation
* removing unecessary scripts
* Resolving comments for build flags and memory leak
* resolved error. options.block_cache_ integration still not working
* added UT's for TC's. removed logs from stats and changed lru
implementation var names
* reverting configs and removing unwanted files
* removing unwanted tst logs
* refactor: rename GetMetrics to UpdateMetrics and improve cache handling
in ResLevelDB
* feat: add LevelDB block cache support and corresponding tests.
* feat: enhance LevelDB block cache with configurable capacity and update
metrics return type
* refactor: update transaction summary variable names for consistency
* build: optimize compilation flags by adding optimization level
* build: remove debugging flags from kv_service and api_tools builds
* build: remove profiling flags from set_random_data binary
* build: remove pyroscope monitoring from kv service startup script
* build: include string header in lru_cache implementation
---
.bazelrc | 2 +-
.gitignore | 7 ++
WORKSPACE | 16 +++
chain/storage/BUILD | 16 +++
chain/storage/kv_storage_test.cpp | 21 +++-
chain/storage/leveldb.cpp | 56 +++++++++-
chain/storage/leveldb.h | 8 ++
chain/storage/leveldb_test.cpp | 117 ++++++++++++++++++++
chain/storage/proto/leveldb_config.proto | 2 +
{chain/storage => common/lru}/BUILD | 49 ++-------
common/lru/lru_cache.cpp | 122 +++++++++++++++++++++
.../leveldb_config.proto => common/lru/lru_cache.h | 37 +++++--
common/lru/lru_cache_test.cpp | 94 ++++++++++++++++
platform/statistic/BUILD | 1 +
platform/statistic/set_random_data.cpp | 16 ++-
platform/statistic/stats.cpp | 71 +++++++++++-
platform/statistic/stats.h | 12 ++
service/kv/BUILD | 3 +-
service/kv/kv_service.cpp | 7 +-
service/tools/config/server/server.config | 2 +
.../kv/server_tools/start_kv_service_monitoring.sh | 3 +-
21 files changed, 590 insertions(+), 72 deletions(-)
diff --git a/.bazelrc b/.bazelrc
index 0566a7c9..cc3d1af5 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -1,4 +1,4 @@
-build --cxxopt='-std=c++17' --copt=-O3 --jobs=40
+build --cxxopt='-std=c++17' --copt=-O3 --jobs=40
#build --action_env=PYTHON_BIN_PATH="/usr/bin/python3.10"
#build --action_env=PYTHON_LIB_PATH="/usr/include/python3.10"
diff --git a/.gitignore b/.gitignore
index 2595903f..08418127 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,10 @@ sdk_validator/venv
__pycache__
MODULE.*
apache_release
+*.out.*
+*.data.*
+*.pb.*
+.cache/
+resdb/
+100*_db/
+gmon.out
\ No newline at end of file
diff --git a/WORKSPACE b/WORKSPACE
index 0b898a36..96e8522d 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,6 +1,22 @@
workspace(name = "com_resdb_nexres")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+ name = "hedron_compile_commands",
+ #Replace the commit hash (4f28899228fb3ad0126897876f147ca15026151e) with
the latest commit hash from the repo
+ url =
"https://github.com/hedronvision/bazel-compile-commands-extractor/archive/4f28899228fb3ad0126897876f147ca15026151e.tar.gz";,
+ strip_prefix =
"bazel-compile-commands-extractor-4f28899228fb3ad0126897876f147ca15026151e",
+)
+load("@hedron_compile_commands//:workspace_setup.bzl",
"hedron_compile_commands_setup")
+hedron_compile_commands_setup()
+load("@hedron_compile_commands//:workspace_setup_transitive.bzl",
"hedron_compile_commands_setup_transitive")
+hedron_compile_commands_setup_transitive()
+load("@hedron_compile_commands//:workspace_setup_transitive_transitive.bzl",
"hedron_compile_commands_setup_transitive_transitive")
+hedron_compile_commands_setup_transitive_transitive()
+load("@hedron_compile_commands//:workspace_setup_transitive_transitive_transitive.bzl",
"hedron_compile_commands_setup_transitive_transitive_transitive")
+hedron_compile_commands_setup_transitive_transitive_transitive()
+
load("//:repositories.bzl", "nexres_repositories")
nexres_repositories()
diff --git a/chain/storage/BUILD b/chain/storage/BUILD
index 19108fa9..a86c2a7f 100644
--- a/chain/storage/BUILD
+++ b/chain/storage/BUILD
@@ -52,6 +52,8 @@ cc_library(
"//chain/storage/proto:kv_cc_proto",
"//chain/storage/proto:leveldb_config_cc_proto",
"//common:comm",
+ "//common/lru:lru_cache",
+ "//platform/statistic:stats",
"//third_party:leveldb",
],
)
@@ -64,4 +66,18 @@ cc_test(
":memory_db",
"//common/test:test_main",
],
+ timeout = "short", # Set the timeout to "short"
+ size = "small", # Set the size to "small"
)
+
+cc_test(
+ name = "leveldb_test",
+ srcs = ["leveldb_test.cpp"],
+ deps = [
+ ":leveldb",
+ "//platform/statistic:stats",
+ "//common/test:test_main",
+ ],
+ timeout = "short", # Set the timeout to "short"
+ size = "small", # Set the size to "small"
+)
\ No newline at end of file
diff --git a/chain/storage/kv_storage_test.cpp
b/chain/storage/kv_storage_test.cpp
index fd8c4bfd..2e55161f 100644
--- a/chain/storage/kv_storage_test.cpp
+++ b/chain/storage/kv_storage_test.cpp
@@ -30,10 +30,7 @@ namespace resdb {
namespace storage {
namespace {
-enum StorageType {
- MEM = 0,
- LEVELDB = 1,
-};
+enum StorageType { MEM = 0, LEVELDB = 1, LEVELDB_WITH_BLOCK_CACHE = 2 };
class KVStorageTest : public ::testing::TestWithParam<StorageType> {
protected:
@@ -47,6 +44,12 @@ class KVStorageTest : public
::testing::TestWithParam<StorageType> {
Reset();
storage = NewResLevelDB(path_);
break;
+ case LEVELDB_WITH_BLOCK_CACHE:
+ Reset();
+ LevelDBInfo config;
+ config.set_enable_block_cache(true);
+ storage = NewResLevelDB(path_, config);
+ break;
}
}
@@ -218,8 +221,16 @@ TEST_P(KVStorageTest, GetHistory) {
}
}
+TEST_P(KVStorageTest, BlockCacheSpecificTest) {
+ if (GetParam() == LEVELDB_WITH_BLOCK_CACHE) {
+ std::cout << "Running BlockCacheSpecificTest for LEVELDB_WITH_BLOCK_CACHE"
+ << std::endl;
+ }
+}
+
INSTANTIATE_TEST_CASE_P(KVStorageTest, KVStorageTest,
- ::testing::Values(MEM, LEVELDB));
+ ::testing::Values(MEM, LEVELDB,
+ LEVELDB_WITH_BLOCK_CACHE));
} // namespace
} // namespace storage
diff --git a/chain/storage/leveldb.cpp b/chain/storage/leveldb.cpp
index 2cc96fb5..f6122b5a 100644
--- a/chain/storage/leveldb.cpp
+++ b/chain/storage/leveldb.cpp
@@ -20,8 +20,12 @@
#include "chain/storage/leveldb.h"
#include <glog/logging.h>
+#include <unistd.h>
+
+#include <cstdint>
#include "chain/storage/proto/kv.pb.h"
+#include "leveldb/options.h"
namespace resdb {
namespace storage {
@@ -50,6 +54,16 @@ ResLevelDB::ResLevelDB(std::optional<LevelDBInfo> config) {
path = (*config).path();
}
}
+ if ((*config).enable_block_cache()) {
+ uint32_t capacity = 1000;
+ if ((*config).has_block_cache_capacity()) {
+ capacity = (*config).block_cache_capacity();
+ }
+ block_cache_ =
+ std::make_unique<LRUCache<std::string, std::string>>(capacity);
+ LOG(ERROR) << "initialized block cache" << std::endl;
+ }
+ global_stats_ = Stats::GetGlobalStats();
CreateDB(path);
}
@@ -74,15 +88,22 @@ ResLevelDB::~ResLevelDB() {
if (db_) {
db_.reset();
}
+ if (block_cache_) {
+ block_cache_->Flush();
+ }
}
int ResLevelDB::SetValue(const std::string& key, const std::string& value) {
+ if (block_cache_) {
+ block_cache_->Put(key, value);
+ }
batch_.Put(key, value);
if (batch_.ApproximateSize() >= write_batch_size_) {
leveldb::Status status = db_->Write(leveldb::WriteOptions(), &batch_);
if (status.ok()) {
batch_.Clear();
+ UpdateMetrics();
return 0;
} else {
LOG(ERROR) << "flush buffer fail:" << status.ToString();
@@ -93,13 +114,23 @@ int ResLevelDB::SetValue(const std::string& key, const
std::string& value) {
}
std::string ResLevelDB::GetValue(const std::string& key) {
- std::string value = "";
- leveldb::Status status = db_->Get(leveldb::ReadOptions(), key, &value);
- if (status.ok()) {
- return value;
- } else {
- return "";
+ std::string value;
+ bool found_in_cache = false;
+
+ if (block_cache_) {
+ value = block_cache_->Get(key);
+ found_in_cache = !value.empty();
}
+
+ if (!found_in_cache) {
+ leveldb::Status status = db_->Get(leveldb::ReadOptions(), key, &value);
+ if (!status.ok()) {
+ value.clear(); // Ensure value is empty if not found in DB
+ }
+ }
+
+ UpdateMetrics();
+ return value;
}
std::string ResLevelDB::GetAllValues(void) {
@@ -134,6 +165,19 @@ std::string ResLevelDB::GetRange(const std::string&
min_key,
return values;
}
+bool ResLevelDB::UpdateMetrics() {
+ if (block_cache_ == nullptr) {
+ return false;
+ }
+ std::string stats;
+ std::string approximate_size;
+ db_->GetProperty("leveldb.stats", &stats);
+ db_->GetProperty("leveldb.approximate-memory-usage", &approximate_size);
+ global_stats_->SetStorageEngineMetrics(block_cache_->GetCacheHitRatio(),
+ stats, approximate_size);
+ return true;
+}
+
bool ResLevelDB::Flush() {
leveldb::Status status = db_->Write(leveldb::WriteOptions(), &batch_);
if (status.ok()) {
diff --git a/chain/storage/leveldb.h b/chain/storage/leveldb.h
index 199f1f27..6103ec20 100644
--- a/chain/storage/leveldb.h
+++ b/chain/storage/leveldb.h
@@ -25,8 +25,10 @@
#include "chain/storage/proto/leveldb_config.pb.h"
#include "chain/storage/storage.h"
+#include "common/lru/lru_cache.h"
#include "leveldb/db.h"
#include "leveldb/write_batch.h"
+#include "platform/statistic/stats.h"
namespace resdb {
namespace storage {
@@ -65,6 +67,8 @@ class ResLevelDB : public Storage {
std::vector<std::pair<std::string, int>> GetTopHistory(
const std::string& key, int top_number) override;
+ bool UpdateMetrics();
+
bool Flush() override;
private:
@@ -75,6 +79,10 @@ class ResLevelDB : public Storage {
::leveldb::WriteBatch batch_;
unsigned int write_buffer_size_ = 64 << 20;
unsigned int write_batch_size_ = 1;
+
+ protected:
+ Stats* global_stats_ = nullptr;
+ std::unique_ptr<LRUCache<std::string, std::string>> block_cache_;
};
} // namespace storage
diff --git a/chain/storage/leveldb_test.cpp b/chain/storage/leveldb_test.cpp
new file mode 100644
index 00000000..87e36d81
--- /dev/null
+++ b/chain/storage/leveldb_test.cpp
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "chain/storage/leveldb.h"
+
+#include <glog/logging.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <filesystem>
+
+namespace resdb {
+namespace storage {
+namespace {
+
+enum class CacheConfig { DISABLED, ENABLED };
+
+class TestableResLevelDB : public ResLevelDB {
+ public:
+ using ResLevelDB::block_cache_;
+ using ResLevelDB::global_stats_;
+ using ResLevelDB::ResLevelDB;
+};
+
+class LevelDBTest : public ::testing::TestWithParam<CacheConfig> {
+ protected:
+ LevelDBTest() {
+ LevelDBInfo config;
+ if (GetParam() == CacheConfig::ENABLED) {
+ config.set_enable_block_cache(true);
+ config.set_block_cache_capacity(1000);
+ } else {
+ Reset();
+ }
+ storage = std::make_unique<TestableResLevelDB>(config);
+ }
+
+ protected:
+ std::unique_ptr<TestableResLevelDB> storage;
+ std::string path_ = "/tmp/leveldb_test";
+
+ private:
+ void Reset() { std::filesystem::remove_all(path_.c_str()); }
+};
+
+TEST_P(LevelDBTest, BlockCacheEnabled) {
+ if (GetParam() == CacheConfig::ENABLED) {
+ EXPECT_TRUE(storage->block_cache_ != nullptr);
+ EXPECT_TRUE(storage->block_cache_->GetCapacity() == 1000);
+ } else {
+ EXPECT_TRUE(storage->block_cache_ == nullptr);
+ EXPECT_FALSE(storage->UpdateMetrics());
+ }
+}
+
+TEST_P(LevelDBTest, AddValueAndCheckCache) {
+ if (GetParam() == CacheConfig::ENABLED) {
+ // Add a value
+ std::string key = "test_key";
+ std::string value = "test_value";
+ EXPECT_EQ(storage->SetValue(key, value), 0);
+
+ // Check if CacheHit is incremented in the Stats class
+ EXPECT_TRUE(storage->block_cache_->Get(key) == "test_value");
+ EXPECT_EQ(storage->block_cache_->GetCacheHits(), 1);
+ }
+}
+
+TEST_P(LevelDBTest, CacheEvictionPolicy) {
+ if (GetParam() == CacheConfig::ENABLED) {
+ // Insert 1000 values
+ for (int i = 1; i <= 1000; ++i) {
+ std::string key = "key_" + std::to_string(i);
+ std::string value = "value_" + std::to_string(i);
+ EXPECT_EQ(storage->SetValue(key, value), 0);
+ }
+
+ // Insert the 1001st value
+ std::string key_1001 = "key_1001";
+ std::string value_1001 = "value_1001";
+ EXPECT_EQ(storage->SetValue(key_1001, value_1001), 0);
+
+ // Check that the 1001st value is not present in the cache
+ EXPECT_TRUE(storage->GetValue("key_1") == "value_1");
+ EXPECT_EQ(storage->block_cache_->GetCacheMisses(), 1);
+
+ // Expect key_2 to be present in cache and hence a cache hit
+ EXPECT_TRUE(storage->GetValue("key_2") == "value_2");
+ EXPECT_EQ(storage->block_cache_->GetCacheHits(), 1);
+
+ EXPECT_TRUE(storage->UpdateMetrics());
+ }
+}
+
+INSTANTIATE_TEST_CASE_P(LevelDBTest, LevelDBTest,
+ ::testing::Values(CacheConfig::ENABLED,
+ CacheConfig::DISABLED));
+
+} // namespace
+} // namespace storage
+} // namespace resdb
\ No newline at end of file
diff --git a/chain/storage/proto/leveldb_config.proto
b/chain/storage/proto/leveldb_config.proto
index d8be2519..28e5a188 100644
--- a/chain/storage/proto/leveldb_config.proto
+++ b/chain/storage/proto/leveldb_config.proto
@@ -25,4 +25,6 @@ message LevelDBInfo {
uint32 write_buffer_size_mb = 2;
uint32 write_batch_size = 3;
string path = 4;
+ optional bool enable_block_cache = 5;
+ optional uint32 block_cache_capacity = 6;
}
diff --git a/chain/storage/BUILD b/common/lru/BUILD
similarity index 55%
copy from chain/storage/BUILD
copy to common/lru/BUILD
index 19108fa9..ad72bd98 100644
--- a/chain/storage/BUILD
+++ b/common/lru/BUILD
@@ -19,49 +19,18 @@
package(default_visibility = ["//visibility:public"])
cc_library(
- name = "storage",
- hdrs = ["storage.h"],
- deps = [
- ],
-)
-
-cc_library(
- name = "mock_storage",
- hdrs = ["mock_storage.h"],
- deps = [
- ":storage",
- ],
-)
-
-cc_library(
- name = "memory_db",
- srcs = ["memory_db.cpp"],
- hdrs = ["memory_db.h"],
- deps = [
- ":storage",
- "//common:comm",
- ],
-)
-
-cc_library(
- name = "leveldb",
- srcs = ["leveldb.cpp"],
- hdrs = ["leveldb.h"],
- deps = [
- ":storage",
- "//chain/storage/proto:kv_cc_proto",
- "//chain/storage/proto:leveldb_config_cc_proto",
- "//common:comm",
- "//third_party:leveldb",
- ],
+ name = "lru_cache",
+ srcs = ["lru_cache.cpp"],
+ hdrs = ["lru_cache.h"],
)
cc_test(
- name = "kv_storage_test",
- srcs = ["kv_storage_test.cpp"],
+ name = "lru_cache_test",
+ srcs = ["lru_cache_test.cpp"],
deps = [
- ":leveldb",
- ":memory_db",
+ "//common/lru:lru_cache",
"//common/test:test_main",
],
-)
+ timeout = "short", # Set the timeout to "short"
+ size = "small", # Set the size to "small"
+)
\ No newline at end of file
diff --git a/common/lru/lru_cache.cpp b/common/lru/lru_cache.cpp
new file mode 100644
index 00000000..75033d46
--- /dev/null
+++ b/common/lru/lru_cache.cpp
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "lru_cache.h"
+
+#include "string"
+
+namespace resdb {
+
+template <typename KeyType, typename ValueType>
+LRUCache<KeyType, ValueType>::LRUCache(int capacity) {
+ capacity_ = capacity;
+ cache_hits_ = 0;
+ cache_misses_ = 0;
+}
+
+template <typename KeyType, typename ValueType>
+LRUCache<KeyType, ValueType>::~LRUCache() {
+ lookup_.clear();
+ key_list_.clear();
+ rlookup_.clear();
+}
+
+template <typename KeyType, typename ValueType>
+ValueType LRUCache<KeyType, ValueType>::Get(KeyType key) {
+ if (lookup_.find(key) == lookup_.end()) {
+ cache_misses_++;
+ return ValueType();
+ }
+
+ // Move accessed key to front of key list. This marks the key as used
+ key_list_.splice(key_list_.begin(), key_list_, rlookup_[key]);
+
+ cache_hits_++;
+ return lookup_[key];
+}
+
+template <typename KeyType, typename ValueType>
+void LRUCache<KeyType, ValueType>::Put(KeyType key, ValueType value) {
+ if (lookup_.find(key) == lookup_.end()) {
+ if (key_list_.size() == capacity_) {
+ KeyType lru_key = key_list_.back();
+ key_list_.pop_back();
+ lookup_.erase(lru_key);
+ rlookup_.erase(lru_key);
+ }
+ key_list_.push_front(key);
+ rlookup_[key] = key_list_.begin();
+ } else {
+ key_list_.splice(key_list_.begin(), key_list_, rlookup_[key]);
+ }
+ lookup_[key] = value; // Set the lookup_ here
+}
+
+template <typename KeyType, typename ValueType>
+void LRUCache<KeyType, ValueType>::SetCapacity(int new_capacity) {
+ if (new_capacity < capacity_) {
+ while (key_list_.size() > new_capacity) {
+ KeyType lru_key = key_list_.back();
+ key_list_.pop_back();
+ lookup_.erase(lru_key);
+ rlookup_.erase(lru_key);
+ }
+ }
+ capacity_ = new_capacity;
+}
+
+template <typename KeyType, typename ValueType>
+int LRUCache<KeyType, ValueType>::GetCapacity() {
+ return capacity_;
+}
+
+template <typename KeyType, typename ValueType>
+void LRUCache<KeyType, ValueType>::Flush() {
+ lookup_.clear();
+ key_list_.clear();
+ rlookup_.clear();
+ cache_hits_ = 0;
+ cache_misses_ = 0;
+}
+
+template <typename KeyType, typename ValueType>
+int LRUCache<KeyType, ValueType>::GetCacheHits() const {
+ return cache_hits_;
+}
+
+template <typename KeyType, typename ValueType>
+int LRUCache<KeyType, ValueType>::GetCacheMisses() const {
+ return cache_misses_;
+}
+
+template <typename KeyType, typename ValueType>
+double LRUCache<KeyType, ValueType>::GetCacheHitRatio() const {
+ int total_accesses = cache_hits_ + cache_misses_;
+ if (total_accesses == 0) {
+ return 0.0;
+ }
+ return static_cast<double>(cache_hits_) / total_accesses;
+}
+
+template class LRUCache<int, int>;
+template class LRUCache<std::string, int>;
+template class LRUCache<int, std::string>;
+template class LRUCache<std::string, std::string>;
+
+} // namespace resdb
\ No newline at end of file
diff --git a/chain/storage/proto/leveldb_config.proto b/common/lru/lru_cache.h
similarity index 50%
copy from chain/storage/proto/leveldb_config.proto
copy to common/lru/lru_cache.h
index d8be2519..aa4a17dc 100644
--- a/chain/storage/proto/leveldb_config.proto
+++ b/common/lru/lru_cache.h
@@ -17,12 +17,35 @@
* under the License.
*/
-syntax = "proto3";
+#include <list>
+#include <unordered_map>
-package resdb.storage;
+namespace resdb {
-message LevelDBInfo {
- uint32 write_buffer_size_mb = 2;
- uint32 write_batch_size = 3;
- string path = 4;
-}
+template <typename KeyType, typename ValueType>
+class LRUCache {
+ public:
+ LRUCache(int capacity);
+ ~LRUCache();
+
+ ValueType Get(KeyType key);
+ void Put(KeyType key, ValueType value);
+ int GetCapacity();
+ void SetCapacity(int new_capacity);
+ void Flush();
+ int GetCacheHits() const;
+ int GetCacheMisses() const;
+ double GetCacheHitRatio() const;
+
+ private:
+ int capacity_;
+ int cache_hits_;
+ int cache_misses_;
+ std::list<KeyType> key_list_; // Doubly linked list to store keys
+ std::unordered_map<KeyType, ValueType>
+ lookup_; // Hash map for key-value pairs
+ std::unordered_map<KeyType, typename std::list<KeyType>::iterator>
+ rlookup_; // Hash map for key-iterator pairs
+};
+
+} // namespace resdb
diff --git a/common/lru/lru_cache_test.cpp b/common/lru/lru_cache_test.cpp
new file mode 100644
index 00000000..aad569f0
--- /dev/null
+++ b/common/lru/lru_cache_test.cpp
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "common/lru/lru_cache.h"
+
+#include <gtest/gtest.h>
+
+namespace resdb {
+namespace {
+
+class LRUCacheTest : public ::testing::Test {
+ protected:
+ LRUCache<int, int> cache_{3};
+};
+
+TEST_F(LRUCacheTest, TestPutAndGet) {
+ cache_.Put(1, 100);
+ EXPECT_EQ(cache_.Get(1), 100);
+ EXPECT_EQ(cache_.GetCacheHits(), 1);
+ EXPECT_EQ(cache_.GetCacheMisses(), 0);
+}
+
+TEST_F(LRUCacheTest, TestUpdateValue) {
+ cache_.Put(1, 100);
+ cache_.Put(1, 200);
+ EXPECT_EQ(cache_.Get(1), 200);
+ EXPECT_EQ(cache_.GetCacheHits(), 1);
+ EXPECT_EQ(cache_.GetCacheMisses(), 0);
+}
+
+TEST_F(LRUCacheTest, TestEviction) {
+ cache_.Put(1, 100);
+ cache_.Put(2, 200);
+ cache_.Put(3, 300);
+ cache_.Put(4, 400); // This should evict key 1
+
+ EXPECT_EQ(cache_.Get(1), 0); // Key 1 should be evicted
+ EXPECT_EQ(cache_.Get(2), 200);
+ EXPECT_EQ(cache_.Get(3), 300);
+ EXPECT_EQ(cache_.Get(4), 400);
+
+ EXPECT_EQ(cache_.GetCacheHits(), 3);
+ EXPECT_EQ(cache_.GetCacheMisses(), 1);
+}
+
+TEST_F(LRUCacheTest, TestUsageOrder) {
+ cache_.Put(1, 100);
+ cache_.Put(2, 200);
+ cache_.Put(3, 300);
+
+ // Access key 1 to update its usage order
+ EXPECT_EQ(cache_.Get(1), 100);
+
+ // Add a new key, which should evict key 2 (least recently used)
+ cache_.Put(4, 400);
+
+ EXPECT_EQ(cache_.Get(1), 100);
+ EXPECT_EQ(cache_.Get(2), 0); // Key 2 should be evicted
+ EXPECT_EQ(cache_.Get(3), 300);
+ EXPECT_EQ(cache_.Get(4), 400);
+
+ EXPECT_EQ(cache_.GetCacheHits(), 4);
+ EXPECT_EQ(cache_.GetCacheMisses(), 1);
+}
+
+TEST_F(LRUCacheTest, TestFlush) {
+ cache_.Put(1, 100);
+ cache_.Put(2, 200);
+ cache_.Flush();
+
+ EXPECT_EQ(cache_.Get(1), 0);
+ EXPECT_EQ(cache_.Get(2), 0);
+ EXPECT_EQ(cache_.GetCacheHits(), 0);
+ EXPECT_EQ(cache_.GetCacheMisses(), 2);
+}
+
+} // namespace
+} // namespace resdb
diff --git a/platform/statistic/BUILD b/platform/statistic/BUILD
index e814f9ff..3094840b 100644
--- a/platform/statistic/BUILD
+++ b/platform/statistic/BUILD
@@ -19,6 +19,7 @@
package(default_visibility = [
"//platform:__subpackages__",
"//service:__subpackages__",
+ "//chain/storage:__subpackages__",
])
cc_library(
diff --git a/platform/statistic/set_random_data.cpp
b/platform/statistic/set_random_data.cpp
index 0b936f58..fba5e327 100644
--- a/platform/statistic/set_random_data.cpp
+++ b/platform/statistic/set_random_data.cpp
@@ -27,6 +27,7 @@
#include <chrono>
#include <fstream>
#include <iostream>
+#include <sstream>
#include <string>
std::string Exec(const char* cmd) {
@@ -63,12 +64,13 @@ int main(int argc, char** argv) {
if (command == "test") {
for (int i = 0; i < std::stoi(value); i++) {
- usleep(1000000);
- std::string test =
- "/home/jyu25/nexres/bazel-bin/example/kv_server_tools "
- "/home/jyu25/nexres/example/kv_client_config.config set " +
- std::to_string(std::rand() % 500) + " " +
- std::to_string(std::rand() % 500);
+ std::stringstream ss;
+ ss << " bazel-bin/service/tools/kv/api_tools/kv_service_tools --config "
+ "service/tools/config/interface/service.config --cmd set "
+ << "--key key" << (std::rand() % 500) << " " << "--value value"
+ << (std::rand() % 500);
+
+ std::string test = ss.str();
output = Exec(test.c_str());
std::cout << i << " " << output << std::endl;
}
@@ -83,4 +85,6 @@ int main(int argc, char** argv) {
std::cout << ++count << " " << output << std::endl;
}
}
+
+ return 0;
}
\ No newline at end of file
diff --git a/platform/statistic/stats.cpp b/platform/statistic/stats.cpp
index 200ea527..21f1524f 100644
--- a/platform/statistic/stats.cpp
+++ b/platform/statistic/stats.cpp
@@ -37,7 +37,7 @@ Stats* Stats::GetGlobalStats(int seconds) {
std::unique_lock<std::mutex> lk(g_mutex);
static Stats stats(seconds);
return &stats;
-}
+} // gets a singelton instance of Stats Class
Stats::Stats(int sleep_time) {
monitor_sleep_time_ = sleep_time;
@@ -96,6 +96,30 @@ Stats::~Stats() {
}
}
+int64_t GetRSS() {
+ int64_t rss = 0;
+ FILE* fp = NULL;
+ if ((fp = fopen("/proc/self/statm", "r")) == NULL) {
+ return 0;
+ }
+
+ uint64_t size, resident, share, text, lib, data, dt;
+ if (fscanf(fp, "%lu %lu %lu %lu %lu %lu %lu", &size, &resident, &share,
&text,
+ &lib, &data, &dt) != 7) {
+ fclose(fp);
+ return 0;
+ }
+ fclose(fp);
+
+ int64_t page_size = sysconf(_SC_PAGESIZE);
+ rss = resident * page_size;
+
+ // Convert to MB
+ rss = rss / (1024 * 1024);
+
+ return rss;
+}
+
void Stats::CrowRoute() {
crow::SimpleApp app;
while (!stop_) {
@@ -151,6 +175,41 @@ void Stats::CrowRoute() {
res.body = "Success";
res.end();
});
+ CROW_ROUTE(app, "/transaction_data")
+ .methods("GET"_method)([this](const crow::request& req,
+ crow::response& res) {
+ LOG(ERROR) << "API 4";
+ res.set_header("Access-Control-Allow-Origin",
+ "*"); // Allow requests from any origin
+ res.set_header("Access-Control-Allow-Methods",
+ "GET, POST, OPTIONS"); // Specify allowed methods
+ res.set_header(
+ "Access-Control-Allow-Headers",
+ "Content-Type, Authorization"); // Specify allowed headers
+
+ nlohmann::json mem_view_json;
+ int status =
+ getrusage(RUSAGE_SELF, &transaction_summary_.process_stats_);
+ if (status == 0) {
+ mem_view_json["resident_set_size"] = GetRSS();
+ mem_view_json["max_resident_set_size"] =
+ transaction_summary_.process_stats_.ru_maxrss;
+ mem_view_json["num_reads"] =
+ transaction_summary_.process_stats_.ru_inblock;
+ mem_view_json["num_writes"] =
+ transaction_summary_.process_stats_.ru_oublock;
+ }
+
+ mem_view_json["ext_cache_hit_ratio"] =
+ transaction_summary_.ext_cache_hit_ratio_;
+ mem_view_json["level_db_stats"] =
+ transaction_summary_.level_db_stats_;
+ mem_view_json["level_db_approx_mem_size"] =
+ transaction_summary_.level_db_approx_mem_size_;
+ res.body = mem_view_json.dump();
+ mem_view_json.clear();
+ res.end();
+ });
app.port(8500 + transaction_summary_.port).multithreaded().run();
sleep(1);
} catch (const std::exception& e) {
@@ -182,6 +241,14 @@ void Stats::SetPrimaryId(int primary_id) {
transaction_summary_.primary_id = primary_id;
}
+void Stats::SetStorageEngineMetrics(double ext_cache_hit_ratio,
+ std::string level_db_stats,
+ std::string level_db_approx_mem_size) {
+ transaction_summary_.ext_cache_hit_ratio_ = ext_cache_hit_ratio;
+ transaction_summary_.level_db_stats_ = level_db_stats;
+ transaction_summary_.level_db_approx_mem_size_ = level_db_approx_mem_size;
+}
+
void Stats::RecordStateTime(std::string state) {
if (!enable_resview) {
return;
@@ -275,6 +342,8 @@ void Stats::SendSummary() {
summary_json_["txn_values"].push_back(transaction_summary_.txn_value[i]);
}
+ summary_json_["ext_cache_hit_ratio"] =
+ transaction_summary_.ext_cache_hit_ratio_;
consensus_history_[std::to_string(transaction_summary_.txn_number)] =
summary_json_;
diff --git a/platform/statistic/stats.h b/platform/statistic/stats.h
index 0ca8dd1e..849c83d1 100644
--- a/platform/statistic/stats.h
+++ b/platform/statistic/stats.h
@@ -31,6 +31,7 @@
#include "platform/proto/resdb.pb.h"
#include "platform/statistic/prometheus_handler.h"
#include "proto/kv/kv.pb.h"
+#include "sys/resource.h"
namespace asio = boost::asio;
namespace beast = boost::beast;
@@ -60,6 +61,14 @@ struct VisualData {
std::vector<std::chrono::system_clock::time_point>
commit_message_count_times_list;
std::chrono::system_clock::time_point execution_time;
+
+ // Storage Engine Stats
+ double ext_cache_hit_ratio_;
+ std::string level_db_stats_;
+ std::string level_db_approx_mem_size_;
+
+ // process stats
+ struct rusage process_stats_;
};
class Stats {
@@ -72,6 +81,9 @@ class Stats {
void SetProps(int replica_id, std::string ip, int port, bool resview_flag,
bool faulty_flag);
void SetPrimaryId(int primary_id);
+ void SetStorageEngineMetrics(double ext_cache_hit_ratio,
+ std::string level_db_stats,
+ std::string level_db_approx_mem_size);
void RecordStateTime(std::string state);
void GetTransactionDetails(BatchUserRequest batch_request);
void SendSummary();
diff --git a/service/kv/BUILD b/service/kv/BUILD
index 0e44111f..e6ac6dd4 100644
--- a/service/kv/BUILD
+++ b/service/kv/BUILD
@@ -27,6 +27,7 @@ cc_binary(
copts = select({
"//chain/storage/setting:enable_leveldb_setting": ["-DENABLE_LEVELDB"],
"//conditions:default": [],
+
}),
deps = [
"//platform/config:resdb_config_utils",
@@ -35,7 +36,7 @@ cc_binary(
"//common:comm",
"//proto/kv:kv_cc_proto",
"//chain/storage:memory_db",
- ] + select({
+ ] + select({
"//chain/storage/setting:enable_leveldb_setting":
["//chain/storage:leveldb"],
"//conditions:default": [],
}),
diff --git a/service/kv/kv_service.cpp b/service/kv/kv_service.cpp
index eda6647c..269fb807 100644
--- a/service/kv/kv_service.cpp
+++ b/service/kv/kv_service.cpp
@@ -39,9 +39,8 @@ std::unique_ptr<Storage> NewStorage(const std::string&
db_path,
const ResConfigData& config_data) {
#ifdef ENABLE_LEVELDB
LOG(INFO) << "use leveldb storage.";
- return NewResLevelDB(db_path, config_data);
+ return NewResLevelDB(db_path, config_data.leveldb_info());
#endif
-
LOG(INFO) << "use memory storage.";
return NewMemoryDB();
}
@@ -64,7 +63,7 @@ int main(int argc, char** argv) {
auto monitor_port = Stats::GetGlobalStats(5);
monitor_port->SetPrometheus(grafana_address);
- LOG(ERROR) << "monitoring prot:" << grafana_address;
+ LOG(ERROR) << "monitoring port:" << grafana_address;
}
std::unique_ptr<ResDBConfig> config =
@@ -78,4 +77,4 @@ int main(int argc, char** argv) {
config_file, private_key_file, cert_file,
std::make_unique<KVExecutor>(NewStorage(db_path, config_data)), nullptr);
server->Run();
-}
+}
\ No newline at end of file
diff --git a/service/tools/config/server/server.config
b/service/tools/config/server/server.config
index b9e57d18..740bae3a 100644
--- a/service/tools/config/server/server.config
+++ b/service/tools/config/server/server.config
@@ -26,6 +26,8 @@
leveldb_info : {
write_buffer_size_mb:128,
write_batch_size:1,
+ enable_block_cache: true,
+ block_cache_capacity: 100
},
require_txn_validation:true,
enable_viewchange:false,
diff --git a/service/tools/kv/server_tools/start_kv_service_monitoring.sh
b/service/tools/kv/server_tools/start_kv_service_monitoring.sh
index 7f47a1b6..696f325f 100755
--- a/service/tools/kv/server_tools/start_kv_service_monitoring.sh
+++ b/service/tools/kv/server_tools/start_kv_service_monitoring.sh
@@ -24,7 +24,8 @@ WORK_PATH=$PWD
CERT_PATH=${WORK_PATH}/service/tools/data/cert/
GRAFANA_PORT=8090
-bazel build //service/kv:kv_service $@
+
+bazel build //service/kv:kv_service --define enable_leveldb=True $@
nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node1.key.pri
$CERT_PATH/cert_1.cert 8090 > server0.log &
nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node2.key.pri
$CERT_PATH/cert_2.cert 8091 > server1.log &
nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node3.key.pri
$CERT_PATH/cert_3.cert 8092 > server2.log &
