This is an automated email from the ASF dual-hosted git repository.
gehafearless pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 1c4fcbb2d refactor(meta_store): Reduce the dependency of meta_store
(#1932)
1c4fcbb2d is described below
commit 1c4fcbb2de063c8a264a3db582391993d15d0eaa
Author: Yingchun Lai <[email protected]>
AuthorDate: Thu Mar 7 14:37:27 2024 +0800
refactor(meta_store): Reduce the dependency of meta_store (#1932)
Make class meta_store not derive from class replica_base and move it
to pegasus_base module, then we can reuse it freely.
---
src/{server => base}/meta_store.cpp | 8 ++++----
src/{server => base}/meta_store.h | 14 +++++++++-----
src/server/pegasus_server_impl.cpp | 25 ++++++++++++-------------
src/server/pegasus_server_impl.h | 3 ---
src/server/pegasus_server_impl_init.cpp | 2 +-
src/server/pegasus_write_service_impl.h | 2 +-
src/server/rocksdb_wrapper.cpp | 2 +-
src/server/test/CMakeLists.txt | 2 +-
src/server/test/pegasus_server_impl_test.cpp | 2 +-
9 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/src/server/meta_store.cpp b/src/base/meta_store.cpp
similarity index 96%
rename from src/server/meta_store.cpp
rename to src/base/meta_store.cpp
index db0b9a0a4..94aaf7db7 100644
--- a/src/server/meta_store.cpp
+++ b/src/base/meta_store.cpp
@@ -23,13 +23,13 @@
#include <rocksdb/status.h>
#include "common/replica_envs.h"
-#include "server/pegasus_server_impl.h"
#include "utils/fmt_logging.h"
#include "utils/string_conv.h"
namespace pegasus {
namespace server {
-
+const std::string meta_store::DATA_COLUMN_FAMILY_NAME = "default";
+const std::string meta_store::META_COLUMN_FAMILY_NAME = "pegasus_meta_cf";
const std::string meta_store::DATA_VERSION = "pegasus_data_version";
const std::string meta_store::LAST_FLUSHED_DECREE =
"pegasus_last_flushed_decree";
const std::string meta_store::LAST_MANUAL_COMPACT_FINISH_TIME =
@@ -38,10 +38,10 @@ const std::string
meta_store::ROCKSDB_ENV_USAGE_SCENARIO_NORMAL = "normal";
const std::string meta_store::ROCKSDB_ENV_USAGE_SCENARIO_PREFER_WRITE =
"prefer_write";
const std::string meta_store::ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD =
"bulk_load";
-meta_store::meta_store(pegasus_server_impl *server,
+meta_store::meta_store(const char *log_prefix,
rocksdb::DB *db,
rocksdb::ColumnFamilyHandle *meta_cf)
- : replica_base(server), _db(db), _meta_cf(meta_cf)
+ : _log_prefix(log_prefix), _db(db), _meta_cf(meta_cf)
{
// disable write ahead logging as replication handles logging instead now
_wt_opts.disableWAL = true;
diff --git a/src/server/meta_store.h b/src/base/meta_store.h
similarity index 92%
rename from src/server/meta_store.h
rename to src/base/meta_store.h
index d8744a1cc..060692d5c 100644
--- a/src/server/meta_store.h
+++ b/src/base/meta_store.h
@@ -24,7 +24,6 @@
#include <stdint.h>
#include <string>
-#include "replica/replica_base.h"
#include "utils/error_code.h"
namespace rocksdb {
@@ -35,16 +34,18 @@ class DB;
namespace pegasus {
namespace server {
-class pegasus_server_impl;
-
// Manage meta data of Pegasus, now support
// - pegasus_data_version
// - pegasus_last_flushed_decree
// - pegasus_last_manual_compact_finish_time
-class meta_store : public dsn::replication::replica_base
+class meta_store
{
public:
- meta_store(pegasus_server_impl *server, rocksdb::DB *db,
rocksdb::ColumnFamilyHandle *meta_cf);
+ // Column family names.
+ static const std::string DATA_COLUMN_FAMILY_NAME;
+ static const std::string META_COLUMN_FAMILY_NAME;
+
+ meta_store(const char *log_prefix, rocksdb::DB *db,
rocksdb::ColumnFamilyHandle *meta_cf);
dsn::error_code get_last_flushed_decree(uint64_t *decree) const;
uint64_t get_decree_from_readonly_db(rocksdb::DB *db,
@@ -85,6 +86,8 @@ private:
FRIEND_TEST(pegasus_server_impl_test, test_open_db_with_latest_options);
FRIEND_TEST(pegasus_server_impl_test, test_open_db_with_app_envs);
+ const char *log_prefix() const { return _log_prefix.c_str(); }
+
// Keys of meta data wrote into meta column family.
static const std::string DATA_VERSION;
static const std::string LAST_FLUSHED_DECREE;
@@ -93,6 +96,7 @@ private:
static const std::string ROCKSDB_ENV_USAGE_SCENARIO_PREFER_WRITE;
static const std::string ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD;
+ const std::string _log_prefix;
rocksdb::DB *_db;
rocksdb::ColumnFamilyHandle *_meta_cf;
rocksdb::WriteOptions _wt_opts;
diff --git a/src/server/pegasus_server_impl.cpp
b/src/server/pegasus_server_impl.cpp
index 5c62455cc..2ccf73f1d 100644
--- a/src/server/pegasus_server_impl.cpp
+++ b/src/server/pegasus_server_impl.cpp
@@ -46,6 +46,7 @@
#include "absl/strings/string_view.h"
#include "base/idl_utils.h" // IWYU pragma: keep
+#include "base/meta_store.h"
#include "base/pegasus_key_schema.h"
#include "base/pegasus_utils.h"
#include "base/pegasus_value_schema.h"
@@ -56,7 +57,6 @@
#include "consensus_types.h"
#include "dsn.layer2_types.h"
#include "hotkey_collector.h"
-#include "meta_store.h"
#include "pegasus_rpc_types.h"
#include "pegasus_server_write.h"
#include "replica_admin_types.h"
@@ -142,8 +142,6 @@ std::shared_ptr<rocksdb::WriteBufferManager>
pegasus_server_impl::_s_write_buffe
METRIC_VAR_DEFINE_gauge_int64(rdb_block_cache_mem_usage_bytes,
pegasus_server_impl);
METRIC_VAR_DEFINE_gauge_int64(rdb_write_rate_limiter_through_bytes_per_sec,
pegasus_server_impl);
const std::string pegasus_server_impl::COMPRESSION_HEADER = "per_level:";
-const std::string pegasus_server_impl::DATA_COLUMN_FAMILY_NAME = "default";
-const std::string pegasus_server_impl::META_COLUMN_FAMILY_NAME =
"pegasus_meta_cf";
const std::chrono::seconds pegasus_server_impl::kServerStatUpdateTimeSec =
std::chrono::seconds(10);
// should be same with items in dsn::backup_restore_constant
@@ -1691,7 +1689,7 @@ dsn::error_code pegasus_server_impl::start(int argc, char
**argv)
if (!has_incompatible_db_options) {
for (int i = 0; i < loaded_cf_descs.size(); ++i) {
- if (loaded_cf_descs[i].name == DATA_COLUMN_FAMILY_NAME) {
+ if (loaded_cf_descs[i].name ==
meta_store::DATA_COLUMN_FAMILY_NAME) {
loaded_data_cf_opts = loaded_cf_descs[i].options;
}
}
@@ -1710,7 +1708,8 @@ dsn::error_code pegasus_server_impl::start(int argc, char
**argv)
}
std::vector<rocksdb::ColumnFamilyDescriptor> column_families(
- {{DATA_COLUMN_FAMILY_NAME, _table_data_cf_opts},
{META_COLUMN_FAMILY_NAME, _meta_cf_opts}});
+ {{meta_store::DATA_COLUMN_FAMILY_NAME, _table_data_cf_opts},
+ {meta_store::META_COLUMN_FAMILY_NAME, _meta_cf_opts}});
rocksdb::ConfigOptions config_options;
config_options.ignore_unknown_options = true;
config_options.ignore_unsupported_options = true;
@@ -1730,13 +1729,13 @@ dsn::error_code pegasus_server_impl::start(int argc,
char **argv)
return dsn::ERR_LOCAL_APP_FAILURE;
}
CHECK_EQ_PREFIX(2, handles_opened.size());
- CHECK_EQ_PREFIX(handles_opened[0]->GetName(), DATA_COLUMN_FAMILY_NAME);
- CHECK_EQ_PREFIX(handles_opened[1]->GetName(), META_COLUMN_FAMILY_NAME);
+ CHECK_EQ_PREFIX(handles_opened[0]->GetName(),
meta_store::DATA_COLUMN_FAMILY_NAME);
+ CHECK_EQ_PREFIX(handles_opened[1]->GetName(),
meta_store::META_COLUMN_FAMILY_NAME);
_data_cf = handles_opened[0];
_meta_cf = handles_opened[1];
// Create _meta_store which provide Pegasus meta data read and write.
- _meta_store = std::make_unique<meta_store>(this, _db, _meta_cf);
+ _meta_store = std::make_unique<meta_store>(replica_name(), _db, _meta_cf);
if (db_exist) {
auto cleanup = dsn::defer([this]() { release_db(); });
@@ -2192,8 +2191,8 @@ private:
// Because of RocksDB's restriction, we have to to open default column
family even though
// not use it
std::vector<rocksdb::ColumnFamilyDescriptor> column_families(
- {{DATA_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()},
- {META_COLUMN_FAMILY_NAME, rocksdb::ColumnFamilyOptions()}});
+ {{meta_store::DATA_COLUMN_FAMILY_NAME,
rocksdb::ColumnFamilyOptions()},
+ {meta_store::META_COLUMN_FAMILY_NAME,
rocksdb::ColumnFamilyOptions()}});
status = rocksdb::DB::OpenForReadOnly(
_db_opts, checkpoint_dir, column_families, &handles_opened,
&snapshot_db);
if (!status.ok()) {
@@ -2204,7 +2203,7 @@ private:
return ::dsn::ERR_LOCAL_APP_FAILURE;
}
CHECK_EQ_PREFIX(handles_opened.size(), 2);
- CHECK_EQ_PREFIX(handles_opened[1]->GetName(), META_COLUMN_FAMILY_NAME);
+ CHECK_EQ_PREFIX(handles_opened[1]->GetName(),
meta_store::META_COLUMN_FAMILY_NAME);
uint64_t last_flushed_decree =
_meta_store->get_decree_from_readonly_db(snapshot_db,
handles_opened[1]);
*checkpoint_decree = last_flushed_decree;
@@ -3322,9 +3321,9 @@ bool pegasus_server_impl::set_options(
}
for (const auto &column_family : column_families) {
- if (column_family == META_COLUMN_FAMILY_NAME) {
+ if (column_family == meta_store::META_COLUMN_FAMILY_NAME) {
*missing_meta_cf = false;
- } else if (column_family == DATA_COLUMN_FAMILY_NAME) {
+ } else if (column_family == meta_store::DATA_COLUMN_FAMILY_NAME) {
*missing_data_cf = false;
} else {
LOG_ERROR_PREFIX("unknown column family name: {}", column_family);
diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h
index 7cac63440..69d02d061 100644
--- a/src/server/pegasus_server_impl.h
+++ b/src/server/pegasus_server_impl.h
@@ -476,9 +476,6 @@ private:
private:
static const std::chrono::seconds kServerStatUpdateTimeSec;
static const std::string COMPRESSION_HEADER;
- // Column family names.
- static const std::string DATA_COLUMN_FAMILY_NAME;
- static const std::string META_COLUMN_FAMILY_NAME;
dsn::gpid _gpid;
std::string _primary_address;
diff --git a/src/server/pegasus_server_impl_init.cpp
b/src/server/pegasus_server_impl_init.cpp
index 4d2e3f1aa..e2321f2e8 100644
--- a/src/server/pegasus_server_impl_init.cpp
+++ b/src/server/pegasus_server_impl_init.cpp
@@ -37,6 +37,7 @@
#include <utility>
#include <vector>
+#include "base/meta_store.h" // IWYU pragma: keep
#include "common/gpid.h"
#include "hashkey_transform.h"
#include "hotkey_collector.h"
@@ -48,7 +49,6 @@
#include "runtime/rpc/rpc_address.h"
#include "server/capacity_unit_calculator.h" // IWYU pragma: keep
#include "server/key_ttl_compaction_filter.h"
-#include "server/meta_store.h" // IWYU pragma: keep
#include "server/pegasus_read_service.h"
#include "server/pegasus_server_write.h" // IWYU pragma: keep
#include "server/range_read_limiter.h"
diff --git a/src/server/pegasus_write_service_impl.h
b/src/server/pegasus_write_service_impl.h
index 3ac6bfe95..df57c59be 100644
--- a/src/server/pegasus_write_service_impl.h
+++ b/src/server/pegasus_write_service_impl.h
@@ -22,9 +22,9 @@
#include <gtest/gtest_prod.h>
#include "base/idl_utils.h"
+#include "base/meta_store.h"
#include "base/pegasus_key_schema.h"
#include "logging_utils.h"
-#include "meta_store.h"
#include "pegasus_server_impl.h"
#include "pegasus_write_service.h"
#include "rocksdb_wrapper.h"
diff --git a/src/server/rocksdb_wrapper.cpp b/src/server/rocksdb_wrapper.cpp
index 8331dbfd9..e4d6e3b21 100644
--- a/src/server/rocksdb_wrapper.cpp
+++ b/src/server/rocksdb_wrapper.cpp
@@ -24,12 +24,12 @@
#include <rocksdb/slice.h>
#include <rocksdb/status.h>
+#include "base/meta_store.h"
#include "base/pegasus_value_schema.h"
#include "pegasus_key_schema.h"
#include "pegasus_utils.h"
#include "pegasus_write_service_impl.h"
#include "server/logging_utils.h"
-#include "server/meta_store.h"
#include "server/pegasus_server_impl.h"
#include "server/pegasus_write_service.h"
#include "utils/autoref_ptr.h"
diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt
index 5907476d3..082a98c1e 100644
--- a/src/server/test/CMakeLists.txt
+++ b/src/server/test/CMakeLists.txt
@@ -26,11 +26,11 @@ set(MY_PROJ_SRC
"../capacity_unit_calculator.cpp"
"../pegasus_mutation_duplicator.cpp"
"../hotspot_partition_calculator.cpp"
- "../meta_store.cpp"
"../hotkey_collector.cpp"
"../rocksdb_wrapper.cpp"
"../compaction_filter_rule.cpp"
"../compaction_operation.cpp")
+
set(MY_SRC_SEARCH_MODE "GLOB")
set(MY_PROJ_LIBS
dsn_replica_server
diff --git a/src/server/test/pegasus_server_impl_test.cpp
b/src/server/test/pegasus_server_impl_test.cpp
index a1cf77db4..229927d35 100644
--- a/src/server/test/pegasus_server_impl_test.cpp
+++ b/src/server/test/pegasus_server_impl_test.cpp
@@ -28,6 +28,7 @@
#include <string>
#include <utility>
+#include "base/meta_store.h"
#include "common/replica_envs.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
@@ -35,7 +36,6 @@
#include "rrdb/rrdb.code.definition.h"
#include "rrdb/rrdb_types.h"
#include "runtime/serverlet.h"
-#include "server/meta_store.h"
#include "server/pegasus_read_service.h"
#include "utils/autoref_ptr.h"
#include "utils/blob.h"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
