This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new e490291 feat: add compaction filter rule creator (#768)
e490291 is described below
commit e4902911fa0c36709b1dd0a4209d84d08dc8621c
Author: zhao liwei <[email protected]>
AuthorDate: Thu Jun 17 19:52:56 2021 +0800
feat: add compaction filter rule creator (#768)
---
src/server/compaction_filter_rule.cpp | 13 +++++++
src/server/compaction_filter_rule.h | 49 ++++++++++++++++++++++++-
src/server/main.cpp | 2 +
src/server/test/compaction_filter_rule_test.cpp | 38 +++++++++++++++++++
src/server/test/main.cpp | 2 +
5 files changed, 102 insertions(+), 2 deletions(-)
diff --git a/src/server/compaction_filter_rule.cpp
b/src/server/compaction_filter_rule.cpp
index a1a66e0..6e37f4b 100644
--- a/src/server/compaction_filter_rule.cpp
+++ b/src/server/compaction_filter_rule.cpp
@@ -51,6 +51,8 @@ bool string_pattern_match(const std::string &value,
}
}
+hashkey_pattern_rule::hashkey_pattern_rule(uint32_t data_version) {}
+
bool hashkey_pattern_rule::match(const std::string &hash_key,
const std::string &sort_key,
const rocksdb::Slice &existing_value) const
@@ -58,6 +60,8 @@ bool hashkey_pattern_rule::match(const std::string &hash_key,
return string_pattern_match(hash_key, match_type, pattern);
}
+sortkey_pattern_rule::sortkey_pattern_rule(uint32_t data_version) {}
+
bool sortkey_pattern_rule::match(const std::string &hash_key,
const std::string &sort_key,
const rocksdb::Slice &existing_value) const
@@ -87,5 +91,14 @@ bool ttl_range_rule::match(const std::string &hash_key,
}
return false;
}
+
+void register_compaction_filter_rules()
+{
+
ttl_range_rule::register_component<ttl_range_rule>(enum_to_string(FRT_TTL_RANGE));
+ sortkey_pattern_rule::register_component<sortkey_pattern_rule>(
+ enum_to_string(FRT_SORTKEY_PATTERN));
+ hashkey_pattern_rule::register_component<hashkey_pattern_rule>(
+ enum_to_string(FRT_HASHKEY_PATTERN));
+}
} // namespace server
} // namespace pegasus
diff --git a/src/server/compaction_filter_rule.h
b/src/server/compaction_filter_rule.h
index bc61039..c6d678c 100644
--- a/src/server/compaction_filter_rule.h
+++ b/src/server/compaction_filter_rule.h
@@ -21,15 +21,50 @@
#include <rocksdb/slice.h>
#include <dsn/utility/enum_helper.h>
+#include <dsn/cpp/json_helper.h>
#include <gtest/gtest.h>
+#include "base/pegasus_value_schema.h"
namespace pegasus {
namespace server {
+enum filter_rule_type
+{
+ FRT_HASHKEY_PATTERN = 0,
+ FRT_SORTKEY_PATTERN,
+ FRT_TTL_RANGE,
+ FRT_INVALID,
+};
+ENUM_BEGIN(filter_rule_type, FRT_INVALID)
+ENUM_REG(FRT_HASHKEY_PATTERN)
+ENUM_REG(FRT_SORTKEY_PATTERN)
+ENUM_REG(FRT_TTL_RANGE)
+ENUM_END(filter_rule_type)
+
+ENUM_TYPE_SERIALIZATION(filter_rule_type, FRT_INVALID)
+
/** compaction_filter_rule represents the compaction rule to filter the keys
which are stored in
* rocksdb. */
class compaction_filter_rule
{
public:
+ template <typename T>
+ static compaction_filter_rule *create(const std::string ¶ms, uint32_t
pegasus_data_version)
+ {
+ T *rule = new T(pegasus_data_version);
+ if (!dsn::json::json_forwarder<T>::decode(
+ dsn::blob::create_from_bytes(params.data(), params.size()),
*rule)) {
+ delete rule;
+ return nullptr;
+ }
+ return rule;
+ }
+
+ template <typename T>
+ static void register_component(const char *name)
+ {
+ dsn::utils::factory_store<compaction_filter_rule>::register_factory(
+ name, create<T>, dsn::PROVIDER_TYPE_MAIN);
+ }
virtual ~compaction_filter_rule() = default;
// TODO(zhaoliwei): we can use `value_filed` to replace existing_value in
the later,
@@ -52,14 +87,17 @@ ENUM_REG(SMT_MATCH_PREFIX)
ENUM_REG(SMT_MATCH_POSTFIX)
ENUM_END(string_match_type)
+ENUM_TYPE_SERIALIZATION(string_match_type, SMT_INVALID)
+
class hashkey_pattern_rule : public compaction_filter_rule
{
public:
- hashkey_pattern_rule() = default;
+ hashkey_pattern_rule(uint32_t data_version = VERSION_MAX);
bool match(const std::string &hash_key,
const std::string &sort_key,
const rocksdb::Slice &existing_value) const;
+ DEFINE_JSON_SERIALIZATION(pattern, match_type)
private:
std::string pattern;
@@ -69,16 +107,18 @@ private:
FRIEND_TEST(delete_key_test, filter);
FRIEND_TEST(update_ttl_test, filter);
FRIEND_TEST(compaction_filter_operation_test, all_rules_match);
+ FRIEND_TEST(compaction_filter_rule_test, create);
};
class sortkey_pattern_rule : public compaction_filter_rule
{
public:
- sortkey_pattern_rule() = default;
+ sortkey_pattern_rule(uint32_t data_version = VERSION_MAX);
bool match(const std::string &hash_key,
const std::string &sort_key,
const rocksdb::Slice &existing_value) const;
+ DEFINE_JSON_SERIALIZATION(pattern, match_type)
private:
std::string pattern;
@@ -86,6 +126,7 @@ private:
FRIEND_TEST(sortkey_pattern_rule_test, match);
FRIEND_TEST(compaction_filter_operation_test, all_rules_match);
+ FRIEND_TEST(compaction_filter_rule_test, create);
};
class ttl_range_rule : public compaction_filter_rule
@@ -96,6 +137,7 @@ public:
bool match(const std::string &hash_key,
const std::string &sort_key,
const rocksdb::Slice &existing_value) const;
+ DEFINE_JSON_SERIALIZATION(start_ttl, stop_ttl)
private:
// = 0 means no limit
@@ -105,6 +147,9 @@ private:
FRIEND_TEST(ttl_range_rule_test, match);
FRIEND_TEST(compaction_filter_operation_test, all_rules_match);
+ FRIEND_TEST(compaction_filter_rule_test, create);
};
+
+void register_compaction_filter_rules();
} // namespace server
} // namespace pegasus
diff --git a/src/server/main.cpp b/src/server/main.cpp
index 44d6758..d12c0e6 100644
--- a/src/server/main.cpp
+++ b/src/server/main.cpp
@@ -21,6 +21,7 @@
#include "pegasus_service_app.h"
#include "info_collector_app.h"
#include "brief_stat.h"
+#include "compaction_filter_rule.h"
#include <pegasus/version.h>
#include <pegasus/git_commit.h>
@@ -87,6 +88,7 @@ void dsn_app_registration_pegasus()
"server-stat - query selected perf counters",
"server-stat",
[](const std::vector<std::string> &args) { return
pegasus::get_brief_stat(); });
+ pegasus::server::register_compaction_filter_rules();
}
int main(int argc, char **argv)
diff --git a/src/server/test/compaction_filter_rule_test.cpp
b/src/server/test/compaction_filter_rule_test.cpp
index a8f436f..40dc6c2 100644
--- a/src/server/test/compaction_filter_rule_test.cpp
+++ b/src/server/test/compaction_filter_rule_test.cpp
@@ -131,5 +131,43 @@ TEST(ttl_range_rule_test, match)
ASSERT_EQ(rule.match("", "", svalue.parts[0]), test.match);
}
}
+
+TEST(compaction_filter_rule_test, create)
+{
+ const uint32_t data_version = 1;
+
+ // ttl_range_rule
+ std::string params_json = R"({"start_ttl":1000,"stop_ttl":2000})";
+ compaction_filter_rule *rule =
+ compaction_filter_rule::create<ttl_range_rule>(params_json,
data_version);
+ ttl_range_rule *expire_rule = static_cast<ttl_range_rule *>(rule);
+ ASSERT_EQ(expire_rule->start_ttl, 1000);
+ ASSERT_EQ(expire_rule->stop_ttl, 2000);
+ delete expire_rule;
+
+ // sortkey_pattern_rule
+ params_json = R"({"pattern":"sortkey","match_type":"SMT_MATCH_PREFIX"})";
+ rule = compaction_filter_rule::create<sortkey_pattern_rule>(params_json,
data_version);
+ sortkey_pattern_rule *sortkey_rule = static_cast<sortkey_pattern_rule
*>(rule);
+ ASSERT_EQ(sortkey_rule->pattern, "sortkey");
+ ASSERT_EQ(sortkey_rule->match_type, SMT_MATCH_PREFIX);
+ delete sortkey_rule;
+
+ // hashkey_pattern_rule
+ params_json = R"({"pattern":"hashkey","match_type":"SMT_MATCH_PREFIX"})";
+ rule = compaction_filter_rule::create<hashkey_pattern_rule>(params_json,
data_version);
+ hashkey_pattern_rule *hashkey_rule = static_cast<hashkey_pattern_rule
*>(rule);
+ ASSERT_EQ(hashkey_rule->pattern, "hashkey");
+ ASSERT_EQ(hashkey_rule->match_type, SMT_MATCH_PREFIX);
+ delete hashkey_rule;
+
+ // invalid sortkey_pattern_rule
+ params_json =
R"({"_patternxxx":"sortkey","match_type":"SMT_MATCH_PREFIX"})";
+ rule = compaction_filter_rule::create<sortkey_pattern_rule>(params_json,
data_version);
+ ASSERT_EQ(rule, nullptr);
+ params_json =
R"({"pattern":"sortkey","_match_typexxx":"SMT_MATCH_PREFIX"})";
+ rule = compaction_filter_rule::create<sortkey_pattern_rule>(params_json,
data_version);
+ ASSERT_EQ(rule, nullptr);
+}
} // namespace server
} // namespace pegasus
diff --git a/src/server/test/main.cpp b/src/server/test/main.cpp
index c40115c..985a3b5 100644
--- a/src/server/test/main.cpp
+++ b/src/server/test/main.cpp
@@ -20,6 +20,7 @@
#include <gtest/gtest.h>
#include <dsn/service_api_cpp.h>
#include <dsn/dist/replication/replication_service_app.h>
+#include "server/compaction_filter_rule.h"
#include "server/pegasus_server_impl.h"
std::atomic_bool gtest_done{false};
@@ -48,6 +49,7 @@ GTEST_API_ int main(int argc, char **argv)
dsn::replication::replication_app_base::register_storage_engine(
"pegasus",
dsn::replication::replication_app_base::create<pegasus::server::pegasus_server_impl>);
+ pegasus::server::register_compaction_filter_rules();
dsn_run_config("config.ini", false);
while (!gtest_done) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
