This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new be072b4f Add unit test for index updater (#2134)
be072b4f is described below
commit be072b4f9f665826699275477da21613698dd69e
Author: Twice <[email protected]>
AuthorDate: Sun Mar 3 11:32:05 2024 +0900
Add unit test for index updater (#2134)
---
src/search/indexer.cc | 20 +-
src/search/indexer.h | 8 +
.../{bitfield_util.cc => bitfield_util_test.cc} | 0
tests/cppunit/indexer_test.cc | 269 +++++++++++++++++++++
4 files changed, 294 insertions(+), 3 deletions(-)
diff --git a/src/search/indexer.cc b/src/search/indexer.cc
index 4576e280..c082133e 100644
--- a/src/search/indexer.cc
+++ b/src/search/indexer.cc
@@ -20,6 +20,7 @@
#include "indexer.h"
+#include <algorithm>
#include <variant>
#include "parse_util.h"
@@ -83,6 +84,9 @@ StatusOr<IndexUpdater::FieldValues>
IndexUpdater::Record(std::string_view key, c
auto s = db.Type(key, &type);
if (!s.ok()) return {Status::NotOK, s.ToString()};
+ // key not exist
+ if (type == kRedisNone) return FieldValues();
+
if (type != static_cast<RedisType>(metadata.on_data_type)) {
// not the expected type, stop record
return {Status::TypeMismatched};
@@ -123,8 +127,18 @@ Status IndexUpdater::UpdateIndex(const std::string &field,
std::string_view key,
auto original_tags = util::Split(original, delim);
auto current_tags = util::Split(current, delim);
- std::set<std::string> tags_to_delete(original_tags.begin(),
original_tags.end());
- std::set<std::string> tags_to_add(current_tags.begin(),
current_tags.end());
+ auto to_tag_set = [](const std::vector<std::string> &tags, bool
case_sensitive) -> std::set<std::string> {
+ if (case_sensitive) {
+ return {tags.begin(), tags.end()};
+ } else {
+ std::set<std::string> res;
+ std::transform(tags.begin(), tags.end(), std::inserter(res,
res.begin()), util::ToLower);
+ return res;
+ }
+ };
+
+ std::set<std::string> tags_to_delete = to_tag_set(original_tags,
tag->case_sensitive);
+ std::set<std::string> tags_to_add = to_tag_set(current_tags,
tag->case_sensitive);
for (auto it = tags_to_delete.begin(); it != tags_to_delete.end();) {
if (auto jt = tags_to_add.find(*it); jt != tags_to_add.end()) {
@@ -210,7 +224,7 @@ Status IndexUpdater::Update(const FieldValues &original,
std::string_view key, c
void GlobalIndexer::Add(IndexUpdater updater) {
auto &up = updaters.emplace_back(std::move(updater));
for (const auto &prefix : up.prefixes) {
- prefix_map.emplace(prefix, &up);
+ prefix_map.insert(prefix, &up);
}
}
diff --git a/src/search/indexer.h b/src/search/indexer.h
index 001ade13..c404ecbb 100644
--- a/src/search/indexer.h
+++ b/src/search/indexer.h
@@ -75,6 +75,14 @@ struct IndexUpdater {
std::map<std::string, std::unique_ptr<SearchFieldMetadata>> fields;
GlobalIndexer *indexer = nullptr;
+ IndexUpdater(const IndexUpdater &) = delete;
+ IndexUpdater(IndexUpdater &&) = default;
+
+ IndexUpdater &operator=(IndexUpdater &&) = default;
+ IndexUpdater &operator=(const IndexUpdater &) = delete;
+
+ ~IndexUpdater() = default;
+
StatusOr<FieldValues> Record(std::string_view key, const std::string &ns);
Status UpdateIndex(const std::string &field, std::string_view key,
std::string_view original,
std::string_view current, const std::string &ns);
diff --git a/tests/cppunit/bitfield_util.cc
b/tests/cppunit/bitfield_util_test.cc
similarity index 100%
rename from tests/cppunit/bitfield_util.cc
rename to tests/cppunit/bitfield_util_test.cc
diff --git a/tests/cppunit/indexer_test.cc b/tests/cppunit/indexer_test.cc
new file mode 100644
index 00000000..f30b45ca
--- /dev/null
+++ b/tests/cppunit/indexer_test.cc
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+#include "search/indexer.h"
+
+#include <gtest/gtest.h>
+#include <test_base.h>
+
+#include <memory>
+
+#include "search/search_encoding.h"
+#include "storage/redis_metadata.h"
+#include "types/redis_hash.h"
+
+struct IndexerTest : TestBase {
+ redis::GlobalIndexer indexer;
+ std::string ns = "index_test";
+
+ IndexerTest() : indexer(storage_.get()) {
+ SearchMetadata hash_field_meta(false);
+ hash_field_meta.on_data_type = SearchOnDataType::HASH;
+
+ std::map<std::string, std::unique_ptr<redis::SearchFieldMetadata>>
hash_fields;
+ hash_fields.emplace("x",
std::make_unique<redis::SearchTagFieldMetadata>());
+ hash_fields.emplace("y",
std::make_unique<redis::SearchNumericFieldMetadata>());
+
+ redis::IndexUpdater hash_updater{"hashtest", hash_field_meta,
{"idxtesthash"}, std::move(hash_fields), &indexer};
+
+ SearchMetadata json_field_meta(false);
+ json_field_meta.on_data_type = SearchOnDataType::JSON;
+
+ std::map<std::string, std::unique_ptr<redis::SearchFieldMetadata>>
json_fields;
+ json_fields.emplace("$.x",
std::make_unique<redis::SearchTagFieldMetadata>());
+ json_fields.emplace("$.y",
std::make_unique<redis::SearchNumericFieldMetadata>());
+
+ redis::IndexUpdater json_updater{"jsontest", json_field_meta,
{"idxtestjson"}, std::move(json_fields), &indexer};
+
+ indexer.Add(std::move(hash_updater));
+ indexer.Add(std::move(json_updater));
+ }
+};
+
+TEST_F(IndexerTest, HashTag) {
+ redis::Hash db(storage_.get(), ns);
+ auto cfhandler = storage_->GetCFHandle("search");
+
+ {
+ auto s = indexer.Record("no_exist", ns);
+ ASSERT_TRUE(s.Is<Status::NoPrefixMatched>());
+ }
+
+ auto key1 = "idxtesthash:k1";
+ auto idxname = "hashtest";
+
+ {
+ auto s = indexer.Record(key1, ns);
+ ASSERT_TRUE(s);
+ ASSERT_EQ(s->first->name, idxname);
+ ASSERT_TRUE(s->second.empty());
+
+ uint64_t cnt = 0;
+ db.Set(key1, "x", "food,kitChen,Beauty", &cnt);
+ ASSERT_EQ(cnt, 1);
+
+ auto s2 = indexer.Update(*s, key1, ns);
+ ASSERT_TRUE(s2);
+
+ auto subkey = redis::ConstructTagFieldSubkey("x", "food", key1);
+ auto nskey = ComposeNamespaceKey(ns, idxname, false);
+ auto key = InternalKey(nskey, subkey, 0, false);
+
+ std::string val;
+ auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+
+ subkey = redis::ConstructTagFieldSubkey("x", "kitchen", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+
+ subkey = redis::ConstructTagFieldSubkey("x", "beauty", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+ }
+
+ {
+ auto s = indexer.Record(key1, ns);
+ ASSERT_TRUE(s);
+ ASSERT_EQ(s->first->name, idxname);
+ ASSERT_EQ(s->second.size(), 1);
+ ASSERT_EQ(s->second["x"], "food,kitChen,Beauty");
+
+ uint64_t cnt = 0;
+ auto s_set = db.Set(key1, "x", "Clothing,FOOD,sport", &cnt);
+ ASSERT_EQ(cnt, 0);
+ ASSERT_TRUE(s_set.ok());
+
+ auto s2 = indexer.Update(*s, key1, ns);
+ ASSERT_TRUE(s2);
+
+ auto subkey = redis::ConstructTagFieldSubkey("x", "food", key1);
+ auto nskey = ComposeNamespaceKey(ns, idxname, false);
+ auto key = InternalKey(nskey, subkey, 0, false);
+
+ std::string val;
+ auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+
+ subkey = redis::ConstructTagFieldSubkey("x", "clothing", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+
+ subkey = redis::ConstructTagFieldSubkey("x", "sport", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+
+ subkey = redis::ConstructTagFieldSubkey("x", "kitchen", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.IsNotFound());
+
+ subkey = redis::ConstructTagFieldSubkey("x", "beauty", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.IsNotFound());
+ }
+}
+
+TEST_F(IndexerTest, JsonTag) {
+ redis::Json db(storage_.get(), ns);
+ auto cfhandler = storage_->GetCFHandle("search");
+
+ {
+ auto s = indexer.Record("no_exist", ns);
+ ASSERT_TRUE(s.Is<Status::NoPrefixMatched>());
+ }
+
+ auto key1 = "idxtestjson:k1";
+ auto idxname = "jsontest";
+
+ {
+ auto s = indexer.Record(key1, ns);
+ ASSERT_TRUE(s);
+ ASSERT_EQ(s->first->name, idxname);
+ ASSERT_TRUE(s->second.empty());
+
+ auto s_set = db.Set(key1, "$", R"({"x": "food,kitChen,Beauty"})");
+ ASSERT_TRUE(s_set.ok());
+
+ auto s2 = indexer.Update(*s, key1, ns);
+ ASSERT_TRUE(s2);
+
+ auto subkey = redis::ConstructTagFieldSubkey("$.x", "food", key1);
+ auto nskey = ComposeNamespaceKey(ns, idxname, false);
+ auto key = InternalKey(nskey, subkey, 0, false);
+
+ std::string val;
+ auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+
+ subkey = redis::ConstructTagFieldSubkey("$.x", "kitchen", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+
+ subkey = redis::ConstructTagFieldSubkey("$.x", "beauty", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+ }
+
+ {
+ auto s = indexer.Record(key1, ns);
+ ASSERT_TRUE(s);
+ ASSERT_EQ(s->first->name, idxname);
+ ASSERT_EQ(s->second.size(), 1);
+ ASSERT_EQ(s->second["$.x"], "food,kitChen,Beauty");
+
+ auto s_set = db.Set(key1, "$.x", "\"Clothing,FOOD,sport\"");
+ ASSERT_TRUE(s_set.ok());
+
+ auto s2 = indexer.Update(*s, key1, ns);
+ ASSERT_TRUE(s2);
+
+ auto subkey = redis::ConstructTagFieldSubkey("$.x", "food", key1);
+ auto nskey = ComposeNamespaceKey(ns, idxname, false);
+ auto key = InternalKey(nskey, subkey, 0, false);
+
+ std::string val;
+ auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+
+ subkey = redis::ConstructTagFieldSubkey("$.x", "clothing", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+
+ subkey = redis::ConstructTagFieldSubkey("$.x", "sport", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.ok());
+ ASSERT_EQ(val, "");
+
+ subkey = redis::ConstructTagFieldSubkey("$.x", "kitchen", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.IsNotFound());
+
+ subkey = redis::ConstructTagFieldSubkey("$.x", "beauty", key1);
+ nskey = ComposeNamespaceKey(ns, idxname, false);
+ key = InternalKey(nskey, subkey, 0, false);
+
+ s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler,
key.Encode(), &val);
+ ASSERT_TRUE(s3.IsNotFound());
+ }
+}
