This is an automated email from the ASF dual-hosted git repository. caipengbo pushed a commit to branch 2.8 in repository https://gitbox.apache.org/repos/asf/kvrocks.git
commit 30dc68ad616ee590af2d30a8ce03a36b5a680739 Author: Twice <[email protected]> AuthorDate: Sun Mar 3 11:32:05 2024 +0900 Add unit test for index updater (#2134) --- src/search/indexer.cc | 20 +- src/search/indexer.h | 8 + .../{bitfield_util.cc => bitfield_util_test.cc} | 0 tests/cppunit/indexer_test.cc | 269 +++++++++++++++++++++ 4 files changed, 294 insertions(+), 3 deletions(-) diff --git a/src/search/indexer.cc b/src/search/indexer.cc index 4576e280..c082133e 100644 --- a/src/search/indexer.cc +++ b/src/search/indexer.cc @@ -20,6 +20,7 @@ #include "indexer.h" +#include <algorithm> #include <variant> #include "parse_util.h" @@ -83,6 +84,9 @@ StatusOr<IndexUpdater::FieldValues> IndexUpdater::Record(std::string_view key, c auto s = db.Type(key, &type); if (!s.ok()) return {Status::NotOK, s.ToString()}; + // key not exist + if (type == kRedisNone) return FieldValues(); + if (type != static_cast<RedisType>(metadata.on_data_type)) { // not the expected type, stop record return {Status::TypeMismatched}; @@ -123,8 +127,18 @@ Status IndexUpdater::UpdateIndex(const std::string &field, std::string_view key, auto original_tags = util::Split(original, delim); auto current_tags = util::Split(current, delim); - std::set<std::string> tags_to_delete(original_tags.begin(), original_tags.end()); - std::set<std::string> tags_to_add(current_tags.begin(), current_tags.end()); + auto to_tag_set = [](const std::vector<std::string> &tags, bool case_sensitive) -> std::set<std::string> { + if (case_sensitive) { + return {tags.begin(), tags.end()}; + } else { + std::set<std::string> res; + std::transform(tags.begin(), tags.end(), std::inserter(res, res.begin()), util::ToLower); + return res; + } + }; + + std::set<std::string> tags_to_delete = to_tag_set(original_tags, tag->case_sensitive); + std::set<std::string> tags_to_add = to_tag_set(current_tags, tag->case_sensitive); for (auto it = tags_to_delete.begin(); it != tags_to_delete.end();) { if (auto jt = tags_to_add.find(*it); jt != tags_to_add.end()) { @@ -210,7 +224,7 @@ Status IndexUpdater::Update(const FieldValues &original, std::string_view key, c void GlobalIndexer::Add(IndexUpdater updater) { auto &up = updaters.emplace_back(std::move(updater)); for (const auto &prefix : up.prefixes) { - prefix_map.emplace(prefix, &up); + prefix_map.insert(prefix, &up); } } diff --git a/src/search/indexer.h b/src/search/indexer.h index 001ade13..c404ecbb 100644 --- a/src/search/indexer.h +++ b/src/search/indexer.h @@ -75,6 +75,14 @@ struct IndexUpdater { std::map<std::string, std::unique_ptr<SearchFieldMetadata>> fields; GlobalIndexer *indexer = nullptr; + IndexUpdater(const IndexUpdater &) = delete; + IndexUpdater(IndexUpdater &&) = default; + + IndexUpdater &operator=(IndexUpdater &&) = default; + IndexUpdater &operator=(const IndexUpdater &) = delete; + + ~IndexUpdater() = default; + StatusOr<FieldValues> Record(std::string_view key, const std::string &ns); Status UpdateIndex(const std::string &field, std::string_view key, std::string_view original, std::string_view current, const std::string &ns); diff --git a/tests/cppunit/bitfield_util.cc b/tests/cppunit/bitfield_util_test.cc similarity index 100% rename from tests/cppunit/bitfield_util.cc rename to tests/cppunit/bitfield_util_test.cc diff --git a/tests/cppunit/indexer_test.cc b/tests/cppunit/indexer_test.cc new file mode 100644 index 00000000..f30b45ca --- /dev/null +++ b/tests/cppunit/indexer_test.cc @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "search/indexer.h" + +#include <gtest/gtest.h> +#include <test_base.h> + +#include <memory> + +#include "search/search_encoding.h" +#include "storage/redis_metadata.h" +#include "types/redis_hash.h" + +struct IndexerTest : TestBase { + redis::GlobalIndexer indexer; + std::string ns = "index_test"; + + IndexerTest() : indexer(storage_.get()) { + SearchMetadata hash_field_meta(false); + hash_field_meta.on_data_type = SearchOnDataType::HASH; + + std::map<std::string, std::unique_ptr<redis::SearchFieldMetadata>> hash_fields; + hash_fields.emplace("x", std::make_unique<redis::SearchTagFieldMetadata>()); + hash_fields.emplace("y", std::make_unique<redis::SearchNumericFieldMetadata>()); + + redis::IndexUpdater hash_updater{"hashtest", hash_field_meta, {"idxtesthash"}, std::move(hash_fields), &indexer}; + + SearchMetadata json_field_meta(false); + json_field_meta.on_data_type = SearchOnDataType::JSON; + + std::map<std::string, std::unique_ptr<redis::SearchFieldMetadata>> json_fields; + json_fields.emplace("$.x", std::make_unique<redis::SearchTagFieldMetadata>()); + json_fields.emplace("$.y", std::make_unique<redis::SearchNumericFieldMetadata>()); + + redis::IndexUpdater json_updater{"jsontest", json_field_meta, {"idxtestjson"}, std::move(json_fields), &indexer}; + + indexer.Add(std::move(hash_updater)); + indexer.Add(std::move(json_updater)); + } +}; + +TEST_F(IndexerTest, HashTag) { + redis::Hash db(storage_.get(), ns); + auto cfhandler = storage_->GetCFHandle("search"); + + { + auto s = indexer.Record("no_exist", ns); + ASSERT_TRUE(s.Is<Status::NoPrefixMatched>()); + } + + auto key1 = "idxtesthash:k1"; + auto idxname = "hashtest"; + + { + auto s = indexer.Record(key1, ns); + ASSERT_TRUE(s); + ASSERT_EQ(s->first->name, idxname); + ASSERT_TRUE(s->second.empty()); + + uint64_t cnt = 0; + db.Set(key1, "x", "food,kitChen,Beauty", &cnt); + ASSERT_EQ(cnt, 1); + + auto s2 = indexer.Update(*s, key1, ns); + ASSERT_TRUE(s2); + + auto subkey = redis::ConstructTagFieldSubkey("x", "food", key1); + auto nskey = ComposeNamespaceKey(ns, idxname, false); + auto key = InternalKey(nskey, subkey, 0, false); + + std::string val; + auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + + subkey = redis::ConstructTagFieldSubkey("x", "kitchen", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + + subkey = redis::ConstructTagFieldSubkey("x", "beauty", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + } + + { + auto s = indexer.Record(key1, ns); + ASSERT_TRUE(s); + ASSERT_EQ(s->first->name, idxname); + ASSERT_EQ(s->second.size(), 1); + ASSERT_EQ(s->second["x"], "food,kitChen,Beauty"); + + uint64_t cnt = 0; + auto s_set = db.Set(key1, "x", "Clothing,FOOD,sport", &cnt); + ASSERT_EQ(cnt, 0); + ASSERT_TRUE(s_set.ok()); + + auto s2 = indexer.Update(*s, key1, ns); + ASSERT_TRUE(s2); + + auto subkey = redis::ConstructTagFieldSubkey("x", "food", key1); + auto nskey = ComposeNamespaceKey(ns, idxname, false); + auto key = InternalKey(nskey, subkey, 0, false); + + std::string val; + auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + + subkey = redis::ConstructTagFieldSubkey("x", "clothing", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + + subkey = redis::ConstructTagFieldSubkey("x", "sport", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + + subkey = redis::ConstructTagFieldSubkey("x", "kitchen", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.IsNotFound()); + + subkey = redis::ConstructTagFieldSubkey("x", "beauty", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.IsNotFound()); + } +} + +TEST_F(IndexerTest, JsonTag) { + redis::Json db(storage_.get(), ns); + auto cfhandler = storage_->GetCFHandle("search"); + + { + auto s = indexer.Record("no_exist", ns); + ASSERT_TRUE(s.Is<Status::NoPrefixMatched>()); + } + + auto key1 = "idxtestjson:k1"; + auto idxname = "jsontest"; + + { + auto s = indexer.Record(key1, ns); + ASSERT_TRUE(s); + ASSERT_EQ(s->first->name, idxname); + ASSERT_TRUE(s->second.empty()); + + auto s_set = db.Set(key1, "$", R"({"x": "food,kitChen,Beauty"})"); + ASSERT_TRUE(s_set.ok()); + + auto s2 = indexer.Update(*s, key1, ns); + ASSERT_TRUE(s2); + + auto subkey = redis::ConstructTagFieldSubkey("$.x", "food", key1); + auto nskey = ComposeNamespaceKey(ns, idxname, false); + auto key = InternalKey(nskey, subkey, 0, false); + + std::string val; + auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + + subkey = redis::ConstructTagFieldSubkey("$.x", "kitchen", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + + subkey = redis::ConstructTagFieldSubkey("$.x", "beauty", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + } + + { + auto s = indexer.Record(key1, ns); + ASSERT_TRUE(s); + ASSERT_EQ(s->first->name, idxname); + ASSERT_EQ(s->second.size(), 1); + ASSERT_EQ(s->second["$.x"], "food,kitChen,Beauty"); + + auto s_set = db.Set(key1, "$.x", "\"Clothing,FOOD,sport\""); + ASSERT_TRUE(s_set.ok()); + + auto s2 = indexer.Update(*s, key1, ns); + ASSERT_TRUE(s2); + + auto subkey = redis::ConstructTagFieldSubkey("$.x", "food", key1); + auto nskey = ComposeNamespaceKey(ns, idxname, false); + auto key = InternalKey(nskey, subkey, 0, false); + + std::string val; + auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + + subkey = redis::ConstructTagFieldSubkey("$.x", "clothing", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + + subkey = redis::ConstructTagFieldSubkey("$.x", "sport", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.ok()); + ASSERT_EQ(val, ""); + + subkey = redis::ConstructTagFieldSubkey("$.x", "kitchen", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.IsNotFound()); + + subkey = redis::ConstructTagFieldSubkey("$.x", "beauty", key1); + nskey = ComposeNamespaceKey(ns, idxname, false); + key = InternalKey(nskey, subkey, 0, false); + + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + ASSERT_TRUE(s3.IsNotFound()); + } +}
