This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 47e6705c feat(search): add a value type system to KQIR (#2369)
47e6705c is described below
commit 47e6705cd1c3939b540f09bc022e2587f56015df
Author: Twice <[email protected]>
AuthorDate: Wed Jun 19 18:38:12 2024 +0900
feat(search): add a value type system to KQIR (#2369)
---
src/commands/cmd_search.cc | 2 +-
src/common/string_util.h | 2 +-
src/search/common_parser.h | 2 +-
src/search/executors/filter_executor.h | 10 +-
src/search/executors/numeric_field_scan_executor.h | 3 +-
src/search/executors/topn_sort_executor.h | 6 +-
src/search/indexer.cc | 96 ++++++++++++++------
src/search/indexer.h | 13 +--
src/search/ir.h | 2 +-
src/search/plan_executor.cc | 9 +-
src/search/plan_executor.h | 8 +-
src/search/redis_query_parser.h | 2 +-
src/search/redis_query_transformer.h | 2 +-
src/search/sql_parser.h | 2 +-
src/search/sql_transformer.h | 4 +-
src/search/value.h | 101 +++++++++++++++++++++
tests/cppunit/indexer_test.cc | 6 +-
tests/cppunit/plan_executor_test.cc | 47 +++++-----
18 files changed, 236 insertions(+), 81 deletions(-)
diff --git a/src/commands/cmd_search.cc b/src/commands/cmd_search.cc
index 618e0cf6..1928672c 100644
--- a/src/commands/cmd_search.cc
+++ b/src/commands/cmd_search.cc
@@ -150,7 +150,7 @@ static void DumpQueryResult(const
std::vector<kqir::ExecutorContext::RowType> &r
output->append(MultiLen(fields.size() * 2));
for (const auto &[info, field] : fields) {
output->append(redis::BulkString(info->name));
- output->append(redis::BulkString(field));
+ output->append(redis::BulkString(field.ToString(info->metadata.get())));
}
}
}
diff --git a/src/common/string_util.h b/src/common/string_util.h
index da345b30..ac3b2904 100644
--- a/src/common/string_util.h
+++ b/src/common/string_util.h
@@ -42,7 +42,7 @@ std::string StringNext(std::string s);
template <typename T, typename F>
std::string StringJoin(
- const T &con, F &&f = [](const auto &v) -> decltype(auto) { return v; },
const std::string &sep = ", ") {
+ const T &con, F &&f = [](const auto &v) -> decltype(auto) { return v; },
std::string_view sep = ", ") {
std::string res;
bool is_first = true;
for (const auto &v : con) {
diff --git a/src/search/common_parser.h b/src/search/common_parser.h
index 15f5bc36..7d617dbd 100644
--- a/src/search/common_parser.h
+++ b/src/search/common_parser.h
@@ -42,7 +42,7 @@ struct UnescapedChar : peg::utf8::range<0x20, 0x10FFFF> {};
struct Char : peg::if_then_else<peg::one<'\\'>, EscapedChar, UnescapedChar> {};
struct StringContent : peg::until<peg::at<peg::one<'"'>>, Char> {};
-struct String : peg::seq<peg::one<'"'>, StringContent, peg::any> {};
+struct StringL : peg::seq<peg::one<'"'>, StringContent, peg::any> {};
struct Identifier : peg::identifier {};
diff --git a/src/search/executors/filter_executor.h
b/src/search/executors/filter_executor.h
index ea860fc6..f668af7d 100644
--- a/src/search/executors/filter_executor.h
+++ b/src/search/executors/filter_executor.h
@@ -74,17 +74,17 @@ struct QueryExprEvaluator {
StatusOr<bool> Visit(TagContainExpr *v) const {
auto val = GET_OR_RET(ctx->Retrieve(row, v->field->info));
- auto meta = v->field->info->MetadataAs<redis::TagFieldMetadata>();
- auto split = util::Split(val, std::string(1, meta->separator));
+ CHECK(val.Is<kqir::StringArray>());
+ auto split = val.Get<kqir::StringArray>();
return std::find(split.begin(), split.end(), v->tag->val) != split.end();
}
StatusOr<bool> Visit(NumericCompareExpr *v) const {
- auto l_str = GET_OR_RET(ctx->Retrieve(row, v->field->info));
+ auto l_val = GET_OR_RET(ctx->Retrieve(row, v->field->info));
- // TODO: reconsider how to handle failure case here
- auto l = GET_OR_RET(ParseFloat(l_str));
+ CHECK(l_val.Is<kqir::Numeric>());
+ auto l = l_val.Get<kqir::Numeric>();
auto r = v->num->val;
switch (v->op) {
diff --git a/src/search/executors/numeric_field_scan_executor.h
b/src/search/executors/numeric_field_scan_executor.h
index e9699aa4..5c997df0 100644
--- a/src/search/executors/numeric_field_scan_executor.h
+++ b/src/search/executors/numeric_field_scan_executor.h
@@ -26,6 +26,7 @@
#include "encoding.h"
#include "search/plan_executor.h"
#include "search/search_encoding.h"
+#include "search/value.h"
#include "storage/redis_db.h"
#include "storage/redis_metadata.h"
#include "storage/storage.h"
@@ -108,7 +109,7 @@ struct NumericFieldScanExecutor : ExecutorNode {
} else {
iter->Prev();
}
- return RowType{key_str, {{scan->field->info, std::to_string(curr)}},
scan->field->info->index};
+ return RowType{key_str, {{scan->field->info,
kqir::MakeValue<kqir::Numeric>(curr)}}, scan->field->info->index};
}
};
diff --git a/src/search/executors/topn_sort_executor.h
b/src/search/executors/topn_sort_executor.h
index 741a9689..163b1bc7 100644
--- a/src/search/executors/topn_sort_executor.h
+++ b/src/search/executors/topn_sort_executor.h
@@ -57,9 +57,9 @@ struct TopNSortExecutor : ExecutorNode {
auto &row = std::get<RowType>(v);
auto get_order = [this](RowType &row) -> StatusOr<double> {
- auto order_str = GET_OR_RET(ctx->Retrieve(row,
sort->order->field->info));
- auto order = GET_OR_RET(ParseFloat(order_str));
- return order;
+ auto order_val = GET_OR_RET(ctx->Retrieve(row,
sort->order->field->info));
+ CHECK(order_val.Is<kqir::Numeric>());
+ return order_val.Get<kqir::Numeric>();
};
if (rows.size() == total) {
diff --git a/src/search/indexer.cc b/src/search/indexer.cc
index 0ee1b39d..80fea7a9 100644
--- a/src/search/indexer.cc
+++ b/src/search/indexer.cc
@@ -26,6 +26,7 @@
#include "db_util.h"
#include "parse_util.h"
#include "search/search_encoding.h"
+#include "search/value.h"
#include "storage/redis_metadata.h"
#include "storage/storage.h"
#include "string_util.h"
@@ -56,25 +57,67 @@ StatusOr<FieldValueRetriever>
FieldValueRetriever::Create(IndexOnDataType type,
}
}
-rocksdb::Status FieldValueRetriever::Retrieve(std::string_view field,
std::string *output) {
+StatusOr<kqir::Value> FieldValueRetriever::Retrieve(std::string_view field,
const redis::IndexFieldMetadata *type) {
if (std::holds_alternative<HashData>(db)) {
auto &[hash, metadata, key] = std::get<HashData>(db);
std::string ns_key = hash.AppendNamespacePrefix(key);
+
LatestSnapShot ss(hash.storage_);
rocksdb::ReadOptions read_options;
read_options.snapshot = ss.GetSnapShot();
std::string sub_key = InternalKey(ns_key, field, metadata.version,
hash.storage_->IsSlotIdEncoded()).Encode();
- return hash.storage_->Get(read_options, sub_key, output);
+ std::string value;
+ auto s = hash.storage_->Get(read_options, sub_key, &value);
+ if (s.IsNotFound()) return {Status::NotFound, s.ToString()};
+ if (!s.ok()) return {Status::NotOK, s.ToString()};
+
+ if (auto numeric [[maybe_unused]] = dynamic_cast<const
redis::NumericFieldMetadata *>(type)) {
+ auto num = GET_OR_RET(ParseFloat(value));
+ return kqir::MakeValue<kqir::Numeric>(num);
+ } else if (auto tag = dynamic_cast<const redis::TagFieldMetadata *>(type))
{
+ const char delim[] = {tag->separator, '\0'};
+ auto vec = util::Split(value, delim);
+ return kqir::MakeValue<kqir::StringArray>(vec);
+ } else {
+ return {Status::NotOK, "unknown field type to retrieve"};
+ }
+
} else if (std::holds_alternative<JsonData>(db)) {
auto &value = std::get<JsonData>(db);
+
auto s = value.Get(field.front() == '$' ? field : fmt::format("$.{}",
field));
- if (!s.IsOK()) return rocksdb::Status::Corruption(s.Msg());
+ if (!s.IsOK()) return {Status::NotOK, s.Msg()};
if (s->value.size() != 1)
- return rocksdb::Status::NotFound("json value specified by the field
(json path) should exist and be unique");
- *output = s->value[0].as_string();
- return rocksdb::Status::OK();
+ return {Status::NotFound, "json value specified by the field (json path)
should exist and be unique"};
+ auto val = s->value[0];
+
+ if (auto numeric [[maybe_unused]] = dynamic_cast<const
redis::NumericFieldMetadata *>(type)) {
+ if (val.is_string()) return {Status::NotOK, "json value cannot be string
for numeric fields"};
+ return kqir::MakeValue<kqir::Numeric>(val.as_double());
+ } else if (auto tag = dynamic_cast<const redis::TagFieldMetadata *>(type))
{
+ if (val.is_string()) {
+ const char delim[] = {tag->separator, '\0'};
+ auto vec = util::Split(val.as_string(), delim);
+ return kqir::MakeValue<kqir::StringArray>(vec);
+ } else if (val.is_array()) {
+ std::vector<std::string> strs;
+ for (size_t i = 0; i < val.size(); ++i) {
+ if (!val[i].is_string())
+ return {Status::NotOK, "json value should be string or array of
strings for tag fields"};
+ strs.push_back(val[i].as_string());
+ }
+ return kqir::MakeValue<kqir::StringArray>(strs);
+ } else {
+ return {Status::NotOK, "json value should be string or array of
strings for tag fields"};
+ }
+ } else {
+ return {Status::NotOK, "unknown field type to retrieve"};
+ }
+
+ return Status::OK();
+
} else {
- __builtin_unreachable();
+ return {Status::NotOK, "unknown redis data type to retrieve"};
}
}
@@ -102,22 +145,22 @@ StatusOr<IndexUpdater::FieldValues>
IndexUpdater::Record(std::string_view key) c
continue;
}
- std::string value;
- auto s = retriever.Retrieve(field, &value);
- if (s.IsNotFound()) continue;
- if (!s.ok()) return {Status::NotOK, s.ToString()};
+ auto s = retriever.Retrieve(field, i.metadata.get());
+ if (s.Is<Status::NotFound>()) continue;
+ if (!s) return s;
- values.emplace(field, value);
+ values.emplace(field, *s);
}
return values;
}
-Status IndexUpdater::UpdateTagIndex(std::string_view key, std::string_view
original, std::string_view current,
+Status IndexUpdater::UpdateTagIndex(std::string_view key, const kqir::Value
&original, const kqir::Value ¤t,
const SearchKey &search_key, const
TagFieldMetadata *tag) const {
- const char delim[] = {tag->separator, '\0'};
- auto original_tags = util::Split(original, delim);
- auto current_tags = util::Split(current, delim);
+ CHECK(original.IsNull() || original.Is<kqir::StringArray>());
+ CHECK(current.IsNull() || current.Is<kqir::StringArray>());
+ auto original_tags = original.IsNull() ? std::vector<std::string>() :
original.Get<kqir::StringArray>();
+ auto current_tags = current.IsNull() ? std::vector<std::string>() :
current.Get<kqir::StringArray>();
auto to_tag_set = [](const std::vector<std::string> &tags, bool
case_sensitive) -> std::set<std::string> {
if (case_sensitive) {
@@ -167,22 +210,23 @@ Status IndexUpdater::UpdateTagIndex(std::string_view key,
std::string_view origi
return Status::OK();
}
-Status IndexUpdater::UpdateNumericIndex(std::string_view key, std::string_view
original, std::string_view current,
+Status IndexUpdater::UpdateNumericIndex(std::string_view key, const
kqir::Value &original, const kqir::Value ¤t,
const SearchKey &search_key, const
NumericFieldMetadata *num) const {
+ CHECK(original.IsNull() || original.Is<kqir::Numeric>());
+ CHECK(original.IsNull() || original.Is<kqir::Numeric>());
+
auto *storage = indexer->storage;
auto batch = storage->GetWriteBatchBase();
auto cf_handle = storage->GetCFHandle(ColumnFamilyID::Search);
- if (!original.empty()) {
- auto original_num = GET_OR_RET(ParseFloat(std::string(original.begin(),
original.end())));
- auto index_key = search_key.ConstructNumericFieldData(original_num, key);
+ if (!original.IsNull()) {
+ auto index_key =
search_key.ConstructNumericFieldData(original.Get<kqir::Numeric>(), key);
batch->Delete(cf_handle, index_key);
}
- if (!current.empty()) {
- auto current_num = GET_OR_RET(ParseFloat(std::string(current.begin(),
current.end())));
- auto index_key = search_key.ConstructNumericFieldData(current_num, key);
+ if (!current.IsNull()) {
+ auto index_key =
search_key.ConstructNumericFieldData(current.Get<kqir::Numeric>(), key);
batch->Put(cf_handle, index_key, Slice());
}
@@ -192,8 +236,8 @@ Status IndexUpdater::UpdateNumericIndex(std::string_view
key, std::string_view o
return Status::OK();
}
-Status IndexUpdater::UpdateIndex(const std::string &field, std::string_view
key, std::string_view original,
- std::string_view current) const {
+Status IndexUpdater::UpdateIndex(const std::string &field, std::string_view
key, const kqir::Value &original,
+ const kqir::Value ¤t) const {
if (original == current) {
// the value of this field is unchanged, no need to update
return Status::OK();
@@ -225,7 +269,7 @@ Status IndexUpdater::Update(const FieldValues &original,
std::string_view key) c
continue;
}
- std::string_view original_val, current_val;
+ kqir::Value original_val, current_val;
if (auto it = original.find(field); it != original.end()) {
original_val = it->second;
diff --git a/src/search/indexer.h b/src/search/indexer.h
index 2bc7471b..029944e2 100644
--- a/src/search/indexer.h
+++ b/src/search/indexer.h
@@ -36,6 +36,7 @@
#include "storage/storage.h"
#include "types/redis_hash.h"
#include "types/redis_json.h"
+#include "value.h"
namespace redis {
@@ -63,11 +64,11 @@ struct FieldValueRetriever {
explicit FieldValueRetriever(JsonValue json) :
db(std::in_place_type<JsonData>, std::move(json)) {}
- rocksdb::Status Retrieve(std::string_view field, std::string *output);
+ StatusOr<kqir::Value> Retrieve(std::string_view field, const
redis::IndexFieldMetadata *type);
};
struct IndexUpdater {
- using FieldValues = std::map<std::string, std::string>;
+ using FieldValues = std::map<std::string, kqir::Value>;
const kqir::IndexInfo *info = nullptr;
GlobalIndexer *indexer = nullptr;
@@ -75,15 +76,15 @@ struct IndexUpdater {
explicit IndexUpdater(const kqir::IndexInfo *info) : info(info) {}
StatusOr<FieldValues> Record(std::string_view key) const;
- Status UpdateIndex(const std::string &field, std::string_view key,
std::string_view original,
- std::string_view current) const;
+ Status UpdateIndex(const std::string &field, std::string_view key, const
kqir::Value &original,
+ const kqir::Value ¤t) const;
Status Update(const FieldValues &original, std::string_view key) const;
Status Build() const;
- Status UpdateTagIndex(std::string_view key, std::string_view original,
std::string_view current,
+ Status UpdateTagIndex(std::string_view key, const kqir::Value &original,
const kqir::Value ¤t,
const SearchKey &search_key, const TagFieldMetadata
*tag) const;
- Status UpdateNumericIndex(std::string_view key, std::string_view original,
std::string_view current,
+ Status UpdateNumericIndex(std::string_view key, const kqir::Value &original,
const kqir::Value ¤t,
const SearchKey &search_key, const
NumericFieldMetadata *num) const;
};
diff --git a/src/search/ir.h b/src/search/ir.h
index 4007890e..72cb7351 100644
--- a/src/search/ir.h
+++ b/src/search/ir.h
@@ -38,7 +38,7 @@
#include "string_util.h"
#include "type_util.h"
-// kqir stands for Kvorcks Query Intermediate Representation
+// kqir stands for Kvrocks Query Intermediate Representation
namespace kqir {
struct Node {
diff --git a/src/search/plan_executor.cc b/src/search/plan_executor.cc
index 7de9f518..9140587e 100644
--- a/src/search/plan_executor.cc
+++ b/src/search/plan_executor.cc
@@ -152,12 +152,11 @@ auto ExecutorContext::Retrieve(RowType &row, const
FieldInfo *field) -> StatusOr
auto retriever = GET_OR_RET(
redis::FieldValueRetriever::Create(field->index->metadata.on_data_type,
row.key, storage, field->index->ns));
- std::string result;
- auto s = retriever.Retrieve(field->name, &result);
- if (!s.ok()) return {Status::NotOK, s.ToString()};
+ auto s = retriever.Retrieve(field->name, field->metadata.get());
+ if (!s) return s;
- row.fields.emplace(field, result);
- return result;
+ row.fields.emplace(field, *s);
+ return *s;
}
} // namespace kqir
diff --git a/src/search/plan_executor.h b/src/search/plan_executor.h
index 82d8e73e..0ead6870 100644
--- a/src/search/plan_executor.h
+++ b/src/search/plan_executor.h
@@ -24,6 +24,7 @@
#include "ir_plan.h"
#include "search/index_info.h"
+#include "search/value.h"
#include "storage/storage.h"
#include "string_util.h"
@@ -33,7 +34,7 @@ struct ExecutorContext;
struct ExecutorNode {
using KeyType = std::string;
- using ValueType = std::string;
+ using ValueType = kqir::Value;
struct RowType {
KeyType key;
std::map<const FieldInfo *, ValueType> fields;
@@ -52,8 +53,9 @@ struct ExecutorNode {
} else {
os << row.key;
}
- return os << " {" << util::StringJoin(row.fields, [](const auto &v) {
return v.first->name + ": " + v.second; })
- << "}";
+ return os << " {" << util::StringJoin(row.fields, [](const auto &v) {
+ return v.first->name + ": " + v.second.ToString();
+ }) << "}";
}
};
diff --git a/src/search/redis_query_parser.h b/src/search/redis_query_parser.h
index fd373830..d64d0bf0 100644
--- a/src/search/redis_query_parser.h
+++ b/src/search/redis_query_parser.h
@@ -32,7 +32,7 @@ using namespace peg;
struct Field : seq<one<'@'>, Identifier> {};
-struct Tag : sor<Identifier, String> {};
+struct Tag : sor<Identifier, StringL> {};
struct TagList : seq<one<'{'>, WSPad<Tag>, star<seq<one<'|'>, WSPad<Tag>>>,
one<'}'>> {};
struct Inf : seq<opt<one<'+', '-'>>, string<'i', 'n', 'f'>> {};
diff --git a/src/search/redis_query_transformer.h
b/src/search/redis_query_transformer.h
index 45734b31..0928ed59 100644
--- a/src/search/redis_query_transformer.h
+++ b/src/search/redis_query_transformer.h
@@ -36,7 +36,7 @@ namespace ir = kqir;
template <typename Rule>
using TreeSelector =
- parse_tree::selector<Rule, parse_tree::store_content::on<Number, String,
Identifier, Inf>,
+ parse_tree::selector<Rule, parse_tree::store_content::on<Number, StringL,
Identifier, Inf>,
parse_tree::remove_content::on<TagList, NumericRange,
ExclusiveNumber, FieldQuery, NotExpr,
AndExpr, OrExpr,
Wildcard>>;
diff --git a/src/search/sql_parser.h b/src/search/sql_parser.h
index 981ea0ac..26e3da6d 100644
--- a/src/search/sql_parser.h
+++ b/src/search/sql_parser.h
@@ -31,7 +31,7 @@ namespace sql {
using namespace peg;
struct HasTag : string<'h', 'a', 's', 't', 'a', 'g'> {};
-struct HasTagExpr : WSPad<seq<Identifier, WSPad<HasTag>, String>> {};
+struct HasTagExpr : WSPad<seq<Identifier, WSPad<HasTag>, StringL>> {};
struct NumericAtomExpr : WSPad<sor<Number, Identifier>> {};
struct NumericCompareOp : sor<string<'!', '='>, string<'<', '='>, string<'>',
'='>, one<'=', '<', '>'>> {};
diff --git a/src/search/sql_transformer.h b/src/search/sql_transformer.h
index 8386a66e..972ae894 100644
--- a/src/search/sql_transformer.h
+++ b/src/search/sql_transformer.h
@@ -38,7 +38,7 @@ namespace ir = kqir;
template <typename Rule>
using TreeSelector = parse_tree::selector<
Rule,
- parse_tree::store_content::on<Boolean, Number, String, Identifier,
NumericCompareOp, AscOrDesc, UnsignedInteger>,
+ parse_tree::store_content::on<Boolean, Number, StringL, Identifier,
NumericCompareOp, AscOrDesc, UnsignedInteger>,
parse_tree::remove_content::on<HasTagExpr, NumericCompareExpr, NotExpr,
AndExpr, OrExpr, Wildcard, SelectExpr,
FromExpr, WhereClause, OrderByClause,
LimitClause, SearchStmt>>;
@@ -58,7 +58,7 @@ struct Transformer : ir::TreeTransformer {
return Node::Create<ir::BoolLiteral>(node->string_view() == "true");
} else if (Is<Number>(node)) {
return Node::Create<ir::NumericLiteral>(*ParseFloat(node->string()));
- } else if (Is<String>(node)) {
+ } else if (Is<StringL>(node)) {
return
Node::Create<ir::StringLiteral>(GET_OR_RET(UnescapeString(node->string_view())));
} else if (Is<HasTagExpr>(node)) {
CHECK(node->children.size() == 2);
diff --git a/src/search/value.h b/src/search/value.h
new file mode 100644
index 00000000..f1a1e8b7
--- /dev/null
+++ b/src/search/value.h
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+#pragma once
+
+#include <string>
+#include <utility>
+#include <variant>
+#include <vector>
+
+#include "fmt/core.h"
+#include "search/search_encoding.h"
+#include "string_util.h"
+
+namespace kqir {
+
+using Null = std::monostate;
+
+using Numeric = double; // used for numeric fields
+
+using String = std::string; // e.g. a single tag
+
+using NumericArray = std::vector<Numeric>; // used for vector fields
+using StringArray = std::vector<String>; // used for tag fields, e.g. a
list for tags
+
+struct Value : std::variant<Null, Numeric, StringArray> {
+ using Base = std::variant<Null, Numeric, StringArray>;
+
+ using Base::Base;
+
+ bool IsNull() const { return Is<Null>(); }
+
+ template <typename T>
+ bool Is() const {
+ return std::holds_alternative<T>(*this);
+ }
+
+ template <typename T>
+ const auto &Get() const {
+ CHECK(Is<T>());
+ return std::get<T>(*this);
+ }
+
+ template <typename T>
+ auto &Get() {
+ CHECK(Is<T>());
+ return std::get<T>(*this);
+ }
+
+ std::string ToString(const std::string &sep = ",") const {
+ if (IsNull()) {
+ return "";
+ } else if (Is<Numeric>()) {
+ return fmt::format("{}", Get<Numeric>());
+ } else if (Is<StringArray>()) {
+ return util::StringJoin(
+ Get<StringArray>(), [](const auto &v) -> decltype(auto) { return v;
}, sep);
+ }
+
+ __builtin_unreachable();
+ }
+
+ std::string ToString(redis::IndexFieldMetadata *meta) const {
+ if (IsNull()) {
+ return "";
+ } else if (Is<Numeric>()) {
+ return fmt::format("{}", Get<Numeric>());
+ } else if (Is<StringArray>()) {
+ auto tag = dynamic_cast<redis::TagFieldMetadata *>(meta);
+ char sep = tag ? tag->separator : ',';
+ return util::StringJoin(
+ Get<StringArray>(), [](const auto &v) -> decltype(auto) { return v;
}, std::string(1, sep));
+ }
+
+ __builtin_unreachable();
+ }
+};
+
+template <typename T, typename... Args>
+auto MakeValue(Args &&...args) {
+ return Value(std::in_place_type<T>, std::forward<Args>(args)...);
+}
+
+} // namespace kqir
diff --git a/tests/cppunit/indexer_test.cc b/tests/cppunit/indexer_test.cc
index d9b3656e..c3a7769e 100644
--- a/tests/cppunit/indexer_test.cc
+++ b/tests/cppunit/indexer_test.cc
@@ -30,6 +30,8 @@
#include "storage/redis_metadata.h"
#include "types/redis_hash.h"
+static auto T(const std::string& v) { return
kqir::MakeValue<kqir::StringArray>(util::Split(v, ",")); }
+
struct IndexerTest : TestBase {
redis::GlobalIndexer indexer;
kqir::IndexMap map;
@@ -115,7 +117,7 @@ TEST_F(IndexerTest, HashTag) {
ASSERT_TRUE(s);
ASSERT_EQ(s->updater.info->name, idxname);
ASSERT_EQ(s->fields.size(), 1);
- ASSERT_EQ(s->fields["x"], "food,kitChen,Beauty");
+ ASSERT_EQ(s->fields["x"], T("food,kitChen,Beauty"));
uint64_t cnt = 0;
auto s_set = db.Set(key1, "x", "Clothing,FOOD,sport", &cnt);
@@ -205,7 +207,7 @@ TEST_F(IndexerTest, JsonTag) {
ASSERT_TRUE(s);
ASSERT_EQ(s->updater.info->name, idxname);
ASSERT_EQ(s->fields.size(), 1);
- ASSERT_EQ(s->fields["$.x"], "food,kitChen,Beauty");
+ ASSERT_EQ(s->fields["$.x"], T("food,kitChen,Beauty"));
auto s_set = db.Set(key1, "$.x", "\"Clothing,FOOD,sport\"");
ASSERT_TRUE(s_set.ok());
diff --git a/tests/cppunit/plan_executor_test.cc
b/tests/cppunit/plan_executor_test.cc
index 20f1f009..00e0c162 100644
--- a/tests/cppunit/plan_executor_test.cc
+++ b/tests/cppunit/plan_executor_test.cc
@@ -29,6 +29,8 @@
#include "search/interval.h"
#include "search/ir.h"
#include "search/ir_plan.h"
+#include "search/value.h"
+#include "string_util.h"
#include "test_base.h"
#include "types/redis_json.h"
@@ -81,12 +83,15 @@ TEST(PlanExecutorTest, Mock) {
static auto IndexI() -> const IndexInfo* { return index_map.Find("ia",
"search_ns")->second.get(); }
static auto FieldI(const std::string& f) -> const FieldInfo* { return
&IndexI()->fields.at(f); }
+static auto N(double n) { return MakeValue<Numeric>(n); }
+static auto T(const std::string& v) { return
MakeValue<StringArray>(util::Split(v, ",")); }
+
TEST(PlanExecutorTest, TopNSort) {
std::vector<ExecutorNode::RowType> data{
- {"a", {{FieldI("f3"), "4"}}, IndexI()}, {"b", {{FieldI("f3"), "2"}},
IndexI()},
- {"c", {{FieldI("f3"), "7"}}, IndexI()}, {"d", {{FieldI("f3"), "3"}},
IndexI()},
- {"e", {{FieldI("f3"), "1"}}, IndexI()}, {"f", {{FieldI("f3"), "6"}},
IndexI()},
- {"g", {{FieldI("f3"), "8"}}, IndexI()},
+ {"a", {{FieldI("f3"), N(4)}}, IndexI()}, {"b", {{FieldI("f3"), N(2)}},
IndexI()},
+ {"c", {{FieldI("f3"), N(7)}}, IndexI()}, {"d", {{FieldI("f3"), N(3)}},
IndexI()},
+ {"e", {{FieldI("f3"), N(1)}}, IndexI()}, {"f", {{FieldI("f3"), N(6)}},
IndexI()},
+ {"g", {{FieldI("f3"), N(8)}}, IndexI()},
};
{
auto op = std::make_unique<TopNSort>(
@@ -118,10 +123,10 @@ TEST(PlanExecutorTest, TopNSort) {
TEST(PlanExecutorTest, Filter) {
std::vector<ExecutorNode::RowType> data{
- {"a", {{FieldI("f3"), "4"}}, IndexI()}, {"b", {{FieldI("f3"), "2"}},
IndexI()},
- {"c", {{FieldI("f3"), "7"}}, IndexI()}, {"d", {{FieldI("f3"), "3"}},
IndexI()},
- {"e", {{FieldI("f3"), "1"}}, IndexI()}, {"f", {{FieldI("f3"), "6"}},
IndexI()},
- {"g", {{FieldI("f3"), "8"}}, IndexI()},
+ {"a", {{FieldI("f3"), N(4)}}, IndexI()}, {"b", {{FieldI("f3"), N(2)}},
IndexI()},
+ {"c", {{FieldI("f3"), N(7)}}, IndexI()}, {"d", {{FieldI("f3"), N(3)}},
IndexI()},
+ {"e", {{FieldI("f3"), N(1)}}, IndexI()}, {"f", {{FieldI("f3"), N(6)}},
IndexI()},
+ {"g", {{FieldI("f3"), N(8)}}, IndexI()},
};
{
auto field = std::make_unique<FieldRef>("f3", FieldI("f3"));
@@ -157,10 +162,10 @@ TEST(PlanExecutorTest, Filter) {
ASSERT_EQ(ctx.Next().GetValue(), exe_end);
}
- data = {{"a", {{FieldI("f1"), "cpp,java"}}, IndexI()}, {"b",
{{FieldI("f1"), "python,cpp,c"}}, IndexI()},
- {"c", {{FieldI("f1"), "c,perl"}}, IndexI()}, {"d",
{{FieldI("f1"), "rust,python"}}, IndexI()},
- {"e", {{FieldI("f1"), "java,kotlin"}}, IndexI()}, {"f",
{{FieldI("f1"), "c,rust"}}, IndexI()},
- {"g", {{FieldI("f1"), "c,cpp,java"}}, IndexI()}};
+ data = {{"a", {{FieldI("f1"), T("cpp,java")}}, IndexI()}, {"b",
{{FieldI("f1"), T("python,cpp,c")}}, IndexI()},
+ {"c", {{FieldI("f1"), T("c,perl")}}, IndexI()}, {"d",
{{FieldI("f1"), T("rust,python")}}, IndexI()},
+ {"e", {{FieldI("f1"), T("java,kotlin")}}, IndexI()}, {"f",
{{FieldI("f1"), T("c,rust")}}, IndexI()},
+ {"g", {{FieldI("f1"), T("c,cpp,java")}}, IndexI()}};
{
auto field = std::make_unique<FieldRef>("f1", FieldI("f1"));
auto op = std::make_unique<Filter>(
@@ -192,10 +197,10 @@ TEST(PlanExecutorTest, Filter) {
TEST(PlanExecutorTest, Limit) {
std::vector<ExecutorNode::RowType> data{
- {"a", {{FieldI("f3"), "4"}}, IndexI()}, {"b", {{FieldI("f3"), "2"}},
IndexI()},
- {"c", {{FieldI("f3"), "7"}}, IndexI()}, {"d", {{FieldI("f3"), "3"}},
IndexI()},
- {"e", {{FieldI("f3"), "1"}}, IndexI()}, {"f", {{FieldI("f3"), "6"}},
IndexI()},
- {"g", {{FieldI("f3"), "8"}}, IndexI()},
+ {"a", {{FieldI("f3"), N(4)}}, IndexI()}, {"b", {{FieldI("f3"), N(2)}},
IndexI()},
+ {"c", {{FieldI("f3"), N(7)}}, IndexI()}, {"d", {{FieldI("f3"), N(3)}},
IndexI()},
+ {"e", {{FieldI("f3"), N(1)}}, IndexI()}, {"f", {{FieldI("f3"), N(6)}},
IndexI()},
+ {"g", {{FieldI("f3"), N(8)}}, IndexI()},
};
{
auto op = std::make_unique<Limit>(std::make_unique<Mock>(data),
std::make_unique<LimitClause>(1, 2));
@@ -219,12 +224,12 @@ TEST(PlanExecutorTest, Limit) {
TEST(PlanExecutorTest, Merge) {
std::vector<ExecutorNode::RowType> data1{
- {"a", {{FieldI("f3"), "4"}}, IndexI()},
- {"b", {{FieldI("f3"), "2"}}, IndexI()},
+ {"a", {{FieldI("f3"), N(4)}}, IndexI()},
+ {"b", {{FieldI("f3"), N(2)}}, IndexI()},
};
- std::vector<ExecutorNode::RowType> data2{{"c", {{FieldI("f3"), "7"}},
IndexI()},
- {"d", {{FieldI("f3"), "3"}},
IndexI()},
- {"e", {{FieldI("f3"), "1"}},
IndexI()}};
+ std::vector<ExecutorNode::RowType> data2{{"c", {{FieldI("f3"), N(7)}},
IndexI()},
+ {"d", {{FieldI("f3"), N(3)}},
IndexI()},
+ {"e", {{FieldI("f3"), N(1)}},
IndexI()}};
{
auto op =
std::make_unique<Merge>(Node::List<PlanOperator>(std::make_unique<Mock>(data1),
std::make_unique<Mock>(data2)));
