This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 1c2ba48423b brach-3.0 cherry-pick [Fix](Variant) fix serialize with
json key contains `.` as name (#51930)
1c2ba48423b is described below
commit 1c2ba48423b93cc2d2b3f6dacfd26e38b5789572
Author: lihangyu <[email protected]>
AuthorDate: Thu Jun 26 10:20:51 2025 +0800
brach-3.0 cherry-pick [Fix](Variant) fix serialize with json key contains
`.` as name (#51930)
cherry-pick from #51857
---
.../olap/rowset/segment_v2/hierarchical_data_reader.h | 8 ++++++++
be/src/vec/columns/column_object.cpp | 8 ++------
be/src/vec/core/field.h | 9 ++-------
be/src/vec/data_types/data_type_object.cpp | 10 ++++++++--
be/src/vec/data_types/serde/data_type_serde.cpp | 2 +-
be/src/vec/json/json_parser.h | 10 ++++++++++
be/src/vec/json/path_in_data.h | 18 ------------------
be/test/vec/columns/column_object_test.cpp | 1 +
gensrc/proto/data.proto | 1 +
regression-test/data/variant_p0/column_name.out | Bin 469 -> 545 bytes
regression-test/suites/variant_p0/column_name.groovy | 15 +++++++++++++--
11 files changed, 46 insertions(+), 36 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
index c6f325aacea..6b0d4d12345 100644
--- a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
+++ b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
@@ -45,6 +45,14 @@
namespace doris::segment_v2 {
+struct PathWithColumnAndType {
+ vectorized::PathInData path;
+ vectorized::ColumnPtr column;
+ vectorized::DataTypePtr type;
+};
+
+using PathsWithColumnAndType = std::vector<PathWithColumnAndType>;
+
// Reader for hierarchical data for variant, merge with root(sparse encoded
columns)
class HierarchicalDataReader : public ColumnIterator {
public:
diff --git a/be/src/vec/columns/column_object.cpp
b/be/src/vec/columns/column_object.cpp
index 4badf5bfaef..f14f0c62deb 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -907,11 +907,7 @@ void ColumnObject::try_insert(const Field& field) {
}
const auto& object = field.get<const VariantMap&>();
size_t old_size = size();
- for (const auto& [key_str, value] : object) {
- PathInData key;
- if (!key_str.empty()) {
- key = PathInData(key_str);
- }
+ for (const auto& [key, value] : object) {
if (!has_subcolumn(key)) {
bool succ = add_sub_column(key, old_size);
if (!succ) {
@@ -1004,7 +1000,7 @@ void ColumnObject::get(size_t n, Field& res) const {
entry->data.get(n, field);
// Notice: we treat null as empty field, since we do not distinguish
null and empty for Variant type.
if (field.get_type() != Field::Types::Null) {
- object.try_emplace(entry->path.get_path(), field);
+ object.try_emplace(entry->path, field);
}
}
if (object.empty()) {
diff --git a/be/src/vec/core/field.h b/be/src/vec/core/field.h
index 8113dc602fb..922f9abb13e 100644
--- a/be/src/vec/core/field.h
+++ b/be/src/vec/core/field.h
@@ -43,6 +43,7 @@
#include "util/quantile_state.h"
#include "vec/common/uint128.h"
#include "vec/core/types.h"
+#include "vec/json/path_in_data.h"
namespace doris {
namespace vectorized {
@@ -153,13 +154,7 @@ DEFINE_FIELD_VECTOR(Tuple);
DEFINE_FIELD_VECTOR(Map);
#undef DEFINE_FIELD_VECTOR
-using FieldMap = std::map<String, Field, std::less<String>>;
-#define DEFINE_FIELD_MAP(X) \
- struct X : public FieldMap { \
- using FieldMap::FieldMap; \
- }
-DEFINE_FIELD_MAP(VariantMap);
-#undef DEFINE_FIELD_MAP
+using VariantMap = std::map<PathInData, Field>;
class JsonbField {
public:
diff --git a/be/src/vec/data_types/data_type_object.cpp
b/be/src/vec/data_types/data_type_object.cpp
index 6dde7df44d5..1eab54fd5e1 100644
--- a/be/src/vec/data_types/data_type_object.cpp
+++ b/be/src/vec/data_types/data_type_object.cpp
@@ -69,6 +69,7 @@ int64_t
DataTypeObject::get_uncompressed_serialized_bytes(const IColumn& column,
}
PColumnMeta column_meta_pb;
column_meta_pb.set_name(entry->path.get_path());
+ entry->path.to_protobuf(column_meta_pb.mutable_column_path(), -1 /*not
used here*/);
type->to_pb_column_meta(&column_meta_pb);
std::string meta_binary;
column_meta_pb.SerializeToString(&meta_binary);
@@ -112,6 +113,7 @@ char* DataTypeObject::serialize(const IColumn& column,
char* buf, int be_exec_ve
++num_of_columns;
PColumnMeta column_meta_pb;
column_meta_pb.set_name(entry->path.get_path());
+ entry->path.to_protobuf(column_meta_pb.mutable_column_path(), -1 /*not
used here*/);
type->to_pb_column_meta(&column_meta_pb);
std::string meta_binary;
column_meta_pb.SerializeToString(&meta_binary);
@@ -157,11 +159,15 @@ const char* DataTypeObject::deserialize(const char* buf,
MutableColumnPtr* colum
MutableColumnPtr sub_column = type->create_column();
buf = type->deserialize(buf, &sub_column, be_exec_version);
- // add subcolumn to column_object
PathInData key;
- if (!column_meta_pb.name().empty()) {
+ if (column_meta_pb.has_column_path()) {
+ // init from path pb
+ key.from_protobuf(column_meta_pb.column_path());
+ } else if (!column_meta_pb.name().empty()) {
+ // init from name for compatible
key = PathInData {column_meta_pb.name()};
}
+ // add subcolumn to column_object
column_object->add_sub_column(key, std::move(sub_column), type);
}
size_t num_rows = 0;
diff --git a/be/src/vec/data_types/serde/data_type_serde.cpp
b/be/src/vec/data_types/serde/data_type_serde.cpp
index 0709df03bb7..d412ce18139 100644
--- a/be/src/vec/data_types/serde/data_type_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_serde.cpp
@@ -55,7 +55,7 @@ void DataTypeSerDe::convert_variant_map_to_rapidjson(
continue;
}
rapidjson::Value key;
- key.SetString(item.first.data(), item.first.size());
+ key.SetString(item.first.get_path().data(),
item.first.get_path().size());
rapidjson::Value val;
convert_field_to_rapidjson(item.second, val, allocator);
if (val.IsNull() && item.first.empty()) {
diff --git a/be/src/vec/json/json_parser.h b/be/src/vec/json/json_parser.h
index af2e452dddc..c1815ae5be4 100644
--- a/be/src/vec/json/json_parser.h
+++ b/be/src/vec/json/json_parser.h
@@ -28,6 +28,7 @@
#include <utility>
#include <vector>
+#include "runtime/primitive_type.h"
#include "util/jsonb_writer.h"
#include "vec/columns/column.h"
#include "vec/common/string_ref.h"
@@ -124,6 +125,15 @@ enum class ExtractType {
struct ParseConfig {
bool enable_flatten_nested = false;
};
+
+/// Result of parsing of a document.
+/// Contains all paths extracted from document
+/// and values which are related to them.
+struct ParseResult {
+ std::vector<PathInData> paths;
+ std::vector<Field> values;
+};
+
template <typename ParserImpl>
class JSONDataParser {
public:
diff --git a/be/src/vec/json/path_in_data.h b/be/src/vec/json/path_in_data.h
index 8d94b02f37a..74ae3458554 100644
--- a/be/src/vec/json/path_in_data.h
+++ b/be/src/vec/json/path_in_data.h
@@ -29,11 +29,8 @@
#include <vector>
#include "gen_cpp/segment_v2.pb.h"
-#include "vec/columns/column.h"
#include "vec/common/uint128.h"
-#include "vec/core/field.h"
#include "vec/core/types.h"
-#include "vec/data_types/data_type.h"
namespace doris::vectorized {
@@ -129,13 +126,6 @@ private:
size_t current_anonymous_array_level = 0;
};
using PathsInData = std::vector<PathInData>;
-/// Result of parsing of a document.
-/// Contains all paths extracted from document
-/// and values which are related to them.
-struct ParseResult {
- std::vector<PathInData> paths;
- std::vector<Field> values;
-};
struct PathInDataRef {
const PathInData* ref;
@@ -148,12 +138,4 @@ struct PathInDataRef {
bool operator==(const PathInDataRef& other) const { return *this->ref ==
*other.ref; }
};
-struct PathWithColumnAndType {
- PathInData path;
- ColumnPtr column;
- DataTypePtr type;
-};
-
-using PathsWithColumnAndType = std::vector<PathWithColumnAndType>;
-
} // namespace doris::vectorized
diff --git a/be/test/vec/columns/column_object_test.cpp
b/be/test/vec/columns/column_object_test.cpp
index 06d987e414e..a7498e82e87 100644
--- a/be/test/vec/columns/column_object_test.cpp
+++ b/be/test/vec/columns/column_object_test.cpp
@@ -22,6 +22,7 @@
#include <gtest/gtest.h>
#include "vec/columns/common_column_test.h"
+#include "vec/json/path_in_data.h"
namespace doris::vectorized {
diff --git a/gensrc/proto/data.proto b/gensrc/proto/data.proto
index 9b3824db3dc..7cb126e973b 100644
--- a/gensrc/proto/data.proto
+++ b/gensrc/proto/data.proto
@@ -63,6 +63,7 @@ message PColumnMeta {
optional bool result_is_nullable = 6;
optional string function_name = 7;
optional int32 be_exec_version = 8;
+ optional segment_v2.ColumnPathInfo column_path = 9;
}
message PBlock {
diff --git a/regression-test/data/variant_p0/column_name.out
b/regression-test/data/variant_p0/column_name.out
index 6ac882d2922..0f54df05d91 100644
Binary files a/regression-test/data/variant_p0/column_name.out and
b/regression-test/data/variant_p0/column_name.out differ
diff --git a/regression-test/suites/variant_p0/column_name.groovy
b/regression-test/suites/variant_p0/column_name.groovy
index 7962112ff75..7cf7fe198b1 100644
--- a/regression-test/suites/variant_p0/column_name.groovy
+++ b/regression-test/suites/variant_p0/column_name.groovy
@@ -25,7 +25,7 @@ suite("regression_test_variant_column_name", "variant_type"){
)
DUPLICATE KEY(`k`)
DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction" =
"true");
+ properties("replication_num" = "1", "disable_auto_compaction" =
"false");
"""
sql """insert into ${table_name} values (1, '{"中文" : "中文",
"\\\u4E2C\\\u6587": "unicode"}')"""
@@ -61,7 +61,18 @@ suite("regression_test_variant_column_name", "variant_type"){
sql """insert into var_column_name values (7, '{"": 1234566}')"""
sql """insert into var_column_name values (7, '{"": 8888888}')"""
- qt_sql "select Tags[''] from var_column_name order by cast(Tags[''] as
string)"
+ qt_sql "select cast(Tags[''] as text) from var_column_name order by
cast(Tags[''] as string)"
+
+ // name with `.`
+ sql "truncate table var_column_name"
+ sql """insert into var_column_name values (7, '{"a.b": "UPPER CASE",
"a.c": "lower case", "a" : {"b" : 123}, "a" : {"c" : 456}}')"""
+ for (int i = 0; i < 7; i++) {
+ sql """insert into var_column_name select * from var_column_name"""
+ }
+ qt_sql_cnt_1 "select count(Tags['a.b']) from var_column_name"
+ qt_sql_cnt_2 "select count(Tags['a.c']) from var_column_name"
+ qt_sql_cnt_3 "select count(Tags['a']['b']) from var_column_name"
+ qt_sql_cnt_4 "select count(Tags['a']['c']) from var_column_name"
try {
sql """insert into var_column_name values (7, '{"": "UPPER CASE", "":
"lower case"}')"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
