This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1de2fd71be1 [refactor](variant) Sync NestedGroup provider interface
and reader guards (#60930)
1de2fd71be1 is described below
commit 1de2fd71be1e634abac2584ea47c96474354ba24
Author: lihangyu <[email protected]>
AuthorDate: Wed Mar 4 19:36:57 2026 +0800
[refactor](variant) Sync NestedGroup provider interface and reader guards
(#60930)
Add more interface and code for NestedGroup
---
be/src/common/config.cpp | 1 +
be/src/common/config.h | 4 +
.../rowset/segment_v2/variant/nested_group_path.h | 4 +
.../segment_v2/variant/nested_group_provider.cpp | 24 +++
.../segment_v2/variant/nested_group_provider.h | 11 ++
.../segment_v2/variant/nested_group_reader.h | 46 +++++
.../variant/nested_group_routing_plan.cpp | 186 +++++++++++++++++++
.../segment_v2/variant/nested_group_routing_plan.h | 82 +++++++++
.../segment_v2/variant/variant_column_reader.cpp | 89 +++++++--
.../segment_v2/variant/variant_column_reader.h | 26 ++-
.../variant/variant_column_writer_impl.cpp | 202 ++++++++++++++-------
.../variant/variant_column_writer_impl.h | 3 +
be/src/vec/columns/column_variant.cpp | 83 ++++++++-
13 files changed, 661 insertions(+), 100 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 36d465d2c77..29ad33ca3ef 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1156,6 +1156,7 @@ DEFINE_mBool(enable_variant_doc_sparse_write_subcolumns,
"true");
// Reserved for future use when NestedGroup expansion moves to storage layer
// Deeper arrays will be stored as JSONB
DEFINE_mInt32(variant_nested_group_max_depth, "3");
+DEFINE_mBool(variant_nested_group_discard_scalar_on_conflict, "true");
DEFINE_Validator(variant_max_json_key_length,
[](const int config) -> bool { return config > 0 && config <=
65535; });
diff --git a/be/src/common/config.h b/be/src/common/config.h
index c1df9a11acb..e2e494af8a9 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1418,6 +1418,10 @@
DECLARE_mBool(enable_variant_doc_sparse_write_subcolumns);
// Maximum depth of nested arrays to track with NestedGroup
// Reserved for future use when NestedGroup expansion moves to storage layer
DECLARE_mInt32(variant_nested_group_max_depth);
+// When true, discard scalar data that conflicts with NestedGroup array<object>
+// data at the same path. This simplifies compaction by always prioritizing
+// nested structure over scalar. When false, report an error on conflict.
+DECLARE_mBool(variant_nested_group_discard_scalar_on_conflict);
DECLARE_mBool(enable_merge_on_write_correctness_check);
// USED FOR DEBUGING
diff --git a/be/src/olap/rowset/segment_v2/variant/nested_group_path.h
b/be/src/olap/rowset/segment_v2/variant/nested_group_path.h
index a27dc4038d6..5c90d1441ad 100644
--- a/be/src/olap/rowset/segment_v2/variant/nested_group_path.h
+++ b/be/src/olap/rowset/segment_v2/variant/nested_group_path.h
@@ -26,6 +26,10 @@ inline constexpr std::string_view kNestedGroupMarker =
"__D0_ng__";
inline constexpr std::string_view kRootNestedGroupPath = "__D0_root__";
inline constexpr std::string_view kNestedGroupOffsetsSuffix = ".__offsets";
+inline bool is_root_nested_group_path(std::string_view path) {
+ return path == kRootNestedGroupPath;
+}
+
inline bool ends_with(std::string_view value, std::string_view suffix) {
return value.size() >= suffix.size() &&
value.compare(value.size() - suffix.size(), suffix.size(), suffix)
== 0;
diff --git a/be/src/olap/rowset/segment_v2/variant/nested_group_provider.cpp
b/be/src/olap/rowset/segment_v2/variant/nested_group_provider.cpp
index 9b664810e3a..91ffb3a2efb 100644
--- a/be/src/olap/rowset/segment_v2/variant/nested_group_provider.cpp
+++ b/be/src/olap/rowset/segment_v2/variant/nested_group_provider.cpp
@@ -19,6 +19,8 @@
#include <string>
+#include "olap/rowset/segment_v2/variant/nested_group_routing_plan.h"
+
namespace doris::segment_v2 {
namespace {
@@ -95,6 +97,17 @@ public:
return Status::NotSupported("NestedGroup element access is not
available in this build");
}
+ Status create_root_merge_iterator(ColumnIteratorUPtr base_iterator,
+ const NestedGroupReaders& /*readers*/,
+ const StorageReadOptions* /*opt*/,
+ ColumnIteratorUPtr* out) override {
+ if (out == nullptr) {
+ return Status::InvalidArgument("out is null");
+ }
+ *out = std::move(base_iterator);
+ return Status::OK();
+ }
+
Status map_elements_to_parent_ords(const std::vector<const
NestedGroupReader*>& /*group_chain*/,
const ColumnIteratorOptions& /*opts*/,
const roaring::Roaring&
/*element_bitmap*/,
@@ -112,6 +125,17 @@ NestedGroupPathMatch find_in_nested_groups(const
NestedGroupReaders& readers,
return {};
}
+Status collect_nested_group_routing_paths_from_variant_jsonb(
+ const vectorized::ColumnVariant& /*variant*/,
std::vector<std::string>* out_ng_paths,
+ std::vector<std::string>* out_conflict_paths) {
+ if (out_ng_paths == nullptr || out_conflict_paths == nullptr) {
+ return Status::InvalidArgument("out_ng_paths or out_conflict_paths is
null");
+ }
+ out_ng_paths->clear();
+ out_conflict_paths->clear();
+ return Status::OK();
+}
+
std::unique_ptr<NestedGroupWriteProvider> create_nested_group_write_provider()
{
return std::make_unique<DefaultNestedGroupWriteProvider>();
}
diff --git a/be/src/olap/rowset/segment_v2/variant/nested_group_provider.h
b/be/src/olap/rowset/segment_v2/variant/nested_group_provider.h
index c0877be2dd1..b9cef3c2aa1 100644
--- a/be/src/olap/rowset/segment_v2/variant/nested_group_provider.h
+++ b/be/src/olap/rowset/segment_v2/variant/nested_group_provider.h
@@ -23,11 +23,14 @@
#include <memory>
#include <optional>
#include <string>
+#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "common/status.h"
#include "olap/rowset/segment_v2/column_reader.h"
+#include "olap/rowset/segment_v2/variant/nested_group_reader.h"
+#include "vec/columns/column.h"
#include "vec/data_types/data_type.h"
#include "vec/json/path_in_data.h"
@@ -214,6 +217,14 @@ public:
uint64_t* total_elements) const = 0;
// Map element-level bitmap to row-level bitmap through the nested group
chain.
+ // Create an iterator that wraps |base_iterator| with root-level NG merge
logic.
+ // For root variant reads, top-level NestedGroup arrays must be merged
back into
+ // the reconstructed variant. CE no-op returns base_iterator unchanged.
+ virtual Status create_root_merge_iterator(ColumnIteratorUPtr base_iterator,
+ const NestedGroupReaders&
readers,
+ const StorageReadOptions* opt,
+ ColumnIteratorUPtr* out) = 0;
+
virtual Status map_elements_to_parent_ords(
const std::vector<const NestedGroupReader*>& group_chain,
const ColumnIteratorOptions& opts, const roaring::Roaring&
element_bitmap,
diff --git a/be/src/olap/rowset/segment_v2/variant/nested_group_reader.h
b/be/src/olap/rowset/segment_v2/variant/nested_group_reader.h
new file mode 100644
index 00000000000..b1b1d62b03f
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/nested_group_reader.h
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+namespace doris::segment_v2 {
+
+class ColumnReader;
+class NestedOffsetsMappingIndex;
+
+struct NestedGroupReader;
+using NestedGroupReaders = std::unordered_map<std::string,
std::unique_ptr<NestedGroupReader>>;
+
+// Holds readers for a single NestedGroup (offsets + child columns + nested
groups)
+struct NestedGroupReader {
+ std::string array_path;
+ size_t depth = 1; // Nesting depth (1 = first level)
+ std::shared_ptr<ColumnReader> offsets_reader;
+ std::shared_ptr<NestedOffsetsMappingIndex> offsets_mapping_index;
+ std::unordered_map<std::string, std::shared_ptr<ColumnReader>>
child_readers;
+ // Nested groups within this group (for multi-level nesting)
+ NestedGroupReaders nested_group_readers;
+
+ bool is_valid() const { return offsets_reader != nullptr; }
+};
+
+} // namespace doris::segment_v2
diff --git
a/be/src/olap/rowset/segment_v2/variant/nested_group_routing_plan.cpp
b/be/src/olap/rowset/segment_v2/variant/nested_group_routing_plan.cpp
new file mode 100644
index 00000000000..9ab078d17a5
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/nested_group_routing_plan.cpp
@@ -0,0 +1,186 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/variant/nested_group_routing_plan.h"
+
+#include <algorithm>
+#include <string_view>
+#include <unordered_set>
+
+#include "common/config.h"
+#include "olap/rowset/segment_v2/variant/nested_group_path.h"
+#include "olap/rowset/segment_v2/variant/nested_group_provider.h"
+#include "vec/columns/column_variant.h"
+#include "vec/common/variant_util.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/json/path_in_data.h"
+
+namespace doris::segment_v2 {
+
+// --------------------------------------------------------------------------
+// Path prefix utilities
+// --------------------------------------------------------------------------
+
+static bool _path_has_prefix(std::string_view path, std::string_view prefix) {
+ return path == prefix ||
+ (path.size() > prefix.size() && path.starts_with(prefix) &&
path[prefix.size()] == '.');
+}
+
+static bool _is_excluded_by_prefixes(std::string_view path,
+ const std::vector<std::string>&
excluded_prefixes,
+ bool exclude_all_paths) {
+ if (exclude_all_paths) {
+ return true;
+ }
+ for (const auto& prefix : excluded_prefixes) {
+ if (_path_has_prefix(path, prefix)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+bool NestedGroupRoutingPlan::is_excluded_subcolumn(const std::string& path)
const {
+ return _is_excluded_by_prefixes(path, ng_only_prefixes,
exclude_all_subcolumns);
+}
+
+// --------------------------------------------------------------------------
+// Routing plan builder helpers
+// --------------------------------------------------------------------------
+
+static std::vector<std::string> _compact_prefixes(std::vector<std::string>
prefixes) {
+ std::sort(prefixes.begin(), prefixes.end());
+ prefixes.erase(std::unique(prefixes.begin(), prefixes.end()),
prefixes.end());
+ std::sort(prefixes.begin(), prefixes.end(), [](const std::string& lhs,
const std::string& rhs) {
+ return lhs.size() < rhs.size();
+ });
+ std::vector<std::string> compacted;
+ for (auto& p : prefixes) {
+ bool redundant = false;
+ for (const auto& c : compacted) {
+ if (_path_has_prefix(p, c)) {
+ redundant = true;
+ break;
+ }
+ }
+ if (!redundant) {
+ compacted.push_back(std::move(p));
+ }
+ }
+ return compacted;
+}
+
+static bool _is_array_variant_type(const vectorized::DataTypePtr& type) {
+ if (!type) return false;
+ auto base_type = vectorized::variant_util::get_base_type_of_array(type);
+ return base_type != nullptr &&
vectorized::remove_nullable(base_type)->get_primitive_type() ==
+ PrimitiveType::TYPE_VARIANT;
+}
+// Routing builder: only NON-conflict NG paths go into ng_only_prefixes.
+// Conflict paths are NOT excluded from subcolumn writes so compaction/write
+// can still preserve conflict-path payload in regular subcolumns.
+static Status _build_ng_routing_from_columns(
+ const vectorized::ColumnVariant& variant,
+ const std::vector<std::string>& ng_candidate_paths,
+ const std::vector<std::string>& conflict_candidate_paths,
+ std::vector<std::string>* ng_only_prefixes, bool*
exclude_all_subcolumns,
+ NestedGroupConflictPolicy* conflict_policy, bool* has_conflict_paths) {
+ if (ng_only_prefixes == nullptr || exclude_all_subcolumns == nullptr ||
+ conflict_policy == nullptr || has_conflict_paths == nullptr) {
+ return Status::InvalidArgument("output argument is null");
+ }
+
+ ng_only_prefixes->clear();
+ *exclude_all_subcolumns = false;
+ *conflict_policy = get_nested_group_conflict_policy();
+ *has_conflict_paths = !conflict_candidate_paths.empty();
+
+ if (ng_candidate_paths.empty()) {
+ return Status::OK();
+ }
+
+ // Under ERROR policy, reject any conflicts immediately.
+ if (*conflict_policy == NestedGroupConflictPolicy::ERROR &&
!conflict_candidate_paths.empty()) {
+ std::string paths_str;
+ for (const auto& p : conflict_candidate_paths) {
+ if (!paths_str.empty()) paths_str += ", ";
+ paths_str += p;
+ }
+ return Status::InvalidArgument("NestedGroup conflict detected
(policy=ERROR) at paths: {}",
+ paths_str);
+ }
+
+ // Build the conflict set for quick lookup.
+ std::unordered_set<std::string>
conflict_set(conflict_candidate_paths.begin(),
+
conflict_candidate_paths.end());
+
+ // Log conflict paths under DISCARD_SCALAR policy.
+ if (!conflict_candidate_paths.empty()) {
+ for (const auto& p : conflict_candidate_paths) {
+ LOG(WARNING) << "NestedGroup conflict at path '" << p
+ << "': policy=DISCARD_SCALAR (prefer nested data;
scalar payload on this "
+ "path may be dropped). The path remains in regular
subcolumns for "
+ "cross-segment compatibility.";
+ }
+ }
+
+ // Only NON-conflict NG paths go into ng_only_prefixes.
+ // Conflict paths are kept as regular subcolumns to avoid data loss.
+ for (const auto& path : ng_candidate_paths) {
+ if (conflict_set.contains(path)) {
+ continue; // Skip conflict paths — they stay as regular subcolumns.
+ }
+ ng_only_prefixes->emplace_back(path);
+ // For root path that is purely array<variant>, exclude all subcolumns.
+ if (is_root_nested_group_path(path) &&
_is_array_variant_type(variant.get_root_type())) {
+ *exclude_all_subcolumns = true;
+ }
+ }
+
+ *ng_only_prefixes = _compact_prefixes(std::move(*ng_only_prefixes));
+ return Status::OK();
+}
+
+// --------------------------------------------------------------------------
+// Public API
+// --------------------------------------------------------------------------
+
+Status build_nested_group_routing_plan(const vectorized::ColumnVariant&
variant,
+ NestedGroupRoutingPlan* plan) {
+ if (plan == nullptr) {
+ return Status::InvalidArgument("plan is null");
+ }
+ *plan = NestedGroupRoutingPlan {};
+
+ std::vector<std::string> ng_candidate_paths;
+ std::vector<std::string> conflict_candidate_paths;
+ RETURN_IF_ERROR(collect_nested_group_routing_paths_from_variant_jsonb(
+ variant, &ng_candidate_paths, &conflict_candidate_paths));
+ RETURN_IF_ERROR(_build_ng_routing_from_columns(
+ variant, ng_candidate_paths, conflict_candidate_paths,
&plan->ng_only_prefixes,
+ &plan->exclude_all_subcolumns, &plan->conflict_policy,
&plan->has_conflict_paths));
+ return Status::OK();
+}
+
+NestedGroupConflictPolicy get_nested_group_conflict_policy() {
+ if (config::variant_nested_group_discard_scalar_on_conflict) {
+ return NestedGroupConflictPolicy::DISCARD_SCALAR;
+ }
+ return NestedGroupConflictPolicy::ERROR;
+}
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/variant/nested_group_routing_plan.h
b/be/src/olap/rowset/segment_v2/variant/nested_group_routing_plan.h
new file mode 100644
index 00000000000..482099c1df5
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/variant/nested_group_routing_plan.h
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "common/status.h"
+
+namespace doris::vectorized {
+class ColumnVariant;
+} // namespace doris::vectorized
+
+namespace doris::segment_v2 {
+
+// Policy for handling NestedGroup vs scalar conflicts.
+// When the same path has both array<object> and scalar data:
+// DISCARD_SCALAR: silently drop scalar data, keep nested data (default)
+// ERROR: report an error when conflict is detected
+enum class NestedGroupConflictPolicy {
+ DISCARD_SCALAR = 0,
+ ERROR = 1,
+};
+
+// Routing plan for NestedGroup write path. Controls which subcolumn paths
+// are excluded from regular writes because they are handled by NestedGroup.
+//
+// Simplified model:
+// - Only NON-conflict NG paths go into ng_only_prefixes.
+// - Conflict paths stay in regular subcolumns (not excluded), so routing can
+// remain compatible with cross-segment compaction where NG payload may
+// become non-JSONB after merge.
+struct NestedGroupRoutingPlan {
+ bool exclude_all_subcolumns = false;
+ bool has_conflict_paths = false;
+ std::vector<std::string> ng_only_prefixes;
+ NestedGroupConflictPolicy conflict_policy =
NestedGroupConflictPolicy::DISCARD_SCALAR;
+
+ // Returns true if |path| should be excluded from regular subcolumn writes.
+ bool is_excluded_subcolumn(const std::string& path) const;
+
+ // Returns true if the plan has any active exclusions (NG paths found).
+ bool has_exclusions() const { return exclude_all_subcolumns ||
!ng_only_prefixes.empty(); }
+
+ // Returns true if root JSONB can be safely replaced with empty defaults.
+ // Only safe when there are NG exclusions AND no conflict paths.
+ // With conflicts, root JSONB may carry data needed by the NG provider.
+ bool can_remove_root_jsonb() const { return has_exclusions() &&
!has_conflict_paths; }
+};
+
+// Build NG routing plan from variant content. Scans the variant for
+// array<object> paths, detects conflicts, and populates the plan.
+Status build_nested_group_routing_plan(const vectorized::ColumnVariant&
variant,
+ NestedGroupRoutingPlan* plan);
+
+// Collect NG routing metadata from variant content:
+// - out_ng_paths: all NG candidate paths
+// - out_conflict_paths: NG paths that have ARRAY<OBJECT> vs non-array
structural conflicts
+// Both outputs are de-duplicated and sorted.
+Status collect_nested_group_routing_paths_from_variant_jsonb(
+ const vectorized::ColumnVariant& variant, std::vector<std::string>*
out_ng_paths,
+ std::vector<std::string>* out_conflict_paths);
+
+// Get the current global conflict policy (driven by config).
+NestedGroupConflictPolicy get_nested_group_conflict_policy();
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
index 6524a22d902..72748a79895 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp
@@ -25,6 +25,7 @@
#include <roaring/roaring.hh>
#include <string>
#include <utility>
+#include <vector>
#include "binary_column_extract_iterator.h"
#include "binary_column_reader.h"
@@ -59,6 +60,15 @@ namespace doris::segment_v2 {
#include "common/compile_check_begin.h"
+namespace {
+
+bool is_compaction_or_checksum_reader(const StorageReadOptions* opts) {
+ return opts != nullptr &&
(ColumnReader::is_compaction_reader_type(opts->io_ctx.reader_type) ||
+ opts->io_ctx.reader_type ==
ReaderType::READER_CHECKSUM);
+}
+
+} // namespace
+
const SubcolumnColumnMetaInfo::Node*
VariantColumnReader::get_subcolumn_meta_by_path(
const vectorized::PathInData& relative_path) const {
std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
@@ -316,10 +326,17 @@ Status VariantColumnReader::_build_read_plan_flat_leaves(
std::shared_lock<std::shared_mutex> lock(_subcolumns_meta_mutex);
DCHECK(opts != nullptr);
+ int32_t col_uid =
+ target_col.unique_id() >= 0 ? target_col.unique_id() :
target_col.parent_unique_id();
auto relative_path = target_col.path_info_ptr()->copy_pop_front();
- // compaction need to read flat leaves nodes data to prevent from
amplification
const auto* node =
target_col.has_path_info() ?
_subcolumns_meta_info->find_leaf(relative_path) : nullptr;
+ if (!relative_path.empty() && _can_use_nested_group_read_path() &&
+ _try_fill_nested_group_plan(plan, target_col, opts, col_uid,
relative_path)) {
+ return Status::OK();
+ }
+
+ // compaction need to read flat leaves nodes data to prevent from
amplification
if (!node) {
// Handle sparse column reads in flat-leaf compaction.
const std::string rel = relative_path.get_path();
@@ -458,8 +475,12 @@ bool VariantColumnReader::_need_read_flat_leaves(const
StorageReadOptions* opts)
return opts != nullptr && opts->tablet_schema != nullptr &&
std::ranges::any_of(opts->tablet_schema->columns(),
[](const auto& column) { return
column->is_extracted_column(); }) &&
- (is_compaction_reader_type(opts->io_ctx.reader_type) ||
- opts->io_ctx.reader_type == ReaderType::READER_CHECKSUM);
+ is_compaction_or_checksum_reader(opts);
+}
+
+bool VariantColumnReader::_can_use_nested_group_read_path() const {
+ return _nested_group_read_provider != nullptr &&
+ _nested_group_read_provider->should_enable_nested_group_read_path();
}
Status VariantColumnReader::_validate_access_paths_debug(
@@ -587,21 +608,11 @@ Status VariantColumnReader::_validate_access_paths_debug(
return Status::OK();
}
-bool VariantColumnReader::_try_build_nested_group_plan(
+bool VariantColumnReader::_try_fill_nested_group_plan(
ReadPlan* plan, const TabletColumn& target_col, const
StorageReadOptions* opt,
int32_t col_uid, const vectorized::PathInData& relative_path) const {
- if (_nested_group_read_provider == nullptr ||
- !_nested_group_read_provider->should_enable_nested_group_read_path()) {
- return false;
- }
- if (_need_read_flat_leaves(opt)) {
- return false;
- }
- // compaction skip read NestedGroup
- if (is_compaction_reader_type(opt->io_ctx.reader_type) ||
- opt->io_ctx.reader_type == ReaderType::READER_CHECKSUM) {
- return false;
- }
+ DCHECK(_nested_group_read_provider != nullptr);
+
bool is_whole = false;
vectorized::DataTypePtr out_type;
vectorized::PathInData out_relative_path;
@@ -626,6 +637,26 @@ bool VariantColumnReader::_try_build_nested_group_plan(
return true;
}
+bool VariantColumnReader::_try_build_nested_group_plan(
+ ReadPlan* plan, const TabletColumn& target_col, const
StorageReadOptions* opt,
+ int32_t col_uid, const vectorized::PathInData& relative_path) const {
+ const bool is_compaction_or_checksum =
is_compaction_or_checksum_reader(opt);
+
+ // Root path in compaction/checksum must reconstruct full Variant rows for
re-write.
+ // Query root reads can still use NestedGroup whole read for top-level
array shape.
+ if (relative_path.empty() && is_compaction_or_checksum) {
+ return false;
+ }
+ if (!_can_use_nested_group_read_path()) {
+ return false;
+ }
+
+ if (_need_read_flat_leaves(opt)) {
+ return false;
+ }
+ return _try_fill_nested_group_plan(plan, target_col, opt, col_uid,
relative_path);
+}
+
Status VariantColumnReader::_try_build_leaf_plan(ReadPlan* plan, int32_t
col_uid,
const vectorized::PathInData&
relative_path,
const
SubcolumnColumnMetaInfo::Node* node,
@@ -705,6 +736,12 @@ Status VariantColumnReader::_build_read_plan(ReadPlan*
plan, const TabletColumn&
node = _subcolumns_meta_info->find_exact(relative_path);
}
+ // NestedGroup path resolution must happen before doc/sparse/hierarchical
fallbacks.
+ // This keeps query/compaction behavior consistent for array<object> paths.
+ if (_try_build_nested_group_plan(plan, target_col, opt, col_uid,
relative_path)) {
+ return Status::OK();
+ }
+
// read root: from doc value column
if (root->path == relative_path &&
_statistics->has_doc_value_column_non_null_size()) {
plan->kind = ReadKind::HIERARCHICAL_DOC;
@@ -838,9 +875,23 @@ Status VariantColumnReader::_create_iterator_from_plan(
case ReadKind::HIERARCHICAL: {
int32_t col_uid = target_col.unique_id() >= 0 ? target_col.unique_id()
:
target_col.parent_unique_id();
+ ColumnIteratorUPtr base_iterator;
RETURN_IF_ERROR(_create_hierarchical_reader(
- iterator, col_uid, plan.relative_path, plan.node, plan.root,
column_reader_cache,
- opt->stats,
HierarchicalDataIterator::ReadType::SUBCOLUMNS_AND_SPARSE));
+ &base_iterator, col_uid, plan.relative_path, plan.node,
plan.root,
+ column_reader_cache, opt->stats,
+ HierarchicalDataIterator::ReadType::SUBCOLUMNS_AND_SPARSE));
+
+ // Root variant reconstruction needs to merge top-level NestedGroup
arrays, because NG leaf
+ // columns are not row-aligned and are skipped by the generic
hierarchical reader.
+ if (plan.relative_path.empty() && _nested_group_read_provider !=
nullptr &&
+ !_nested_group_readers.empty()) {
+ ColumnIteratorUPtr merged_iterator;
+
RETURN_IF_ERROR(_nested_group_read_provider->create_root_merge_iterator(
+ std::move(base_iterator), _nested_group_readers, opt,
&merged_iterator));
+ *iterator = std::move(merged_iterator);
+ return Status::OK();
+ }
+ *iterator = std::move(base_iterator);
return Status::OK();
}
case ReadKind::LEAF: {
@@ -1167,7 +1218,7 @@ Status VariantColumnReader::init(const
ColumnReaderOptions& opts, ColumnMetaAcce
// NestedGroup initialization is provider-driven. Disabled providers keep
fallback behavior,
// while enabled providers populate nested group readers from segment
footer.
- if (_nested_group_read_provider->should_enable_nested_group_read_path()) {
+ if (_can_use_nested_group_read_path()) {
RETURN_IF_ERROR(_nested_group_read_provider->init_readers(opts,
footer, file_reader,
num_rows,
_nested_group_readers));
}
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h
b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h
index 1318fc9402c..038764684c8 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.h
@@ -29,6 +29,7 @@
#include <vector>
#include "nested_group_provider.h"
+#include "nested_group_reader.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/indexed_column_reader.h"
#include "olap/rowset/segment_v2/page_handle.h"
@@ -179,21 +180,7 @@ using BinaryColumnCacheSPtr =
std::shared_ptr<BinaryColumnCache>;
using PathToBinaryColumnCache = std::unordered_map<std::string,
BinaryColumnCacheSPtr>;
using PathToBinaryColumnCacheUPtr = std::unique_ptr<PathToBinaryColumnCache>;
-// Forward declaration
-struct NestedGroupReader;
-
-// Holds readers for a single NestedGroup (offsets + child columns + nested
groups)
-struct NestedGroupReader {
- std::string array_path;
- size_t depth = 1; // Nesting depth (1 = first level)
- std::shared_ptr<ColumnReader> offsets_reader;
- std::shared_ptr<NestedOffsetsMappingIndex> offsets_mapping_index;
- std::unordered_map<std::string, std::shared_ptr<ColumnReader>>
child_readers;
- // Nested groups within this group (for multi-level nesting)
- NestedGroupReaders nested_group_readers;
-
- bool is_valid() const { return offsets_reader != nullptr; }
-};
+// NestedGroupReader is defined in nested_group_reader.h
class VariantColumnReader : public ColumnReader {
public:
@@ -219,6 +206,11 @@ public:
const VariantStatistics* get_stats() const { return _statistics.get(); }
+ // Expose raw root column reader for test assertions (e.g., dedup checks).
+ const std::shared_ptr<ColumnReader>& get_root_column_reader() const {
+ return _root_column_reader;
+ }
+
int64_t get_metadata_size() const override;
// Return shared_ptr to ensure the lifetime of TabletIndex objects
@@ -361,9 +353,13 @@ private:
PathToBinaryColumnCache* binary_column_cache_ptr);
static bool _need_read_flat_leaves(const StorageReadOptions* opts);
+ bool _can_use_nested_group_read_path() const;
Status _validate_access_paths_debug(const TabletColumn& target_col,
const StorageReadOptions* opt, int32_t
col_uid,
const vectorized::PathInData&
relative_path) const;
+ bool _try_fill_nested_group_plan(ReadPlan* plan, const TabletColumn&
target_col,
+ const StorageReadOptions* opt, int32_t
col_uid,
+ const vectorized::PathInData&
relative_path) const;
bool _try_build_nested_group_plan(ReadPlan* plan, const TabletColumn&
target_col,
const StorageReadOptions* opt, int32_t
col_uid,
const vectorized::PathInData&
relative_path) const;
diff --git
a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
index d50fc75e434..f5de9c1e651 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
@@ -19,23 +19,41 @@
#include <gen_cpp/segment_v2.pb.h>
#include <algorithm>
+#include <iostream>
#include <memory>
+#include <string_view>
+#include <unordered_map>
+#include <unordered_set>
+#include "common/cast_set.h"
#include "common/status.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
#include "olap/rowset/rowset_writer_context.h"
#include "olap/rowset/segment_v2/column_writer.h"
#include "olap/rowset/segment_v2/indexed_column_writer.h"
+#include "olap/rowset/segment_v2/variant/nested_group_path.h"
+#include "olap/rowset/segment_v2/variant/nested_group_routing_plan.h"
#include "olap/tablet_schema.h"
#include "olap/types.h"
+#include "runtime/runtime_state.h"
#include "vec/columns/column.h"
+#include "vec/columns/column_const.h"
+#include "vec/columns/column_map.h"
#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
#include "vec/columns/column_variant.h"
#include "vec/common/variant_util.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
#include "vec/data_types/data_type.h"
#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_jsonb.h"
#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_variant.h"
+#include "vec/exprs/function_context.h"
+#include "vec/functions/simple_function_factory.h"
#include "vec/json/path_in_data.h"
#include "vec/olap/olap_data_convertor.h"
@@ -678,11 +696,15 @@ Status UnifiedSparseColumnWriter::append_single_sparse(
size_t limit = parent_column.variant_max_sparse_column_statistics_size();
for (size_t i = 0; i != paths->size(); ++i) {
auto k = paths->get_data_at(i);
- if (auto it = path_counts.find(k); it != path_counts.end())
+ if (auto it = path_counts.find(k); it != path_counts.end()) {
++it->second;
- else if (path_counts.size() < limit)
+ } else if (path_counts.size() < limit) {
path_counts.emplace(k, 1);
+ }
}
+
+ // Build path frequency statistics with upper bound limit to avoid
+ // large memory and metadata size. Persist to meta for readers.
segment_v2::VariantStatistics sparse_stats;
for (const auto& [k, cnt] : path_counts) {
sparse_stats.sparse_column_non_null_size.emplace(k.to_string(),
static_cast<uint32_t>(cnt));
@@ -1064,6 +1086,24 @@ Status
VariantColumnWriterImpl::_process_root_column(vectorized::ColumnVariant*
auto& nullable_column =
assert_cast<vectorized::ColumnNullable&>(*ptr->get_root()->assume_mutable());
auto root_column = nullable_column.get_nested_column_ptr();
+
+ // Simplified dedup logic:
+ // If we have NG paths that cover the root data, replace root JSONB with
+ // empty defaults — the actual data lives in NG columns.
+ // Conflict scalar data is discarded per the conflict policy.
+ if (_nested_group_routing_plan.can_remove_root_jsonb()) {
+ const bool has_root_ng = std::ranges::any_of(
+ _nested_group_routing_plan.ng_only_prefixes,
+ [](const std::string& p) { return
is_root_nested_group_path(p); });
+ if (has_root_ng) {
+ // Replace with empty JSONB defaults — the actual data is in NG
columns.
+ auto bare_jsonb_type =
std::make_shared<vectorized::ColumnVariant::MostCommonType>();
+ auto bare_jsonb_col = bare_jsonb_type->create_column();
+ bare_jsonb_col->insert_many_defaults(num_rows);
+ root_column = std::move(bare_jsonb_col);
+ }
+ }
+
// If the root variant is nullable, then update the root column null
column with the outer null column.
if (_tablet_column->is_nullable()) {
// use outer null column as final null column
@@ -1093,66 +1133,59 @@ Status
VariantColumnWriterImpl::_process_root_column(vectorized::ColumnVariant*
Status VariantColumnWriterImpl::_process_subcolumns(vectorized::ColumnVariant*
ptr,
vectorized::OlapBlockDataConvertor* converter,
size_t num_rows, int&
column_id) {
- // generate column info by entry info
- auto generate_column_info = [&](const auto& entry) {
- const std::string& column_name =
- _tablet_column->name_lower_case() + "." +
entry->path.get_path();
- const vectorized::DataTypePtr& final_data_type_from_object =
- entry->data.get_least_common_type();
+ auto generate_column_info = [&](const vectorized::PathInData&
relative_path,
+ const vectorized::DataTypePtr&
final_data_type) {
+ const std::string column_name =
+ _tablet_column->name_lower_case() + "." +
relative_path.get_path();
vectorized::PathInData full_path;
- if (entry->path.has_nested_part()) {
+ if (relative_path.has_nested_part()) {
vectorized::PathInDataBuilder full_path_builder;
full_path =
full_path_builder.append(_tablet_column->name_lower_case(), false)
- .append(entry->path.get_parts(), false)
+ .append(relative_path.get_parts(), false)
.build();
} else {
full_path = vectorized::PathInData(column_name);
}
- // set unique_id and parent_unique_id, will use unique_id to get
iterator correct
- auto column = vectorized::variant_util::get_column_by_type(
- final_data_type_from_object, column_name,
+ return vectorized::variant_util::get_column_by_type(
+ final_data_type, column_name,
vectorized::variant_util::ExtraInfo {
.unique_id = -1,
.parent_unique_id = _tablet_column->unique_id(),
.path_info = full_path});
- return column;
};
_subcolumns_indexes.resize(ptr->get_subcolumns().size());
- // convert sub column data from engine format to storage layer format
- // NOTE: We only keep up to variant_max_subcolumns_count as extracted
columns; others are externalized.
- for (const auto& entry :
-
vectorized::variant_util::get_sorted_subcolumns(ptr->get_subcolumns())) {
- const auto& least_common_type = entry->data.get_least_common_type();
- if (vectorized::variant_util::get_base_type_of_array(least_common_type)
- ->get_primitive_type() == PrimitiveType::INVALID_TYPE) {
- continue;
- }
- if (entry->path.empty()) {
- // already handled
- continue;
- }
- CHECK(entry->data.is_finalized());
- // create subcolumn writer if under limit; otherwise externalize
ColumnMetaPB via IndexedColumn
+ auto write_one_subcolumn =
+ [&](const std::string& current_path, const vectorized::PathInData&
relative_path,
+ const vectorized::DataTypePtr& current_type,
+ const vectorized::ColumnPtr& current_column, size_t
non_null_count,
+ bool check_storage_type, bool use_existing_subcolumn_info) ->
Status {
int current_column_id = column_id++;
+ if (_subcolumns_indexes.size() <= cast_set<size_t>(current_column_id))
{
+ _subcolumns_indexes.resize(cast_set<size_t>(current_column_id) +
1);
+ }
+
TabletColumn tablet_column;
- int64_t none_null_value_size = entry->data.get_non_null_value_size();
- vectorized::ColumnPtr current_column =
entry->data.get_finalized_column_ptr()->get_ptr();
- vectorized::DataTypePtr current_type =
entry->data.get_least_common_type();
- if (auto current_path = entry->path.get_path();
- _subcolumns_info.find(current_path) != _subcolumns_info.end()) {
- tablet_column = std::move(_subcolumns_info[current_path].column);
- _subcolumns_indexes[current_column_id] =
- std::move(_subcolumns_info[current_path].indexes);
- if (auto storage_type =
-
vectorized::DataTypeFactory::instance().create_data_type(tablet_column);
- !storage_type->equals(*current_type)) {
- return Status::InvalidArgument("Storage type {} is not equal
to current type {}",
- storage_type->get_name(),
current_type->get_name());
+ if (use_existing_subcolumn_info) {
+ if (auto it = _subcolumns_info.find(current_path); it !=
_subcolumns_info.end()) {
+ tablet_column = it->second.column;
+ _subcolumns_indexes[current_column_id] = it->second.indexes;
+ if (check_storage_type) {
+ auto storage_type =
+
vectorized::DataTypeFactory::instance().create_data_type(tablet_column);
+ if (!storage_type->equals(*current_type)) {
+ return Status::InvalidArgument(
+ "Storage type {} is not equal to current type
{} for path {}",
+ storage_type->get_name(),
current_type->get_name(), current_path);
+ }
+ }
+ } else {
+ tablet_column = generate_column_info(relative_path,
current_type);
}
} else {
- tablet_column = generate_column_info(entry);
+ tablet_column = generate_column_info(relative_path, current_type);
}
+
ColumnWriterOptions opts;
opts.meta = _opts.footer->add_columns();
opts.index_file_writer = _opts.index_file_writer;
@@ -1171,14 +1204,49 @@ Status
VariantColumnWriterImpl::_process_subcolumns(vectorized::ColumnVariant* p
RETURN_IF_ERROR(_create_column_writer(
current_column_id, tablet_column,
_opts.rowset_ctx->tablet_schema,
_opts.index_file_writer, &writer,
_subcolumns_indexes[current_column_id], &opts,
- none_null_value_size, need_record_none_null_value_size));
+ non_null_count, need_record_none_null_value_size));
_subcolumn_writers.push_back(std::move(writer));
_subcolumn_opts.push_back(opts);
- _subcolumn_opts[current_column_id - 1].meta->set_num_rows(num_rows);
+ _subcolumn_opts.back().meta->set_num_rows(num_rows);
RETURN_IF_ERROR(convert_and_write_column(converter, tablet_column,
current_type,
-
_subcolumn_writers[current_column_id - 1].get(),
- current_column, ptr->rows(),
current_column_id));
+
_subcolumn_writers.back().get(), current_column,
+ ptr->rows(),
current_column_id));
+ return Status::OK();
+ };
+
+ // convert sub column data from engine format to storage layer format
+ // NOTE: We only keep up to variant_max_subcolumns_count as extracted
columns; others are externalized.
+ for (const auto& entry :
+
vectorized::variant_util::get_sorted_subcolumns(ptr->get_subcolumns())) {
+ if (entry->path.empty()) {
+ continue;
+ }
+ const auto& least_common_type = entry->data.get_least_common_type();
+ if (least_common_type == nullptr) {
+ continue;
+ }
+ auto base_type =
vectorized::variant_util::get_base_type_of_array(least_common_type);
+ if (base_type != nullptr &&
+ base_type->get_primitive_type() == PrimitiveType::INVALID_TYPE) {
+ continue;
+ }
+ // Skip Array(Variant) subcolumns — these represent NG (nested group)
data
+ // that should be handled by the NG writer, not as regular subcolumns.
+ if (base_type != nullptr &&
+ typeid_cast<const vectorized::DataTypeVariant*>(base_type.get())
!= nullptr) {
+ continue;
+ }
+ const std::string current_path = entry->path.get_path();
+ if (_nested_group_routing_plan.is_excluded_subcolumn(current_path)) {
+ continue;
+ }
+ CHECK(entry->data.is_finalized());
+ RETURN_IF_ERROR(write_one_subcolumn(current_path, entry->path,
least_common_type,
+
entry->data.get_finalized_column_ptr()->get_ptr(),
+
entry->data.get_non_null_value_size(),
+ true /* check_storage_type */,
+ true /*
use_existing_subcolumn_info */));
}
return Status::OK();
}
@@ -1204,7 +1272,9 @@ Status VariantColumnWriterImpl::_process_binary_column(
Status VariantColumnWriterImpl::finalize() {
auto* ptr = _column.get();
ptr->set_max_subcolumns_count(_tablet_column->variant_max_subcolumns_count());
+
ptr->finalize(vectorized::ColumnVariant::FinalizeMode::WRITE_MODE);
+
// convert each subcolumns to storage format and add data to sub columns
writers buffer
auto olap_data_convertor =
std::make_unique<vectorized::OlapBlockDataConvertor>();
@@ -1229,6 +1299,21 @@ Status VariantColumnWriterImpl::finalize() {
}
RETURN_IF_ERROR(ptr->convert_typed_path_to_storage_type(_subcolumns_info));
+ _nested_group_routing_plan = NestedGroupRoutingPlan {};
+
+ const int current_variant_uid = _tablet_column->unique_id();
+ const bool has_extracted_columns = std::ranges::any_of(
+ _opts.rowset_ctx->tablet_schema->columns(),
[current_variant_uid](const auto& column) {
+ return column->is_extracted_column() &&
+ column->parent_unique_id() == current_variant_uid;
+ });
+ if (!has_extracted_columns) {
+ RETURN_IF_ERROR(build_nested_group_routing_plan(*ptr,
&_nested_group_routing_plan));
+
+ // Root NG dedup is handled in _process_root_column() — see the
+ // has_root_ng check there. We intentionally do NOT modify the
in-memory
+ // root data here because _nested_group_provider->prepare() needs it.
+ }
RETURN_IF_ERROR(ptr->pick_subcolumns_to_sparse_column(
_subcolumns_info,
_tablet_column->variant_enable_typed_paths_to_sparse()));
@@ -1243,12 +1328,7 @@ Status VariantColumnWriterImpl::finalize() {
// convert root column data from engine format to storage layer format
RETURN_IF_ERROR(_process_root_column(ptr, olap_data_convertor.get(),
num_rows, column_id));
- auto has_extracted_columns = [this]() {
- return std::ranges::any_of(
- _opts.rowset_ctx->tablet_schema->columns(),
- [](const auto& column) { return column->is_extracted_column();
});
- };
- if (!has_extracted_columns()) {
+ if (!has_extracted_columns) {
if (!_tablet_column->variant_enable_doc_mode()) {
// process and append each subcolumns to sub columns writers buffer
RETURN_IF_ERROR(
@@ -1258,16 +1338,16 @@ Status VariantColumnWriterImpl::finalize() {
// process sparse column and append to sparse writer buffer
RETURN_IF_ERROR(
_process_binary_column(ptr, olap_data_convertor.get(),
num_rows, column_id));
+ }
- // NestedGroup write behavior is determined by the injected provider
implementation.
- RETURN_IF_ERROR(_nested_group_provider->prepare(
- *ptr, /*include_jsonb_subcolumns=*/true, _tablet_column, _opts,
- olap_data_convertor.get(), num_rows, &column_id,
&_statistics));
- if (_binary_writer) {
- _binary_writer->merge_stats_to(&_statistics);
- }
- _statistics.to_pb(_opts.meta->mutable_variant_statistics());
+ // NestedGroup write behavior is determined by the injected provider
implementation.
+ RETURN_IF_ERROR(_nested_group_provider->prepare(
+ *ptr, /*include_jsonb_subcolumns=*/true, _tablet_column, _opts,
+ olap_data_convertor.get(), num_rows, &column_id, &_statistics));
+ if (_binary_writer) {
+ _binary_writer->merge_stats_to(&_statistics);
}
+ _statistics.to_pb(_opts.meta->mutable_variant_statistics());
_is_finalized = true;
return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h
b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h
index f135105da18..824e9a5ed44 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.h
@@ -21,12 +21,14 @@
#include <functional>
#include <unordered_map>
+#include <unordered_set>
#include <vector>
#include "common/status.h"
#include "olap/rowset/segment_v2/column_writer.h"
#include "olap/rowset/segment_v2/indexed_column_writer.h"
#include "olap/rowset/segment_v2/variant/nested_group_provider.h"
+#include "olap/rowset/segment_v2/variant/nested_group_routing_plan.h"
#include "olap/rowset/segment_v2/variant/variant_statistics.h"
#include "olap/tablet_schema.h"
#include "vec/columns/column.h"
@@ -216,6 +218,7 @@ private:
std::unordered_map<std::string, TabletSchema::SubColumnInfo>
_subcolumns_info;
std::unique_ptr<NestedGroupWriteProvider> _nested_group_provider;
VariantStatistics _statistics;
+ NestedGroupRoutingPlan _nested_group_routing_plan;
};
class VariantDocCompactWriter : public ColumnWriter {
diff --git a/be/src/vec/columns/column_variant.cpp
b/be/src/vec/columns/column_variant.cpp
index b46025c1021..bd07e84b5fe 100644
--- a/be/src/vec/columns/column_variant.cpp
+++ b/be/src/vec/columns/column_variant.cpp
@@ -40,6 +40,7 @@
#include <vector>
#include "common/compiler_util.h" // IWYU pragma: keep
+#include "common/config.h"
#include "common/exception.h"
#include "common/logging.h"
#include "common/status.h"
@@ -129,6 +130,45 @@ size_t get_number_of_dimensions(const IDataType& type) {
}
return num_dimensions;
}
+
+// ============================================================================
+// NestedGroup (NG) type-conflict helpers
+// ============================================================================
+// These helpers encapsulate the NG-specific logic that must run inside
+// Subcolumn::insert_range_from and Subcolumn::finalize. Keeping them here
+// avoids spreading NG semantics throughout the generic column code.
+
+// Returns true if the base element type of `type` is DataTypeVariant,
+// which indicates NG-originated array<object> data.
+bool is_nested_group_type(const DataTypePtr& type) {
+ auto base = get_base_type_of_array(type);
+ return typeid_cast<const DataTypeVariant*>(base.get()) != nullptr;
+}
+
+// Resolve a type conflict between dst (current LCT) and src types when one
+// side is an NG type (Array<Variant>) and the other is a scalar type.
+//
+// Under DISCARD_SCALAR policy: returns the NG side's type (NG wins).
+// Under ERROR policy: throws an exception.
+//
+// Returns nullptr if neither side is an NG type (caller should fall through
+// to the normal get_least_supertype_jsonb path).
+DataTypePtr resolve_ng_type_conflict(const DataTypePtr& dst_type, const
DataTypePtr& src_type) {
+ bool dst_is_ng = is_nested_group_type(dst_type);
+ bool src_is_ng = is_nested_group_type(src_type);
+ if (!dst_is_ng && !src_is_ng) {
+ return nullptr; // Not an NG conflict — use normal type resolution.
+ }
+ if (!config::variant_nested_group_discard_scalar_on_conflict) {
+ throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
+ "NestedGroup type conflict: cannot merge
Array<Variant> with "
+ "scalar type. dst={}, src={}",
+ dst_type->get_name(), src_type->get_name());
+ }
+ // NG wins: keep whichever side is the NG type.
+ return dst_is_ng ? dst_type : src_type;
+}
+
} // namespace
// current nested level is 2, inside column object
@@ -324,9 +364,14 @@ void ColumnVariant::Subcolumn::insert_range_from(const
Subcolumn& src, size_t st
if (data.empty()) {
add_new_column_part(src.get_least_common_type());
} else if (!least_common_type.get()->equals(*src.get_least_common_type()))
{
- DataTypePtr new_least_common_type;
- get_least_supertype_jsonb(DataTypes {least_common_type.get(),
src.get_least_common_type()},
- &new_least_common_type);
+ DataTypePtr new_least_common_type =
+ resolve_ng_type_conflict(least_common_type.get(),
src.get_least_common_type());
+ if (new_least_common_type == nullptr) {
+ // Normal (non-NG) type promotion.
+ get_least_supertype_jsonb(
+ DataTypes {least_common_type.get(),
src.get_least_common_type()},
+ &new_least_common_type);
+ }
if (!new_least_common_type->equals(*least_common_type.get())) {
add_new_column_part(std::move(new_least_common_type));
}
@@ -350,6 +395,18 @@ void ColumnVariant::Subcolumn::insert_range_from(const
Subcolumn& src, size_t st
data.back()->insert_range_from(*column, from, n);
return;
}
+ // When LCT is Array<Variant> (NG data) and the part is scalar, cast
+ // would crash. Under DISCARD_SCALAR the scalar part becomes defaults.
+ if (is_nested_group_type(least_common_type.get())) {
+ if (!config::variant_nested_group_discard_scalar_on_conflict) {
+ throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
+ "NestedGroup type conflict: cannot cast
scalar type {} to "
+ "Array<Variant>",
+ column_type->get_name());
+ }
+ data.back()->insert_many_defaults(n);
+ return;
+ }
/// If we need to insert large range, there is no sense to cut part of
column and cast it.
/// Casting of all column and inserting from it can be faster.
/// Threshold is just a guess.
@@ -488,6 +545,19 @@ void ColumnVariant::Subcolumn::finalize(FinalizeMode mode)
{
part = part->convert_to_full_column_if_const();
size_t part_size = part->size();
if (!from_type->equals(*to_type)) {
+ // NG vs scalar mismatch: casting Array(Variant) ↔ scalar is not
+ // supported. Under DISCARD_SCALAR the non-NG part becomes
defaults.
+ if (is_nested_group_type(to_type) !=
is_nested_group_type(from_type)) {
+ if (!config::variant_nested_group_discard_scalar_on_conflict) {
+ throw doris::Exception(
+ ErrorCode::INVALID_ARGUMENT,
+ "NestedGroup type conflict in finalize: cannot
cast {} to {}",
+ from_type->get_name(), to_type->get_name());
+ }
+ result_column->insert_many_defaults(part_size);
+ continue;
+ }
+
ColumnPtr ptr;
Status st = variant_util::cast_column({part, from_type, ""},
to_type, &ptr);
if (!st.ok()) {
@@ -1820,6 +1890,8 @@ Status ColumnVariant::serialize_sparse_columns(
return Status::OK();
}
+/// @deprecated This function is deprecated. Array<Variant> subcolumns are now
handled
+/// directly as NestedGroup data by the writer (VariantColumnWriterImpl).
void ColumnVariant::unnest(Subcolumns::NodePtr& entry, Subcolumns&
res_subcolumns) const {
entry->data.finalize();
auto nested_column =
entry->data.get_finalized_column_ptr()->assume_mutable();
@@ -2006,9 +2078,10 @@ void ColumnVariant::finalize(FinalizeMode mode) {
continue;
}
- // unnest all nested columns, add them to new_subcolumns
+ // [DEPRECATED] unnest path - Array<Variant> subcolumns are now handled
+ // directly as NestedGroup by the writer. This branch exists only for
+ // backward compatibility with the exact NESTED_TYPE wrapper.
if (mode == FinalizeMode::WRITE_MODE &&
least_common_type->equals(*NESTED_TYPE)) {
- // reset counter
unnest(entry, new_subcolumns);
continue;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
