This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 04cae614aaa [feat](Variant) Support NestedGroup public config (#64680)
04cae614aaa is described below
commit 04cae614aaa967f41c25a4520f32bd116eb6e2f1
Author: lihangyu <[email protected]>
AuthorDate: Wed Jun 24 10:50:24 2026 +0800
[feat](Variant) Support NestedGroup public config (#64680)
This PR exposes `variant_enable_nested_group` as a public VARIANT
property and wires the related configuration through parser/type
serialization.
Main changes:
- Allow `variant_enable_nested_group` in VARIANT predefined fields.
- Disable doc mode and sparse-column related options when NestedGroup is
enabled.
- Serialize `variant_enable_nested_group` in `VariantType#toSql`.
- Add `variant_nested_group_max_depth` config and make the default
NestedGroup write provider explicitly return not-supported status when
the write path is unavailable.
- Update FE/BE tests for parser behavior, type serialization, and
disabled NestedGroup write-path handling.
---
be/src/common/config.cpp | 6 +++++-
.../segment/variant/nested_group_provider.cpp | 12 ++++++++----
.../storage/segment/nested_group_provider_test.cpp | 20 +++++++++++++++++++-
.../segment/variant_column_writer_reader_test.cpp | 14 ++++++++++++++
.../doris/nereids/parser/LogicalPlanBuilder.java | 7 +++++--
.../java/org/apache/doris/catalog/TypeTest.java | 4 ++--
.../doris/nereids/parser/NereidsParserTest.java | 21 +++++++++++++++++++--
.../java/org/apache/doris/catalog/VariantType.java | 5 +++++
.../variant_p0/test_variant_search_subcolumn.groovy | 2 ++
9 files changed, 79 insertions(+), 12 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 86b7d12e0aa..b521fa57ec3 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1172,7 +1172,11 @@ DEFINE_mBool(variant_enable_duplicate_json_path_check,
"false");
DEFINE_mInt32(variant_storage_parse_mode, "0");
DEFINE_mBool(enable_vertical_compact_variant_subcolumns, "true");
DEFINE_mBool(enable_variant_doc_sparse_write_subcolumns, "true");
-DEFINE_mBool(variant_nested_group_discard_scalar_on_conflict, "false");
+// Maximum depth of nested arrays to track with NestedGroup
+// Reserved for future use when NestedGroup expansion moves to storage layer
+// Deeper arrays will be stored as JSONB
+DEFINE_mInt32(variant_nested_group_max_depth, "10");
+DEFINE_mBool(variant_nested_group_discard_scalar_on_conflict, "true");
DEFINE_Validator(variant_max_json_key_length,
[](const int config) -> bool { return config > 0 && config <=
65535; });
diff --git a/be/src/storage/segment/variant/nested_group_provider.cpp
b/be/src/storage/segment/variant/nested_group_provider.cpp
index b2271bc6943..27a8abf681e 100644
--- a/be/src/storage/segment/variant/nested_group_provider.cpp
+++ b/be/src/storage/segment/variant/nested_group_provider.cpp
@@ -17,6 +17,10 @@
#include "storage/segment/variant/nested_group_provider.h"
+#include <algorithm>
+#include <string>
+#include <utility>
+
namespace doris::segment_v2 {
namespace {
@@ -123,7 +127,7 @@ public:
statistics == nullptr) {
return Status::InvalidArgument("NestedGroup provider input is
null");
}
- return Status::OK();
+ return Status::NotSupported("NestedGroup write path is not available
in this build");
}
Status prepare_with_built_groups(const NestedGroupsMap& /*nested_groups*/,
@@ -135,7 +139,7 @@ public:
statistics == nullptr) {
return Status::InvalidArgument("NestedGroup provider input is
null");
}
- return Status::OK();
+ return Status::NotSupported("NestedGroup write path is not available
in this build");
}
Status init_with_plan(const NestedGroupStreamingWritePlan& /*plan*/,
@@ -144,12 +148,12 @@ public:
if (tablet_column == nullptr || column_id == nullptr || statistics ==
nullptr) {
return Status::InvalidArgument("NestedGroup streaming init input
is null");
}
- return Status::OK();
+ return Status::NotSupported("NestedGroup write path is not available
in this build");
}
Status append_chunk(const NestedGroupStreamingWritePlan& /*plan*/,
const ColumnVariant& /*variant*/) override {
- return Status::OK();
+ return Status::NotSupported("NestedGroup write path is not available
in this build");
}
uint64_t estimate_buffer_size() const override { return 0; }
diff --git a/be/test/storage/segment/nested_group_provider_test.cpp
b/be/test/storage/segment/nested_group_provider_test.cpp
index 7baa6f51fcb..ef55d38cd5d 100644
--- a/be/test/storage/segment/nested_group_provider_test.cpp
+++ b/be/test/storage/segment/nested_group_provider_test.cpp
@@ -29,6 +29,7 @@
#include <roaring/roaring.hh>
#include "core/column/column_variant.h"
+#include "storage/iterator/olap_data_convertor.h"
#include "storage/segment/column_writer.h"
#include "storage/segment/variant/variant_column_reader.h"
#include "storage/segment/variant/variant_statistics.h"
@@ -48,7 +49,7 @@ TEST(NestedGroupProviderTest, DefaultReadProviderIsDisabled) {
EXPECT_FALSE(provider->should_enable_nested_group_read_path());
}
-TEST(NestedGroupProviderTest, DefaultWriteProviderIsNoOp) {
+TEST(NestedGroupProviderTest, DefaultWriteProviderRejectsNestedGroupWritePath)
{
auto write_provider = create_nested_group_write_provider();
ASSERT_TRUE(write_provider != nullptr);
@@ -66,6 +67,23 @@ TEST(NestedGroupProviderTest, DefaultWriteProviderIsNoOp) {
write_provider->prepare(*column_variant, nullptr, opts, nullptr,
nullptr, &statistics);
EXPECT_FALSE(status.ok());
EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>());
+
+ TabletColumn tablet_column;
+ OlapBlockDataConvertor converter;
+ int column_id = 0;
+ status = write_provider->prepare(*column_variant, &tablet_column, opts,
&converter, &column_id,
+ &statistics);
+ EXPECT_FALSE(status.ok());
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>());
+ EXPECT_NE(status.to_string().find("not available"), std::string::npos);
+
+ NestedGroupsMap nested_groups;
+ status = write_provider->prepare_with_built_groups(nested_groups,
&tablet_column, opts,
+ &converter, &column_id,
&statistics);
+ EXPECT_FALSE(status.ok());
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>());
+ EXPECT_NE(status.to_string().find("not available"), std::string::npos);
+
EXPECT_EQ(0, write_provider->estimate_buffer_size());
EXPECT_TRUE(write_provider->finish().ok());
EXPECT_TRUE(write_provider->write_data().ok());
diff --git a/be/test/storage/segment/variant_column_writer_reader_test.cpp
b/be/test/storage/segment/variant_column_writer_reader_test.cpp
index 737326ffbcd..8b92a1983af 100644
--- a/be/test/storage/segment/variant_column_writer_reader_test.cpp
+++ b/be/test/storage/segment/variant_column_writer_reader_test.cpp
@@ -34,6 +34,7 @@
#include "storage/segment/variant/binary_column_extract_iterator.h"
#include "storage/segment/variant/hierarchical_data_iterator.h"
#include "storage/segment/variant/nested_group_path.h"
+#include "storage/segment/variant/nested_group_provider.h"
#include "storage/segment/variant/nested_group_streaming_write_plan.h"
#include "storage/segment/variant/sparse_column_merge_iterator.h"
#include "storage/segment/variant/variant_column_reader.h"
@@ -85,6 +86,11 @@ static void construct_tablet_index(TabletIndexPB*
tablet_index, int64_t index_id
tablet_index->add_col_unique_id(col_unique_id);
}
+static bool nested_group_write_path_available() {
+ auto provider = segment_v2::create_nested_group_read_provider();
+ return provider != nullptr &&
provider->should_enable_nested_group_read_path();
+}
+
static void fill_nullable_variant_block(Block* block,
std::unordered_map<int, std::string>*
inserted_jsonstr,
variant_util::PathToNoneNullValues*
path_with_size) {
@@ -5236,6 +5242,10 @@ TEST_F(VariantColumnWriterReaderTest,
test_concurrent_load_external_meta_and_get
TEST_F(VariantColumnWriterReaderTest,
test_streaming_write_plan_collects_regular_paths_from_rowset_metadata) {
+ if (!nested_group_write_path_available()) {
+ GTEST_SKIP() << "NestedGroup write path is not available in this
build";
+ }
+
init_variant_tablet(41000, 10, true);
std::vector<RowsetSharedPtr> input_rowsets;
@@ -5267,6 +5277,10 @@ TEST_F(VariantColumnWriterReaderTest,
TEST_F(VariantColumnWriterReaderTest,
test_streaming_compaction_writer_streams_regular_array_paths_across_batches) {
+ if (!nested_group_write_path_available()) {
+ GTEST_SKIP() << "NestedGroup write path is not available in this
build";
+ }
+
init_variant_tablet(41001, 10, true);
std::vector<RowsetSharedPtr> input_rowsets;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index b01b62db5f3..34c69e97eee 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -5329,8 +5329,11 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
}
if (enableNestedGroup) {
- throw new NotSupportedException(
- "variant_enable_nested_group is not supported now");
+ enableVariantDocMode = false;
+ variantMaxSubcolumnsCount = 0;
+ enableTypedPathsToSparse = false;
+ variantMaxSparseColumnStatisticsSize = 0;
+ variantSparseHashShardCount = 0;
}
// When doc mode is enabled, disable subcolumn extraction and sparse
column features
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java
index 44e973e851e..7713519b383 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java
@@ -129,11 +129,11 @@ public class TypeTest {
}
@Test
- public void
testVariantToSqlDoesNotSerializeUnsupportedNestedGroupProperty() {
+ public void testVariantToSqlSerializesNestedGroupProperty() {
VariantType variantType = new VariantType(new ArrayList<>(), 0, false,
10000, 0,
false, 0L, 64, true);
-
Assert.assertFalse(variantType.toSql().contains("variant_enable_nested_group"));
+
Assert.assertTrue(variantType.toSql().contains("\"variant_enable_nested_group\"
= \"true\""));
}
// ===================== Mixed Nesting & Precision =====================
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index a64415480db..c87981ed368 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -1444,14 +1444,31 @@ public class NereidsParserTest extends ParserTestBase {
}
@Test
- public void testCreateTableVariantNestedGroupPropertyIsRejected() {
+ public void testCreateTableVariantNestedGroupPropertyIsAccepted() {
NereidsParser parser = new NereidsParser();
String sql = "CREATE TABLE t_variant_ng (k1 INT, v VARIANT<PROPERTIES("
+ "\"variant_enable_nested_group\" = \"true\")>) "
+ "DISTRIBUTED BY HASH(k1) BUCKETS 1";
+ LogicalPlan logicalPlan = parser.parseSingle(sql);
+ Assertions.assertInstanceOf(CreateTableCommand.class, logicalPlan);
+ CreateTableCommand createTableCommand = (CreateTableCommand)
logicalPlan;
+ org.apache.doris.nereids.types.VariantType variantType =
+ (org.apache.doris.nereids.types.VariantType)
createTableCommand.getCreateTableInfo()
+ .getColumnDefinitions().get(1).getType();
+ Assertions.assertTrue(variantType.getEnableNestedGroup());
+ }
+
+ @Test
+ public void
testCreateTableVariantNestedGroupPropertyConflictsWithDocMode() {
+ NereidsParser parser = new NereidsParser();
+ String sql = "CREATE TABLE t_variant_ng (k1 INT, v VARIANT<PROPERTIES("
+ + "\"variant_enable_nested_group\" = \"true\", "
+ + "\"variant_enable_doc_mode\" = \"true\")>) "
+ + "DISTRIBUTED BY HASH(k1) BUCKETS 1";
NotSupportedException exception =
Assertions.assertThrowsExactly(NotSupportedException.class, ()
-> parser.parseSingle(sql));
-
Assertions.assertTrue(exception.getMessage().contains("variant_enable_nested_group
is not supported now"));
+ Assertions.assertTrue(exception.getMessage()
+ .contains("variant_enable_nested_group and
variant_enable_doc_mode cannot both be true"));
}
@Test
diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/VariantType.java
b/fe/fe-type/src/main/java/org/apache/doris/catalog/VariantType.java
index b3b34b2a045..d93ef9894dc 100644
--- a/fe/fe-type/src/main/java/org/apache/doris/catalog/VariantType.java
+++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/VariantType.java
@@ -187,6 +187,11 @@ public class VariantType extends ScalarType {
sb.append("\"variant_sparse_hash_shard_count\" = \"")
.append(String.valueOf(Math.max(1,
variantSparseHashShardCount))).append("\"");
}
+ if (enableNestedGroup) {
+ sb.append(",");
+ sb.append("\"variant_enable_nested_group\" = \"")
+ .append(String.valueOf(enableNestedGroup)).append("\"");
+ }
sb.append(")>");
return sb.toString();
}
diff --git
a/regression-test/suites/variant_p0/test_variant_search_subcolumn.groovy
b/regression-test/suites/variant_p0/test_variant_search_subcolumn.groovy
index 095d3fb84d6..3da6de106e1 100644
--- a/regression-test/suites/variant_p0/test_variant_search_subcolumn.groovy
+++ b/regression-test/suites/variant_p0/test_variant_search_subcolumn.groovy
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// DORIS-25891: Variant SEARCH must bind subcolumn predicates to the real
stored
+// field names for direct, nested, and special-character paths.
suite("test_variant_search_subcolumn") {
def table_name = "test_variant_search_subcolumn"
sql "set default_variant_doc_materialization_min_rows = 0"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
