This is an automated email from the ASF dual-hosted git repository. suxiaogang223 pushed a commit to branch codex/paimon-variant-write in repository https://gitbox.apache.org/repos/asf/doris.git
commit 20fbee9a1080913a4964d0b506cd40bc8de73227 Author: suxiaogang <[email protected]> AuthorDate: Tue Aug 4 19:06:39 2026 +0800 [fix](paimon) Address Variant write review feedback --- .../cast/variant_v2/cast_array_to_variant.cpp | 6 ++++-- .../function/cast/cast_variant_v2_from_test.cpp | 24 ++++++++++++++++++++++ .../java-udf/src/main/resources/package.xml | 1 + .../test_paimon_write_variant_table_modes.groovy | 6 ++++-- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/be/src/exprs/function/cast/variant_v2/cast_array_to_variant.cpp b/be/src/exprs/function/cast/variant_v2/cast_array_to_variant.cpp index e35c9b1983b..66a61f2da96 100644 --- a/be/src/exprs/function/cast/variant_v2/cast_array_to_variant.cpp +++ b/be/src/exprs/function/cast/variant_v2/cast_array_to_variant.cpp @@ -137,7 +137,9 @@ Status build_array_node_plan(const ColumnPtr& source, const DataTypePtr& source_ Status build_array_leaf_plan(const ColumnPtr& source, PrimitiveType primitive, ArrayEncodePlan* plan) { - if (primitive == INVALID_TYPE && source->empty()) { + if (primitive == INVALID_TYPE) { + // DataTypeNothing is represented by the element null map, including non-empty + // expressions such as array(NULL). return Status::OK(); } else if (primitive == TYPE_VARIANT) { const auto* variant = check_and_get_column<ColumnVariantV2>(source.get()); @@ -196,7 +198,7 @@ void append_array_value(const ArrayEncodePlan& plan, size_t index, VariantBatchB } else if (plan.jsonb_leaf != nullptr) { jsonb_to_variant(plan.jsonb_leaf->get_data_at(index), *row); } else { - DORIS_CHECK(false) << "empty Array leaf unexpectedly contains a value"; + DORIS_CHECK(false) << "Array Variant V2 leaf has no encoder"; } return; } diff --git a/be/test/exprs/function/cast/cast_variant_v2_from_test.cpp b/be/test/exprs/function/cast/cast_variant_v2_from_test.cpp index a7858522cf1..c28a3adb1e1 100644 --- a/be/test/exprs/function/cast/cast_variant_v2_from_test.cpp +++ b/be/test/exprs/function/cast/cast_variant_v2_from_test.cpp @@ -29,6 +29,7 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_jsonb.h" +#include "core/data_type/data_type_nothing.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" @@ -484,6 +485,29 @@ TEST(CastVariantV2FromTest, NestedArrayRoundTripPreservesNullAndEmptyArray) { EXPECT_EQ(assert_cast<const ColumnInt32&>(values.get_nested_column()).get_data()[0], 1); } +TEST(CastVariantV2FromTest, NullOnlyArrayEncodesNonEmptyElements) { + auto array_type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeNothing>()); + MutableColumnPtr source = array_type->create_column(); + Array values {Field::create_field<TYPE_NULL>(Null()), Field::create_field<TYPE_NULL>(Null())}; + source->insert(Field::create_field<TYPE_ARRAY>(std::move(values))); + + auto variant_type = std::make_shared<DataTypeVariantV2>(); + Block block {{source->get_ptr(), array_type, "source"}, + {variant_type->create_column(), variant_type, "result"}}; + RuntimeState state; + auto context = FunctionContext::create_context(&state, {}, {}); + Status status = + create_cast_to_variant_v2_wrapper(array_type)(context.get(), block, {0}, 1, 1, nullptr); + ASSERT_TRUE(status.ok()) << status; + + VariantRef encoded = + assert_cast<const ColumnVariantV2&>(*block.get_by_position(1).column).get_value_ref(0); + ASSERT_EQ(encoded.basic_type(), VariantBasicType::ARRAY); + ASSERT_EQ(encoded.num_elements(), 2); + EXPECT_TRUE(encoded.array_at(0).is_null()); + EXPECT_TRUE(encoded.array_at(1).is_null()); +} + TEST(CastVariantV2FromTest, DecimalScale38CastsAndScale39IsRejectedAtEncodingBoundary) { VariantBatchBuilder builder(VariantBatchBuilder::ReserveHint {.rows = 1}); auto row = builder.begin_row(); diff --git a/fe/be-java-extensions/java-udf/src/main/resources/package.xml b/fe/be-java-extensions/java-udf/src/main/resources/package.xml index 16062019419..53d7bed90c3 100644 --- a/fe/be-java-extensions/java-udf/src/main/resources/package.xml +++ b/fe/be-java-extensions/java-udf/src/main/resources/package.xml @@ -47,6 +47,7 @@ under the License. <!-- ServiceLoader also consults parent resources. Do not retain service descriptors whose providers were removed with the embedded SDK. --> <exclude>META-INF/services/org.apache.paimon*</exclude> + <exclude>META-INF/services/java.time.chrono.Chronology</exclude> </excludes> </unpackOptions> </dependencySet> diff --git a/regression-test/suites/paimon_write/test_paimon_write_variant_table_modes.groovy b/regression-test/suites/paimon_write/test_paimon_write_variant_table_modes.groovy index 177570ad8eb..f7976d21bb9 100644 --- a/regression-test/suites/paimon_write/test_paimon_write_variant_table_modes.groovy +++ b/regression-test/suites/paimon_write/test_paimon_write_variant_table_modes.groovy @@ -126,10 +126,12 @@ suite("test_paimon_write_variant_table_modes", "p0,external,paimon") { """ def dynamicRows = spark_paimon """ SELECT COUNT(*), - SUM(variant_get(payload, '${root}.id', 'int')) + SUM(variant_get(payload, '${root}.id', 'int')), + SUM(CASE WHEN id <> variant_get(payload, '${root}.id', 'int') + THEN 1 ELSE 0 END) FROM paimon.${dbName}.t_variant_dynamic_bucket """ - assertEquals(["32", "496"], + assertEquals(["32", "496", "0"], dynamicRows[0].collect { value -> value.toString() }) // Schema evolution: add Variant, write it, then add a normal column and continue writing. --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
