github-actions[bot] commented on code in PR #68667:
URL: https://github.com/apache/doris/pull/68667#discussion_r4141922700
##########
be/src/core/data_type_serde/data_type_variant_serde.cpp:
##########
@@ -157,6 +165,101 @@ Status DataTypeVariantSerDe::write_column_to_arrow(const
IColumn& column, const
int64_t start, int64_t end,
const cctz::time_zone& ctz)
const {
const auto* var = check_and_get_column<ColumnVariant>(column);
+ if (array_builder->type()->id() == arrow::Type::STRUCT) {
+ // Legacy documents need JSON conversion; typed scalar roots can keep
their type.
+ // The outer null map must remain SQL NULL on the wire.
+ if (start < 0 || end < start || end > column.size() ||
+ (null_map != nullptr && end > null_map->size())) {
+ return Status::InvalidArgument("Invalid Variant Arrow row range
[{}, {})", start, end);
+ }
+ if (var->is_scalar_variant()) {
+ auto scalar_type = remove_nullable(var->get_root_type());
+ if (scalar_type->get_primitive_type() == TYPE_DECIMAL256) {
+ return Status::NotSupported(
+ "Native Arrow Variant does not support Decimal256
roots");
+ }
+ if
(is_supported_variant_typed_identity(scalar_type->get_primitive_type())) {
+ // Avoid a JSON round trip that would turn exact decimal roots
into doubles.
+ auto typed =
+
ColumnVariantV2::create_typed(make_nullable(var->get_root()), scalar_type);
+ return DataTypeVariantV2SerDe().write_column_to_arrow(
+ *typed, null_map, array_builder, start, end, ctz);
+ }
+ }
+ const size_t rows = end - start;
+ NullMap selected_nulls(rows, 0);
+ NullMap root_mask(rows, 1);
+ bool has_roots = false;
+ bool has_documents = false;
+ for (size_t row = 0; row < rows; ++row) {
+ selected_nulls[row] = null_map != nullptr && (*null_map)[start +
row];
+ if (selected_nulls[row]) {
+ continue;
+ }
+ const bool root_visible = var->is_scalar_variant()
+ ?
!var->get_root()->is_null_at(start + row)
+ :
var->is_visible_root_value(start + row);
+ root_mask[row] = !root_visible;
+ has_roots |= root_visible;
+ has_documents |= !root_visible;
+ }
+ ColumnPtr roots;
+ if (has_roots) {
+ // JSON reparsing loses decimal/temporal identities and rejects
non-finite numbers.
+ // Reuse typed CAST for every visible root, including arrays and
mixed-path batches.
+ auto root_type = remove_nullable(var->get_root_type());
+ auto target_type = std::make_shared<DataTypeVariantV2>();
+ // Invisible roots, including nullable roots, are already masked
above.
+ Block root_block {
+ {remove_nullable(var->get_root())->cut(start, rows),
root_type, "root"},
+ {target_type->create_column(), target_type, "encoded"}};
+ auto encode_root =
CastWrapper::create_cast_to_variant_v2_wrapper(root_type);
+ RETURN_IF_ERROR(encode_root(nullptr, root_block, {0}, 1, rows,
root_mask.data()));
Review Comment:
[P1] Cover all legal legacy roots before invoking V2 CAST. FE permits MAP,
STRUCT, TIMEV2, and ARRAY values to be cast to legacy VARIANT, but native
Flight sends every visible root to this encoder. V2 CAST rejects
MAP/STRUCT/TIMEV2 roots and ARRAY leaves containing MAP, STRUCT, TIMEV2, or
legacy VARIANT, so queries that work in UTF8 mode fail in native mode. Extend
the encoder or use a compatible fallback for unsupported source families, and
test root and nested cases.
##########
be/src/core/data_type_serde/data_type_variant_serde.cpp:
##########
@@ -157,6 +165,101 @@ Status DataTypeVariantSerDe::write_column_to_arrow(const
IColumn& column, const
int64_t start, int64_t end,
const cctz::time_zone& ctz)
const {
const auto* var = check_and_get_column<ColumnVariant>(column);
+ if (array_builder->type()->id() == arrow::Type::STRUCT) {
+ // Legacy documents need JSON conversion; typed scalar roots can keep
their type.
+ // The outer null map must remain SQL NULL on the wire.
+ if (start < 0 || end < start || end > column.size() ||
+ (null_map != nullptr && end > null_map->size())) {
+ return Status::InvalidArgument("Invalid Variant Arrow row range
[{}, {})", start, end);
+ }
+ if (var->is_scalar_variant()) {
+ auto scalar_type = remove_nullable(var->get_root_type());
+ if (scalar_type->get_primitive_type() == TYPE_DECIMAL256) {
+ return Status::NotSupported(
+ "Native Arrow Variant does not support Decimal256
roots");
+ }
+ if
(is_supported_variant_typed_identity(scalar_type->get_primitive_type())) {
+ // Avoid a JSON round trip that would turn exact decimal roots
into doubles.
+ auto typed =
+
ColumnVariantV2::create_typed(make_nullable(var->get_root()), scalar_type);
Review Comment:
[P1] Preserve the legacy null-root value in the scalar shortcut. A legacy
column parsed from `42` and JSON `null` remains scalar; UTF8 output serializes
the null root as `{}`, while `create_typed` keeps its inner null map and native
output emits Parquet Variant null. The value therefore changes with native mode
(and with a mixed-path batch). Apply root visibility before this shortcut and
test the scalar-only null row.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
