github-actions[bot] commented on code in PR #68667:
URL: https://github.com/apache/doris/pull/68667#discussion_r4152611380
##########
be/src/core/data_type_serde/data_type_variant_serde.cpp:
##########
@@ -19,28 +19,268 @@
#include <arrow/array/builder_binary.h>
+#include <algorithm>
+#include <cmath>
#include <cstdint>
#include <string>
+#include <string_view>
+#include <vector>
#include "common/cast_set.h"
#include "common/config.h"
#include "common/exception.h"
#include "common/status.h"
#include "core/assert_cast.h"
#include "core/column/column.h"
+#include "core/column/column_array.h"
+#include "core/column/column_map.h"
+#include "core/column/column_struct.h"
#include "core/column/column_variant.h"
+#include "core/column/variant_v2/column_variant_v2.h"
+#include "core/column/variant_v2/column_variant_v2_typed_column.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_struct.h"
#include "core/data_type_serde/data_type_serde.h"
+#include "core/data_type_serde/data_type_variant_v2_serde.h"
#include "core/field.h"
#include "core/string_ref.h"
#include "core/types.h"
#include "core/value/jsonb_value.h"
#include "exec/common/variant_util.h"
+#include "exprs/function/parse/variant_jsonb_parse.h"
+#include "exprs/function/parse/variant_string_parse.h"
#include "util/json/json_parser.h"
#include "util/jsonb_writer.h"
namespace doris {
namespace {
+Status append_legacy_arrow_document(const ColumnVariant& column, size_t index,
+ VariantBatchBuilder::Row& output,
+ const DataTypeSerDe::FormatOptions&
options, size_t depth);
+
+// Legacy CAST accepts more root families than V2 CAST. Encode their structure
here so
+// Flight output does not reject valid roots or lose typed leaves through JSON
reparsing.
+Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr&
type, size_t index,
+ VariantBatchBuilder::Row& output,
+ const DataTypeSerDe::FormatOptions& options,
size_t depth = 0) {
+ if (depth > VARIANT_MAX_NESTING_DEPTH) {
+ return Status::NotSupported(
+ "Native Arrow Variant nesting exceeds {}; "
+ "use enable_arrow_flight_sql_native_variant=false for UTF8
output",
+ VARIANT_MAX_NESTING_DEPTH);
+ }
+ if (const auto* constant = check_and_get_column<ColumnConst>(column)) {
+ return append_legacy_arrow_value(constant->get_data_column(), type, 0,
output, options,
+ depth);
+ }
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(column)) {
+ if (nullable->is_null_at(index)) {
+ output.add_null();
+ return Status::OK();
+ }
+ return append_legacy_arrow_value(nullable->get_nested_column(),
remove_nullable(type),
+ index, output, options, depth);
+ }
+ const auto primitive = type->get_primitive_type();
+ if (is_supported_variant_typed_identity(primitive)) {
+ dispatch_variant_typed_column(
+ column, primitive, [&]<PrimitiveType Type>(const auto& scalar)
{
+ with_variant_typed_scalar<Type>(
+ scalar, index,
cast_set<uint8_t>(type->get_scale()),
+ [&](const VariantScalarRef& value) {
output.add_scalar(value); });
+ });
+ } else if (primitive == TYPE_TIMEV2) {
+ // TIMEV2 already stores microseconds; treating its physical double as
a number loses its type.
+ const double micros = assert_cast<const
ColumnTimeV2&>(column).get_data()[index];
+ // Parquet TIME is a time of day, whereas Doris TIME also represents
signed durations.
+ // Reject unrepresentable durations instead of wrapping them or
emitting invalid TIME values.
+ constexpr int64_t micros_per_day = 86400000000;
+ if (!std::isfinite(micros) || micros < 0 || micros >= micros_per_day ||
+ std::llround(micros) >= micros_per_day) {
+ return Status::NotSupported(
+ "Native Arrow Variant TIMEV2 requires a time in [00:00:00,
24:00:00); "
+ "use enable_arrow_flight_sql_native_variant=false for UTF8
output");
+ }
+ output.add_time_ntz_micros(std::llround(micros));
+ } else if (primitive == TYPE_VARBINARY) {
+ // Binary leaves must retain arbitrary bytes, including NUL and
non-UTF8 data.
+ output.add_binary(column.get_data_at(index));
+ } else if (primitive == TYPE_JSONB) {
+ // JSONB leaves retain their own depth limit, but also consume the
enclosing Variant depth.
+ try {
+ jsonb_to_variant(column.get_data_at(index), output,
cast_set<uint32_t>(depth));
+ } catch (const Exception& e) {
+ if (e.code() != ErrorCode::INVALID_ARGUMENT) {
+ return e.to_status();
+ }
+ return Status::NotSupported(
+ "Native Arrow Variant cannot encode JSONB leaf: {}; "
+ "use enable_arrow_flight_sql_native_variant=false for UTF8
output",
+ e.what());
+ }
+ } else if (primitive == TYPE_ARRAY) {
+ const auto& array = assert_cast<const ColumnArray&>(column);
+ const auto& array_type = assert_cast<const DataTypeArray&>(*type);
+ auto scope = output.start_array();
+ for (size_t element = array.offset_at(index); element <
array.get_offsets()[index];
+ ++element) {
+ RETURN_IF_ERROR(append_legacy_arrow_value(array.get_data(),
+
array_type.get_nested_type(), element, output,
+ options, depth + 1));
+ }
+ scope.finish();
+ } else if (primitive == TYPE_MAP) {
+ const auto& map = assert_cast<const ColumnMap&>(column);
+ const auto& map_type = assert_cast<const DataTypeMap&>(*type);
+ auto scope = output.start_object();
+ for (size_t element = map.get_offsets()[static_cast<ssize_t>(index) -
1];
+ element < map.get_offsets()[index]; ++element) {
+ // Variant object keys cannot distinguish SQL NULL from the
literal string "null".
+ if (map.get_keys().is_null_at(element)) {
+ return Status::NotSupported(
+ "Native Arrow Variant cannot represent MAP with NULL
keys; "
+ "use enable_arrow_flight_sql_native_variant=false for
UTF8 output");
+ }
+ auto key = map_type.get_key_type()->to_string(map.get_keys(),
element, options);
+ scope.add_key({key.data(), key.size()});
+ RETURN_IF_ERROR(append_legacy_arrow_value(map.get_values(),
map_type.get_value_type(),
+ element, output,
options, depth + 1));
+ }
+ scope.finish();
+ } else if (primitive == TYPE_STRUCT) {
+ const auto& structure = assert_cast<const ColumnStruct&>(column);
+ const auto& struct_type = assert_cast<const DataTypeStruct&>(*type);
+ auto scope = output.start_object();
+ for (size_t field = 0; field < struct_type.get_elements().size();
++field) {
+ const auto& name = struct_type.get_element_names()[field];
+ scope.add_key({name.data(), name.size()});
+
RETURN_IF_ERROR(append_legacy_arrow_value(structure.get_column(field),
+
struct_type.get_element(field), index, output,
+ options, depth + 1));
+ }
+ scope.finish();
+ } else if (primitive == TYPE_VARIANT) {
+ if (const auto* legacy = check_and_get_column<ColumnVariant>(column)) {
+ const bool visible = legacy->is_scalar_variant()
+ ?
!legacy->get_root()->is_null_at(index)
+ :
legacy->is_visible_root_value(index);
+ if (visible) {
+ return append_legacy_arrow_value(*legacy->get_root(),
legacy->get_root_type(),
+ index, output, options,
depth);
+ }
+ RETURN_IF_ERROR(append_legacy_arrow_document(*legacy, index,
output, options, depth));
+ } else {
+ visit_variant_v2_values(
+ column, index, index + 1, {}, [&](size_t) {
output.add_null(); },
+ [&](size_t, VariantRef value) { output.add_value(value);
});
Review Comment:
[P2] Avoid validating the shared V2 dictionary for every nested leaf. A
legacy ARRAY root can contain N encoded V2 values whose rows share one N-key
metadata dictionary. This branch calls Row::add_value for each element; that
re-runs metadata.validate over all N keys, so a single native Flight row needs
O(N squared) key scans before writing the values. Validate shared metadata once
per source dictionary or use a trusted selected-value import, and cover a large
ARRAY<VARIANT_V2> root.
##########
be/src/core/data_type_serde/data_type_variant_serde.cpp:
##########
@@ -19,28 +19,268 @@
#include <arrow/array/builder_binary.h>
+#include <algorithm>
+#include <cmath>
#include <cstdint>
#include <string>
+#include <string_view>
+#include <vector>
#include "common/cast_set.h"
#include "common/config.h"
#include "common/exception.h"
#include "common/status.h"
#include "core/assert_cast.h"
#include "core/column/column.h"
+#include "core/column/column_array.h"
+#include "core/column/column_map.h"
+#include "core/column/column_struct.h"
#include "core/column/column_variant.h"
+#include "core/column/variant_v2/column_variant_v2.h"
+#include "core/column/variant_v2/column_variant_v2_typed_column.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_struct.h"
#include "core/data_type_serde/data_type_serde.h"
+#include "core/data_type_serde/data_type_variant_v2_serde.h"
#include "core/field.h"
#include "core/string_ref.h"
#include "core/types.h"
#include "core/value/jsonb_value.h"
#include "exec/common/variant_util.h"
+#include "exprs/function/parse/variant_jsonb_parse.h"
+#include "exprs/function/parse/variant_string_parse.h"
#include "util/json/json_parser.h"
#include "util/jsonb_writer.h"
namespace doris {
namespace {
+Status append_legacy_arrow_document(const ColumnVariant& column, size_t index,
+ VariantBatchBuilder::Row& output,
+ const DataTypeSerDe::FormatOptions&
options, size_t depth);
+
+// Legacy CAST accepts more root families than V2 CAST. Encode their structure
here so
+// Flight output does not reject valid roots or lose typed leaves through JSON
reparsing.
+Status append_legacy_arrow_value(const IColumn& column, const DataTypePtr&
type, size_t index,
+ VariantBatchBuilder::Row& output,
+ const DataTypeSerDe::FormatOptions& options,
size_t depth = 0) {
+ if (depth > VARIANT_MAX_NESTING_DEPTH) {
+ return Status::NotSupported(
+ "Native Arrow Variant nesting exceeds {}; "
+ "use enable_arrow_flight_sql_native_variant=false for UTF8
output",
+ VARIANT_MAX_NESTING_DEPTH);
+ }
+ if (const auto* constant = check_and_get_column<ColumnConst>(column)) {
+ return append_legacy_arrow_value(constant->get_data_column(), type, 0,
output, options,
+ depth);
+ }
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(column)) {
+ if (nullable->is_null_at(index)) {
+ output.add_null();
+ return Status::OK();
+ }
+ return append_legacy_arrow_value(nullable->get_nested_column(),
remove_nullable(type),
+ index, output, options, depth);
+ }
+ const auto primitive = type->get_primitive_type();
+ if (is_supported_variant_typed_identity(primitive)) {
+ dispatch_variant_typed_column(
+ column, primitive, [&]<PrimitiveType Type>(const auto& scalar)
{
+ with_variant_typed_scalar<Type>(
+ scalar, index,
cast_set<uint8_t>(type->get_scale()),
+ [&](const VariantScalarRef& value) {
output.add_scalar(value); });
+ });
+ } else if (primitive == TYPE_TIMEV2) {
+ // TIMEV2 already stores microseconds; treating its physical double as
a number loses its type.
+ const double micros = assert_cast<const
ColumnTimeV2&>(column).get_data()[index];
+ // Parquet TIME is a time of day, whereas Doris TIME also represents
signed durations.
+ // Reject unrepresentable durations instead of wrapping them or
emitting invalid TIME values.
+ constexpr int64_t micros_per_day = 86400000000;
+ if (!std::isfinite(micros) || micros < 0 || micros >= micros_per_day ||
+ std::llround(micros) >= micros_per_day) {
+ return Status::NotSupported(
+ "Native Arrow Variant TIMEV2 requires a time in [00:00:00,
24:00:00); "
+ "use enable_arrow_flight_sql_native_variant=false for UTF8
output");
+ }
+ output.add_time_ntz_micros(std::llround(micros));
+ } else if (primitive == TYPE_VARBINARY) {
+ // Binary leaves must retain arbitrary bytes, including NUL and
non-UTF8 data.
+ output.add_binary(column.get_data_at(index));
+ } else if (primitive == TYPE_JSONB) {
+ // JSONB leaves retain their own depth limit, but also consume the
enclosing Variant depth.
+ try {
+ jsonb_to_variant(column.get_data_at(index), output,
cast_set<uint32_t>(depth));
+ } catch (const Exception& e) {
+ if (e.code() != ErrorCode::INVALID_ARGUMENT) {
+ return e.to_status();
+ }
+ return Status::NotSupported(
+ "Native Arrow Variant cannot encode JSONB leaf: {}; "
+ "use enable_arrow_flight_sql_native_variant=false for UTF8
output",
+ e.what());
+ }
+ } else if (primitive == TYPE_ARRAY) {
+ const auto& array = assert_cast<const ColumnArray&>(column);
+ const auto& array_type = assert_cast<const DataTypeArray&>(*type);
+ auto scope = output.start_array();
+ for (size_t element = array.offset_at(index); element <
array.get_offsets()[index];
+ ++element) {
+ RETURN_IF_ERROR(append_legacy_arrow_value(array.get_data(),
+
array_type.get_nested_type(), element, output,
+ options, depth + 1));
+ }
+ scope.finish();
+ } else if (primitive == TYPE_MAP) {
+ const auto& map = assert_cast<const ColumnMap&>(column);
+ const auto& map_type = assert_cast<const DataTypeMap&>(*type);
+ auto scope = output.start_object();
+ for (size_t element = map.get_offsets()[static_cast<ssize_t>(index) -
1];
+ element < map.get_offsets()[index]; ++element) {
+ // Variant object keys cannot distinguish SQL NULL from the
literal string "null".
+ if (map.get_keys().is_null_at(element)) {
+ return Status::NotSupported(
+ "Native Arrow Variant cannot represent MAP with NULL
keys; "
+ "use enable_arrow_flight_sql_native_variant=false for
UTF8 output");
+ }
+ auto key = map_type.get_key_type()->to_string(map.get_keys(),
element, options);
+ scope.add_key({key.data(), key.size()});
+ RETURN_IF_ERROR(append_legacy_arrow_value(map.get_values(),
map_type.get_value_type(),
+ element, output,
options, depth + 1));
+ }
+ scope.finish();
+ } else if (primitive == TYPE_STRUCT) {
+ const auto& structure = assert_cast<const ColumnStruct&>(column);
+ const auto& struct_type = assert_cast<const DataTypeStruct&>(*type);
+ auto scope = output.start_object();
+ for (size_t field = 0; field < struct_type.get_elements().size();
++field) {
+ const auto& name = struct_type.get_element_names()[field];
+ scope.add_key({name.data(), name.size()});
+
RETURN_IF_ERROR(append_legacy_arrow_value(structure.get_column(field),
+
struct_type.get_element(field), index, output,
+ options, depth + 1));
+ }
+ scope.finish();
+ } else if (primitive == TYPE_VARIANT) {
+ if (const auto* legacy = check_and_get_column<ColumnVariant>(column)) {
+ const bool visible = legacy->is_scalar_variant()
+ ?
!legacy->get_root()->is_null_at(index)
+ :
legacy->is_visible_root_value(index);
+ if (visible) {
+ return append_legacy_arrow_value(*legacy->get_root(),
legacy->get_root_type(),
+ index, output, options,
depth);
+ }
+ RETURN_IF_ERROR(append_legacy_arrow_document(*legacy, index,
output, options, depth));
+ } else {
+ visit_variant_v2_values(
+ column, index, index + 1, {}, [&](size_t) {
output.add_null(); },
+ [&](size_t, VariantRef value) { output.add_value(value);
});
Review Comment:
[P2] Return the native-depth fallback error for nested V2 leaves. A legal
depth-128 V2 value inside a legacy ARRAY is imported here while the array scope
is open, so Row::add_value reaches depth 129 and throws INVALID_ARGUMENT.
ArrowBlockConvertor turns that into a generic InternalError, although the value
remains readable in UTF8 mode and other depth branches return NotSupported with
the setting to disable native Variant. Convert this import-depth case to the
same actionable error and cover a nested V2 leaf at the boundary.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
