github-actions[bot] commented on code in PR #68883:
URL: https://github.com/apache/doris/pull/68883#discussion_r4239837762
##########
be/src/format/arrow/arrow_row_batch.cpp:
##########
@@ -196,59 +205,124 @@ Status convert_to_arrow_type(const DataTypePtr&
origin_type,
return Status::OK();
}
-// Helper function to create an Arrow Field with type metadata if applicable,
such as IP types
+// Logical types sharing Arrow storage need the same marker at the root and
every child field.
std::shared_ptr<arrow::Field> create_arrow_field_with_metadata(
const std::string& field_name, const std::shared_ptr<arrow::DataType>&
arrow_type,
bool is_nullable, PrimitiveType primitive_type) {
- if (primitive_type == PrimitiveType::TYPE_IPV4) {
- auto metadata = arrow::KeyValueMetadata::Make({"doris_type"},
{"IPV4"});
- return std::make_shared<arrow::Field>(field_name, arrow_type,
is_nullable, metadata);
- } else if (primitive_type == PrimitiveType::TYPE_IPV6) {
- auto metadata = arrow::KeyValueMetadata::Make({"doris_type"},
{"IPV6"});
- return std::make_shared<arrow::Field>(field_name, arrow_type,
is_nullable, metadata);
- } else if (primitive_type == PrimitiveType::TYPE_UUID) {
- auto metadata = arrow::KeyValueMetadata::Make({"doris_type"},
{"UUID"});
- return std::make_shared<arrow::Field>(field_name, arrow_type,
is_nullable, metadata);
- } else if (primitive_type == PrimitiveType::TYPE_LARGEINT) {
- auto metadata = arrow::KeyValueMetadata::Make({"doris_type"},
{"LARGEINT"});
- return std::make_shared<arrow::Field>(field_name, arrow_type,
is_nullable, metadata);
- } else {
+ const char* type_name;
+ switch (primitive_type) {
+ case TYPE_UUID:
+ type_name = "UUID";
+ break;
+ case TYPE_IPV4:
+ type_name = "IPV4";
+ break;
+ case TYPE_IPV6:
+ type_name = "IPV6";
+ break;
+ case TYPE_LARGEINT:
+ type_name = "LARGEINT";
+ break;
+ case TYPE_JSONB:
+ type_name = "JSON";
+ break;
+ case TYPE_VARIANT:
+ type_name = "VARIANT";
+ break;
+ default:
return std::make_shared<arrow::Field>(field_name, arrow_type,
is_nullable);
}
+ auto metadata = arrow::KeyValueMetadata::Make({"doris_type"}, {type_name});
+ return std::make_shared<arrow::Field>(field_name, arrow_type, is_nullable,
metadata);
}
-Status get_arrow_schema_from_block(const Block& block,
std::shared_ptr<arrow::Schema>* result,
- const std::string& timezone, bool
datetime_naive) {
+Status DorisArrowSchemaConvertor::get_arrow_schema_from_block(
+ const Block& block, std::shared_ptr<arrow::Schema>* result) const {
std::vector<std::shared_ptr<arrow::Field>> fields;
for (const auto& type_and_name : block) {
std::shared_ptr<arrow::DataType> arrow_type;
- RETURN_IF_ERROR(
- convert_to_arrow_type(type_and_name.type, &arrow_type,
timezone, datetime_naive));
- auto field = create_arrow_field_with_metadata(type_and_name.name,
arrow_type,
-
type_and_name.type->is_nullable(),
-
type_and_name.type->get_primitive_type());
+ RETURN_IF_ERROR(convert_to_arrow_type(type_and_name.type,
&arrow_type));
+ auto field = make_field(type_and_name.name, arrow_type,
type_and_name.type->is_nullable(),
+ type_and_name.type->get_primitive_type());
fields.push_back(field);
}
*result = arrow::schema(std::move(fields));
return Status::OK();
}
-Status get_arrow_schema_from_expr_ctxs(const VExprContextSPtrs&
output_vexpr_ctxs,
- std::shared_ptr<arrow::Schema>* result,
- const std::string& timezone, bool
datetime_naive) {
+Status
DorisArrowSchemaConvertor::get_arrow_schema(std::shared_ptr<arrow::Schema>*
result) const {
+ return get_arrow_schema_from_block(_header, result);
+}
+
+std::string DorisArrowSchemaConvertor::timestamp_timezone(PrimitiveType) const
{
+ // Arrow clients expect a timezone name rather than the ISO-8601 UTC alias.
+ return _timezone == "Z" ? "UTC" : _timezone;
+}
+
+Status ArrowFlightSchemaConvertor::convert_to_arrow_type(
+ const DataTypePtr& type, std::shared_ptr<arrow::DataType>* result)
const {
+ // Flight always uses native Variant, including recursively converted
children.
+ if (type->get_primitive_type() == TYPE_VARIANT) {
Review Comment:
[P1] Preserve UTF8 Variant for an FE without the Flight capability. An old
FE omits `enable_arrow_type_metadata`, so `ResultSinkOperatorX` selects
`LegacyArrowFlightSchemaConvertor`, but it inherits this native Variant
mapping. The new BE consequently advertises and sends
`arrow.parquet.variant`/STRUCT through both Flight readers, while the old FE
GetTables still describes Variant as UTF8 and pre-upgrade clients expect JSON
text. During an old-FE/new-BE rolling upgrade, discovery and query schemas
disagree and the result wire type changes. Gate native Variant output on an
explicit FE capability and keep the UTF8 path for older plans. This is separate
from the existing memory-scratch scanner thread.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
