Gabriel39 commented on code in PR #70808: URL: https://github.com/apache/doris/pull/70808#discussion_r4642508414
########## be/src/format_v2/avro/avro_reader.cpp: ########## @@ -0,0 +1,1074 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/avro/avro_reader.h" + +#include <algorithm> +#include <array> +#include <avro/Allocation.hh> +#include <avro/DataFile.hh> +#include <avro/Generic.hh> +#include <avro/NodeImpl.hh> +#include <charconv> +#include <limits> +#include <mutex> +#include <type_traits> +#include <unordered_set> +#include <utility> + +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/custom_allocator.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_uuid.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/value/timestamp_ns_value.h" +#include "core/value/timestamptz_value.h" +#include "format_v2/column_mapper.h" +#include "format_v2/materialized_reader_util.h" +#include "io/fs/file_reader.h" +#include "io/io_common.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "storage/utils.h" + +namespace doris::format::avro { +namespace { + +// Avro must use the same filesystem, credentials, cache and cancellation state as other readers. +// The library owns its stream; the shared physical reader remains alive until the stream closes. +class NativeInputStream final : public ::avro::SeekableInputStream { +public: + NativeInputStream(io::FileReaderSPtr reader, std::shared_ptr<io::IOContext> io_ctx) + : _reader(std::move(reader)), _io_ctx(std::move(io_ctx)) {} + + bool next(const uint8_t** data, size_t* length) override { + if (_position == _reader->size()) { + return false; + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + throw Exception(ErrorCode::CANCELLED, "Avro scan cancelled"); + } + const auto requested = std::min(_buffer.size(), _reader->size() - _position); + size_t read = 0; + auto status = + _reader->read_at(_position, Slice(_buffer.data(), requested), &read, _io_ctx.get()); + if (!status.ok()) { + throw Exception(status.code(), status.to_string()); + } + if (read == 0) { + throw ::avro::Exception("Truncated Avro input"); + } + *data = reinterpret_cast<const uint8_t*>(_buffer.data()); + *length = read; + _position += read; + _last_read = read; + return true; + } + + void backup(size_t length) override { + if (length > _last_read) { + throw ::avro::Exception("Invalid Avro stream backup"); + } + _position -= length; + _last_read -= length; + } + + void skip(size_t length) override { + if (length > _reader->size() - _position) { + throw ::avro::Exception("Truncated Avro input"); + } + _position += length; + _last_read = 0; + } + + size_t byteCount() const override { return _position; } + + void seek(int64_t position) override { + if (position < 0 || static_cast<uint64_t>(position) > _reader->size()) { + throw ::avro::Exception("Invalid Avro stream offset"); + } + _position = position; + _last_read = 0; + } + +private: + io::FileReaderSPtr _reader; + std::shared_ptr<io::IOContext> _io_ctx; + std::array<char, 64 * 1024> _buffer; + size_t _position = 0; + size_t _last_read = 0; +}; + +::avro::NodePtr resolved_node(const ::avro::NodePtr& node) { + return node->type() == ::avro::AVRO_SYMBOLIC ? ::avro::resolveSymbol(node) : node; +} + +std::string lowercase(std::string name) { + for (auto& c : name) { + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + } + return name; +} + +Status set_identifier(const ::avro::NodePtr& node, size_t attributes_index, const char* attribute, + ColumnDefinition* column) { + if (attributes_index >= node->customAttributes()) { + return Status::OK(); + } + const auto value = node->customAttributesAt(attributes_index).getAttribute(attribute); + if (!value.has_value()) { + return Status::OK(); + } + int32_t id = 0; + const auto parsed = std::from_chars(value->data(), value->data() + value->size(), id); + if (parsed.ec != std::errc() || parsed.ptr != value->data() + value->size() || id <= 0) { + return Status::InvalidArgument("Invalid Avro field identifier"); + } + column->identifier = Field::create_field<TYPE_INT>(id); + return Status::OK(); +} + +bool is_logical_map(const ::avro::NodePtr& node) { + const auto logical = node->logicalType(); + return node->type() == ::avro::AVRO_ARRAY && logical.type() == ::avro::LogicalType::CUSTOM && + logical.customLogicalType()->name() == "map"; +} + +// Keep logical and physical alternatives in one visitor so nullable handling is shared. +// NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity) +Status build_schema(const ::avro::NodePtr& input, const std::string& name, int32_t ordinal, + bool timestamp_tz, bool varbinary, + std::unordered_set<const ::avro::Node*>* ancestors, ColumnDefinition* column) { + const auto node = resolved_node(input); + // A recursive Avro record cannot be represented by a finite Doris STRUCT type. + if (ancestors->size() >= 64 || !ancestors->insert(node.get()).second) { + return Status::NotSupported("Recursive or excessively nested Avro schemas are unsupported"); + } + struct Pop { + std::unordered_set<const ::avro::Node*>* nodes; + const ::avro::Node* node; + ~Pop() { nodes->erase(node); } + } pop {.nodes = ancestors, .node = node.get()}; + if (node->leaves() > static_cast<size_t>(std::numeric_limits<int32_t>::max())) { + return Status::NotSupported("Avro schema has too many fields"); + } + column->name = lowercase(name); + column->local_id = ordinal; + column->identifier = Field::create_field<TYPE_STRING>(column->name); + if (node->type() == ::avro::AVRO_UNION) { + ::avro::NodePtr value; + for (size_t i = 0; i < node->leaves(); ++i) { + if (node->leafAt(i)->type() != ::avro::AVRO_NULL) { + if (value != nullptr) { + return Status::NotSupported( + "Avro unions with multiple non-null types are unsupported"); + } + value = node->leafAt(i); + } + } + if (value != nullptr) { + return build_schema(value, name, ordinal, timestamp_tz, varbinary, ancestors, column); + } + column->type = make_nullable(std::make_shared<DataTypeString>()); + return Status::OK(); + } + + DataTypePtr type; + const auto logical = node->logicalType(); + using Logical = ::avro::LogicalType; + switch (logical.type()) { + case Logical::DECIMAL: + if (logical.precision() > 76) { + return Status::NotSupported("Avro decimal precision exceeds 76"); + } + type = create_decimal(logical.precision(), logical.scale(), false); + break; + case Logical::UUID: + type = std::make_shared<DataTypeUUID>(); + break; + case Logical::DATE: + type = std::make_shared<DataTypeDateV2>(); + break; + case Logical::TIME_MILLIS: + case Logical::TIME_MICROS: + type = std::make_shared<DataTypeTimeV2>(logical.type() == Logical::TIME_MILLIS ? 3 : 6); + break; + case Logical::TIMESTAMP_NANOS: + case Logical::LOCAL_TIMESTAMP_NANOS: + type = std::make_shared<DataTypeTimeStampNs>(); + column->timestamp_is_adjusted_to_utc = logical.type() == Logical::TIMESTAMP_NANOS; + break; + case Logical::TIMESTAMP_MILLIS: + case Logical::TIMESTAMP_MICROS: + case Logical::LOCAL_TIMESTAMP_MILLIS: + case Logical::LOCAL_TIMESTAMP_MICROS: { + const bool adjusted = logical.type() == Logical::TIMESTAMP_MILLIS || + logical.type() == Logical::TIMESTAMP_MICROS; + const int scale = logical.type() == Logical::TIMESTAMP_MILLIS || + logical.type() == Logical::LOCAL_TIMESTAMP_MILLIS + ? 3 + : 6; + type = timestamp_tz && adjusted ? DataTypePtr(std::make_shared<DataTypeTimeStampTz>(scale)) + : DataTypePtr(std::make_shared<DataTypeDateTimeV2>(scale)); + column->timestamp_is_adjusted_to_utc = adjusted; + break; + } + default: + break; + } + if (type == nullptr) { + switch (node->type()) { + case ::avro::AVRO_NULL: + type = std::make_shared<DataTypeString>(); + break; + case ::avro::AVRO_BOOL: + type = std::make_shared<DataTypeUInt8>(); + break; + case ::avro::AVRO_INT: + type = std::make_shared<DataTypeInt32>(); + break; + case ::avro::AVRO_LONG: + type = std::make_shared<DataTypeInt64>(); + break; + case ::avro::AVRO_FLOAT: + type = std::make_shared<DataTypeFloat32>(); + break; + case ::avro::AVRO_DOUBLE: + type = std::make_shared<DataTypeFloat64>(); + break; + case ::avro::AVRO_BYTES: + case ::avro::AVRO_FIXED: + type = varbinary ? DataTypePtr(std::make_shared<DataTypeVarbinary>()) + : DataTypePtr(std::make_shared<DataTypeString>()); + break; + case ::avro::AVRO_STRING: + case ::avro::AVRO_ENUM: + type = std::make_shared<DataTypeString>(); + break; + case ::avro::AVRO_RECORD: { + DataTypes types; + Strings names; + for (size_t i = 0; i < node->leaves(); ++i) { + ColumnDefinition child; + RETURN_IF_ERROR(build_schema(node->leafAt(i), node->nameAt(i), + static_cast<int32_t>(i), timestamp_tz, varbinary, + ancestors, &child)); + RETURN_IF_ERROR(set_identifier(node, i, "field-id", &child)); + types.push_back(child.type); + names.push_back(child.name); + column->children.push_back(std::move(child)); + } + if (types.empty()) { + return Status::NotSupported("Empty Avro records are unsupported"); + } + type = std::make_shared<DataTypeStruct>(types, names); + break; + } + case ::avro::AVRO_ARRAY: + case ::avro::AVRO_MAP: { + const bool map = node->type() == ::avro::AVRO_MAP || is_logical_map(node); + if (is_logical_map(node)) { + const auto entry = resolved_node(node->leafAt(0)); + if (entry->type() != ::avro::AVRO_RECORD || entry->leaves() != 2 || + entry->nameAt(0) != "key" || entry->nameAt(1) != "value") { + return Status::InvalidArgument("Invalid Avro logical map entry schema"); + } + ColumnDefinition entries; + RETURN_IF_ERROR(build_schema(entry, "entry", 0, timestamp_tz, varbinary, ancestors, + &entries)); + column->children = std::move(entries.children); + } else { + for (size_t i = 0; std::cmp_less(i, (map ? 2 : 1)); ++i) { + ColumnDefinition child; + RETURN_IF_ERROR(build_schema( + node->leafAt(i), map ? (i == 0 ? "key" : "value") : "element", + static_cast<int32_t>(i), timestamp_tz, varbinary, ancestors, &child)); + RETURN_IF_ERROR(set_identifier( + node, 0, map ? (i == 0 ? "key-id" : "value-id") : "element-id", + &child)); + column->children.push_back(std::move(child)); + } + } + type = map ? DataTypePtr(std::make_shared<DataTypeMap>(column->children[0].type, + column->children[1].type)) + : DataTypePtr(std::make_shared<DataTypeArray>(column->children[0].type)); + break; + } + default: + return Status::NotSupported("Unsupported Avro schema type"); + } + } + column->type = make_nullable(type); + return Status::OK(); +} + +::avro::NodePtr clone_projected_schema( + const ::avro::NodePtr& input, + std::unordered_map<const ::avro::Node*, ::avro::NodePtr>* copies, + const std::unordered_set<int32_t>* selected_roots = nullptr) { + const auto source = resolved_node(input); + if (const auto found = copies->find(source.get()); found != copies->end()) { + return found->second; + } + ::avro::NodePtr result; + switch (source->type()) { + case ::avro::AVRO_RECORD: + result = std::make_shared<::avro::NodeRecord>(); + break; + case ::avro::AVRO_ENUM: + result = std::make_shared<::avro::NodeEnum>(); + break; + case ::avro::AVRO_FIXED: + result = std::make_shared<::avro::NodeFixed>(); + break; + case ::avro::AVRO_ARRAY: + result = std::make_shared<::avro::NodeArray>(); + break; + case ::avro::AVRO_MAP: + result = std::make_shared<::avro::NodeMap>(); + break; + case ::avro::AVRO_UNION: + result = std::make_shared<::avro::NodeUnion>(); + break; + default: + result = std::make_shared<::avro::NodePrimitive>(source->type()); + break; + } + copies->emplace(source.get(), result); + if (source->hasName()) { + result->setName(source->name()); + } + if (source->type() == ::avro::AVRO_FIXED) { + result->setFixedSize(source->fixedSize()); + } + for (size_t i = source->type() == ::avro::AVRO_MAP ? 1 : 0; i < source->leaves(); ++i) { + if (selected_roots != nullptr && !selected_roots->contains(static_cast<int32_t>(i))) { + continue; + } + result->addLeaf(clone_projected_schema(source->leafAt(i), copies)); + if (source->type() == ::avro::AVRO_RECORD) { + result->addName(source->nameAt(i)); + } + } + if (source->type() == ::avro::AVRO_ENUM) { + for (size_t i = 0; i < source->names(); ++i) { + result->addName(source->nameAt(i)); + } + } + // Rebuild named definitions instead of retaining symbolic references to omitted roots. + // The ValidSchema validator replaces repeated definitions with links within this schema. + result->setLogicalType(source->logicalType()); + return result; +} + +void apply_timestamp_projection(ColumnDefinition* column, const LocalColumnIndex& projection) { + const auto type = remove_nullable(column->type); + if (projection.timestamp_is_adjusted_to_utc.has_value() && + (type->get_primitive_type() == TYPE_DATETIMEV2 || + type->get_primitive_type() == TYPE_TIMESTAMPTZ)) { + column->timestamp_is_adjusted_to_utc = projection.timestamp_is_adjusted_to_utc; + column->type = make_nullable( Review Comment: Fixed in 3ae183546dd. Timestamp materialization now follows the actual file-block DataType at every nested leaf; the projection only carries temporal semantics. The mapper-generated load regression failed with Field::get type mismatch before the fix and now passes, together with the Iceberg TIMESTAMPTZ/equality-delete cases. ########## be/src/format_v2/avro/avro_reader.cpp: ########## @@ -0,0 +1,1074 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/avro/avro_reader.h" + +#include <algorithm> +#include <array> +#include <avro/Allocation.hh> +#include <avro/DataFile.hh> +#include <avro/Generic.hh> +#include <avro/NodeImpl.hh> +#include <charconv> +#include <limits> +#include <mutex> +#include <type_traits> +#include <unordered_set> +#include <utility> + +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/custom_allocator.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_uuid.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/value/timestamp_ns_value.h" +#include "core/value/timestamptz_value.h" +#include "format_v2/column_mapper.h" +#include "format_v2/materialized_reader_util.h" +#include "io/fs/file_reader.h" +#include "io/io_common.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "storage/utils.h" + +namespace doris::format::avro { +namespace { + +// Avro must use the same filesystem, credentials, cache and cancellation state as other readers. +// The library owns its stream; the shared physical reader remains alive until the stream closes. +class NativeInputStream final : public ::avro::SeekableInputStream { +public: + NativeInputStream(io::FileReaderSPtr reader, std::shared_ptr<io::IOContext> io_ctx) + : _reader(std::move(reader)), _io_ctx(std::move(io_ctx)) {} + + bool next(const uint8_t** data, size_t* length) override { + if (_position == _reader->size()) { + return false; + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + throw Exception(ErrorCode::CANCELLED, "Avro scan cancelled"); + } + const auto requested = std::min(_buffer.size(), _reader->size() - _position); + size_t read = 0; + auto status = + _reader->read_at(_position, Slice(_buffer.data(), requested), &read, _io_ctx.get()); + if (!status.ok()) { + throw Exception(status.code(), status.to_string()); + } + if (read == 0) { + throw ::avro::Exception("Truncated Avro input"); + } + *data = reinterpret_cast<const uint8_t*>(_buffer.data()); + *length = read; + _position += read; + _last_read = read; + return true; + } + + void backup(size_t length) override { + if (length > _last_read) { + throw ::avro::Exception("Invalid Avro stream backup"); + } + _position -= length; + _last_read -= length; + } + + void skip(size_t length) override { + if (length > _reader->size() - _position) { + throw ::avro::Exception("Truncated Avro input"); + } + _position += length; + _last_read = 0; + } + + size_t byteCount() const override { return _position; } + + void seek(int64_t position) override { + if (position < 0 || static_cast<uint64_t>(position) > _reader->size()) { + throw ::avro::Exception("Invalid Avro stream offset"); + } + _position = position; + _last_read = 0; + } + +private: + io::FileReaderSPtr _reader; + std::shared_ptr<io::IOContext> _io_ctx; + std::array<char, 64 * 1024> _buffer; + size_t _position = 0; + size_t _last_read = 0; +}; + +::avro::NodePtr resolved_node(const ::avro::NodePtr& node) { + return node->type() == ::avro::AVRO_SYMBOLIC ? ::avro::resolveSymbol(node) : node; +} + +std::string lowercase(std::string name) { + for (auto& c : name) { + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + } + return name; +} + +Status set_identifier(const ::avro::NodePtr& node, size_t attributes_index, const char* attribute, + ColumnDefinition* column) { + if (attributes_index >= node->customAttributes()) { + return Status::OK(); + } + const auto value = node->customAttributesAt(attributes_index).getAttribute(attribute); + if (!value.has_value()) { + return Status::OK(); + } + int32_t id = 0; + const auto parsed = std::from_chars(value->data(), value->data() + value->size(), id); + if (parsed.ec != std::errc() || parsed.ptr != value->data() + value->size() || id <= 0) { + return Status::InvalidArgument("Invalid Avro field identifier"); + } + column->identifier = Field::create_field<TYPE_INT>(id); + return Status::OK(); +} + +bool is_logical_map(const ::avro::NodePtr& node) { + const auto logical = node->logicalType(); + return node->type() == ::avro::AVRO_ARRAY && logical.type() == ::avro::LogicalType::CUSTOM && + logical.customLogicalType()->name() == "map"; +} + +// Keep logical and physical alternatives in one visitor so nullable handling is shared. +// NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity) +Status build_schema(const ::avro::NodePtr& input, const std::string& name, int32_t ordinal, + bool timestamp_tz, bool varbinary, + std::unordered_set<const ::avro::Node*>* ancestors, ColumnDefinition* column) { + const auto node = resolved_node(input); + // A recursive Avro record cannot be represented by a finite Doris STRUCT type. + if (ancestors->size() >= 64 || !ancestors->insert(node.get()).second) { + return Status::NotSupported("Recursive or excessively nested Avro schemas are unsupported"); + } + struct Pop { + std::unordered_set<const ::avro::Node*>* nodes; + const ::avro::Node* node; + ~Pop() { nodes->erase(node); } + } pop {.nodes = ancestors, .node = node.get()}; + if (node->leaves() > static_cast<size_t>(std::numeric_limits<int32_t>::max())) { + return Status::NotSupported("Avro schema has too many fields"); + } + column->name = lowercase(name); + column->local_id = ordinal; + column->identifier = Field::create_field<TYPE_STRING>(column->name); + if (node->type() == ::avro::AVRO_UNION) { + ::avro::NodePtr value; + for (size_t i = 0; i < node->leaves(); ++i) { + if (node->leafAt(i)->type() != ::avro::AVRO_NULL) { + if (value != nullptr) { + return Status::NotSupported( + "Avro unions with multiple non-null types are unsupported"); + } + value = node->leafAt(i); + } + } + if (value != nullptr) { + return build_schema(value, name, ordinal, timestamp_tz, varbinary, ancestors, column); + } + column->type = make_nullable(std::make_shared<DataTypeString>()); + return Status::OK(); + } + + DataTypePtr type; + const auto logical = node->logicalType(); + using Logical = ::avro::LogicalType; + switch (logical.type()) { + case Logical::DECIMAL: + if (logical.precision() > 76) { + return Status::NotSupported("Avro decimal precision exceeds 76"); + } + type = create_decimal(logical.precision(), logical.scale(), false); + break; + case Logical::UUID: + type = std::make_shared<DataTypeUUID>(); + break; + case Logical::DATE: + type = std::make_shared<DataTypeDateV2>(); + break; + case Logical::TIME_MILLIS: + case Logical::TIME_MICROS: + type = std::make_shared<DataTypeTimeV2>(logical.type() == Logical::TIME_MILLIS ? 3 : 6); + break; + case Logical::TIMESTAMP_NANOS: + case Logical::LOCAL_TIMESTAMP_NANOS: Review Comment: Fixed in 3ae183546dd. Instant timestamp-nanos now returns an explicit unsupported error: Doris TIMESTAMP_NS is a civil clock and TIMESTAMPTZ cannot retain nanosecond precision. local-timestamp-nanos remains supported. The new rejection test covers both timestamp mapping settings. ########## be/src/format_v2/table/hive_reader.cpp: ########## @@ -132,6 +132,9 @@ format::TableColumnMappingMode HiveReader::mapping_mode() const { use_column_names = query_options.hive_orc_use_column_names; } else if (_format == format::FileFormat::PARQUET) { use_column_names = query_options.hive_parquet_use_column_names; + } else if (_format == format::FileFormat::AVRO) { Review Comment: Fixed in 3ae183546dd. Hive now passes a dedicated temporal policy to the reader, restores writer.time.zone and keeps Hive TIMESTAMP civil rather than applying the query timezone through TIMESTAMPTZ. Tests cover America/Los_Angeles and normalized Java fixed offsets including the +/-18-hour endpoints. Metadata-free pre-1900 timestamps and legacy writer conversion fail explicitly instead of returning plausible wrong values. ########## be/src/format_v2/avro/avro_reader.cpp: ########## @@ -0,0 +1,1074 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/avro/avro_reader.h" + +#include <algorithm> +#include <array> +#include <avro/Allocation.hh> +#include <avro/DataFile.hh> +#include <avro/Generic.hh> +#include <avro/NodeImpl.hh> +#include <charconv> +#include <limits> +#include <mutex> +#include <type_traits> +#include <unordered_set> +#include <utility> + +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/custom_allocator.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_uuid.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/value/timestamp_ns_value.h" +#include "core/value/timestamptz_value.h" +#include "format_v2/column_mapper.h" +#include "format_v2/materialized_reader_util.h" +#include "io/fs/file_reader.h" +#include "io/io_common.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "storage/utils.h" + +namespace doris::format::avro { +namespace { + +// Avro must use the same filesystem, credentials, cache and cancellation state as other readers. +// The library owns its stream; the shared physical reader remains alive until the stream closes. +class NativeInputStream final : public ::avro::SeekableInputStream { +public: + NativeInputStream(io::FileReaderSPtr reader, std::shared_ptr<io::IOContext> io_ctx) + : _reader(std::move(reader)), _io_ctx(std::move(io_ctx)) {} + + bool next(const uint8_t** data, size_t* length) override { + if (_position == _reader->size()) { + return false; + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + throw Exception(ErrorCode::CANCELLED, "Avro scan cancelled"); + } + const auto requested = std::min(_buffer.size(), _reader->size() - _position); + size_t read = 0; + auto status = + _reader->read_at(_position, Slice(_buffer.data(), requested), &read, _io_ctx.get()); + if (!status.ok()) { + throw Exception(status.code(), status.to_string()); + } + if (read == 0) { + throw ::avro::Exception("Truncated Avro input"); + } + *data = reinterpret_cast<const uint8_t*>(_buffer.data()); + *length = read; + _position += read; + _last_read = read; + return true; + } + + void backup(size_t length) override { + if (length > _last_read) { + throw ::avro::Exception("Invalid Avro stream backup"); + } + _position -= length; + _last_read -= length; + } + + void skip(size_t length) override { + if (length > _reader->size() - _position) { + throw ::avro::Exception("Truncated Avro input"); + } + _position += length; + _last_read = 0; + } + + size_t byteCount() const override { return _position; } + + void seek(int64_t position) override { + if (position < 0 || static_cast<uint64_t>(position) > _reader->size()) { + throw ::avro::Exception("Invalid Avro stream offset"); + } + _position = position; + _last_read = 0; + } + +private: + io::FileReaderSPtr _reader; + std::shared_ptr<io::IOContext> _io_ctx; + std::array<char, 64 * 1024> _buffer; + size_t _position = 0; + size_t _last_read = 0; +}; + +::avro::NodePtr resolved_node(const ::avro::NodePtr& node) { + return node->type() == ::avro::AVRO_SYMBOLIC ? ::avro::resolveSymbol(node) : node; +} + +std::string lowercase(std::string name) { + for (auto& c : name) { + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + } + return name; +} + +Status set_identifier(const ::avro::NodePtr& node, size_t attributes_index, const char* attribute, + ColumnDefinition* column) { + if (attributes_index >= node->customAttributes()) { + return Status::OK(); + } + const auto value = node->customAttributesAt(attributes_index).getAttribute(attribute); + if (!value.has_value()) { + return Status::OK(); + } + int32_t id = 0; + const auto parsed = std::from_chars(value->data(), value->data() + value->size(), id); + if (parsed.ec != std::errc() || parsed.ptr != value->data() + value->size() || id <= 0) { + return Status::InvalidArgument("Invalid Avro field identifier"); + } + column->identifier = Field::create_field<TYPE_INT>(id); + return Status::OK(); +} + +bool is_logical_map(const ::avro::NodePtr& node) { + const auto logical = node->logicalType(); + return node->type() == ::avro::AVRO_ARRAY && logical.type() == ::avro::LogicalType::CUSTOM && + logical.customLogicalType()->name() == "map"; +} + +// Keep logical and physical alternatives in one visitor so nullable handling is shared. +// NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity) +Status build_schema(const ::avro::NodePtr& input, const std::string& name, int32_t ordinal, + bool timestamp_tz, bool varbinary, + std::unordered_set<const ::avro::Node*>* ancestors, ColumnDefinition* column) { + const auto node = resolved_node(input); + // A recursive Avro record cannot be represented by a finite Doris STRUCT type. + if (ancestors->size() >= 64 || !ancestors->insert(node.get()).second) { + return Status::NotSupported("Recursive or excessively nested Avro schemas are unsupported"); + } + struct Pop { + std::unordered_set<const ::avro::Node*>* nodes; + const ::avro::Node* node; + ~Pop() { nodes->erase(node); } + } pop {.nodes = ancestors, .node = node.get()}; + if (node->leaves() > static_cast<size_t>(std::numeric_limits<int32_t>::max())) { + return Status::NotSupported("Avro schema has too many fields"); + } + column->name = lowercase(name); + column->local_id = ordinal; + column->identifier = Field::create_field<TYPE_STRING>(column->name); + if (node->type() == ::avro::AVRO_UNION) { + ::avro::NodePtr value; + for (size_t i = 0; i < node->leaves(); ++i) { + if (node->leafAt(i)->type() != ::avro::AVRO_NULL) { + if (value != nullptr) { + return Status::NotSupported( + "Avro unions with multiple non-null types are unsupported"); + } + value = node->leafAt(i); + } + } + if (value != nullptr) { + return build_schema(value, name, ordinal, timestamp_tz, varbinary, ancestors, column); + } + column->type = make_nullable(std::make_shared<DataTypeString>()); + return Status::OK(); + } + + DataTypePtr type; + const auto logical = node->logicalType(); + using Logical = ::avro::LogicalType; + switch (logical.type()) { + case Logical::DECIMAL: + if (logical.precision() > 76) { + return Status::NotSupported("Avro decimal precision exceeds 76"); + } + type = create_decimal(logical.precision(), logical.scale(), false); + break; + case Logical::UUID: + type = std::make_shared<DataTypeUUID>(); + break; + case Logical::DATE: + type = std::make_shared<DataTypeDateV2>(); + break; + case Logical::TIME_MILLIS: + case Logical::TIME_MICROS: + type = std::make_shared<DataTypeTimeV2>(logical.type() == Logical::TIME_MILLIS ? 3 : 6); + break; + case Logical::TIMESTAMP_NANOS: + case Logical::LOCAL_TIMESTAMP_NANOS: + type = std::make_shared<DataTypeTimeStampNs>(); + column->timestamp_is_adjusted_to_utc = logical.type() == Logical::TIMESTAMP_NANOS; + break; + case Logical::TIMESTAMP_MILLIS: + case Logical::TIMESTAMP_MICROS: + case Logical::LOCAL_TIMESTAMP_MILLIS: + case Logical::LOCAL_TIMESTAMP_MICROS: { + const bool adjusted = logical.type() == Logical::TIMESTAMP_MILLIS || + logical.type() == Logical::TIMESTAMP_MICROS; + const int scale = logical.type() == Logical::TIMESTAMP_MILLIS || + logical.type() == Logical::LOCAL_TIMESTAMP_MILLIS + ? 3 + : 6; + type = timestamp_tz && adjusted ? DataTypePtr(std::make_shared<DataTypeTimeStampTz>(scale)) + : DataTypePtr(std::make_shared<DataTypeDateTimeV2>(scale)); + column->timestamp_is_adjusted_to_utc = adjusted; + break; + } + default: + break; + } + if (type == nullptr) { + switch (node->type()) { + case ::avro::AVRO_NULL: + type = std::make_shared<DataTypeString>(); + break; + case ::avro::AVRO_BOOL: + type = std::make_shared<DataTypeUInt8>(); + break; + case ::avro::AVRO_INT: + type = std::make_shared<DataTypeInt32>(); + break; + case ::avro::AVRO_LONG: + type = std::make_shared<DataTypeInt64>(); + break; + case ::avro::AVRO_FLOAT: + type = std::make_shared<DataTypeFloat32>(); + break; + case ::avro::AVRO_DOUBLE: + type = std::make_shared<DataTypeFloat64>(); + break; + case ::avro::AVRO_BYTES: + case ::avro::AVRO_FIXED: + type = varbinary ? DataTypePtr(std::make_shared<DataTypeVarbinary>()) + : DataTypePtr(std::make_shared<DataTypeString>()); + break; + case ::avro::AVRO_STRING: + case ::avro::AVRO_ENUM: + type = std::make_shared<DataTypeString>(); + break; + case ::avro::AVRO_RECORD: { + DataTypes types; + Strings names; + for (size_t i = 0; i < node->leaves(); ++i) { + ColumnDefinition child; + RETURN_IF_ERROR(build_schema(node->leafAt(i), node->nameAt(i), + static_cast<int32_t>(i), timestamp_tz, varbinary, + ancestors, &child)); + RETURN_IF_ERROR(set_identifier(node, i, "field-id", &child)); + types.push_back(child.type); + names.push_back(child.name); + column->children.push_back(std::move(child)); + } + if (types.empty()) { + return Status::NotSupported("Empty Avro records are unsupported"); + } + type = std::make_shared<DataTypeStruct>(types, names); + break; + } + case ::avro::AVRO_ARRAY: + case ::avro::AVRO_MAP: { + const bool map = node->type() == ::avro::AVRO_MAP || is_logical_map(node); + if (is_logical_map(node)) { + const auto entry = resolved_node(node->leafAt(0)); + if (entry->type() != ::avro::AVRO_RECORD || entry->leaves() != 2 || + entry->nameAt(0) != "key" || entry->nameAt(1) != "value") { + return Status::InvalidArgument("Invalid Avro logical map entry schema"); + } + ColumnDefinition entries; + RETURN_IF_ERROR(build_schema(entry, "entry", 0, timestamp_tz, varbinary, ancestors, + &entries)); + column->children = std::move(entries.children); + } else { + for (size_t i = 0; std::cmp_less(i, (map ? 2 : 1)); ++i) { + ColumnDefinition child; + RETURN_IF_ERROR(build_schema( + node->leafAt(i), map ? (i == 0 ? "key" : "value") : "element", + static_cast<int32_t>(i), timestamp_tz, varbinary, ancestors, &child)); + RETURN_IF_ERROR(set_identifier( + node, 0, map ? (i == 0 ? "key-id" : "value-id") : "element-id", + &child)); + column->children.push_back(std::move(child)); + } + } + type = map ? DataTypePtr(std::make_shared<DataTypeMap>(column->children[0].type, + column->children[1].type)) + : DataTypePtr(std::make_shared<DataTypeArray>(column->children[0].type)); + break; + } + default: + return Status::NotSupported("Unsupported Avro schema type"); + } + } + column->type = make_nullable(type); + return Status::OK(); +} + +::avro::NodePtr clone_projected_schema( + const ::avro::NodePtr& input, + std::unordered_map<const ::avro::Node*, ::avro::NodePtr>* copies, + const std::unordered_set<int32_t>* selected_roots = nullptr) { + const auto source = resolved_node(input); + if (const auto found = copies->find(source.get()); found != copies->end()) { + return found->second; + } + ::avro::NodePtr result; + switch (source->type()) { + case ::avro::AVRO_RECORD: + result = std::make_shared<::avro::NodeRecord>(); + break; + case ::avro::AVRO_ENUM: + result = std::make_shared<::avro::NodeEnum>(); + break; + case ::avro::AVRO_FIXED: + result = std::make_shared<::avro::NodeFixed>(); + break; + case ::avro::AVRO_ARRAY: + result = std::make_shared<::avro::NodeArray>(); + break; + case ::avro::AVRO_MAP: + result = std::make_shared<::avro::NodeMap>(); + break; + case ::avro::AVRO_UNION: + result = std::make_shared<::avro::NodeUnion>(); + break; + default: + result = std::make_shared<::avro::NodePrimitive>(source->type()); + break; + } + copies->emplace(source.get(), result); + if (source->hasName()) { + result->setName(source->name()); + } + if (source->type() == ::avro::AVRO_FIXED) { + result->setFixedSize(source->fixedSize()); + } + for (size_t i = source->type() == ::avro::AVRO_MAP ? 1 : 0; i < source->leaves(); ++i) { + if (selected_roots != nullptr && !selected_roots->contains(static_cast<int32_t>(i))) { + continue; + } + result->addLeaf(clone_projected_schema(source->leafAt(i), copies)); + if (source->type() == ::avro::AVRO_RECORD) { + result->addName(source->nameAt(i)); + } + } + if (source->type() == ::avro::AVRO_ENUM) { + for (size_t i = 0; i < source->names(); ++i) { + result->addName(source->nameAt(i)); + } + } + // Rebuild named definitions instead of retaining symbolic references to omitted roots. + // The ValidSchema validator replaces repeated definitions with links within this schema. + result->setLogicalType(source->logicalType()); + return result; +} + +void apply_timestamp_projection(ColumnDefinition* column, const LocalColumnIndex& projection) { + const auto type = remove_nullable(column->type); + if (projection.timestamp_is_adjusted_to_utc.has_value() && + (type->get_primitive_type() == TYPE_DATETIMEV2 || + type->get_primitive_type() == TYPE_TIMESTAMPTZ)) { + column->timestamp_is_adjusted_to_utc = projection.timestamp_is_adjusted_to_utc; + column->type = make_nullable( + *projection.timestamp_is_adjusted_to_utc + ? DataTypePtr(std::make_shared<DataTypeTimeStampTz>(type->get_scale())) + : DataTypePtr(std::make_shared<DataTypeDateTimeV2>(type->get_scale()))); + return; + } + for (const auto& child_projection : projection.children) { + for (auto& child : column->children) { + if (child.local_id == child_projection.local_id()) { + apply_timestamp_projection(&child, child_projection); + break; + } + } + } + if (type->get_primitive_type() == TYPE_ARRAY) { + column->type = make_nullable(std::make_shared<DataTypeArray>(column->children[0].type)); + } else if (type->get_primitive_type() == TYPE_MAP) { + column->type = make_nullable( + std::make_shared<DataTypeMap>(column->children[0].type, column->children[1].type)); + } else if (type->get_primitive_type() == TYPE_STRUCT) { + DataTypes types; + Strings names; + for (const auto& child : column->children) { + types.push_back(child.type); + names.push_back(child.name); + } + column->type = make_nullable(std::make_shared<DataTypeStruct>(types, names)); + } +} + +Status append_decimal(const std::vector<uint8_t>& bytes, const DataTypePtr& type, IColumn* column) { + // Avro decimals are signed, big-endian two's-complement integers, including FIXED values. + size_t start = 0; + // FIXED may be wider than its precision requires; discard only redundant sign extension. + while (start + 1 < bytes.size() && ((bytes[start] == 0 && (bytes[start + 1] & 0x80) == 0) || + (bytes[start] == 255 && (bytes[start + 1] & 0x80) != 0))) { + ++start; + } + if (bytes.empty() || bytes.size() - start > 32) { + return Status::InvalidArgument("Invalid Avro decimal width"); + } + Int256 value = (bytes[start] & 0x80) != 0 ? -1 : 0; + for (; start < bytes.size(); ++start) { + value = value * 256 + bytes[start]; + } + Int256 limit = 1; + for (unsigned i = 0; i < type->get_precision(); ++i) { + limit *= 10; + } + if (value <= -limit || value >= limit) { + return Status::InvalidArgument("Avro decimal exceeds its precision"); + } + switch (type->get_primitive_type()) { + case TYPE_DECIMAL32: + column->insert(Field::create_field<TYPE_DECIMAL32>(Decimal32(static_cast<int32_t>(value)))); + break; + case TYPE_DECIMAL64: + column->insert(Field::create_field<TYPE_DECIMAL64>(Decimal64(static_cast<int64_t>(value)))); + break; + case TYPE_DECIMAL128I: + column->insert( + Field::create_field<TYPE_DECIMAL128I>(Decimal128V3(static_cast<int128_t>(value)))); + break; + case TYPE_DECIMAL256: + column->insert(Field::create_field<TYPE_DECIMAL256>(Decimal256(value))); + break; + default: + return Status::InternalError("Invalid Avro decimal output type"); + } + return Status::OK(); +} + +// One visitor preserves the same null/child-offset rules for all Avro value alternatives. +// NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity) +Status append_value(const ::avro::GenericDatum& datum, const ColumnDefinition& schema, + IColumn* output, const cctz::time_zone& timezone) { + if (datum.type() == ::avro::AVRO_NULL) { + output->insert_default(); + return Status::OK(); + } + auto& nullable = assert_cast<ColumnNullable&>(*output); + auto* column = &nullable.get_nested_column(); + const auto type = remove_nullable(schema.type); + using Logical = ::avro::LogicalType; + const auto logical = datum.logicalType().type(); + if (logical == Logical::DECIMAL) { + const auto& bytes = datum.type() == ::avro::AVRO_FIXED + ? datum.value<::avro::GenericFixed>().value() + : datum.value<std::vector<uint8_t>>(); + RETURN_IF_ERROR(append_decimal(bytes, type, column)); + } else if (logical == Logical::UUID) { + UUIDValueType value; + if (datum.type() == ::avro::AVRO_FIXED) { + const auto& bytes = datum.value<::avro::GenericFixed>().value(); + if (bytes.size() != UUIDValue::BINARY_LENGTH) { + return Status::InvalidArgument("Invalid Avro UUID width"); + } + value = UUIDValue::from_big_endian(bytes.data()); + } else if (!UUIDValue::from_string(value, datum.value<std::string>())) { + return Status::InvalidArgument("Invalid Avro UUID value"); + } + column->insert(Field::create_field<TYPE_UUID>(value)); + } else if (logical == Logical::DATE) { + DateV2Value<DateV2ValueType> date; + const int64_t day = static_cast<int64_t>(datum.value<int32_t>()) + 719528; Review Comment: Fixed in 3ae183546dd. Hive-only calendar policy now rebases hybrid Julian/Gregorian dates using the writer metadata/version conventions. The regression checks Hive 2 defaults, Hive 3 proleptic behavior and Hive 4 writer.proleptic=false across the 1582 cutover, older century offsets and Julian-only leap-day normalization. Ordinary Avro and Iceberg stay proleptic. ########## be/src/format_v2/avro/avro_reader.cpp: ########## @@ -0,0 +1,1074 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/avro/avro_reader.h" + +#include <algorithm> +#include <array> +#include <avro/Allocation.hh> +#include <avro/DataFile.hh> +#include <avro/Generic.hh> +#include <avro/NodeImpl.hh> +#include <charconv> +#include <limits> +#include <mutex> +#include <type_traits> +#include <unordered_set> +#include <utility> + +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/custom_allocator.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_uuid.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/value/timestamp_ns_value.h" +#include "core/value/timestamptz_value.h" +#include "format_v2/column_mapper.h" +#include "format_v2/materialized_reader_util.h" +#include "io/fs/file_reader.h" +#include "io/io_common.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "storage/utils.h" + +namespace doris::format::avro { +namespace { + +// Avro must use the same filesystem, credentials, cache and cancellation state as other readers. +// The library owns its stream; the shared physical reader remains alive until the stream closes. +class NativeInputStream final : public ::avro::SeekableInputStream { +public: + NativeInputStream(io::FileReaderSPtr reader, std::shared_ptr<io::IOContext> io_ctx) + : _reader(std::move(reader)), _io_ctx(std::move(io_ctx)) {} + + bool next(const uint8_t** data, size_t* length) override { + if (_position == _reader->size()) { + return false; + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + throw Exception(ErrorCode::CANCELLED, "Avro scan cancelled"); + } + const auto requested = std::min(_buffer.size(), _reader->size() - _position); + size_t read = 0; + auto status = + _reader->read_at(_position, Slice(_buffer.data(), requested), &read, _io_ctx.get()); + if (!status.ok()) { + throw Exception(status.code(), status.to_string()); + } + if (read == 0) { + throw ::avro::Exception("Truncated Avro input"); + } + *data = reinterpret_cast<const uint8_t*>(_buffer.data()); + *length = read; + _position += read; + _last_read = read; + return true; + } + + void backup(size_t length) override { + if (length > _last_read) { + throw ::avro::Exception("Invalid Avro stream backup"); + } + _position -= length; + _last_read -= length; + } + + void skip(size_t length) override { + if (length > _reader->size() - _position) { + throw ::avro::Exception("Truncated Avro input"); + } + _position += length; + _last_read = 0; + } + + size_t byteCount() const override { return _position; } + + void seek(int64_t position) override { + if (position < 0 || static_cast<uint64_t>(position) > _reader->size()) { + throw ::avro::Exception("Invalid Avro stream offset"); + } + _position = position; + _last_read = 0; + } + +private: + io::FileReaderSPtr _reader; + std::shared_ptr<io::IOContext> _io_ctx; + std::array<char, 64 * 1024> _buffer; + size_t _position = 0; + size_t _last_read = 0; +}; + +::avro::NodePtr resolved_node(const ::avro::NodePtr& node) { + return node->type() == ::avro::AVRO_SYMBOLIC ? ::avro::resolveSymbol(node) : node; +} + +std::string lowercase(std::string name) { + for (auto& c : name) { + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + } + return name; +} + +Status set_identifier(const ::avro::NodePtr& node, size_t attributes_index, const char* attribute, + ColumnDefinition* column) { + if (attributes_index >= node->customAttributes()) { + return Status::OK(); + } + const auto value = node->customAttributesAt(attributes_index).getAttribute(attribute); + if (!value.has_value()) { + return Status::OK(); + } + int32_t id = 0; + const auto parsed = std::from_chars(value->data(), value->data() + value->size(), id); + if (parsed.ec != std::errc() || parsed.ptr != value->data() + value->size() || id <= 0) { + return Status::InvalidArgument("Invalid Avro field identifier"); + } + column->identifier = Field::create_field<TYPE_INT>(id); + return Status::OK(); +} + +bool is_logical_map(const ::avro::NodePtr& node) { + const auto logical = node->logicalType(); + return node->type() == ::avro::AVRO_ARRAY && logical.type() == ::avro::LogicalType::CUSTOM && + logical.customLogicalType()->name() == "map"; +} + +// Keep logical and physical alternatives in one visitor so nullable handling is shared. +// NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity) +Status build_schema(const ::avro::NodePtr& input, const std::string& name, int32_t ordinal, + bool timestamp_tz, bool varbinary, + std::unordered_set<const ::avro::Node*>* ancestors, ColumnDefinition* column) { + const auto node = resolved_node(input); + // A recursive Avro record cannot be represented by a finite Doris STRUCT type. + if (ancestors->size() >= 64 || !ancestors->insert(node.get()).second) { + return Status::NotSupported("Recursive or excessively nested Avro schemas are unsupported"); + } + struct Pop { + std::unordered_set<const ::avro::Node*>* nodes; + const ::avro::Node* node; + ~Pop() { nodes->erase(node); } + } pop {.nodes = ancestors, .node = node.get()}; + if (node->leaves() > static_cast<size_t>(std::numeric_limits<int32_t>::max())) { + return Status::NotSupported("Avro schema has too many fields"); + } + column->name = lowercase(name); + column->local_id = ordinal; + column->identifier = Field::create_field<TYPE_STRING>(column->name); + if (node->type() == ::avro::AVRO_UNION) { + ::avro::NodePtr value; + for (size_t i = 0; i < node->leaves(); ++i) { + if (node->leafAt(i)->type() != ::avro::AVRO_NULL) { + if (value != nullptr) { + return Status::NotSupported( + "Avro unions with multiple non-null types are unsupported"); + } + value = node->leafAt(i); + } + } + if (value != nullptr) { + return build_schema(value, name, ordinal, timestamp_tz, varbinary, ancestors, column); + } + column->type = make_nullable(std::make_shared<DataTypeString>()); + return Status::OK(); + } + + DataTypePtr type; + const auto logical = node->logicalType(); + using Logical = ::avro::LogicalType; + switch (logical.type()) { + case Logical::DECIMAL: + if (logical.precision() > 76) { + return Status::NotSupported("Avro decimal precision exceeds 76"); + } + type = create_decimal(logical.precision(), logical.scale(), false); + break; + case Logical::UUID: + type = std::make_shared<DataTypeUUID>(); + break; + case Logical::DATE: + type = std::make_shared<DataTypeDateV2>(); + break; + case Logical::TIME_MILLIS: + case Logical::TIME_MICROS: + type = std::make_shared<DataTypeTimeV2>(logical.type() == Logical::TIME_MILLIS ? 3 : 6); + break; + case Logical::TIMESTAMP_NANOS: + case Logical::LOCAL_TIMESTAMP_NANOS: + type = std::make_shared<DataTypeTimeStampNs>(); + column->timestamp_is_adjusted_to_utc = logical.type() == Logical::TIMESTAMP_NANOS; + break; + case Logical::TIMESTAMP_MILLIS: + case Logical::TIMESTAMP_MICROS: + case Logical::LOCAL_TIMESTAMP_MILLIS: + case Logical::LOCAL_TIMESTAMP_MICROS: { + const bool adjusted = logical.type() == Logical::TIMESTAMP_MILLIS || + logical.type() == Logical::TIMESTAMP_MICROS; + const int scale = logical.type() == Logical::TIMESTAMP_MILLIS || + logical.type() == Logical::LOCAL_TIMESTAMP_MILLIS + ? 3 + : 6; + type = timestamp_tz && adjusted ? DataTypePtr(std::make_shared<DataTypeTimeStampTz>(scale)) + : DataTypePtr(std::make_shared<DataTypeDateTimeV2>(scale)); + column->timestamp_is_adjusted_to_utc = adjusted; + break; + } + default: + break; + } + if (type == nullptr) { + switch (node->type()) { + case ::avro::AVRO_NULL: + type = std::make_shared<DataTypeString>(); + break; + case ::avro::AVRO_BOOL: + type = std::make_shared<DataTypeUInt8>(); + break; + case ::avro::AVRO_INT: + type = std::make_shared<DataTypeInt32>(); + break; + case ::avro::AVRO_LONG: + type = std::make_shared<DataTypeInt64>(); + break; + case ::avro::AVRO_FLOAT: + type = std::make_shared<DataTypeFloat32>(); + break; + case ::avro::AVRO_DOUBLE: + type = std::make_shared<DataTypeFloat64>(); + break; + case ::avro::AVRO_BYTES: + case ::avro::AVRO_FIXED: + type = varbinary ? DataTypePtr(std::make_shared<DataTypeVarbinary>()) + : DataTypePtr(std::make_shared<DataTypeString>()); + break; + case ::avro::AVRO_STRING: + case ::avro::AVRO_ENUM: + type = std::make_shared<DataTypeString>(); + break; + case ::avro::AVRO_RECORD: { + DataTypes types; + Strings names; + for (size_t i = 0; i < node->leaves(); ++i) { + ColumnDefinition child; + RETURN_IF_ERROR(build_schema(node->leafAt(i), node->nameAt(i), + static_cast<int32_t>(i), timestamp_tz, varbinary, + ancestors, &child)); Review Comment: Fixed in 3ae183546dd. Schema initialization now estimates expanded column-tree size with memoized named-node costs, rejects excessive expansion and reserves/accounts the retained schema before constructing it. The repeated named-record expansion regression fails within a small query budget before building the large tree. Fixed-value payload admission is deferred until an actual owned row, so empty containers remain readable. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
