github-actions[bot] commented on code in PR #70808:
URL: https://github.com/apache/doris/pull/70808#discussion_r4640552935
##########
be/src/format_v2/table/hive_reader.cpp:
##########
@@ -132,6 +132,9 @@ format::TableColumnMappingMode HiveReader::mapping_mode()
const {
use_column_names = query_options.hive_orc_use_column_names;
} else if (_format == format::FileFormat::PARQUET) {
use_column_names = query_options.hive_parquet_use_column_names;
+ } else if (_format == format::FileFormat::AVRO) {
Review Comment:
[P2] Restore Hive Avro timestamps using the writer timezone
Enabling Avro name mapping also needs Hive's temporal interpretation. Hive 3
writes civil TIMESTAMP values in UTC and records `writer.time.zone`; its reader
converts back using that stored writer zone. Here the Avro reader ignores that
metadata. Current catalog requests produce a TIMESTAMPTZ file column, while
plain Hive TIMESTAMP remains DATETIMEV2, so the final EvolutionCast uses the
query's timezone. For example, a Hive writer in America/Los_Angeles storing
`2024-01-02 00:00:00` writes millis `1704182400000`; Doris in UTC returns
`08:00:00` instead of the original midnight. Carry the writer-zone semantics
through the Hive adapter while retaining ordinary Avro/Iceberg instant
behavior. The [Hive reader's restoration
logic](https://github.com/apache/hive/blob/rel/release-3.1.3/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java#L320-L347)
documents this contract.
##########
be/src/format_v2/avro/avro_reader.cpp:
##########
@@ -0,0 +1,1074 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format_v2/avro/avro_reader.h"
+
+#include <algorithm>
+#include <array>
+#include <avro/Allocation.hh>
+#include <avro/DataFile.hh>
+#include <avro/Generic.hh>
+#include <avro/NodeImpl.hh>
+#include <charconv>
+#include <limits>
+#include <mutex>
+#include <type_traits>
+#include <unordered_set>
+#include <utility>
+
+#include "common/exception.h"
+#include "core/assert_cast.h"
+#include "core/block/block.h"
+#include "core/column/column_array.h"
+#include "core/column/column_map.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
+#include "core/custom_allocator.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_decimal.h"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
+#include "core/data_type/data_type_timestamptz.h"
+#include "core/data_type/data_type_uuid.h"
+#include "core/data_type/data_type_varbinary.h"
+#include "core/value/timestamp_ns_value.h"
+#include "core/value/timestamptz_value.h"
+#include "format_v2/column_mapper.h"
+#include "format_v2/materialized_reader_util.h"
+#include "io/fs/file_reader.h"
+#include "io/io_common.h"
+#include "runtime/memory/mem_tracker_limiter.h"
+#include "runtime/memory/thread_mem_tracker_mgr.h"
+#include "runtime/runtime_state.h"
+#include "runtime/thread_context.h"
+#include "storage/utils.h"
+
+namespace doris::format::avro {
+namespace {
+
+// Avro must use the same filesystem, credentials, cache and cancellation
state as other readers.
+// The library owns its stream; the shared physical reader remains alive until
the stream closes.
+class NativeInputStream final : public ::avro::SeekableInputStream {
+public:
+ NativeInputStream(io::FileReaderSPtr reader,
std::shared_ptr<io::IOContext> io_ctx)
+ : _reader(std::move(reader)), _io_ctx(std::move(io_ctx)) {}
+
+ bool next(const uint8_t** data, size_t* length) override {
+ if (_position == _reader->size()) {
+ return false;
+ }
+ if (_io_ctx != nullptr && _io_ctx->should_stop) {
+ throw Exception(ErrorCode::CANCELLED, "Avro scan cancelled");
+ }
+ const auto requested = std::min(_buffer.size(), _reader->size() -
_position);
+ size_t read = 0;
+ auto status =
+ _reader->read_at(_position, Slice(_buffer.data(), requested),
&read, _io_ctx.get());
+ if (!status.ok()) {
+ throw Exception(status.code(), status.to_string());
+ }
+ if (read == 0) {
+ throw ::avro::Exception("Truncated Avro input");
+ }
+ *data = reinterpret_cast<const uint8_t*>(_buffer.data());
+ *length = read;
+ _position += read;
+ _last_read = read;
+ return true;
+ }
+
+ void backup(size_t length) override {
+ if (length > _last_read) {
+ throw ::avro::Exception("Invalid Avro stream backup");
+ }
+ _position -= length;
+ _last_read -= length;
+ }
+
+ void skip(size_t length) override {
+ if (length > _reader->size() - _position) {
+ throw ::avro::Exception("Truncated Avro input");
+ }
+ _position += length;
+ _last_read = 0;
+ }
+
+ size_t byteCount() const override { return _position; }
+
+ void seek(int64_t position) override {
+ if (position < 0 || static_cast<uint64_t>(position) > _reader->size())
{
+ throw ::avro::Exception("Invalid Avro stream offset");
+ }
+ _position = position;
+ _last_read = 0;
+ }
+
+private:
+ io::FileReaderSPtr _reader;
+ std::shared_ptr<io::IOContext> _io_ctx;
+ std::array<char, 64 * 1024> _buffer;
+ size_t _position = 0;
+ size_t _last_read = 0;
+};
+
+::avro::NodePtr resolved_node(const ::avro::NodePtr& node) {
+ return node->type() == ::avro::AVRO_SYMBOLIC ? ::avro::resolveSymbol(node)
: node;
+}
+
+std::string lowercase(std::string name) {
+ for (auto& c : name) {
+ if (c >= 'A' && c <= 'Z') {
+ c += 'a' - 'A';
+ }
+ }
+ return name;
+}
+
+Status set_identifier(const ::avro::NodePtr& node, size_t attributes_index,
const char* attribute,
+ ColumnDefinition* column) {
+ if (attributes_index >= node->customAttributes()) {
+ return Status::OK();
+ }
+ const auto value =
node->customAttributesAt(attributes_index).getAttribute(attribute);
+ if (!value.has_value()) {
+ return Status::OK();
+ }
+ int32_t id = 0;
+ const auto parsed = std::from_chars(value->data(), value->data() +
value->size(), id);
+ if (parsed.ec != std::errc() || parsed.ptr != value->data() +
value->size() || id <= 0) {
+ return Status::InvalidArgument("Invalid Avro field identifier");
+ }
+ column->identifier = Field::create_field<TYPE_INT>(id);
+ return Status::OK();
+}
+
+bool is_logical_map(const ::avro::NodePtr& node) {
+ const auto logical = node->logicalType();
+ return node->type() == ::avro::AVRO_ARRAY && logical.type() ==
::avro::LogicalType::CUSTOM &&
+ logical.customLogicalType()->name() == "map";
+}
+
+// Keep logical and physical alternatives in one visitor so nullable handling
is shared.
+//
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity)
+Status build_schema(const ::avro::NodePtr& input, const std::string& name,
int32_t ordinal,
+ bool timestamp_tz, bool varbinary,
+ std::unordered_set<const ::avro::Node*>* ancestors,
ColumnDefinition* column) {
+ const auto node = resolved_node(input);
+ // A recursive Avro record cannot be represented by a finite Doris STRUCT
type.
+ if (ancestors->size() >= 64 || !ancestors->insert(node.get()).second) {
+ return Status::NotSupported("Recursive or excessively nested Avro
schemas are unsupported");
+ }
+ struct Pop {
+ std::unordered_set<const ::avro::Node*>* nodes;
+ const ::avro::Node* node;
+ ~Pop() { nodes->erase(node); }
+ } pop {.nodes = ancestors, .node = node.get()};
+ if (node->leaves() >
static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
+ return Status::NotSupported("Avro schema has too many fields");
+ }
+ column->name = lowercase(name);
+ column->local_id = ordinal;
+ column->identifier = Field::create_field<TYPE_STRING>(column->name);
+ if (node->type() == ::avro::AVRO_UNION) {
+ ::avro::NodePtr value;
+ for (size_t i = 0; i < node->leaves(); ++i) {
+ if (node->leafAt(i)->type() != ::avro::AVRO_NULL) {
+ if (value != nullptr) {
+ return Status::NotSupported(
+ "Avro unions with multiple non-null types are
unsupported");
+ }
+ value = node->leafAt(i);
+ }
+ }
+ if (value != nullptr) {
+ return build_schema(value, name, ordinal, timestamp_tz, varbinary,
ancestors, column);
+ }
+ column->type = make_nullable(std::make_shared<DataTypeString>());
+ return Status::OK();
+ }
+
+ DataTypePtr type;
+ const auto logical = node->logicalType();
+ using Logical = ::avro::LogicalType;
+ switch (logical.type()) {
+ case Logical::DECIMAL:
+ if (logical.precision() > 76) {
+ return Status::NotSupported("Avro decimal precision exceeds 76");
+ }
+ type = create_decimal(logical.precision(), logical.scale(), false);
+ break;
+ case Logical::UUID:
+ type = std::make_shared<DataTypeUUID>();
+ break;
+ case Logical::DATE:
+ type = std::make_shared<DataTypeDateV2>();
+ break;
+ case Logical::TIME_MILLIS:
+ case Logical::TIME_MICROS:
+ type = std::make_shared<DataTypeTimeV2>(logical.type() ==
Logical::TIME_MILLIS ? 3 : 6);
+ break;
+ case Logical::TIMESTAMP_NANOS:
+ case Logical::LOCAL_TIMESTAMP_NANOS:
Review Comment:
[P2] Preserve instant semantics for timestamp-nanos
`timestamp-nanos` represents a UTC instant, but this branch assigns it the
same civil TIMESTAMP_NS type as `local-timestamp-nanos`. Its append branch
copies raw nanoseconds into TimeStampNsValue, whose formatting and casts use
epoch/civil arithmetic without a timezone. Consequently a TVF reading value `0`
with session `+08:00` returns unzoned `1970-01-01 00:00:00.000000000`, and even
the current mapping=true request never exposes TIMESTAMPTZ. The existing
Parquet path reserves TIMESTAMP_NS for unadjusted nanos. Separate the instant
and local cases and honor instant mapping/timezone semantics, or reject the
unsupported instant representation explicitly. The added nanos test exercises
only the local case. See the [Avro timestamp
specification](https://avro.apache.org/docs/1.12.0/specification/#timestamps).
##########
be/src/format_v2/avro/avro_reader.cpp:
##########
@@ -0,0 +1,1074 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format_v2/avro/avro_reader.h"
+
+#include <algorithm>
+#include <array>
+#include <avro/Allocation.hh>
+#include <avro/DataFile.hh>
+#include <avro/Generic.hh>
+#include <avro/NodeImpl.hh>
+#include <charconv>
+#include <limits>
+#include <mutex>
+#include <type_traits>
+#include <unordered_set>
+#include <utility>
+
+#include "common/exception.h"
+#include "core/assert_cast.h"
+#include "core/block/block.h"
+#include "core/column/column_array.h"
+#include "core/column/column_map.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
+#include "core/custom_allocator.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_decimal.h"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
+#include "core/data_type/data_type_timestamptz.h"
+#include "core/data_type/data_type_uuid.h"
+#include "core/data_type/data_type_varbinary.h"
+#include "core/value/timestamp_ns_value.h"
+#include "core/value/timestamptz_value.h"
+#include "format_v2/column_mapper.h"
+#include "format_v2/materialized_reader_util.h"
+#include "io/fs/file_reader.h"
+#include "io/io_common.h"
+#include "runtime/memory/mem_tracker_limiter.h"
+#include "runtime/memory/thread_mem_tracker_mgr.h"
+#include "runtime/runtime_state.h"
+#include "runtime/thread_context.h"
+#include "storage/utils.h"
+
+namespace doris::format::avro {
+namespace {
+
+// Avro must use the same filesystem, credentials, cache and cancellation
state as other readers.
+// The library owns its stream; the shared physical reader remains alive until
the stream closes.
+class NativeInputStream final : public ::avro::SeekableInputStream {
+public:
+ NativeInputStream(io::FileReaderSPtr reader,
std::shared_ptr<io::IOContext> io_ctx)
+ : _reader(std::move(reader)), _io_ctx(std::move(io_ctx)) {}
+
+ bool next(const uint8_t** data, size_t* length) override {
+ if (_position == _reader->size()) {
+ return false;
+ }
+ if (_io_ctx != nullptr && _io_ctx->should_stop) {
+ throw Exception(ErrorCode::CANCELLED, "Avro scan cancelled");
+ }
+ const auto requested = std::min(_buffer.size(), _reader->size() -
_position);
+ size_t read = 0;
+ auto status =
+ _reader->read_at(_position, Slice(_buffer.data(), requested),
&read, _io_ctx.get());
+ if (!status.ok()) {
+ throw Exception(status.code(), status.to_string());
+ }
+ if (read == 0) {
+ throw ::avro::Exception("Truncated Avro input");
+ }
+ *data = reinterpret_cast<const uint8_t*>(_buffer.data());
+ *length = read;
+ _position += read;
+ _last_read = read;
+ return true;
+ }
+
+ void backup(size_t length) override {
+ if (length > _last_read) {
+ throw ::avro::Exception("Invalid Avro stream backup");
+ }
+ _position -= length;
+ _last_read -= length;
+ }
+
+ void skip(size_t length) override {
+ if (length > _reader->size() - _position) {
+ throw ::avro::Exception("Truncated Avro input");
+ }
+ _position += length;
+ _last_read = 0;
+ }
+
+ size_t byteCount() const override { return _position; }
+
+ void seek(int64_t position) override {
+ if (position < 0 || static_cast<uint64_t>(position) > _reader->size())
{
+ throw ::avro::Exception("Invalid Avro stream offset");
+ }
+ _position = position;
+ _last_read = 0;
+ }
+
+private:
+ io::FileReaderSPtr _reader;
+ std::shared_ptr<io::IOContext> _io_ctx;
+ std::array<char, 64 * 1024> _buffer;
+ size_t _position = 0;
+ size_t _last_read = 0;
+};
+
+::avro::NodePtr resolved_node(const ::avro::NodePtr& node) {
+ return node->type() == ::avro::AVRO_SYMBOLIC ? ::avro::resolveSymbol(node)
: node;
+}
+
+std::string lowercase(std::string name) {
+ for (auto& c : name) {
+ if (c >= 'A' && c <= 'Z') {
+ c += 'a' - 'A';
+ }
+ }
+ return name;
+}
+
+Status set_identifier(const ::avro::NodePtr& node, size_t attributes_index,
const char* attribute,
+ ColumnDefinition* column) {
+ if (attributes_index >= node->customAttributes()) {
+ return Status::OK();
+ }
+ const auto value =
node->customAttributesAt(attributes_index).getAttribute(attribute);
+ if (!value.has_value()) {
+ return Status::OK();
+ }
+ int32_t id = 0;
+ const auto parsed = std::from_chars(value->data(), value->data() +
value->size(), id);
+ if (parsed.ec != std::errc() || parsed.ptr != value->data() +
value->size() || id <= 0) {
+ return Status::InvalidArgument("Invalid Avro field identifier");
+ }
+ column->identifier = Field::create_field<TYPE_INT>(id);
+ return Status::OK();
+}
+
+bool is_logical_map(const ::avro::NodePtr& node) {
+ const auto logical = node->logicalType();
+ return node->type() == ::avro::AVRO_ARRAY && logical.type() ==
::avro::LogicalType::CUSTOM &&
+ logical.customLogicalType()->name() == "map";
+}
+
+// Keep logical and physical alternatives in one visitor so nullable handling
is shared.
+//
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity)
+Status build_schema(const ::avro::NodePtr& input, const std::string& name,
int32_t ordinal,
+ bool timestamp_tz, bool varbinary,
+ std::unordered_set<const ::avro::Node*>* ancestors,
ColumnDefinition* column) {
+ const auto node = resolved_node(input);
+ // A recursive Avro record cannot be represented by a finite Doris STRUCT
type.
+ if (ancestors->size() >= 64 || !ancestors->insert(node.get()).second) {
+ return Status::NotSupported("Recursive or excessively nested Avro
schemas are unsupported");
+ }
+ struct Pop {
+ std::unordered_set<const ::avro::Node*>* nodes;
+ const ::avro::Node* node;
+ ~Pop() { nodes->erase(node); }
+ } pop {.nodes = ancestors, .node = node.get()};
+ if (node->leaves() >
static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
+ return Status::NotSupported("Avro schema has too many fields");
+ }
+ column->name = lowercase(name);
+ column->local_id = ordinal;
+ column->identifier = Field::create_field<TYPE_STRING>(column->name);
+ if (node->type() == ::avro::AVRO_UNION) {
+ ::avro::NodePtr value;
+ for (size_t i = 0; i < node->leaves(); ++i) {
+ if (node->leafAt(i)->type() != ::avro::AVRO_NULL) {
+ if (value != nullptr) {
+ return Status::NotSupported(
+ "Avro unions with multiple non-null types are
unsupported");
+ }
+ value = node->leafAt(i);
+ }
+ }
+ if (value != nullptr) {
+ return build_schema(value, name, ordinal, timestamp_tz, varbinary,
ancestors, column);
+ }
+ column->type = make_nullable(std::make_shared<DataTypeString>());
+ return Status::OK();
+ }
+
+ DataTypePtr type;
+ const auto logical = node->logicalType();
+ using Logical = ::avro::LogicalType;
+ switch (logical.type()) {
+ case Logical::DECIMAL:
+ if (logical.precision() > 76) {
+ return Status::NotSupported("Avro decimal precision exceeds 76");
+ }
+ type = create_decimal(logical.precision(), logical.scale(), false);
+ break;
+ case Logical::UUID:
+ type = std::make_shared<DataTypeUUID>();
+ break;
+ case Logical::DATE:
+ type = std::make_shared<DataTypeDateV2>();
+ break;
+ case Logical::TIME_MILLIS:
+ case Logical::TIME_MICROS:
+ type = std::make_shared<DataTypeTimeV2>(logical.type() ==
Logical::TIME_MILLIS ? 3 : 6);
+ break;
+ case Logical::TIMESTAMP_NANOS:
+ case Logical::LOCAL_TIMESTAMP_NANOS:
+ type = std::make_shared<DataTypeTimeStampNs>();
+ column->timestamp_is_adjusted_to_utc = logical.type() ==
Logical::TIMESTAMP_NANOS;
+ break;
+ case Logical::TIMESTAMP_MILLIS:
+ case Logical::TIMESTAMP_MICROS:
+ case Logical::LOCAL_TIMESTAMP_MILLIS:
+ case Logical::LOCAL_TIMESTAMP_MICROS: {
+ const bool adjusted = logical.type() == Logical::TIMESTAMP_MILLIS ||
+ logical.type() == Logical::TIMESTAMP_MICROS;
+ const int scale = logical.type() == Logical::TIMESTAMP_MILLIS ||
+ logical.type() ==
Logical::LOCAL_TIMESTAMP_MILLIS
+ ? 3
+ : 6;
+ type = timestamp_tz && adjusted ?
DataTypePtr(std::make_shared<DataTypeTimeStampTz>(scale))
+ :
DataTypePtr(std::make_shared<DataTypeDateTimeV2>(scale));
+ column->timestamp_is_adjusted_to_utc = adjusted;
+ break;
+ }
+ default:
+ break;
+ }
+ if (type == nullptr) {
+ switch (node->type()) {
+ case ::avro::AVRO_NULL:
+ type = std::make_shared<DataTypeString>();
+ break;
+ case ::avro::AVRO_BOOL:
+ type = std::make_shared<DataTypeUInt8>();
+ break;
+ case ::avro::AVRO_INT:
+ type = std::make_shared<DataTypeInt32>();
+ break;
+ case ::avro::AVRO_LONG:
+ type = std::make_shared<DataTypeInt64>();
+ break;
+ case ::avro::AVRO_FLOAT:
+ type = std::make_shared<DataTypeFloat32>();
+ break;
+ case ::avro::AVRO_DOUBLE:
+ type = std::make_shared<DataTypeFloat64>();
+ break;
+ case ::avro::AVRO_BYTES:
+ case ::avro::AVRO_FIXED:
+ type = varbinary ?
DataTypePtr(std::make_shared<DataTypeVarbinary>())
+ : DataTypePtr(std::make_shared<DataTypeString>());
+ break;
+ case ::avro::AVRO_STRING:
+ case ::avro::AVRO_ENUM:
+ type = std::make_shared<DataTypeString>();
+ break;
+ case ::avro::AVRO_RECORD: {
+ DataTypes types;
+ Strings names;
+ for (size_t i = 0; i < node->leaves(); ++i) {
+ ColumnDefinition child;
+ RETURN_IF_ERROR(build_schema(node->leafAt(i), node->nameAt(i),
+ static_cast<int32_t>(i),
timestamp_tz, varbinary,
+ ancestors, &child));
+ RETURN_IF_ERROR(set_identifier(node, i, "field-id", &child));
+ types.push_back(child.type);
+ names.push_back(child.name);
+ column->children.push_back(std::move(child));
+ }
+ if (types.empty()) {
+ return Status::NotSupported("Empty Avro records are
unsupported");
+ }
+ type = std::make_shared<DataTypeStruct>(types, names);
+ break;
+ }
+ case ::avro::AVRO_ARRAY:
+ case ::avro::AVRO_MAP: {
+ const bool map = node->type() == ::avro::AVRO_MAP ||
is_logical_map(node);
+ if (is_logical_map(node)) {
+ const auto entry = resolved_node(node->leafAt(0));
+ if (entry->type() != ::avro::AVRO_RECORD || entry->leaves() !=
2 ||
+ entry->nameAt(0) != "key" || entry->nameAt(1) != "value") {
+ return Status::InvalidArgument("Invalid Avro logical map
entry schema");
+ }
+ ColumnDefinition entries;
+ RETURN_IF_ERROR(build_schema(entry, "entry", 0, timestamp_tz,
varbinary, ancestors,
+ &entries));
+ column->children = std::move(entries.children);
+ } else {
+ for (size_t i = 0; std::cmp_less(i, (map ? 2 : 1)); ++i) {
+ ColumnDefinition child;
+ RETURN_IF_ERROR(build_schema(
+ node->leafAt(i), map ? (i == 0 ? "key" : "value")
: "element",
+ static_cast<int32_t>(i), timestamp_tz, varbinary,
ancestors, &child));
+ RETURN_IF_ERROR(set_identifier(
+ node, 0, map ? (i == 0 ? "key-id" : "value-id") :
"element-id",
+ &child));
+ column->children.push_back(std::move(child));
+ }
+ }
+ type = map ?
DataTypePtr(std::make_shared<DataTypeMap>(column->children[0].type,
+
column->children[1].type))
+ :
DataTypePtr(std::make_shared<DataTypeArray>(column->children[0].type));
+ break;
+ }
+ default:
+ return Status::NotSupported("Unsupported Avro schema type");
+ }
+ }
+ column->type = make_nullable(type);
+ return Status::OK();
+}
+
+::avro::NodePtr clone_projected_schema(
+ const ::avro::NodePtr& input,
+ std::unordered_map<const ::avro::Node*, ::avro::NodePtr>* copies,
+ const std::unordered_set<int32_t>* selected_roots = nullptr) {
+ const auto source = resolved_node(input);
+ if (const auto found = copies->find(source.get()); found != copies->end())
{
+ return found->second;
+ }
+ ::avro::NodePtr result;
+ switch (source->type()) {
+ case ::avro::AVRO_RECORD:
+ result = std::make_shared<::avro::NodeRecord>();
+ break;
+ case ::avro::AVRO_ENUM:
+ result = std::make_shared<::avro::NodeEnum>();
+ break;
+ case ::avro::AVRO_FIXED:
+ result = std::make_shared<::avro::NodeFixed>();
+ break;
+ case ::avro::AVRO_ARRAY:
+ result = std::make_shared<::avro::NodeArray>();
+ break;
+ case ::avro::AVRO_MAP:
+ result = std::make_shared<::avro::NodeMap>();
+ break;
+ case ::avro::AVRO_UNION:
+ result = std::make_shared<::avro::NodeUnion>();
+ break;
+ default:
+ result = std::make_shared<::avro::NodePrimitive>(source->type());
+ break;
+ }
+ copies->emplace(source.get(), result);
+ if (source->hasName()) {
+ result->setName(source->name());
+ }
+ if (source->type() == ::avro::AVRO_FIXED) {
+ result->setFixedSize(source->fixedSize());
+ }
+ for (size_t i = source->type() == ::avro::AVRO_MAP ? 1 : 0; i <
source->leaves(); ++i) {
+ if (selected_roots != nullptr &&
!selected_roots->contains(static_cast<int32_t>(i))) {
+ continue;
+ }
+ result->addLeaf(clone_projected_schema(source->leafAt(i), copies));
+ if (source->type() == ::avro::AVRO_RECORD) {
+ result->addName(source->nameAt(i));
+ }
+ }
+ if (source->type() == ::avro::AVRO_ENUM) {
+ for (size_t i = 0; i < source->names(); ++i) {
+ result->addName(source->nameAt(i));
+ }
+ }
+ // Rebuild named definitions instead of retaining symbolic references to
omitted roots.
+ // The ValidSchema validator replaces repeated definitions with links
within this schema.
+ result->setLogicalType(source->logicalType());
+ return result;
+}
+
+void apply_timestamp_projection(ColumnDefinition* column, const
LocalColumnIndex& projection) {
+ const auto type = remove_nullable(column->type);
+ if (projection.timestamp_is_adjusted_to_utc.has_value() &&
+ (type->get_primitive_type() == TYPE_DATETIMEV2 ||
+ type->get_primitive_type() == TYPE_TIMESTAMPTZ)) {
+ column->timestamp_is_adjusted_to_utc =
projection.timestamp_is_adjusted_to_utc;
+ column->type = make_nullable(
+ *projection.timestamp_is_adjusted_to_utc
+ ?
DataTypePtr(std::make_shared<DataTypeTimeStampTz>(type->get_scale()))
+ :
DataTypePtr(std::make_shared<DataTypeDateTimeV2>(type->get_scale())));
+ return;
+ }
+ for (const auto& child_projection : projection.children) {
+ for (auto& child : column->children) {
+ if (child.local_id == child_projection.local_id()) {
+ apply_timestamp_projection(&child, child_projection);
+ break;
+ }
+ }
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ column->type =
make_nullable(std::make_shared<DataTypeArray>(column->children[0].type));
+ } else if (type->get_primitive_type() == TYPE_MAP) {
+ column->type = make_nullable(
+ std::make_shared<DataTypeMap>(column->children[0].type,
column->children[1].type));
+ } else if (type->get_primitive_type() == TYPE_STRUCT) {
+ DataTypes types;
+ Strings names;
+ for (const auto& child : column->children) {
+ types.push_back(child.type);
+ names.push_back(child.name);
+ }
+ column->type = make_nullable(std::make_shared<DataTypeStruct>(types,
names));
+ }
+}
+
+Status append_decimal(const std::vector<uint8_t>& bytes, const DataTypePtr&
type, IColumn* column) {
+ // Avro decimals are signed, big-endian two's-complement integers,
including FIXED values.
+ size_t start = 0;
+ // FIXED may be wider than its precision requires; discard only redundant
sign extension.
+ while (start + 1 < bytes.size() && ((bytes[start] == 0 && (bytes[start +
1] & 0x80) == 0) ||
+ (bytes[start] == 255 && (bytes[start +
1] & 0x80) != 0))) {
+ ++start;
+ }
+ if (bytes.empty() || bytes.size() - start > 32) {
+ return Status::InvalidArgument("Invalid Avro decimal width");
+ }
+ Int256 value = (bytes[start] & 0x80) != 0 ? -1 : 0;
+ for (; start < bytes.size(); ++start) {
+ value = value * 256 + bytes[start];
+ }
+ Int256 limit = 1;
+ for (unsigned i = 0; i < type->get_precision(); ++i) {
+ limit *= 10;
+ }
+ if (value <= -limit || value >= limit) {
+ return Status::InvalidArgument("Avro decimal exceeds its precision");
+ }
+ switch (type->get_primitive_type()) {
+ case TYPE_DECIMAL32:
+
column->insert(Field::create_field<TYPE_DECIMAL32>(Decimal32(static_cast<int32_t>(value))));
+ break;
+ case TYPE_DECIMAL64:
+
column->insert(Field::create_field<TYPE_DECIMAL64>(Decimal64(static_cast<int64_t>(value))));
+ break;
+ case TYPE_DECIMAL128I:
+ column->insert(
+
Field::create_field<TYPE_DECIMAL128I>(Decimal128V3(static_cast<int128_t>(value))));
+ break;
+ case TYPE_DECIMAL256:
+
column->insert(Field::create_field<TYPE_DECIMAL256>(Decimal256(value)));
+ break;
+ default:
+ return Status::InternalError("Invalid Avro decimal output type");
+ }
+ return Status::OK();
+}
+
+// One visitor preserves the same null/child-offset rules for all Avro value
alternatives.
+//
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity)
+Status append_value(const ::avro::GenericDatum& datum, const ColumnDefinition&
schema,
+ IColumn* output, const cctz::time_zone& timezone) {
+ if (datum.type() == ::avro::AVRO_NULL) {
+ output->insert_default();
+ return Status::OK();
+ }
+ auto& nullable = assert_cast<ColumnNullable&>(*output);
+ auto* column = &nullable.get_nested_column();
+ const auto type = remove_nullable(schema.type);
+ using Logical = ::avro::LogicalType;
+ const auto logical = datum.logicalType().type();
+ if (logical == Logical::DECIMAL) {
+ const auto& bytes = datum.type() == ::avro::AVRO_FIXED
+ ?
datum.value<::avro::GenericFixed>().value()
+ : datum.value<std::vector<uint8_t>>();
+ RETURN_IF_ERROR(append_decimal(bytes, type, column));
+ } else if (logical == Logical::UUID) {
+ UUIDValueType value;
+ if (datum.type() == ::avro::AVRO_FIXED) {
+ const auto& bytes = datum.value<::avro::GenericFixed>().value();
+ if (bytes.size() != UUIDValue::BINARY_LENGTH) {
+ return Status::InvalidArgument("Invalid Avro UUID width");
+ }
+ value = UUIDValue::from_big_endian(bytes.data());
+ } else if (!UUIDValue::from_string(value, datum.value<std::string>()))
{
+ return Status::InvalidArgument("Invalid Avro UUID value");
+ }
+ column->insert(Field::create_field<TYPE_UUID>(value));
+ } else if (logical == Logical::DATE) {
+ DateV2Value<DateV2ValueType> date;
+ const int64_t day = static_cast<int64_t>(datum.value<int32_t>()) +
719528;
Review Comment:
[P2] Rebase legacy Hive Avro dates before materializing them
Adding the raw day count directly to Doris's proleptic Gregorian epoch
misreads supported Hive 2 Avro files: their DATE writer uses
java.sql.Date/DateWritable with the hybrid Julian/Gregorian calendar. A Hive
DATE `1582-10-04` is consequently returned here as `1582-10-14`. Hive 4 can
explicitly write the same representation with `writer.proleptic=false`, which
is also ignored. This is reachable with a DATE-only schema and is independent
of the timestamp findings. Preserve a Hive-specific writer/calendar policy and
rebase before producing the column, while leaving standard Avro/Iceberg
proleptic dates unchanged. The new 1969/2024 fixtures miss this boundary; see
the [Hive 2 date
writer](https://github.com/apache/hive/blob/rel/release-2.3.9/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java#L211-L213)
and [Hive 4 calendar
restoration](https://github.com/apache/hive/blob/rel/release-4.0.1/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java#L323-
L341).
##########
be/src/format_v2/avro/avro_reader.cpp:
##########
@@ -0,0 +1,1074 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format_v2/avro/avro_reader.h"
+
+#include <algorithm>
+#include <array>
+#include <avro/Allocation.hh>
+#include <avro/DataFile.hh>
+#include <avro/Generic.hh>
+#include <avro/NodeImpl.hh>
+#include <charconv>
+#include <limits>
+#include <mutex>
+#include <type_traits>
+#include <unordered_set>
+#include <utility>
+
+#include "common/exception.h"
+#include "core/assert_cast.h"
+#include "core/block/block.h"
+#include "core/column/column_array.h"
+#include "core/column/column_map.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
+#include "core/custom_allocator.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_decimal.h"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
+#include "core/data_type/data_type_timestamptz.h"
+#include "core/data_type/data_type_uuid.h"
+#include "core/data_type/data_type_varbinary.h"
+#include "core/value/timestamp_ns_value.h"
+#include "core/value/timestamptz_value.h"
+#include "format_v2/column_mapper.h"
+#include "format_v2/materialized_reader_util.h"
+#include "io/fs/file_reader.h"
+#include "io/io_common.h"
+#include "runtime/memory/mem_tracker_limiter.h"
+#include "runtime/memory/thread_mem_tracker_mgr.h"
+#include "runtime/runtime_state.h"
+#include "runtime/thread_context.h"
+#include "storage/utils.h"
+
+namespace doris::format::avro {
+namespace {
+
+// Avro must use the same filesystem, credentials, cache and cancellation
state as other readers.
+// The library owns its stream; the shared physical reader remains alive until
the stream closes.
+class NativeInputStream final : public ::avro::SeekableInputStream {
+public:
+ NativeInputStream(io::FileReaderSPtr reader,
std::shared_ptr<io::IOContext> io_ctx)
+ : _reader(std::move(reader)), _io_ctx(std::move(io_ctx)) {}
+
+ bool next(const uint8_t** data, size_t* length) override {
+ if (_position == _reader->size()) {
+ return false;
+ }
+ if (_io_ctx != nullptr && _io_ctx->should_stop) {
+ throw Exception(ErrorCode::CANCELLED, "Avro scan cancelled");
+ }
+ const auto requested = std::min(_buffer.size(), _reader->size() -
_position);
+ size_t read = 0;
+ auto status =
+ _reader->read_at(_position, Slice(_buffer.data(), requested),
&read, _io_ctx.get());
+ if (!status.ok()) {
+ throw Exception(status.code(), status.to_string());
+ }
+ if (read == 0) {
+ throw ::avro::Exception("Truncated Avro input");
+ }
+ *data = reinterpret_cast<const uint8_t*>(_buffer.data());
+ *length = read;
+ _position += read;
+ _last_read = read;
+ return true;
+ }
+
+ void backup(size_t length) override {
+ if (length > _last_read) {
+ throw ::avro::Exception("Invalid Avro stream backup");
+ }
+ _position -= length;
+ _last_read -= length;
+ }
+
+ void skip(size_t length) override {
+ if (length > _reader->size() - _position) {
+ throw ::avro::Exception("Truncated Avro input");
+ }
+ _position += length;
+ _last_read = 0;
+ }
+
+ size_t byteCount() const override { return _position; }
+
+ void seek(int64_t position) override {
+ if (position < 0 || static_cast<uint64_t>(position) > _reader->size())
{
+ throw ::avro::Exception("Invalid Avro stream offset");
+ }
+ _position = position;
+ _last_read = 0;
+ }
+
+private:
+ io::FileReaderSPtr _reader;
+ std::shared_ptr<io::IOContext> _io_ctx;
+ std::array<char, 64 * 1024> _buffer;
+ size_t _position = 0;
+ size_t _last_read = 0;
+};
+
+::avro::NodePtr resolved_node(const ::avro::NodePtr& node) {
+ return node->type() == ::avro::AVRO_SYMBOLIC ? ::avro::resolveSymbol(node)
: node;
+}
+
+std::string lowercase(std::string name) {
+ for (auto& c : name) {
+ if (c >= 'A' && c <= 'Z') {
+ c += 'a' - 'A';
+ }
+ }
+ return name;
+}
+
+Status set_identifier(const ::avro::NodePtr& node, size_t attributes_index,
const char* attribute,
+ ColumnDefinition* column) {
+ if (attributes_index >= node->customAttributes()) {
+ return Status::OK();
+ }
+ const auto value =
node->customAttributesAt(attributes_index).getAttribute(attribute);
+ if (!value.has_value()) {
+ return Status::OK();
+ }
+ int32_t id = 0;
+ const auto parsed = std::from_chars(value->data(), value->data() +
value->size(), id);
+ if (parsed.ec != std::errc() || parsed.ptr != value->data() +
value->size() || id <= 0) {
+ return Status::InvalidArgument("Invalid Avro field identifier");
+ }
+ column->identifier = Field::create_field<TYPE_INT>(id);
+ return Status::OK();
+}
+
+bool is_logical_map(const ::avro::NodePtr& node) {
+ const auto logical = node->logicalType();
+ return node->type() == ::avro::AVRO_ARRAY && logical.type() ==
::avro::LogicalType::CUSTOM &&
+ logical.customLogicalType()->name() == "map";
+}
+
+// Keep logical and physical alternatives in one visitor so nullable handling
is shared.
+//
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity)
+Status build_schema(const ::avro::NodePtr& input, const std::string& name,
int32_t ordinal,
+ bool timestamp_tz, bool varbinary,
+ std::unordered_set<const ::avro::Node*>* ancestors,
ColumnDefinition* column) {
+ const auto node = resolved_node(input);
+ // A recursive Avro record cannot be represented by a finite Doris STRUCT
type.
+ if (ancestors->size() >= 64 || !ancestors->insert(node.get()).second) {
+ return Status::NotSupported("Recursive or excessively nested Avro
schemas are unsupported");
+ }
+ struct Pop {
+ std::unordered_set<const ::avro::Node*>* nodes;
+ const ::avro::Node* node;
+ ~Pop() { nodes->erase(node); }
+ } pop {.nodes = ancestors, .node = node.get()};
+ if (node->leaves() >
static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
+ return Status::NotSupported("Avro schema has too many fields");
+ }
+ column->name = lowercase(name);
+ column->local_id = ordinal;
+ column->identifier = Field::create_field<TYPE_STRING>(column->name);
+ if (node->type() == ::avro::AVRO_UNION) {
+ ::avro::NodePtr value;
+ for (size_t i = 0; i < node->leaves(); ++i) {
+ if (node->leafAt(i)->type() != ::avro::AVRO_NULL) {
+ if (value != nullptr) {
+ return Status::NotSupported(
+ "Avro unions with multiple non-null types are
unsupported");
+ }
+ value = node->leafAt(i);
+ }
+ }
+ if (value != nullptr) {
+ return build_schema(value, name, ordinal, timestamp_tz, varbinary,
ancestors, column);
+ }
+ column->type = make_nullable(std::make_shared<DataTypeString>());
+ return Status::OK();
+ }
+
+ DataTypePtr type;
+ const auto logical = node->logicalType();
+ using Logical = ::avro::LogicalType;
+ switch (logical.type()) {
+ case Logical::DECIMAL:
+ if (logical.precision() > 76) {
+ return Status::NotSupported("Avro decimal precision exceeds 76");
+ }
+ type = create_decimal(logical.precision(), logical.scale(), false);
+ break;
+ case Logical::UUID:
+ type = std::make_shared<DataTypeUUID>();
+ break;
+ case Logical::DATE:
+ type = std::make_shared<DataTypeDateV2>();
+ break;
+ case Logical::TIME_MILLIS:
+ case Logical::TIME_MICROS:
+ type = std::make_shared<DataTypeTimeV2>(logical.type() ==
Logical::TIME_MILLIS ? 3 : 6);
+ break;
+ case Logical::TIMESTAMP_NANOS:
+ case Logical::LOCAL_TIMESTAMP_NANOS:
+ type = std::make_shared<DataTypeTimeStampNs>();
+ column->timestamp_is_adjusted_to_utc = logical.type() ==
Logical::TIMESTAMP_NANOS;
+ break;
+ case Logical::TIMESTAMP_MILLIS:
+ case Logical::TIMESTAMP_MICROS:
+ case Logical::LOCAL_TIMESTAMP_MILLIS:
+ case Logical::LOCAL_TIMESTAMP_MICROS: {
+ const bool adjusted = logical.type() == Logical::TIMESTAMP_MILLIS ||
+ logical.type() == Logical::TIMESTAMP_MICROS;
+ const int scale = logical.type() == Logical::TIMESTAMP_MILLIS ||
+ logical.type() ==
Logical::LOCAL_TIMESTAMP_MILLIS
+ ? 3
+ : 6;
+ type = timestamp_tz && adjusted ?
DataTypePtr(std::make_shared<DataTypeTimeStampTz>(scale))
+ :
DataTypePtr(std::make_shared<DataTypeDateTimeV2>(scale));
+ column->timestamp_is_adjusted_to_utc = adjusted;
+ break;
+ }
+ default:
+ break;
+ }
+ if (type == nullptr) {
+ switch (node->type()) {
+ case ::avro::AVRO_NULL:
+ type = std::make_shared<DataTypeString>();
+ break;
+ case ::avro::AVRO_BOOL:
+ type = std::make_shared<DataTypeUInt8>();
+ break;
+ case ::avro::AVRO_INT:
+ type = std::make_shared<DataTypeInt32>();
+ break;
+ case ::avro::AVRO_LONG:
+ type = std::make_shared<DataTypeInt64>();
+ break;
+ case ::avro::AVRO_FLOAT:
+ type = std::make_shared<DataTypeFloat32>();
+ break;
+ case ::avro::AVRO_DOUBLE:
+ type = std::make_shared<DataTypeFloat64>();
+ break;
+ case ::avro::AVRO_BYTES:
+ case ::avro::AVRO_FIXED:
+ type = varbinary ?
DataTypePtr(std::make_shared<DataTypeVarbinary>())
+ : DataTypePtr(std::make_shared<DataTypeString>());
+ break;
+ case ::avro::AVRO_STRING:
+ case ::avro::AVRO_ENUM:
+ type = std::make_shared<DataTypeString>();
+ break;
+ case ::avro::AVRO_RECORD: {
+ DataTypes types;
+ Strings names;
+ for (size_t i = 0; i < node->leaves(); ++i) {
+ ColumnDefinition child;
+ RETURN_IF_ERROR(build_schema(node->leafAt(i), node->nameAt(i),
+ static_cast<int32_t>(i),
timestamp_tz, varbinary,
+ ancestors, &child));
+ RETURN_IF_ERROR(set_identifier(node, i, "field-id", &child));
+ types.push_back(child.type);
+ names.push_back(child.name);
+ column->children.push_back(std::move(child));
+ }
+ if (types.empty()) {
+ return Status::NotSupported("Empty Avro records are
unsupported");
+ }
+ type = std::make_shared<DataTypeStruct>(types, names);
+ break;
+ }
+ case ::avro::AVRO_ARRAY:
+ case ::avro::AVRO_MAP: {
+ const bool map = node->type() == ::avro::AVRO_MAP ||
is_logical_map(node);
+ if (is_logical_map(node)) {
+ const auto entry = resolved_node(node->leafAt(0));
+ if (entry->type() != ::avro::AVRO_RECORD || entry->leaves() !=
2 ||
+ entry->nameAt(0) != "key" || entry->nameAt(1) != "value") {
+ return Status::InvalidArgument("Invalid Avro logical map
entry schema");
+ }
+ ColumnDefinition entries;
+ RETURN_IF_ERROR(build_schema(entry, "entry", 0, timestamp_tz,
varbinary, ancestors,
+ &entries));
+ column->children = std::move(entries.children);
+ } else {
+ for (size_t i = 0; std::cmp_less(i, (map ? 2 : 1)); ++i) {
+ ColumnDefinition child;
+ RETURN_IF_ERROR(build_schema(
+ node->leafAt(i), map ? (i == 0 ? "key" : "value")
: "element",
+ static_cast<int32_t>(i), timestamp_tz, varbinary,
ancestors, &child));
+ RETURN_IF_ERROR(set_identifier(
+ node, 0, map ? (i == 0 ? "key-id" : "value-id") :
"element-id",
+ &child));
+ column->children.push_back(std::move(child));
+ }
+ }
+ type = map ?
DataTypePtr(std::make_shared<DataTypeMap>(column->children[0].type,
+
column->children[1].type))
+ :
DataTypePtr(std::make_shared<DataTypeArray>(column->children[0].type));
+ break;
+ }
+ default:
+ return Status::NotSupported("Unsupported Avro schema type");
+ }
+ }
+ column->type = make_nullable(type);
+ return Status::OK();
+}
+
+::avro::NodePtr clone_projected_schema(
+ const ::avro::NodePtr& input,
+ std::unordered_map<const ::avro::Node*, ::avro::NodePtr>* copies,
+ const std::unordered_set<int32_t>* selected_roots = nullptr) {
+ const auto source = resolved_node(input);
+ if (const auto found = copies->find(source.get()); found != copies->end())
{
+ return found->second;
+ }
+ ::avro::NodePtr result;
+ switch (source->type()) {
+ case ::avro::AVRO_RECORD:
+ result = std::make_shared<::avro::NodeRecord>();
+ break;
+ case ::avro::AVRO_ENUM:
+ result = std::make_shared<::avro::NodeEnum>();
+ break;
+ case ::avro::AVRO_FIXED:
+ result = std::make_shared<::avro::NodeFixed>();
+ break;
+ case ::avro::AVRO_ARRAY:
+ result = std::make_shared<::avro::NodeArray>();
+ break;
+ case ::avro::AVRO_MAP:
+ result = std::make_shared<::avro::NodeMap>();
+ break;
+ case ::avro::AVRO_UNION:
+ result = std::make_shared<::avro::NodeUnion>();
+ break;
+ default:
+ result = std::make_shared<::avro::NodePrimitive>(source->type());
+ break;
+ }
+ copies->emplace(source.get(), result);
+ if (source->hasName()) {
+ result->setName(source->name());
+ }
+ if (source->type() == ::avro::AVRO_FIXED) {
+ result->setFixedSize(source->fixedSize());
+ }
+ for (size_t i = source->type() == ::avro::AVRO_MAP ? 1 : 0; i <
source->leaves(); ++i) {
+ if (selected_roots != nullptr &&
!selected_roots->contains(static_cast<int32_t>(i))) {
+ continue;
+ }
+ result->addLeaf(clone_projected_schema(source->leafAt(i), copies));
+ if (source->type() == ::avro::AVRO_RECORD) {
+ result->addName(source->nameAt(i));
+ }
+ }
+ if (source->type() == ::avro::AVRO_ENUM) {
+ for (size_t i = 0; i < source->names(); ++i) {
+ result->addName(source->nameAt(i));
+ }
+ }
+ // Rebuild named definitions instead of retaining symbolic references to
omitted roots.
+ // The ValidSchema validator replaces repeated definitions with links
within this schema.
+ result->setLogicalType(source->logicalType());
+ return result;
+}
+
+void apply_timestamp_projection(ColumnDefinition* column, const
LocalColumnIndex& projection) {
+ const auto type = remove_nullable(column->type);
+ if (projection.timestamp_is_adjusted_to_utc.has_value() &&
+ (type->get_primitive_type() == TYPE_DATETIMEV2 ||
+ type->get_primitive_type() == TYPE_TIMESTAMPTZ)) {
+ column->timestamp_is_adjusted_to_utc =
projection.timestamp_is_adjusted_to_utc;
+ column->type = make_nullable(
Review Comment:
[P1] Keep the timestamp decoder type aligned with the load block
An Avro stream/broker load containing a non-null `timestamp-millis` or
`timestamp-micros` reaches this with `enable_mapping_timestamp_tz=false`:
`NereidsLoadPlanInfoCollector.toFileScanRangeParams()` leaves that optional
field unset. Schema discovery therefore creates DATETIMEV2, and the mapper
builds a DATETIMEV2 file column while copying the physical `adjusted=true`
marker into its projection. This hook then changes only the reader schema to
TIMESTAMPTZ. `append_value()` inserts a TIMESTAMPTZ Field into
ColumnDateTimeV2, which throws `Field::get type mismatch` and fails the load.
Preserve the selected file-block type unless a table annotation has changed
both contracts consistently, and cover the actual mapper-generated load
request. Current TVF/catalog requests force mapping=true and do not exercise
this failure.
##########
be/src/format_v2/avro/avro_reader.cpp:
##########
@@ -0,0 +1,1074 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format_v2/avro/avro_reader.h"
+
+#include <algorithm>
+#include <array>
+#include <avro/Allocation.hh>
+#include <avro/DataFile.hh>
+#include <avro/Generic.hh>
+#include <avro/NodeImpl.hh>
+#include <charconv>
+#include <limits>
+#include <mutex>
+#include <type_traits>
+#include <unordered_set>
+#include <utility>
+
+#include "common/exception.h"
+#include "core/assert_cast.h"
+#include "core/block/block.h"
+#include "core/column/column_array.h"
+#include "core/column/column_map.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
+#include "core/custom_allocator.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_decimal.h"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
+#include "core/data_type/data_type_timestamptz.h"
+#include "core/data_type/data_type_uuid.h"
+#include "core/data_type/data_type_varbinary.h"
+#include "core/value/timestamp_ns_value.h"
+#include "core/value/timestamptz_value.h"
+#include "format_v2/column_mapper.h"
+#include "format_v2/materialized_reader_util.h"
+#include "io/fs/file_reader.h"
+#include "io/io_common.h"
+#include "runtime/memory/mem_tracker_limiter.h"
+#include "runtime/memory/thread_mem_tracker_mgr.h"
+#include "runtime/runtime_state.h"
+#include "runtime/thread_context.h"
+#include "storage/utils.h"
+
+namespace doris::format::avro {
+namespace {
+
+// Avro must use the same filesystem, credentials, cache and cancellation
state as other readers.
+// The library owns its stream; the shared physical reader remains alive until
the stream closes.
+class NativeInputStream final : public ::avro::SeekableInputStream {
+public:
+ NativeInputStream(io::FileReaderSPtr reader,
std::shared_ptr<io::IOContext> io_ctx)
+ : _reader(std::move(reader)), _io_ctx(std::move(io_ctx)) {}
+
+ bool next(const uint8_t** data, size_t* length) override {
+ if (_position == _reader->size()) {
+ return false;
+ }
+ if (_io_ctx != nullptr && _io_ctx->should_stop) {
+ throw Exception(ErrorCode::CANCELLED, "Avro scan cancelled");
+ }
+ const auto requested = std::min(_buffer.size(), _reader->size() -
_position);
+ size_t read = 0;
+ auto status =
+ _reader->read_at(_position, Slice(_buffer.data(), requested),
&read, _io_ctx.get());
+ if (!status.ok()) {
+ throw Exception(status.code(), status.to_string());
+ }
+ if (read == 0) {
+ throw ::avro::Exception("Truncated Avro input");
+ }
+ *data = reinterpret_cast<const uint8_t*>(_buffer.data());
+ *length = read;
+ _position += read;
+ _last_read = read;
+ return true;
+ }
+
+ void backup(size_t length) override {
+ if (length > _last_read) {
+ throw ::avro::Exception("Invalid Avro stream backup");
+ }
+ _position -= length;
+ _last_read -= length;
+ }
+
+ void skip(size_t length) override {
+ if (length > _reader->size() - _position) {
+ throw ::avro::Exception("Truncated Avro input");
+ }
+ _position += length;
+ _last_read = 0;
+ }
+
+ size_t byteCount() const override { return _position; }
+
+ void seek(int64_t position) override {
+ if (position < 0 || static_cast<uint64_t>(position) > _reader->size())
{
+ throw ::avro::Exception("Invalid Avro stream offset");
+ }
+ _position = position;
+ _last_read = 0;
+ }
+
+private:
+ io::FileReaderSPtr _reader;
+ std::shared_ptr<io::IOContext> _io_ctx;
+ std::array<char, 64 * 1024> _buffer;
+ size_t _position = 0;
+ size_t _last_read = 0;
+};
+
+::avro::NodePtr resolved_node(const ::avro::NodePtr& node) {
+ return node->type() == ::avro::AVRO_SYMBOLIC ? ::avro::resolveSymbol(node)
: node;
+}
+
+std::string lowercase(std::string name) {
+ for (auto& c : name) {
+ if (c >= 'A' && c <= 'Z') {
+ c += 'a' - 'A';
+ }
+ }
+ return name;
+}
+
+Status set_identifier(const ::avro::NodePtr& node, size_t attributes_index,
const char* attribute,
+ ColumnDefinition* column) {
+ if (attributes_index >= node->customAttributes()) {
+ return Status::OK();
+ }
+ const auto value =
node->customAttributesAt(attributes_index).getAttribute(attribute);
+ if (!value.has_value()) {
+ return Status::OK();
+ }
+ int32_t id = 0;
+ const auto parsed = std::from_chars(value->data(), value->data() +
value->size(), id);
+ if (parsed.ec != std::errc() || parsed.ptr != value->data() +
value->size() || id <= 0) {
+ return Status::InvalidArgument("Invalid Avro field identifier");
+ }
+ column->identifier = Field::create_field<TYPE_INT>(id);
+ return Status::OK();
+}
+
+bool is_logical_map(const ::avro::NodePtr& node) {
+ const auto logical = node->logicalType();
+ return node->type() == ::avro::AVRO_ARRAY && logical.type() ==
::avro::LogicalType::CUSTOM &&
+ logical.customLogicalType()->name() == "map";
+}
+
+// Keep logical and physical alternatives in one visitor so nullable handling
is shared.
+//
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity)
+Status build_schema(const ::avro::NodePtr& input, const std::string& name,
int32_t ordinal,
+ bool timestamp_tz, bool varbinary,
+ std::unordered_set<const ::avro::Node*>* ancestors,
ColumnDefinition* column) {
+ const auto node = resolved_node(input);
+ // A recursive Avro record cannot be represented by a finite Doris STRUCT
type.
+ if (ancestors->size() >= 64 || !ancestors->insert(node.get()).second) {
+ return Status::NotSupported("Recursive or excessively nested Avro
schemas are unsupported");
+ }
+ struct Pop {
+ std::unordered_set<const ::avro::Node*>* nodes;
+ const ::avro::Node* node;
+ ~Pop() { nodes->erase(node); }
+ } pop {.nodes = ancestors, .node = node.get()};
+ if (node->leaves() >
static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
+ return Status::NotSupported("Avro schema has too many fields");
+ }
+ column->name = lowercase(name);
+ column->local_id = ordinal;
+ column->identifier = Field::create_field<TYPE_STRING>(column->name);
+ if (node->type() == ::avro::AVRO_UNION) {
+ ::avro::NodePtr value;
+ for (size_t i = 0; i < node->leaves(); ++i) {
+ if (node->leafAt(i)->type() != ::avro::AVRO_NULL) {
+ if (value != nullptr) {
+ return Status::NotSupported(
+ "Avro unions with multiple non-null types are
unsupported");
+ }
+ value = node->leafAt(i);
+ }
+ }
+ if (value != nullptr) {
+ return build_schema(value, name, ordinal, timestamp_tz, varbinary,
ancestors, column);
+ }
+ column->type = make_nullable(std::make_shared<DataTypeString>());
+ return Status::OK();
+ }
+
+ DataTypePtr type;
+ const auto logical = node->logicalType();
+ using Logical = ::avro::LogicalType;
+ switch (logical.type()) {
+ case Logical::DECIMAL:
+ if (logical.precision() > 76) {
+ return Status::NotSupported("Avro decimal precision exceeds 76");
+ }
+ type = create_decimal(logical.precision(), logical.scale(), false);
+ break;
+ case Logical::UUID:
+ type = std::make_shared<DataTypeUUID>();
+ break;
+ case Logical::DATE:
+ type = std::make_shared<DataTypeDateV2>();
+ break;
+ case Logical::TIME_MILLIS:
+ case Logical::TIME_MICROS:
+ type = std::make_shared<DataTypeTimeV2>(logical.type() ==
Logical::TIME_MILLIS ? 3 : 6);
+ break;
+ case Logical::TIMESTAMP_NANOS:
+ case Logical::LOCAL_TIMESTAMP_NANOS:
+ type = std::make_shared<DataTypeTimeStampNs>();
+ column->timestamp_is_adjusted_to_utc = logical.type() ==
Logical::TIMESTAMP_NANOS;
+ break;
+ case Logical::TIMESTAMP_MILLIS:
+ case Logical::TIMESTAMP_MICROS:
+ case Logical::LOCAL_TIMESTAMP_MILLIS:
+ case Logical::LOCAL_TIMESTAMP_MICROS: {
+ const bool adjusted = logical.type() == Logical::TIMESTAMP_MILLIS ||
+ logical.type() == Logical::TIMESTAMP_MICROS;
+ const int scale = logical.type() == Logical::TIMESTAMP_MILLIS ||
+ logical.type() ==
Logical::LOCAL_TIMESTAMP_MILLIS
+ ? 3
+ : 6;
+ type = timestamp_tz && adjusted ?
DataTypePtr(std::make_shared<DataTypeTimeStampTz>(scale))
+ :
DataTypePtr(std::make_shared<DataTypeDateTimeV2>(scale));
+ column->timestamp_is_adjusted_to_utc = adjusted;
+ break;
+ }
+ default:
+ break;
+ }
+ if (type == nullptr) {
+ switch (node->type()) {
+ case ::avro::AVRO_NULL:
+ type = std::make_shared<DataTypeString>();
+ break;
+ case ::avro::AVRO_BOOL:
+ type = std::make_shared<DataTypeUInt8>();
+ break;
+ case ::avro::AVRO_INT:
+ type = std::make_shared<DataTypeInt32>();
+ break;
+ case ::avro::AVRO_LONG:
+ type = std::make_shared<DataTypeInt64>();
+ break;
+ case ::avro::AVRO_FLOAT:
+ type = std::make_shared<DataTypeFloat32>();
+ break;
+ case ::avro::AVRO_DOUBLE:
+ type = std::make_shared<DataTypeFloat64>();
+ break;
+ case ::avro::AVRO_BYTES:
+ case ::avro::AVRO_FIXED:
+ type = varbinary ?
DataTypePtr(std::make_shared<DataTypeVarbinary>())
+ : DataTypePtr(std::make_shared<DataTypeString>());
+ break;
+ case ::avro::AVRO_STRING:
+ case ::avro::AVRO_ENUM:
+ type = std::make_shared<DataTypeString>();
+ break;
+ case ::avro::AVRO_RECORD: {
+ DataTypes types;
+ Strings names;
+ for (size_t i = 0; i < node->leaves(); ++i) {
+ ColumnDefinition child;
+ RETURN_IF_ERROR(build_schema(node->leafAt(i), node->nameAt(i),
+ static_cast<int32_t>(i),
timestamp_tz, varbinary,
+ ancestors, &child));
Review Comment:
[P2] Bound and account the expanded schema before allocating it
This recursive expansion is outside the Avro allocation checks and
retained-scratch accounting. A small valid schema can define `N0={v:long}` and
then `Ni={a:N(i-1),b:N(i-1)}` using previously defined names. Avro keeps those
references shared, but this visitor rebuilds both subtrees independently: N22
alone produces over four million leaf definitions at depth below 64. All these
vectors/type objects use ordinary allocation, and `reconcile_scratch()`
excludes `_impl->schema`, so a small query memory limit cannot stop the
expansion. It happens during `init()` even for a header-only container or an
unprojected complex field. Bound the expanded size and reserve/account schema
construction and retained ownership before allocating, returning a
memory/status error when it exceeds the budget.
##########
be/src/format_v2/avro/avro_reader.cpp:
##########
@@ -0,0 +1,1074 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format_v2/avro/avro_reader.h"
+
+#include <algorithm>
+#include <array>
+#include <avro/Allocation.hh>
+#include <avro/DataFile.hh>
+#include <avro/Generic.hh>
+#include <avro/NodeImpl.hh>
+#include <charconv>
+#include <limits>
+#include <mutex>
+#include <type_traits>
+#include <unordered_set>
+#include <utility>
+
+#include "common/exception.h"
+#include "core/assert_cast.h"
+#include "core/block/block.h"
+#include "core/column/column_array.h"
+#include "core/column/column_map.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
+#include "core/custom_allocator.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_decimal.h"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
+#include "core/data_type/data_type_timestamptz.h"
+#include "core/data_type/data_type_uuid.h"
+#include "core/data_type/data_type_varbinary.h"
+#include "core/value/timestamp_ns_value.h"
+#include "core/value/timestamptz_value.h"
+#include "format_v2/column_mapper.h"
+#include "format_v2/materialized_reader_util.h"
+#include "io/fs/file_reader.h"
+#include "io/io_common.h"
+#include "runtime/memory/mem_tracker_limiter.h"
+#include "runtime/memory/thread_mem_tracker_mgr.h"
+#include "runtime/runtime_state.h"
+#include "runtime/thread_context.h"
+#include "storage/utils.h"
+
+namespace doris::format::avro {
+namespace {
+
+// Avro must use the same filesystem, credentials, cache and cancellation
state as other readers.
+// The library owns its stream; the shared physical reader remains alive until
the stream closes.
+class NativeInputStream final : public ::avro::SeekableInputStream {
+public:
+ NativeInputStream(io::FileReaderSPtr reader,
std::shared_ptr<io::IOContext> io_ctx)
+ : _reader(std::move(reader)), _io_ctx(std::move(io_ctx)) {}
+
+ bool next(const uint8_t** data, size_t* length) override {
+ if (_position == _reader->size()) {
+ return false;
+ }
+ if (_io_ctx != nullptr && _io_ctx->should_stop) {
+ throw Exception(ErrorCode::CANCELLED, "Avro scan cancelled");
+ }
+ const auto requested = std::min(_buffer.size(), _reader->size() -
_position);
+ size_t read = 0;
+ auto status =
+ _reader->read_at(_position, Slice(_buffer.data(), requested),
&read, _io_ctx.get());
+ if (!status.ok()) {
+ throw Exception(status.code(), status.to_string());
+ }
+ if (read == 0) {
+ throw ::avro::Exception("Truncated Avro input");
+ }
+ *data = reinterpret_cast<const uint8_t*>(_buffer.data());
+ *length = read;
+ _position += read;
+ _last_read = read;
+ return true;
+ }
+
+ void backup(size_t length) override {
+ if (length > _last_read) {
+ throw ::avro::Exception("Invalid Avro stream backup");
+ }
+ _position -= length;
+ _last_read -= length;
+ }
+
+ void skip(size_t length) override {
+ if (length > _reader->size() - _position) {
+ throw ::avro::Exception("Truncated Avro input");
+ }
+ _position += length;
+ _last_read = 0;
+ }
+
+ size_t byteCount() const override { return _position; }
+
+ void seek(int64_t position) override {
+ if (position < 0 || static_cast<uint64_t>(position) > _reader->size())
{
+ throw ::avro::Exception("Invalid Avro stream offset");
+ }
+ _position = position;
+ _last_read = 0;
+ }
+
+private:
+ io::FileReaderSPtr _reader;
+ std::shared_ptr<io::IOContext> _io_ctx;
+ std::array<char, 64 * 1024> _buffer;
+ size_t _position = 0;
+ size_t _last_read = 0;
+};
+
+::avro::NodePtr resolved_node(const ::avro::NodePtr& node) {
+ return node->type() == ::avro::AVRO_SYMBOLIC ? ::avro::resolveSymbol(node)
: node;
+}
+
+std::string lowercase(std::string name) {
+ for (auto& c : name) {
+ if (c >= 'A' && c <= 'Z') {
+ c += 'a' - 'A';
+ }
+ }
+ return name;
+}
+
+Status set_identifier(const ::avro::NodePtr& node, size_t attributes_index,
const char* attribute,
+ ColumnDefinition* column) {
+ if (attributes_index >= node->customAttributes()) {
+ return Status::OK();
+ }
+ const auto value =
node->customAttributesAt(attributes_index).getAttribute(attribute);
+ if (!value.has_value()) {
+ return Status::OK();
+ }
+ int32_t id = 0;
+ const auto parsed = std::from_chars(value->data(), value->data() +
value->size(), id);
+ if (parsed.ec != std::errc() || parsed.ptr != value->data() +
value->size() || id <= 0) {
+ return Status::InvalidArgument("Invalid Avro field identifier");
+ }
+ column->identifier = Field::create_field<TYPE_INT>(id);
+ return Status::OK();
+}
+
+bool is_logical_map(const ::avro::NodePtr& node) {
+ const auto logical = node->logicalType();
+ return node->type() == ::avro::AVRO_ARRAY && logical.type() ==
::avro::LogicalType::CUSTOM &&
+ logical.customLogicalType()->name() == "map";
+}
+
+// Keep logical and physical alternatives in one visitor so nullable handling
is shared.
+//
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity)
+Status build_schema(const ::avro::NodePtr& input, const std::string& name,
int32_t ordinal,
+ bool timestamp_tz, bool varbinary,
+ std::unordered_set<const ::avro::Node*>* ancestors,
ColumnDefinition* column) {
+ const auto node = resolved_node(input);
+ // A recursive Avro record cannot be represented by a finite Doris STRUCT
type.
+ if (ancestors->size() >= 64 || !ancestors->insert(node.get()).second) {
+ return Status::NotSupported("Recursive or excessively nested Avro
schemas are unsupported");
+ }
+ struct Pop {
+ std::unordered_set<const ::avro::Node*>* nodes;
+ const ::avro::Node* node;
+ ~Pop() { nodes->erase(node); }
+ } pop {.nodes = ancestors, .node = node.get()};
+ if (node->leaves() >
static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
+ return Status::NotSupported("Avro schema has too many fields");
+ }
+ column->name = lowercase(name);
+ column->local_id = ordinal;
+ column->identifier = Field::create_field<TYPE_STRING>(column->name);
+ if (node->type() == ::avro::AVRO_UNION) {
+ ::avro::NodePtr value;
+ for (size_t i = 0; i < node->leaves(); ++i) {
+ if (node->leafAt(i)->type() != ::avro::AVRO_NULL) {
+ if (value != nullptr) {
+ return Status::NotSupported(
+ "Avro unions with multiple non-null types are
unsupported");
+ }
+ value = node->leafAt(i);
+ }
+ }
+ if (value != nullptr) {
+ return build_schema(value, name, ordinal, timestamp_tz, varbinary,
ancestors, column);
+ }
+ column->type = make_nullable(std::make_shared<DataTypeString>());
+ return Status::OK();
+ }
+
+ DataTypePtr type;
+ const auto logical = node->logicalType();
+ using Logical = ::avro::LogicalType;
+ switch (logical.type()) {
+ case Logical::DECIMAL:
+ if (logical.precision() > 76) {
+ return Status::NotSupported("Avro decimal precision exceeds 76");
+ }
+ type = create_decimal(logical.precision(), logical.scale(), false);
+ break;
+ case Logical::UUID:
+ type = std::make_shared<DataTypeUUID>();
+ break;
+ case Logical::DATE:
+ type = std::make_shared<DataTypeDateV2>();
+ break;
+ case Logical::TIME_MILLIS:
+ case Logical::TIME_MICROS:
+ type = std::make_shared<DataTypeTimeV2>(logical.type() ==
Logical::TIME_MILLIS ? 3 : 6);
+ break;
+ case Logical::TIMESTAMP_NANOS:
+ case Logical::LOCAL_TIMESTAMP_NANOS:
+ type = std::make_shared<DataTypeTimeStampNs>();
+ column->timestamp_is_adjusted_to_utc = logical.type() ==
Logical::TIMESTAMP_NANOS;
+ break;
+ case Logical::TIMESTAMP_MILLIS:
+ case Logical::TIMESTAMP_MICROS:
+ case Logical::LOCAL_TIMESTAMP_MILLIS:
+ case Logical::LOCAL_TIMESTAMP_MICROS: {
+ const bool adjusted = logical.type() == Logical::TIMESTAMP_MILLIS ||
+ logical.type() == Logical::TIMESTAMP_MICROS;
+ const int scale = logical.type() == Logical::TIMESTAMP_MILLIS ||
+ logical.type() ==
Logical::LOCAL_TIMESTAMP_MILLIS
+ ? 3
+ : 6;
+ type = timestamp_tz && adjusted ?
DataTypePtr(std::make_shared<DataTypeTimeStampTz>(scale))
+ :
DataTypePtr(std::make_shared<DataTypeDateTimeV2>(scale));
+ column->timestamp_is_adjusted_to_utc = adjusted;
+ break;
+ }
+ default:
+ break;
+ }
+ if (type == nullptr) {
+ switch (node->type()) {
+ case ::avro::AVRO_NULL:
+ type = std::make_shared<DataTypeString>();
+ break;
+ case ::avro::AVRO_BOOL:
+ type = std::make_shared<DataTypeUInt8>();
+ break;
+ case ::avro::AVRO_INT:
+ type = std::make_shared<DataTypeInt32>();
+ break;
+ case ::avro::AVRO_LONG:
+ type = std::make_shared<DataTypeInt64>();
+ break;
+ case ::avro::AVRO_FLOAT:
+ type = std::make_shared<DataTypeFloat32>();
+ break;
+ case ::avro::AVRO_DOUBLE:
+ type = std::make_shared<DataTypeFloat64>();
+ break;
+ case ::avro::AVRO_BYTES:
+ case ::avro::AVRO_FIXED:
+ type = varbinary ?
DataTypePtr(std::make_shared<DataTypeVarbinary>())
+ : DataTypePtr(std::make_shared<DataTypeString>());
+ break;
+ case ::avro::AVRO_STRING:
+ case ::avro::AVRO_ENUM:
+ type = std::make_shared<DataTypeString>();
+ break;
+ case ::avro::AVRO_RECORD: {
+ DataTypes types;
+ Strings names;
+ for (size_t i = 0; i < node->leaves(); ++i) {
+ ColumnDefinition child;
+ RETURN_IF_ERROR(build_schema(node->leafAt(i), node->nameAt(i),
+ static_cast<int32_t>(i),
timestamp_tz, varbinary,
+ ancestors, &child));
+ RETURN_IF_ERROR(set_identifier(node, i, "field-id", &child));
+ types.push_back(child.type);
+ names.push_back(child.name);
+ column->children.push_back(std::move(child));
+ }
+ if (types.empty()) {
+ return Status::NotSupported("Empty Avro records are
unsupported");
+ }
+ type = std::make_shared<DataTypeStruct>(types, names);
+ break;
+ }
+ case ::avro::AVRO_ARRAY:
+ case ::avro::AVRO_MAP: {
+ const bool map = node->type() == ::avro::AVRO_MAP ||
is_logical_map(node);
+ if (is_logical_map(node)) {
+ const auto entry = resolved_node(node->leafAt(0));
+ if (entry->type() != ::avro::AVRO_RECORD || entry->leaves() !=
2 ||
+ entry->nameAt(0) != "key" || entry->nameAt(1) != "value") {
+ return Status::InvalidArgument("Invalid Avro logical map
entry schema");
+ }
+ ColumnDefinition entries;
+ RETURN_IF_ERROR(build_schema(entry, "entry", 0, timestamp_tz,
varbinary, ancestors,
+ &entries));
+ column->children = std::move(entries.children);
+ } else {
+ for (size_t i = 0; std::cmp_less(i, (map ? 2 : 1)); ++i) {
+ ColumnDefinition child;
+ RETURN_IF_ERROR(build_schema(
+ node->leafAt(i), map ? (i == 0 ? "key" : "value")
: "element",
+ static_cast<int32_t>(i), timestamp_tz, varbinary,
ancestors, &child));
+ RETURN_IF_ERROR(set_identifier(
+ node, 0, map ? (i == 0 ? "key-id" : "value-id") :
"element-id",
+ &child));
+ column->children.push_back(std::move(child));
+ }
+ }
+ type = map ?
DataTypePtr(std::make_shared<DataTypeMap>(column->children[0].type,
+
column->children[1].type))
+ :
DataTypePtr(std::make_shared<DataTypeArray>(column->children[0].type));
+ break;
+ }
+ default:
+ return Status::NotSupported("Unsupported Avro schema type");
+ }
+ }
+ column->type = make_nullable(type);
+ return Status::OK();
+}
+
+::avro::NodePtr clone_projected_schema(
+ const ::avro::NodePtr& input,
+ std::unordered_map<const ::avro::Node*, ::avro::NodePtr>* copies,
+ const std::unordered_set<int32_t>* selected_roots = nullptr) {
+ const auto source = resolved_node(input);
+ if (const auto found = copies->find(source.get()); found != copies->end())
{
+ return found->second;
+ }
+ ::avro::NodePtr result;
+ switch (source->type()) {
+ case ::avro::AVRO_RECORD:
+ result = std::make_shared<::avro::NodeRecord>();
+ break;
+ case ::avro::AVRO_ENUM:
+ result = std::make_shared<::avro::NodeEnum>();
+ break;
+ case ::avro::AVRO_FIXED:
+ result = std::make_shared<::avro::NodeFixed>();
+ break;
+ case ::avro::AVRO_ARRAY:
+ result = std::make_shared<::avro::NodeArray>();
+ break;
+ case ::avro::AVRO_MAP:
+ result = std::make_shared<::avro::NodeMap>();
+ break;
+ case ::avro::AVRO_UNION:
+ result = std::make_shared<::avro::NodeUnion>();
+ break;
+ default:
+ result = std::make_shared<::avro::NodePrimitive>(source->type());
+ break;
+ }
+ copies->emplace(source.get(), result);
+ if (source->hasName()) {
+ result->setName(source->name());
+ }
+ if (source->type() == ::avro::AVRO_FIXED) {
+ result->setFixedSize(source->fixedSize());
+ }
+ for (size_t i = source->type() == ::avro::AVRO_MAP ? 1 : 0; i <
source->leaves(); ++i) {
+ if (selected_roots != nullptr &&
!selected_roots->contains(static_cast<int32_t>(i))) {
+ continue;
+ }
+ result->addLeaf(clone_projected_schema(source->leafAt(i), copies));
+ if (source->type() == ::avro::AVRO_RECORD) {
+ result->addName(source->nameAt(i));
+ }
+ }
+ if (source->type() == ::avro::AVRO_ENUM) {
+ for (size_t i = 0; i < source->names(); ++i) {
+ result->addName(source->nameAt(i));
+ }
+ }
+ // Rebuild named definitions instead of retaining symbolic references to
omitted roots.
+ // The ValidSchema validator replaces repeated definitions with links
within this schema.
+ result->setLogicalType(source->logicalType());
+ return result;
+}
+
+void apply_timestamp_projection(ColumnDefinition* column, const
LocalColumnIndex& projection) {
+ const auto type = remove_nullable(column->type);
+ if (projection.timestamp_is_adjusted_to_utc.has_value() &&
+ (type->get_primitive_type() == TYPE_DATETIMEV2 ||
+ type->get_primitive_type() == TYPE_TIMESTAMPTZ)) {
+ column->timestamp_is_adjusted_to_utc =
projection.timestamp_is_adjusted_to_utc;
+ column->type = make_nullable(
+ *projection.timestamp_is_adjusted_to_utc
+ ?
DataTypePtr(std::make_shared<DataTypeTimeStampTz>(type->get_scale()))
+ :
DataTypePtr(std::make_shared<DataTypeDateTimeV2>(type->get_scale())));
+ return;
+ }
+ for (const auto& child_projection : projection.children) {
+ for (auto& child : column->children) {
+ if (child.local_id == child_projection.local_id()) {
+ apply_timestamp_projection(&child, child_projection);
+ break;
+ }
+ }
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ column->type =
make_nullable(std::make_shared<DataTypeArray>(column->children[0].type));
+ } else if (type->get_primitive_type() == TYPE_MAP) {
+ column->type = make_nullable(
+ std::make_shared<DataTypeMap>(column->children[0].type,
column->children[1].type));
+ } else if (type->get_primitive_type() == TYPE_STRUCT) {
+ DataTypes types;
+ Strings names;
+ for (const auto& child : column->children) {
+ types.push_back(child.type);
+ names.push_back(child.name);
+ }
+ column->type = make_nullable(std::make_shared<DataTypeStruct>(types,
names));
+ }
+}
+
+Status append_decimal(const std::vector<uint8_t>& bytes, const DataTypePtr&
type, IColumn* column) {
+ // Avro decimals are signed, big-endian two's-complement integers,
including FIXED values.
+ size_t start = 0;
+ // FIXED may be wider than its precision requires; discard only redundant
sign extension.
+ while (start + 1 < bytes.size() && ((bytes[start] == 0 && (bytes[start +
1] & 0x80) == 0) ||
+ (bytes[start] == 255 && (bytes[start +
1] & 0x80) != 0))) {
+ ++start;
+ }
+ if (bytes.empty() || bytes.size() - start > 32) {
+ return Status::InvalidArgument("Invalid Avro decimal width");
+ }
+ Int256 value = (bytes[start] & 0x80) != 0 ? -1 : 0;
+ for (; start < bytes.size(); ++start) {
+ value = value * 256 + bytes[start];
+ }
+ Int256 limit = 1;
+ for (unsigned i = 0; i < type->get_precision(); ++i) {
+ limit *= 10;
+ }
+ if (value <= -limit || value >= limit) {
+ return Status::InvalidArgument("Avro decimal exceeds its precision");
+ }
+ switch (type->get_primitive_type()) {
+ case TYPE_DECIMAL32:
+
column->insert(Field::create_field<TYPE_DECIMAL32>(Decimal32(static_cast<int32_t>(value))));
+ break;
+ case TYPE_DECIMAL64:
+
column->insert(Field::create_field<TYPE_DECIMAL64>(Decimal64(static_cast<int64_t>(value))));
+ break;
+ case TYPE_DECIMAL128I:
+ column->insert(
+
Field::create_field<TYPE_DECIMAL128I>(Decimal128V3(static_cast<int128_t>(value))));
+ break;
+ case TYPE_DECIMAL256:
+
column->insert(Field::create_field<TYPE_DECIMAL256>(Decimal256(value)));
+ break;
+ default:
+ return Status::InternalError("Invalid Avro decimal output type");
+ }
+ return Status::OK();
+}
+
+// One visitor preserves the same null/child-offset rules for all Avro value
alternatives.
+//
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity)
+Status append_value(const ::avro::GenericDatum& datum, const ColumnDefinition&
schema,
+ IColumn* output, const cctz::time_zone& timezone) {
+ if (datum.type() == ::avro::AVRO_NULL) {
+ output->insert_default();
+ return Status::OK();
+ }
+ auto& nullable = assert_cast<ColumnNullable&>(*output);
+ auto* column = &nullable.get_nested_column();
+ const auto type = remove_nullable(schema.type);
+ using Logical = ::avro::LogicalType;
+ const auto logical = datum.logicalType().type();
+ if (logical == Logical::DECIMAL) {
+ const auto& bytes = datum.type() == ::avro::AVRO_FIXED
+ ?
datum.value<::avro::GenericFixed>().value()
+ : datum.value<std::vector<uint8_t>>();
+ RETURN_IF_ERROR(append_decimal(bytes, type, column));
+ } else if (logical == Logical::UUID) {
+ UUIDValueType value;
+ if (datum.type() == ::avro::AVRO_FIXED) {
+ const auto& bytes = datum.value<::avro::GenericFixed>().value();
+ if (bytes.size() != UUIDValue::BINARY_LENGTH) {
+ return Status::InvalidArgument("Invalid Avro UUID width");
+ }
+ value = UUIDValue::from_big_endian(bytes.data());
+ } else if (!UUIDValue::from_string(value, datum.value<std::string>()))
{
+ return Status::InvalidArgument("Invalid Avro UUID value");
+ }
+ column->insert(Field::create_field<TYPE_UUID>(value));
+ } else if (logical == Logical::DATE) {
+ DateV2Value<DateV2ValueType> date;
+ const int64_t day = static_cast<int64_t>(datum.value<int32_t>()) +
719528;
+ if (day < 0 || !date.get_date_from_daynr(day)) {
+ return Status::InvalidArgument("Avro date is out of range");
+ }
+ column->insert(Field::create_field<TYPE_DATEV2>(date));
+ } else if (logical == Logical::TIME_MILLIS || logical ==
Logical::TIME_MICROS) {
+ const int64_t micros = logical == Logical::TIME_MILLIS
+ ?
static_cast<int64_t>(datum.value<int32_t>()) * 1000
+ : datum.value<int64_t>();
+ if (micros < 0 || micros >= 86400000000LL) {
+ return Status::InvalidArgument("Avro time is out of range");
+ }
+
column->insert(Field::create_field<TYPE_TIMEV2>(static_cast<double>(micros)));
+ } else if (logical == Logical::TIMESTAMP_NANOS || logical ==
Logical::LOCAL_TIMESTAMP_NANOS) {
+ column->insert(
+
Field::create_field<TYPE_TIMESTAMP_NS>(TimeStampNsValue(datum.value<int64_t>())));
+ } else if (logical == Logical::TIMESTAMP_MILLIS || logical ==
Logical::TIMESTAMP_MICROS ||
+ logical == Logical::LOCAL_TIMESTAMP_MILLIS ||
+ logical == Logical::LOCAL_TIMESTAMP_MICROS) {
+ const int64_t units =
+ logical == Logical::TIMESTAMP_MILLIS || logical ==
Logical::LOCAL_TIMESTAMP_MILLIS
+ ? 1000
+ : 1000000;
+ const auto raw = datum.value<int64_t>();
+ int64_t seconds = raw / units;
+ int64_t fraction = raw % units;
+ if (fraction < 0) {
+ --seconds;
+ fraction += units;
+ }
+ const bool adjusted = schema.timestamp_is_adjusted_to_utc.value_or(
+ logical == Logical::TIMESTAMP_MILLIS || logical ==
Logical::TIMESTAMP_MICROS);
+ if (seconds < -62167305600LL || seconds > 253402387199LL) {
+ return Status::InvalidArgument("Avro timestamp is out of range");
+ }
+ DateV2Value<DateTimeV2ValueType> value;
+ // Local timestamps encode a civil clock. UTC is only its arithmetic
epoch anchor;
+ // applying the session zone would change that clock (for example
00:00 into 08:00).
+ const auto zone = adjusted && type->get_primitive_type() !=
TYPE_TIMESTAMPTZ
+ ? timezone
+ : cctz::utc_time_zone();
+ // DateTimeV2's fractional argument is nanoseconds, even when the
output scale is six.
+ value.from_unixtime(seconds, static_cast<int32_t>(fraction *
(1000000000 / units)), zone,
+ 6);
+ if (!value.is_valid_date()) {
+ return Status::InvalidArgument("Avro timestamp is out of range");
+ }
+ if (type->get_primitive_type() == TYPE_TIMESTAMPTZ) {
+
column->insert(Field::create_field<TYPE_TIMESTAMPTZ>(TimestampTzValue(value)));
+ } else {
+ column->insert(Field::create_field<TYPE_DATETIMEV2>(value));
+ }
+ } else {
+ switch (datum.type()) {
+ case ::avro::AVRO_BOOL:
+
column->insert(Field::create_field<TYPE_BOOLEAN>(datum.value<bool>()));
+ break;
+ case ::avro::AVRO_INT:
+
column->insert(Field::create_field<TYPE_INT>(datum.value<int32_t>()));
+ break;
+ case ::avro::AVRO_LONG:
+
column->insert(Field::create_field<TYPE_BIGINT>(datum.value<int64_t>()));
+ break;
+ case ::avro::AVRO_FLOAT:
+
column->insert(Field::create_field<TYPE_FLOAT>(datum.value<float>()));
+ break;
+ case ::avro::AVRO_DOUBLE:
+
column->insert(Field::create_field<TYPE_DOUBLE>(datum.value<double>()));
+ break;
+ case ::avro::AVRO_STRING: {
+ const auto& value = datum.value<std::string>();
+ column->insert_data(value.data(), value.size());
+ break;
+ }
+ case ::avro::AVRO_ENUM: {
+ const auto& value = datum.value<::avro::GenericEnum>().symbol();
+ column->insert_data(value.data(), value.size());
+ break;
+ }
+ case ::avro::AVRO_BYTES:
+ case ::avro::AVRO_FIXED: {
+ const auto& value = datum.type() == ::avro::AVRO_FIXED
+ ?
datum.value<::avro::GenericFixed>().value()
+ : datum.value<std::vector<uint8_t>>();
+ column->insert_data(reinterpret_cast<const char*>(value.data()),
value.size());
+ break;
+ }
+ case ::avro::AVRO_RECORD: {
+ const auto& record = datum.value<::avro::GenericRecord>();
+ auto& target = assert_cast<ColumnStruct&>(*column);
+ for (size_t i = 0; i < schema.children.size(); ++i) {
+ RETURN_IF_ERROR(append_value(record.fieldAt(i),
schema.children[i],
+ &target.get_column(i), timezone));
+ }
+ break;
+ }
+ case ::avro::AVRO_ARRAY: {
+ const auto& values = datum.value<::avro::GenericArray>().value();
+ if (type->get_primitive_type() == TYPE_MAP) {
+ auto& target = assert_cast<ColumnMap&>(*column);
+ for (const auto& entry : values) {
+ const auto& record = entry.value<::avro::GenericRecord>();
+ RETURN_IF_ERROR(append_value(record.fieldAt(0),
schema.children[0],
+ &target.get_keys(),
timezone));
+ RETURN_IF_ERROR(append_value(record.fieldAt(1),
schema.children[1],
+ &target.get_values(),
timezone));
+ }
+ target.get_offsets().push_back(target.get_keys().size());
+ } else {
+ auto& target = assert_cast<ColumnArray&>(*column);
+ for (const auto& value : values) {
+ RETURN_IF_ERROR(
+ append_value(value, schema.children[0],
&target.get_data(), timezone));
+ }
+ target.get_offsets().push_back(target.get_data().size());
+ }
+ break;
+ }
+ case ::avro::AVRO_MAP: {
+ auto& target = assert_cast<ColumnMap&>(*column);
+ for (const auto& [key, value] :
datum.value<::avro::GenericMap>().value()) {
+ auto& keys = assert_cast<ColumnNullable&>(target.get_keys());
+ keys.get_nested_column().insert_data(key.data(), key.size());
+ keys.get_null_map_data().push_back(0);
+ RETURN_IF_ERROR(
+ append_value(value, schema.children[1],
&target.get_values(), timezone));
+ }
+ target.get_offsets().push_back(target.get_keys().size());
+ break;
+ }
+ default:
+ return Status::NotSupported("Unsupported Avro value type");
+ }
+ }
+ nullable.get_null_map_data().push_back(0);
+ return Status::OK();
+}
+
+size_t datum_scratch_bytes(const ::avro::GenericDatum& datum) {
+ // type()/value() unwrap a union, but its std::any-owned wrapper remains
allocated.
+ const size_t wrapper = datum.isUnion() ? sizeof(::avro::GenericUnion) : 0;
+ switch (datum.type()) {
+ case ::avro::AVRO_STRING:
+ return wrapper + sizeof(std::string) +
+ (datum.value<std::string>().capacity() >
std::string().capacity()
+ ? datum.value<std::string>().capacity() + 1
+ : 0);
+ case ::avro::AVRO_BYTES:
+ return wrapper + sizeof(std::vector<uint8_t>) +
+ datum.value<std::vector<uint8_t>>().capacity();
+ case ::avro::AVRO_FIXED:
+ return wrapper + sizeof(::avro::GenericFixed) +
+ datum.value<::avro::GenericFixed>().value().capacity();
+ case ::avro::AVRO_RECORD: {
+ const auto& record = datum.value<::avro::GenericRecord>();
+ size_t bytes =
+ wrapper + sizeof(record) + record.fieldCount() *
sizeof(::avro::GenericDatum);
+ for (size_t i = 0; i < record.fieldCount(); ++i) {
+ bytes += datum_scratch_bytes(record.fieldAt(i));
+ }
+ return bytes;
+ }
+ case ::avro::AVRO_ARRAY: {
+ const auto& array = datum.value<::avro::GenericArray>().value();
+ size_t bytes = wrapper + sizeof(::avro::GenericArray) +
+ array.capacity() * sizeof(::avro::GenericDatum);
+ for (const auto& value : array) {
+ bytes += datum_scratch_bytes(value);
+ }
+ return bytes;
+ }
+ case ::avro::AVRO_MAP: {
+ const auto& map = datum.value<::avro::GenericMap>().value();
+ size_t bytes = wrapper + sizeof(::avro::GenericMap) +
+ map.capacity() *
sizeof(std::remove_reference_t<decltype(map)>::value_type);
+ for (const auto& [key, value] : map) {
+ bytes += key.capacity() + datum_scratch_bytes(value);
+ }
+ return bytes;
+ }
+ case ::avro::AVRO_ENUM:
+ return wrapper + sizeof(::avro::GenericEnum);
+ default:
+ return wrapper;
+ }
+}
+
+} // namespace
+
+struct AvroReader::Impl {
+ std::shared_ptr<MemTrackerLimiter> memory_tracker;
+ DorisVector<ReservedMemoryToken> reservations;
+ int64_t accounted_scratch = 0;
+ RuntimeProfile::Counter* scratch_accounting_passes = nullptr;
+ size_t small_reservation_available = 0;
+ size_t collection_scratch_before_reset = 0;
+
+ static void reserve_callback(void* context, size_t bytes) {
+ auto* self = static_cast<Impl*>(context);
+ if (bytes > static_cast<size_t>(std::numeric_limits<int64_t>::max())) {
+ throw Exception(ErrorCode::MEM_LIMIT_EXCEEDED, "Avro allocation
exceeds Int64");
+ }
+ if (bytes <= self->small_reservation_available) {
+ self->small_reservation_available -= bytes;
+ return;
+ }
+ // Batch tiny std::any allocations so large nullable arrays do not
acquire one token per item.
+ const size_t reservation = std::max(bytes, size_t(64 * 1024));
+ auto* manager = thread_context()->thread_mem_tracker_mgr.get();
+ // Preserve the scanner's existing reservation; each Avro growth owns
an independent token.
+ auto previous = manager->take_reserved_memory();
+ Defer restore([&] {
manager->adopt_reserved_memory(std::move(previous)); });
+ const auto status =
manager->try_reserve(static_cast<int64_t>(reservation));
+ if (!status.ok()) {
+ throw Exception(status);
+ }
+ self->reservations.push_back(manager->take_reserved_memory());
+ if (bytes < reservation) {
+ self->small_reservation_available = reservation - bytes;
+ }
+ }
+
+ static void checkpoint_callback(void* context, const ::avro::GenericDatum&
collection,
+ bool before) {
+ auto* self = static_cast<Impl*>(context);
+ const size_t retained = datum_scratch_bytes(collection);
+ if (before) {
+ self->collection_scratch_before_reset = retained;
+ } else {
+ // Reset only destroys children from the previously accounted row.
These paired callbacks
+ // cannot nest, and newly constructed collections have no children
to release.
+ const auto released =
+ static_cast<int64_t>(self->collection_scratch_before_reset
- retained);
+ DCHECK_LE(released, self->accounted_scratch);
+ self->memory_tracker->release(released);
+ self->accounted_scratch -= released;
+ }
+ }
+
+ void reconcile_scratch() {
+ COUNTER_UPDATE(scratch_accounting_passes, 1);
+ const size_t retained = (reader ? reader->scratchBytes() +
sizeof(NativeInputStream) : 0) +
+ (datum ? datum_scratch_bytes(*datum) : 0);
+ // Replace conservative transient-growth reservations with retained
capacities after every
+ // record. Retained codec and datum buffers remain charged between
batches until close.
+ memory_tracker->consume(static_cast<int64_t>(retained) -
accounted_scratch);
+ accounted_scratch = static_cast<int64_t>(retained);
+ reservations.clear();
+ small_reservation_available = 0;
+ }
+
+ bool timestamp_tz;
+ bool varbinary;
+ cctz::time_zone timezone = cctz::utc_time_zone();
+ size_t batch_size = 4096;
+ size_t max_block_bytes = std::numeric_limits<size_t>::max();
+ std::unordered_set<int32_t> count_placeholders;
+ std::unordered_map<int32_t, size_t> projected_ordinals;
+ std::unique_ptr<::avro::DataFileReaderBase> reader;
+ std::unique_ptr<::avro::GenericDatum> datum;
+ std::vector<ColumnDefinition> schema;
+ int64_t header_end = 0;
+ int64_t range_end = std::numeric_limits<int64_t>::max();
+ int64_t row_position = 0;
+ size_t next_requested_row = 0;
+ std::optional<GlobalRowIdContext> global_rowid_context;
+};
+
+AvroReader::AvroReader(std::shared_ptr<io::FileSystemProperties>&
system_properties,
+ std::unique_ptr<io::FileDescription>& file_description,
+ std::shared_ptr<io::IOContext> io_ctx, RuntimeProfile*
profile,
+ bool enable_mapping_timestamp_tz, bool
enable_mapping_varbinary,
+ std::optional<GlobalRowIdContext> global_rowid_context)
+ : FileReader(system_properties, file_description, std::move(io_ctx),
profile),
+ _impl(std::make_unique<Impl>()) {
+ _impl->timestamp_tz = enable_mapping_timestamp_tz;
+ _impl->varbinary = enable_mapping_varbinary;
+ _impl->global_rowid_context = global_rowid_context;
+}
+
+AvroReader::~AvroReader() {
+ (void)close();
+}
+
+Status AvroReader::init(RuntimeState* state) {
+ _impl->memory_tracker =
thread_context()->thread_mem_tracker_mgr->limiter_mem_tracker_sptr();
+ RETURN_IF_ERROR(FileReader::init(state));
+ _impl->scratch_accounting_passes =
+ ADD_COUNTER(_profile, "AvroScratchAccountingPasses", TUnit::UNIT);
+ if (state != nullptr) {
+ _impl->timezone = state->timezone_obj();
+ _impl->max_block_bytes = state->preferred_block_size_bytes();
+ }
+ static std::once_flag register_map;
+ std::call_once(register_map, [] {
+ // Iceberg's non-string-key maps are arrays of key/value records with
logicalType=map.
+ ::avro::CustomLogicalTypeRegistry::instance().registerType("map",
[](const std::string&) {
+ return std::make_shared<::avro::CustomLogicalType>("map");
+ });
+ });
+ try {
+ ::avro::AllocationCheckScope budget(&Impl::reserve_callback,
_impl.get(),
+ &Impl::checkpoint_callback);
+ Impl::reserve_callback(_impl.get(), sizeof(NativeInputStream));
+ _impl->reader = std::make_unique<::avro::DataFileReaderBase>(
+ std::make_unique<NativeInputStream>(_tracing_file_reader,
_io_ctx));
+ const auto root = _impl->reader->dataSchema().root();
+ if (root->type() != ::avro::AVRO_RECORD) {
+ return Status::NotSupported("Avro container root must be a
record");
+ }
+ _impl->header_end = _impl->reader->previousSync();
+ std::unordered_set<const ::avro::Node*> ancestors;
+ ColumnDefinition record;
+ RETURN_IF_ERROR(build_schema(root, "row", 0, _impl->timestamp_tz,
_impl->varbinary,
+ &ancestors, &record));
+ _impl->schema = std::move(record.children);
+ _impl->reconcile_scratch();
+ return Status::OK();
+ } catch (const doris::Exception& e) {
+ return e.to_status();
+ } catch (const std::bad_alloc&) {
+ return Status::MemoryLimitExceeded("Avro allocation failed");
+ } catch (const std::exception& e) {
+ return Status::InvalidArgument("Cannot read Avro schema: {}",
e.what());
+ }
+}
+
+Status AvroReader::get_schema(std::vector<ColumnDefinition>* file_schema)
const {
+ *file_schema = _impl->schema;
+ if (_impl->global_rowid_context.has_value()) {
+ auto rowid = global_rowid_column_definition();
+ rowid.type = make_nullable(rowid.type);
+ file_schema->push_back(std::move(rowid));
+ }
+ return Status::OK();
+}
+
+std::unique_ptr<TableColumnMapper> AvroReader::create_column_mapper(
+ TableColumnMapperOptions options) const {
+ return std::make_unique<MaterializedColumnMapper>(options);
+}
+
+// Range ownership and initialization update the same absolute row-position
state.
+//
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity)
+Status AvroReader::open(std::shared_ptr<FileScanRequest> request) {
+ RETURN_IF_ERROR(FileReader::open(std::move(request)));
+ try {
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_impl->memory_tracker);
+ ::avro::AllocationCheckScope budget(&Impl::reserve_callback,
_impl.get(),
+ &Impl::checkpoint_callback);
+ // The mapper communicates per-leaf physical decoding semantics
through local projections.
+ // Apply them before writing into the file block, whose types already
carry these markers.
+ auto apply_projection = [&](const LocalColumnIndex& projection) {
+ const auto id = projection.local_id();
+ if (id >= 0 && static_cast<size_t>(id) < _impl->schema.size()) {
+ apply_timestamp_projection(&_impl->schema[id], projection);
+ }
+ };
+ for (const auto& projection : _request->predicate_columns) {
+ apply_projection(projection);
+ }
+ for (const auto& projection : _request->non_predicate_columns) {
+ apply_projection(projection);
+ }
+ // Equality-delete dependencies may promote a COUNT carrier after it
was marked.
+ // Such columns still need their actual values for the predicate to
remove rows.
+ for (const auto id : _request->count_star_placeholder_columns) {
+ if (std::ranges::none_of(_request->predicate_columns, [&](const
auto& projection) {
+ return projection.column_id() == id;
+ })) {
+ _impl->count_placeholders.insert(id.value());
+ }
+ }
+ std::unordered_set<int32_t> selected;
+ for (const auto& [id, position] : _request->local_positions) {
+ if (id.value() >= 0 &&
!_impl->count_placeholders.contains(id.value())) {
+ if (static_cast<size_t>(id.value()) >= _impl->schema.size()) {
+ return Status::InvalidArgument("Unknown Avro file-local
column");
+ }
+ selected.insert(id.value());
+ }
+ }
+ size_t projected = 0;
+ for (size_t i = 0; i < _impl->schema.size(); ++i) {
+ if (selected.contains(static_cast<int32_t>(i))) {
+ _impl->projected_ordinals.emplace(static_cast<int32_t>(i),
projected++);
+ }
+ }
+ const int64_t start = _file_description->range_start_offset;
+ const int64_t size = _file_description->range_size;
+ if (start < 0 || size < -1 ||
+ (size >= 0 && start > std::numeric_limits<int64_t>::max() - size))
{
+ return Status::InvalidArgument("Invalid Avro file range");
+ }
+ _impl->range_end = size < 0 ? std::numeric_limits<int64_t>::max() :
start + size;
+ // A range owns blocks whose start lies in [start,end). Count
preceding blocks from their
+ // OCF headers, without decoding payloads, so position deletes retain
absolute file rows.
+ int64_t offset = _impl->header_end;
+ while (offset < start && std::cmp_less(offset,
_tracing_file_reader->size())) {
+ std::array<uint8_t, 20> bytes;
+ size_t read = 0;
+ RETURN_IF_ERROR(_tracing_file_reader->read_at(
Review Comment:
[P2] Avoid rescanning the entire prefix for every Avro split
Every nonzero range walks from `header_end` using one synchronous 20-byte
read per preceding OCF block, even when no row-position column or delete needs
absolute coordinates. Iceberg splits Avro files, and the large-object RANDOM
reader has no prefetch buffer; with file cache disabled each call becomes an
object-store GET. A valid 1 GiB file with roughly 64 KiB blocks and eight 128
MiB ranges adds about 57,000 prefix GETs, with roughly 14,000 serial requests
before the last range can return its first row. These are counts inferred from
the loop, not benchmark results. Skip prefix enumeration when coordinates are
unnecessary and share/coalesce a block-to-row index when they are required.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]