Gabriel39 commented on code in PR #70808: URL: https://github.com/apache/doris/pull/70808#discussion_r4642510162
########## be/src/format_v2/avro/avro_reader.cpp: ########## @@ -0,0 +1,1074 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/avro/avro_reader.h" + +#include <algorithm> +#include <array> +#include <avro/Allocation.hh> +#include <avro/DataFile.hh> +#include <avro/Generic.hh> +#include <avro/NodeImpl.hh> +#include <charconv> +#include <limits> +#include <mutex> +#include <type_traits> +#include <unordered_set> +#include <utility> + +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/custom_allocator.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_uuid.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/value/timestamp_ns_value.h" +#include "core/value/timestamptz_value.h" +#include "format_v2/column_mapper.h" +#include "format_v2/materialized_reader_util.h" +#include "io/fs/file_reader.h" +#include "io/io_common.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "storage/utils.h" + +namespace doris::format::avro { +namespace { + +// Avro must use the same filesystem, credentials, cache and cancellation state as other readers. +// The library owns its stream; the shared physical reader remains alive until the stream closes. +class NativeInputStream final : public ::avro::SeekableInputStream { +public: + NativeInputStream(io::FileReaderSPtr reader, std::shared_ptr<io::IOContext> io_ctx) + : _reader(std::move(reader)), _io_ctx(std::move(io_ctx)) {} + + bool next(const uint8_t** data, size_t* length) override { + if (_position == _reader->size()) { + return false; + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + throw Exception(ErrorCode::CANCELLED, "Avro scan cancelled"); + } + const auto requested = std::min(_buffer.size(), _reader->size() - _position); + size_t read = 0; + auto status = + _reader->read_at(_position, Slice(_buffer.data(), requested), &read, _io_ctx.get()); + if (!status.ok()) { + throw Exception(status.code(), status.to_string()); + } + if (read == 0) { + throw ::avro::Exception("Truncated Avro input"); + } + *data = reinterpret_cast<const uint8_t*>(_buffer.data()); + *length = read; + _position += read; + _last_read = read; + return true; + } + + void backup(size_t length) override { + if (length > _last_read) { + throw ::avro::Exception("Invalid Avro stream backup"); + } + _position -= length; + _last_read -= length; + } + + void skip(size_t length) override { + if (length > _reader->size() - _position) { + throw ::avro::Exception("Truncated Avro input"); + } + _position += length; + _last_read = 0; + } + + size_t byteCount() const override { return _position; } + + void seek(int64_t position) override { + if (position < 0 || static_cast<uint64_t>(position) > _reader->size()) { + throw ::avro::Exception("Invalid Avro stream offset"); + } + _position = position; + _last_read = 0; + } + +private: + io::FileReaderSPtr _reader; + std::shared_ptr<io::IOContext> _io_ctx; + std::array<char, 64 * 1024> _buffer; + size_t _position = 0; + size_t _last_read = 0; +}; + +::avro::NodePtr resolved_node(const ::avro::NodePtr& node) { + return node->type() == ::avro::AVRO_SYMBOLIC ? ::avro::resolveSymbol(node) : node; +} + +std::string lowercase(std::string name) { + for (auto& c : name) { + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + } + return name; +} + +Status set_identifier(const ::avro::NodePtr& node, size_t attributes_index, const char* attribute, + ColumnDefinition* column) { + if (attributes_index >= node->customAttributes()) { + return Status::OK(); + } + const auto value = node->customAttributesAt(attributes_index).getAttribute(attribute); + if (!value.has_value()) { + return Status::OK(); + } + int32_t id = 0; + const auto parsed = std::from_chars(value->data(), value->data() + value->size(), id); + if (parsed.ec != std::errc() || parsed.ptr != value->data() + value->size() || id <= 0) { + return Status::InvalidArgument("Invalid Avro field identifier"); + } + column->identifier = Field::create_field<TYPE_INT>(id); + return Status::OK(); +} + +bool is_logical_map(const ::avro::NodePtr& node) { + const auto logical = node->logicalType(); + return node->type() == ::avro::AVRO_ARRAY && logical.type() == ::avro::LogicalType::CUSTOM && + logical.customLogicalType()->name() == "map"; +} + +// Keep logical and physical alternatives in one visitor so nullable handling is shared. +// NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity) +Status build_schema(const ::avro::NodePtr& input, const std::string& name, int32_t ordinal, + bool timestamp_tz, bool varbinary, + std::unordered_set<const ::avro::Node*>* ancestors, ColumnDefinition* column) { + const auto node = resolved_node(input); + // A recursive Avro record cannot be represented by a finite Doris STRUCT type. + if (ancestors->size() >= 64 || !ancestors->insert(node.get()).second) { + return Status::NotSupported("Recursive or excessively nested Avro schemas are unsupported"); + } + struct Pop { + std::unordered_set<const ::avro::Node*>* nodes; + const ::avro::Node* node; + ~Pop() { nodes->erase(node); } + } pop {.nodes = ancestors, .node = node.get()}; + if (node->leaves() > static_cast<size_t>(std::numeric_limits<int32_t>::max())) { + return Status::NotSupported("Avro schema has too many fields"); + } + column->name = lowercase(name); + column->local_id = ordinal; + column->identifier = Field::create_field<TYPE_STRING>(column->name); + if (node->type() == ::avro::AVRO_UNION) { + ::avro::NodePtr value; + for (size_t i = 0; i < node->leaves(); ++i) { + if (node->leafAt(i)->type() != ::avro::AVRO_NULL) { + if (value != nullptr) { + return Status::NotSupported( + "Avro unions with multiple non-null types are unsupported"); + } + value = node->leafAt(i); + } + } + if (value != nullptr) { + return build_schema(value, name, ordinal, timestamp_tz, varbinary, ancestors, column); + } + column->type = make_nullable(std::make_shared<DataTypeString>()); + return Status::OK(); + } + + DataTypePtr type; + const auto logical = node->logicalType(); + using Logical = ::avro::LogicalType; + switch (logical.type()) { + case Logical::DECIMAL: + if (logical.precision() > 76) { + return Status::NotSupported("Avro decimal precision exceeds 76"); + } + type = create_decimal(logical.precision(), logical.scale(), false); + break; + case Logical::UUID: + type = std::make_shared<DataTypeUUID>(); + break; + case Logical::DATE: + type = std::make_shared<DataTypeDateV2>(); + break; + case Logical::TIME_MILLIS: + case Logical::TIME_MICROS: + type = std::make_shared<DataTypeTimeV2>(logical.type() == Logical::TIME_MILLIS ? 3 : 6); + break; + case Logical::TIMESTAMP_NANOS: + case Logical::LOCAL_TIMESTAMP_NANOS: + type = std::make_shared<DataTypeTimeStampNs>(); + column->timestamp_is_adjusted_to_utc = logical.type() == Logical::TIMESTAMP_NANOS; + break; + case Logical::TIMESTAMP_MILLIS: + case Logical::TIMESTAMP_MICROS: + case Logical::LOCAL_TIMESTAMP_MILLIS: + case Logical::LOCAL_TIMESTAMP_MICROS: { + const bool adjusted = logical.type() == Logical::TIMESTAMP_MILLIS || + logical.type() == Logical::TIMESTAMP_MICROS; + const int scale = logical.type() == Logical::TIMESTAMP_MILLIS || + logical.type() == Logical::LOCAL_TIMESTAMP_MILLIS + ? 3 + : 6; + type = timestamp_tz && adjusted ? DataTypePtr(std::make_shared<DataTypeTimeStampTz>(scale)) + : DataTypePtr(std::make_shared<DataTypeDateTimeV2>(scale)); + column->timestamp_is_adjusted_to_utc = adjusted; + break; + } + default: + break; + } + if (type == nullptr) { + switch (node->type()) { + case ::avro::AVRO_NULL: + type = std::make_shared<DataTypeString>(); + break; + case ::avro::AVRO_BOOL: + type = std::make_shared<DataTypeUInt8>(); + break; + case ::avro::AVRO_INT: + type = std::make_shared<DataTypeInt32>(); + break; + case ::avro::AVRO_LONG: + type = std::make_shared<DataTypeInt64>(); + break; + case ::avro::AVRO_FLOAT: + type = std::make_shared<DataTypeFloat32>(); + break; + case ::avro::AVRO_DOUBLE: + type = std::make_shared<DataTypeFloat64>(); + break; + case ::avro::AVRO_BYTES: + case ::avro::AVRO_FIXED: + type = varbinary ? DataTypePtr(std::make_shared<DataTypeVarbinary>()) + : DataTypePtr(std::make_shared<DataTypeString>()); + break; + case ::avro::AVRO_STRING: + case ::avro::AVRO_ENUM: + type = std::make_shared<DataTypeString>(); + break; + case ::avro::AVRO_RECORD: { + DataTypes types; + Strings names; + for (size_t i = 0; i < node->leaves(); ++i) { + ColumnDefinition child; + RETURN_IF_ERROR(build_schema(node->leafAt(i), node->nameAt(i), + static_cast<int32_t>(i), timestamp_tz, varbinary, + ancestors, &child)); + RETURN_IF_ERROR(set_identifier(node, i, "field-id", &child)); + types.push_back(child.type); + names.push_back(child.name); + column->children.push_back(std::move(child)); + } + if (types.empty()) { + return Status::NotSupported("Empty Avro records are unsupported"); + } + type = std::make_shared<DataTypeStruct>(types, names); + break; + } + case ::avro::AVRO_ARRAY: + case ::avro::AVRO_MAP: { + const bool map = node->type() == ::avro::AVRO_MAP || is_logical_map(node); + if (is_logical_map(node)) { + const auto entry = resolved_node(node->leafAt(0)); + if (entry->type() != ::avro::AVRO_RECORD || entry->leaves() != 2 || + entry->nameAt(0) != "key" || entry->nameAt(1) != "value") { + return Status::InvalidArgument("Invalid Avro logical map entry schema"); + } + ColumnDefinition entries; + RETURN_IF_ERROR(build_schema(entry, "entry", 0, timestamp_tz, varbinary, ancestors, + &entries)); + column->children = std::move(entries.children); + } else { + for (size_t i = 0; std::cmp_less(i, (map ? 2 : 1)); ++i) { + ColumnDefinition child; + RETURN_IF_ERROR(build_schema( + node->leafAt(i), map ? (i == 0 ? "key" : "value") : "element", + static_cast<int32_t>(i), timestamp_tz, varbinary, ancestors, &child)); + RETURN_IF_ERROR(set_identifier( + node, 0, map ? (i == 0 ? "key-id" : "value-id") : "element-id", + &child)); + column->children.push_back(std::move(child)); + } + } + type = map ? DataTypePtr(std::make_shared<DataTypeMap>(column->children[0].type, + column->children[1].type)) + : DataTypePtr(std::make_shared<DataTypeArray>(column->children[0].type)); + break; + } + default: + return Status::NotSupported("Unsupported Avro schema type"); + } + } + column->type = make_nullable(type); + return Status::OK(); +} + +::avro::NodePtr clone_projected_schema( + const ::avro::NodePtr& input, + std::unordered_map<const ::avro::Node*, ::avro::NodePtr>* copies, + const std::unordered_set<int32_t>* selected_roots = nullptr) { + const auto source = resolved_node(input); + if (const auto found = copies->find(source.get()); found != copies->end()) { + return found->second; + } + ::avro::NodePtr result; + switch (source->type()) { + case ::avro::AVRO_RECORD: + result = std::make_shared<::avro::NodeRecord>(); + break; + case ::avro::AVRO_ENUM: + result = std::make_shared<::avro::NodeEnum>(); + break; + case ::avro::AVRO_FIXED: + result = std::make_shared<::avro::NodeFixed>(); + break; + case ::avro::AVRO_ARRAY: + result = std::make_shared<::avro::NodeArray>(); + break; + case ::avro::AVRO_MAP: + result = std::make_shared<::avro::NodeMap>(); + break; + case ::avro::AVRO_UNION: + result = std::make_shared<::avro::NodeUnion>(); + break; + default: + result = std::make_shared<::avro::NodePrimitive>(source->type()); + break; + } + copies->emplace(source.get(), result); + if (source->hasName()) { + result->setName(source->name()); + } + if (source->type() == ::avro::AVRO_FIXED) { + result->setFixedSize(source->fixedSize()); + } + for (size_t i = source->type() == ::avro::AVRO_MAP ? 1 : 0; i < source->leaves(); ++i) { + if (selected_roots != nullptr && !selected_roots->contains(static_cast<int32_t>(i))) { + continue; + } + result->addLeaf(clone_projected_schema(source->leafAt(i), copies)); + if (source->type() == ::avro::AVRO_RECORD) { + result->addName(source->nameAt(i)); + } + } + if (source->type() == ::avro::AVRO_ENUM) { + for (size_t i = 0; i < source->names(); ++i) { + result->addName(source->nameAt(i)); + } + } + // Rebuild named definitions instead of retaining symbolic references to omitted roots. + // The ValidSchema validator replaces repeated definitions with links within this schema. + result->setLogicalType(source->logicalType()); + return result; +} + +void apply_timestamp_projection(ColumnDefinition* column, const LocalColumnIndex& projection) { + const auto type = remove_nullable(column->type); + if (projection.timestamp_is_adjusted_to_utc.has_value() && + (type->get_primitive_type() == TYPE_DATETIMEV2 || + type->get_primitive_type() == TYPE_TIMESTAMPTZ)) { + column->timestamp_is_adjusted_to_utc = projection.timestamp_is_adjusted_to_utc; + column->type = make_nullable( + *projection.timestamp_is_adjusted_to_utc + ? DataTypePtr(std::make_shared<DataTypeTimeStampTz>(type->get_scale())) + : DataTypePtr(std::make_shared<DataTypeDateTimeV2>(type->get_scale()))); + return; + } + for (const auto& child_projection : projection.children) { + for (auto& child : column->children) { + if (child.local_id == child_projection.local_id()) { + apply_timestamp_projection(&child, child_projection); + break; + } + } + } + if (type->get_primitive_type() == TYPE_ARRAY) { + column->type = make_nullable(std::make_shared<DataTypeArray>(column->children[0].type)); + } else if (type->get_primitive_type() == TYPE_MAP) { + column->type = make_nullable( + std::make_shared<DataTypeMap>(column->children[0].type, column->children[1].type)); + } else if (type->get_primitive_type() == TYPE_STRUCT) { + DataTypes types; + Strings names; + for (const auto& child : column->children) { + types.push_back(child.type); + names.push_back(child.name); + } + column->type = make_nullable(std::make_shared<DataTypeStruct>(types, names)); + } +} + +Status append_decimal(const std::vector<uint8_t>& bytes, const DataTypePtr& type, IColumn* column) { + // Avro decimals are signed, big-endian two's-complement integers, including FIXED values. + size_t start = 0; + // FIXED may be wider than its precision requires; discard only redundant sign extension. + while (start + 1 < bytes.size() && ((bytes[start] == 0 && (bytes[start + 1] & 0x80) == 0) || + (bytes[start] == 255 && (bytes[start + 1] & 0x80) != 0))) { + ++start; + } + if (bytes.empty() || bytes.size() - start > 32) { + return Status::InvalidArgument("Invalid Avro decimal width"); + } + Int256 value = (bytes[start] & 0x80) != 0 ? -1 : 0; + for (; start < bytes.size(); ++start) { + value = value * 256 + bytes[start]; + } + Int256 limit = 1; + for (unsigned i = 0; i < type->get_precision(); ++i) { + limit *= 10; + } + if (value <= -limit || value >= limit) { + return Status::InvalidArgument("Avro decimal exceeds its precision"); + } + switch (type->get_primitive_type()) { + case TYPE_DECIMAL32: + column->insert(Field::create_field<TYPE_DECIMAL32>(Decimal32(static_cast<int32_t>(value)))); + break; + case TYPE_DECIMAL64: + column->insert(Field::create_field<TYPE_DECIMAL64>(Decimal64(static_cast<int64_t>(value)))); + break; + case TYPE_DECIMAL128I: + column->insert( + Field::create_field<TYPE_DECIMAL128I>(Decimal128V3(static_cast<int128_t>(value)))); + break; + case TYPE_DECIMAL256: + column->insert(Field::create_field<TYPE_DECIMAL256>(Decimal256(value))); + break; + default: + return Status::InternalError("Invalid Avro decimal output type"); + } + return Status::OK(); +} + +// One visitor preserves the same null/child-offset rules for all Avro value alternatives. +// NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity) +Status append_value(const ::avro::GenericDatum& datum, const ColumnDefinition& schema, + IColumn* output, const cctz::time_zone& timezone) { + if (datum.type() == ::avro::AVRO_NULL) { + output->insert_default(); + return Status::OK(); + } + auto& nullable = assert_cast<ColumnNullable&>(*output); + auto* column = &nullable.get_nested_column(); + const auto type = remove_nullable(schema.type); + using Logical = ::avro::LogicalType; + const auto logical = datum.logicalType().type(); + if (logical == Logical::DECIMAL) { + const auto& bytes = datum.type() == ::avro::AVRO_FIXED + ? datum.value<::avro::GenericFixed>().value() + : datum.value<std::vector<uint8_t>>(); + RETURN_IF_ERROR(append_decimal(bytes, type, column)); + } else if (logical == Logical::UUID) { + UUIDValueType value; + if (datum.type() == ::avro::AVRO_FIXED) { + const auto& bytes = datum.value<::avro::GenericFixed>().value(); + if (bytes.size() != UUIDValue::BINARY_LENGTH) { + return Status::InvalidArgument("Invalid Avro UUID width"); + } + value = UUIDValue::from_big_endian(bytes.data()); + } else if (!UUIDValue::from_string(value, datum.value<std::string>())) { + return Status::InvalidArgument("Invalid Avro UUID value"); + } + column->insert(Field::create_field<TYPE_UUID>(value)); + } else if (logical == Logical::DATE) { + DateV2Value<DateV2ValueType> date; + const int64_t day = static_cast<int64_t>(datum.value<int32_t>()) + 719528; + if (day < 0 || !date.get_date_from_daynr(day)) { + return Status::InvalidArgument("Avro date is out of range"); + } + column->insert(Field::create_field<TYPE_DATEV2>(date)); + } else if (logical == Logical::TIME_MILLIS || logical == Logical::TIME_MICROS) { + const int64_t micros = logical == Logical::TIME_MILLIS + ? static_cast<int64_t>(datum.value<int32_t>()) * 1000 + : datum.value<int64_t>(); + if (micros < 0 || micros >= 86400000000LL) { + return Status::InvalidArgument("Avro time is out of range"); + } + column->insert(Field::create_field<TYPE_TIMEV2>(static_cast<double>(micros))); + } else if (logical == Logical::TIMESTAMP_NANOS || logical == Logical::LOCAL_TIMESTAMP_NANOS) { + column->insert( + Field::create_field<TYPE_TIMESTAMP_NS>(TimeStampNsValue(datum.value<int64_t>()))); + } else if (logical == Logical::TIMESTAMP_MILLIS || logical == Logical::TIMESTAMP_MICROS || + logical == Logical::LOCAL_TIMESTAMP_MILLIS || + logical == Logical::LOCAL_TIMESTAMP_MICROS) { + const int64_t units = + logical == Logical::TIMESTAMP_MILLIS || logical == Logical::LOCAL_TIMESTAMP_MILLIS + ? 1000 + : 1000000; + const auto raw = datum.value<int64_t>(); + int64_t seconds = raw / units; + int64_t fraction = raw % units; + if (fraction < 0) { + --seconds; + fraction += units; + } + const bool adjusted = schema.timestamp_is_adjusted_to_utc.value_or( + logical == Logical::TIMESTAMP_MILLIS || logical == Logical::TIMESTAMP_MICROS); + if (seconds < -62167305600LL || seconds > 253402387199LL) { + return Status::InvalidArgument("Avro timestamp is out of range"); + } + DateV2Value<DateTimeV2ValueType> value; + // Local timestamps encode a civil clock. UTC is only its arithmetic epoch anchor; + // applying the session zone would change that clock (for example 00:00 into 08:00). + const auto zone = adjusted && type->get_primitive_type() != TYPE_TIMESTAMPTZ + ? timezone + : cctz::utc_time_zone(); + // DateTimeV2's fractional argument is nanoseconds, even when the output scale is six. + value.from_unixtime(seconds, static_cast<int32_t>(fraction * (1000000000 / units)), zone, + 6); + if (!value.is_valid_date()) { + return Status::InvalidArgument("Avro timestamp is out of range"); + } + if (type->get_primitive_type() == TYPE_TIMESTAMPTZ) { + column->insert(Field::create_field<TYPE_TIMESTAMPTZ>(TimestampTzValue(value))); + } else { + column->insert(Field::create_field<TYPE_DATETIMEV2>(value)); + } + } else { + switch (datum.type()) { + case ::avro::AVRO_BOOL: + column->insert(Field::create_field<TYPE_BOOLEAN>(datum.value<bool>())); + break; + case ::avro::AVRO_INT: + column->insert(Field::create_field<TYPE_INT>(datum.value<int32_t>())); + break; + case ::avro::AVRO_LONG: + column->insert(Field::create_field<TYPE_BIGINT>(datum.value<int64_t>())); + break; + case ::avro::AVRO_FLOAT: + column->insert(Field::create_field<TYPE_FLOAT>(datum.value<float>())); + break; + case ::avro::AVRO_DOUBLE: + column->insert(Field::create_field<TYPE_DOUBLE>(datum.value<double>())); + break; + case ::avro::AVRO_STRING: { + const auto& value = datum.value<std::string>(); + column->insert_data(value.data(), value.size()); + break; + } + case ::avro::AVRO_ENUM: { + const auto& value = datum.value<::avro::GenericEnum>().symbol(); + column->insert_data(value.data(), value.size()); + break; + } + case ::avro::AVRO_BYTES: + case ::avro::AVRO_FIXED: { + const auto& value = datum.type() == ::avro::AVRO_FIXED + ? datum.value<::avro::GenericFixed>().value() + : datum.value<std::vector<uint8_t>>(); + column->insert_data(reinterpret_cast<const char*>(value.data()), value.size()); + break; + } + case ::avro::AVRO_RECORD: { + const auto& record = datum.value<::avro::GenericRecord>(); + auto& target = assert_cast<ColumnStruct&>(*column); + for (size_t i = 0; i < schema.children.size(); ++i) { + RETURN_IF_ERROR(append_value(record.fieldAt(i), schema.children[i], + &target.get_column(i), timezone)); + } + break; + } + case ::avro::AVRO_ARRAY: { + const auto& values = datum.value<::avro::GenericArray>().value(); + if (type->get_primitive_type() == TYPE_MAP) { + auto& target = assert_cast<ColumnMap&>(*column); + for (const auto& entry : values) { + const auto& record = entry.value<::avro::GenericRecord>(); + RETURN_IF_ERROR(append_value(record.fieldAt(0), schema.children[0], + &target.get_keys(), timezone)); + RETURN_IF_ERROR(append_value(record.fieldAt(1), schema.children[1], + &target.get_values(), timezone)); + } + target.get_offsets().push_back(target.get_keys().size()); + } else { + auto& target = assert_cast<ColumnArray&>(*column); + for (const auto& value : values) { + RETURN_IF_ERROR( + append_value(value, schema.children[0], &target.get_data(), timezone)); + } + target.get_offsets().push_back(target.get_data().size()); + } + break; + } + case ::avro::AVRO_MAP: { + auto& target = assert_cast<ColumnMap&>(*column); + for (const auto& [key, value] : datum.value<::avro::GenericMap>().value()) { + auto& keys = assert_cast<ColumnNullable&>(target.get_keys()); + keys.get_nested_column().insert_data(key.data(), key.size()); + keys.get_null_map_data().push_back(0); + RETURN_IF_ERROR( + append_value(value, schema.children[1], &target.get_values(), timezone)); + } + target.get_offsets().push_back(target.get_keys().size()); + break; + } + default: + return Status::NotSupported("Unsupported Avro value type"); + } + } + nullable.get_null_map_data().push_back(0); + return Status::OK(); +} + +size_t datum_scratch_bytes(const ::avro::GenericDatum& datum) { + // type()/value() unwrap a union, but its std::any-owned wrapper remains allocated. + const size_t wrapper = datum.isUnion() ? sizeof(::avro::GenericUnion) : 0; + switch (datum.type()) { + case ::avro::AVRO_STRING: + return wrapper + sizeof(std::string) + + (datum.value<std::string>().capacity() > std::string().capacity() + ? datum.value<std::string>().capacity() + 1 + : 0); + case ::avro::AVRO_BYTES: + return wrapper + sizeof(std::vector<uint8_t>) + + datum.value<std::vector<uint8_t>>().capacity(); + case ::avro::AVRO_FIXED: + return wrapper + sizeof(::avro::GenericFixed) + + datum.value<::avro::GenericFixed>().value().capacity(); + case ::avro::AVRO_RECORD: { + const auto& record = datum.value<::avro::GenericRecord>(); + size_t bytes = + wrapper + sizeof(record) + record.fieldCount() * sizeof(::avro::GenericDatum); + for (size_t i = 0; i < record.fieldCount(); ++i) { + bytes += datum_scratch_bytes(record.fieldAt(i)); + } + return bytes; + } + case ::avro::AVRO_ARRAY: { + const auto& array = datum.value<::avro::GenericArray>().value(); + size_t bytes = wrapper + sizeof(::avro::GenericArray) + + array.capacity() * sizeof(::avro::GenericDatum); + for (const auto& value : array) { + bytes += datum_scratch_bytes(value); + } + return bytes; + } + case ::avro::AVRO_MAP: { + const auto& map = datum.value<::avro::GenericMap>().value(); + size_t bytes = wrapper + sizeof(::avro::GenericMap) + + map.capacity() * sizeof(std::remove_reference_t<decltype(map)>::value_type); + for (const auto& [key, value] : map) { + bytes += key.capacity() + datum_scratch_bytes(value); + } + return bytes; + } + case ::avro::AVRO_ENUM: + return wrapper + sizeof(::avro::GenericEnum); + default: + return wrapper; + } +} + +} // namespace + +struct AvroReader::Impl { + std::shared_ptr<MemTrackerLimiter> memory_tracker; + DorisVector<ReservedMemoryToken> reservations; + int64_t accounted_scratch = 0; + RuntimeProfile::Counter* scratch_accounting_passes = nullptr; + size_t small_reservation_available = 0; + size_t collection_scratch_before_reset = 0; + + static void reserve_callback(void* context, size_t bytes) { + auto* self = static_cast<Impl*>(context); + if (bytes > static_cast<size_t>(std::numeric_limits<int64_t>::max())) { + throw Exception(ErrorCode::MEM_LIMIT_EXCEEDED, "Avro allocation exceeds Int64"); + } + if (bytes <= self->small_reservation_available) { + self->small_reservation_available -= bytes; + return; + } + // Batch tiny std::any allocations so large nullable arrays do not acquire one token per item. + const size_t reservation = std::max(bytes, size_t(64 * 1024)); + auto* manager = thread_context()->thread_mem_tracker_mgr.get(); + // Preserve the scanner's existing reservation; each Avro growth owns an independent token. + auto previous = manager->take_reserved_memory(); + Defer restore([&] { manager->adopt_reserved_memory(std::move(previous)); }); + const auto status = manager->try_reserve(static_cast<int64_t>(reservation)); + if (!status.ok()) { + throw Exception(status); + } + self->reservations.push_back(manager->take_reserved_memory()); + if (bytes < reservation) { + self->small_reservation_available = reservation - bytes; + } + } + + static void checkpoint_callback(void* context, const ::avro::GenericDatum& collection, + bool before) { + auto* self = static_cast<Impl*>(context); + const size_t retained = datum_scratch_bytes(collection); + if (before) { + self->collection_scratch_before_reset = retained; + } else { + // Reset only destroys children from the previously accounted row. These paired callbacks + // cannot nest, and newly constructed collections have no children to release. + const auto released = + static_cast<int64_t>(self->collection_scratch_before_reset - retained); + DCHECK_LE(released, self->accounted_scratch); + self->memory_tracker->release(released); + self->accounted_scratch -= released; + } + } + + void reconcile_scratch() { + COUNTER_UPDATE(scratch_accounting_passes, 1); + const size_t retained = (reader ? reader->scratchBytes() + sizeof(NativeInputStream) : 0) + + (datum ? datum_scratch_bytes(*datum) : 0); + // Replace conservative transient-growth reservations with retained capacities after every + // record. Retained codec and datum buffers remain charged between batches until close. + memory_tracker->consume(static_cast<int64_t>(retained) - accounted_scratch); + accounted_scratch = static_cast<int64_t>(retained); + reservations.clear(); + small_reservation_available = 0; + } + + bool timestamp_tz; + bool varbinary; + cctz::time_zone timezone = cctz::utc_time_zone(); + size_t batch_size = 4096; + size_t max_block_bytes = std::numeric_limits<size_t>::max(); + std::unordered_set<int32_t> count_placeholders; + std::unordered_map<int32_t, size_t> projected_ordinals; + std::unique_ptr<::avro::DataFileReaderBase> reader; + std::unique_ptr<::avro::GenericDatum> datum; + std::vector<ColumnDefinition> schema; + int64_t header_end = 0; + int64_t range_end = std::numeric_limits<int64_t>::max(); + int64_t row_position = 0; + size_t next_requested_row = 0; + std::optional<GlobalRowIdContext> global_rowid_context; +}; + +AvroReader::AvroReader(std::shared_ptr<io::FileSystemProperties>& system_properties, + std::unique_ptr<io::FileDescription>& file_description, + std::shared_ptr<io::IOContext> io_ctx, RuntimeProfile* profile, + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary, + std::optional<GlobalRowIdContext> global_rowid_context) + : FileReader(system_properties, file_description, std::move(io_ctx), profile), + _impl(std::make_unique<Impl>()) { + _impl->timestamp_tz = enable_mapping_timestamp_tz; + _impl->varbinary = enable_mapping_varbinary; + _impl->global_rowid_context = global_rowid_context; +} + +AvroReader::~AvroReader() { + (void)close(); +} + +Status AvroReader::init(RuntimeState* state) { + _impl->memory_tracker = thread_context()->thread_mem_tracker_mgr->limiter_mem_tracker_sptr(); + RETURN_IF_ERROR(FileReader::init(state)); + _impl->scratch_accounting_passes = + ADD_COUNTER(_profile, "AvroScratchAccountingPasses", TUnit::UNIT); + if (state != nullptr) { + _impl->timezone = state->timezone_obj(); + _impl->max_block_bytes = state->preferred_block_size_bytes(); + } + static std::once_flag register_map; + std::call_once(register_map, [] { + // Iceberg's non-string-key maps are arrays of key/value records with logicalType=map. + ::avro::CustomLogicalTypeRegistry::instance().registerType("map", [](const std::string&) { + return std::make_shared<::avro::CustomLogicalType>("map"); + }); + }); + try { + ::avro::AllocationCheckScope budget(&Impl::reserve_callback, _impl.get(), + &Impl::checkpoint_callback); + Impl::reserve_callback(_impl.get(), sizeof(NativeInputStream)); + _impl->reader = std::make_unique<::avro::DataFileReaderBase>( + std::make_unique<NativeInputStream>(_tracing_file_reader, _io_ctx)); + const auto root = _impl->reader->dataSchema().root(); + if (root->type() != ::avro::AVRO_RECORD) { + return Status::NotSupported("Avro container root must be a record"); + } + _impl->header_end = _impl->reader->previousSync(); + std::unordered_set<const ::avro::Node*> ancestors; + ColumnDefinition record; + RETURN_IF_ERROR(build_schema(root, "row", 0, _impl->timestamp_tz, _impl->varbinary, + &ancestors, &record)); + _impl->schema = std::move(record.children); + _impl->reconcile_scratch(); + return Status::OK(); + } catch (const doris::Exception& e) { + return e.to_status(); + } catch (const std::bad_alloc&) { + return Status::MemoryLimitExceeded("Avro allocation failed"); + } catch (const std::exception& e) { + return Status::InvalidArgument("Cannot read Avro schema: {}", e.what()); + } +} + +Status AvroReader::get_schema(std::vector<ColumnDefinition>* file_schema) const { + *file_schema = _impl->schema; + if (_impl->global_rowid_context.has_value()) { + auto rowid = global_rowid_column_definition(); + rowid.type = make_nullable(rowid.type); + file_schema->push_back(std::move(rowid)); + } + return Status::OK(); +} + +std::unique_ptr<TableColumnMapper> AvroReader::create_column_mapper( + TableColumnMapperOptions options) const { + return std::make_unique<MaterializedColumnMapper>(options); +} + +// Range ownership and initialization update the same absolute row-position state. +// NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity) +Status AvroReader::open(std::shared_ptr<FileScanRequest> request) { + RETURN_IF_ERROR(FileReader::open(std::move(request))); + try { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_impl->memory_tracker); + ::avro::AllocationCheckScope budget(&Impl::reserve_callback, _impl.get(), + &Impl::checkpoint_callback); + // The mapper communicates per-leaf physical decoding semantics through local projections. + // Apply them before writing into the file block, whose types already carry these markers. + auto apply_projection = [&](const LocalColumnIndex& projection) { + const auto id = projection.local_id(); + if (id >= 0 && static_cast<size_t>(id) < _impl->schema.size()) { + apply_timestamp_projection(&_impl->schema[id], projection); + } + }; + for (const auto& projection : _request->predicate_columns) { + apply_projection(projection); + } + for (const auto& projection : _request->non_predicate_columns) { + apply_projection(projection); + } + // Equality-delete dependencies may promote a COUNT carrier after it was marked. + // Such columns still need their actual values for the predicate to remove rows. + for (const auto id : _request->count_star_placeholder_columns) { + if (std::ranges::none_of(_request->predicate_columns, [&](const auto& projection) { + return projection.column_id() == id; + })) { + _impl->count_placeholders.insert(id.value()); + } + } + std::unordered_set<int32_t> selected; + for (const auto& [id, position] : _request->local_positions) { + if (id.value() >= 0 && !_impl->count_placeholders.contains(id.value())) { + if (static_cast<size_t>(id.value()) >= _impl->schema.size()) { + return Status::InvalidArgument("Unknown Avro file-local column"); + } + selected.insert(id.value()); + } + } + size_t projected = 0; + for (size_t i = 0; i < _impl->schema.size(); ++i) { + if (selected.contains(static_cast<int32_t>(i))) { + _impl->projected_ordinals.emplace(static_cast<int32_t>(i), projected++); + } + } + const int64_t start = _file_description->range_start_offset; + const int64_t size = _file_description->range_size; + if (start < 0 || size < -1 || + (size >= 0 && start > std::numeric_limits<int64_t>::max() - size)) { + return Status::InvalidArgument("Invalid Avro file range"); + } + _impl->range_end = size < 0 ? std::numeric_limits<int64_t>::max() : start + size; + // A range owns blocks whose start lies in [start,end). Count preceding blocks from their + // OCF headers, without decoding payloads, so position deletes retain absolute file rows. + int64_t offset = _impl->header_end; + while (offset < start && std::cmp_less(offset, _tracing_file_reader->size())) { + std::array<uint8_t, 20> bytes; + size_t read = 0; + RETURN_IF_ERROR(_tracing_file_reader->read_at( Review Comment: Fixed in 3ae183546dd. Ordinary ranges now seek directly to the next sync marker. Ranges requiring absolute coordinates scan prefix headers through a coalesced 1 MiB buffer. LateRangesAvoidTinyPrefixReads asserts bounded physical read calls on 1,024 small blocks and verifies the absolute positions and values. Adjacent-range and delete tests still pass. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
