morningman commented on code in PR #69581: URL: https://github.com/apache/doris/pull/69581#discussion_r4389151884
########## be/src/format_v2/lance/lance_reader_helper.cpp: ########## @@ -0,0 +1,802 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_reader_helper.h" + +#include <arrow/array.h> +#include <arrow/builder.h> +#include <arrow/c/bridge.h> +#include <arrow/compute/cast.h> +#include <arrow/extension_type.h> +#include <arrow/type.h> +#include <arrow/util/key_value_metadata.h> +#include <fmt/format.h> +#include <lance/lance.h> + +#include <bit> +#include <limits> +#include <optional> +#include <unordered_set> + +#include "common/config.h" +#include "common/logging.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nothing.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type_serde/arrow_validation.h" +#include "exec/common/endian.h" + +namespace doris::format::lance { + +RuntimeProfile::Counter* add_lance_counter(RuntimeProfile* profile, const char* name, + TUnit::type unit, const char* group) { + // These counters label sections; they do not represent time or a sum of their children. + ADD_CHILD_COUNTER_WITH_LEVEL(profile, group, TUnit::NONE, LANCE_READER_PROFILE, 1); + return ADD_CHILD_COUNTER_WITH_LEVEL(profile, name, unit, group, 1); +} + +namespace { + +constexpr std::string_view ARROW_EXTENSION_NAME = "ARROW:extension:name"; +constexpr std::string_view ARROW_JSON_EXTENSION = "arrow.json"; +constexpr std::string_view LANCE_JSON_EXTENSION = "lance.json"; +constexpr std::string_view LANCE_BFLOAT16_EXTENSION = "lance.bfloat16"; + +enum class LanceExtensionKind { + NONE, + JSON, + BFLOAT16, +}; + +int arrow_time_precision(arrow::TimeUnit::type unit) { + switch (unit) { + case arrow::TimeUnit::SECOND: + return 0; + case arrow::TimeUnit::MILLI: + return 3; + case arrow::TimeUnit::MICRO: + case arrow::TimeUnit::NANO: + return 6; + } + return 6; +} + +// Extract and validate extension names from metadata and registered Arrow types. +Status get_lance_extension(const std::shared_ptr<arrow::Field>& field, + LanceExtensionKind* extension_kind, + std::shared_ptr<arrow::DataType>* storage_type) { + DORIS_CHECK(field != nullptr); + DORIS_CHECK(extension_kind != nullptr); + DORIS_CHECK(storage_type != nullptr); + *extension_kind = LanceExtensionKind::NONE; + *storage_type = field->type(); + + std::optional<std::string> extension_name; + if (field->HasMetadata()) { + const auto metadata_name = field->metadata()->Get(ARROW_EXTENSION_NAME); + if (metadata_name.ok() && !metadata_name.ValueUnsafe().empty()) { + extension_name = metadata_name.ValueUnsafe(); + } + } + if (field->type()->id() == arrow::Type::EXTENSION) { + const auto extension_type = std::dynamic_pointer_cast<arrow::ExtensionType>(field->type()); + if (extension_type == nullptr) { + return Status::InvalidArgument("invalid Arrow extension type for Lance field '{}'", + field->name()); + } + if (extension_name.has_value() && *extension_name != extension_type->extension_name()) { + return Status::InvalidArgument( + "conflicting Arrow extension names for Lance field '{}': '{}' and '{}'", + field->name(), *extension_name, extension_type->extension_name()); + } + extension_name = extension_type->extension_name(); + *storage_type = extension_type->storage_type(); + } + if (!extension_name.has_value()) { + return Status::OK(); + } + + if (*extension_name == ARROW_JSON_EXTENSION) { + if ((*storage_type)->id() != arrow::Type::STRING && + (*storage_type)->id() != arrow::Type::LARGE_STRING) { + return Status::NotSupported( + "Arrow JSON extension for Lance field '{}' requires UTF8 storage, got {}", + field->name(), (*storage_type)->ToString()); + } + *extension_kind = LanceExtensionKind::JSON; + return Status::OK(); + } + if (*extension_name == LANCE_JSON_EXTENSION) { + if ((*storage_type)->id() != arrow::Type::LARGE_BINARY) { + return Status::NotSupported( + "Lance JSON extension for field '{}' requires LARGE_BINARY storage, got {}", + field->name(), (*storage_type)->ToString()); + } + *extension_kind = LanceExtensionKind::JSON; + return Status::OK(); + } + if (*extension_name == LANCE_BFLOAT16_EXTENSION) { + if ((*storage_type)->id() != arrow::Type::FIXED_SIZE_BINARY || + std::static_pointer_cast<arrow::FixedSizeBinaryType>(*storage_type)->byte_width() != + 2) { + return Status::NotSupported( + "Lance BFloat16 extension for field '{}' requires FIXED_SIZE_BINARY(2) " + "storage, got {}", + field->name(), (*storage_type)->ToString()); + } + *extension_kind = LanceExtensionKind::BFLOAT16; + return Status::OK(); + } + return Status::NotSupported("unsupported Lance Arrow extension type '{}' for field '{}'", + *extension_name, field->name()); +} + +// Map Arrow fields recursively, preserving Null leaves in complex types. +Status arrow_field_to_doris_type(const std::shared_ptr<arrow::Field>& field, + DataTypePtr* doris_type) { + const auto nullable_primitive = [&](PrimitiveType type, int precision = 0, int scale = 0, + int len = -1) { + *doris_type = + DataTypeFactory::instance().create_data_type(type, true, precision, scale, len); + return Status::OK(); + }; + + LanceExtensionKind extension_kind; + std::shared_ptr<arrow::DataType> arrow_type; + RETURN_IF_ERROR(get_lance_extension(field, &extension_kind, &arrow_type)); + switch (extension_kind) { + case LanceExtensionKind::JSON: + return nullable_primitive(TYPE_JSONB); + case LanceExtensionKind::BFLOAT16: + return nullable_primitive(TYPE_FLOAT); + case LanceExtensionKind::NONE: + break; + } + if (arrow_type->id() == arrow::Type::DICTIONARY) { + return Status::NotSupported("unsupported Lance Arrow dictionary type for field '{}': {}", + field->name(), arrow_type->ToString()); + } + + switch (arrow_type->id()) { + case arrow::Type::NA: + // Required Null leaves can bypass NullableSerDe through FE schema reconstruction. + // Reject them before Arrow NA buffers reach the Boolean-backed physical SerDe. + if (!field->nullable()) { + return Status::NotSupported("non-nullable Lance Arrow Null field: {}", field->name()); + } + return nullable_primitive(TYPE_NULL); + case arrow::Type::BOOL: + return nullable_primitive(TYPE_BOOLEAN); + case arrow::Type::INT8: + return nullable_primitive(TYPE_TINYINT); + case arrow::Type::UINT8: + case arrow::Type::INT16: + return nullable_primitive(TYPE_SMALLINT); + case arrow::Type::UINT16: + case arrow::Type::INT32: + return nullable_primitive(TYPE_INT); + case arrow::Type::UINT32: + case arrow::Type::INT64: + return nullable_primitive(TYPE_BIGINT); + case arrow::Type::UINT64: + return nullable_primitive(TYPE_LARGEINT); + case arrow::Type::HALF_FLOAT: + case arrow::Type::FLOAT: + return nullable_primitive(TYPE_FLOAT); + case arrow::Type::DOUBLE: + return nullable_primitive(TYPE_DOUBLE); + case arrow::Type::STRING: + case arrow::Type::LARGE_STRING: + return nullable_primitive(TYPE_STRING); + case arrow::Type::BINARY: + case arrow::Type::LARGE_BINARY: + return nullable_primitive(TYPE_VARBINARY, 0, 0, std::numeric_limits<int32_t>::max()); + case arrow::Type::FIXED_SIZE_BINARY: { + const auto binary = std::static_pointer_cast<arrow::FixedSizeBinaryType>(arrow_type); + return nullable_primitive(TYPE_VARBINARY, 0, 0, binary->byte_width()); + } + case arrow::Type::DATE32: + case arrow::Type::DATE64: + return nullable_primitive(TYPE_DATEV2); + case arrow::Type::TIME32: + case arrow::Type::TIME64: { + const auto time = std::static_pointer_cast<arrow::TimeType>(arrow_type); + return nullable_primitive(TYPE_TIMEV2, 0, arrow_time_precision(time->unit())); + } + case arrow::Type::TIMESTAMP: { + const auto timestamp = std::static_pointer_cast<arrow::TimestampType>(arrow_type); + const auto timestamp_type = + timestamp->timezone().empty() ? TYPE_DATETIMEV2 : TYPE_TIMESTAMPTZ; + return nullable_primitive(timestamp_type, 0, arrow_time_precision(timestamp->unit())); + } + case arrow::Type::DURATION: + return nullable_primitive(TYPE_BIGINT); + case arrow::Type::DECIMAL128: + case arrow::Type::DECIMAL256: { + const auto decimal = std::static_pointer_cast<arrow::DecimalType>(arrow_type); + const int precision = decimal->precision(); + const int scale = decimal->scale(); + if (precision <= 0 || precision > arrow::Decimal256Type::kMaxPrecision || scale < 0 || + scale > precision) { + return Status::NotSupported( + "unsupported Lance Arrow decimal type for field '{}': precision={}, scale={}", + field->name(), precision, scale); + } + const PrimitiveType doris_decimal_type = precision <= 9 ? TYPE_DECIMAL32 + : precision <= 18 ? TYPE_DECIMAL64 + : precision <= 38 ? TYPE_DECIMAL128I + : TYPE_DECIMAL256; + return nullable_primitive(doris_decimal_type, precision, scale); + } + case arrow::Type::LIST: + case arrow::Type::LARGE_LIST: + case arrow::Type::FIXED_SIZE_LIST: { + const auto list = std::static_pointer_cast<arrow::BaseListType>(arrow_type); + DataTypePtr value_type; + RETURN_IF_ERROR(arrow_field_to_doris_type(list->value_field(), &value_type)); + *doris_type = make_nullable(std::make_shared<DataTypeArray>(value_type)); + return Status::OK(); + } + case arrow::Type::MAP: { + const auto map = std::static_pointer_cast<arrow::MapType>(arrow_type); + DataTypePtr key_type; + DataTypePtr item_type; + RETURN_IF_ERROR(arrow_field_to_doris_type(map->key_field(), &key_type)); + RETURN_IF_ERROR(arrow_field_to_doris_type(map->item_field(), &item_type)); + *doris_type = make_nullable(std::make_shared<DataTypeMap>(key_type, item_type)); + return Status::OK(); + } + case arrow::Type::STRUCT: { + const auto struct_type = std::static_pointer_cast<arrow::StructType>(arrow_type); + DataTypes field_types; + Strings field_names; + field_types.reserve(struct_type->num_fields()); + field_names.reserve(struct_type->num_fields()); + for (const auto& child : struct_type->fields()) { + DataTypePtr field_type; + RETURN_IF_ERROR(arrow_field_to_doris_type(child, &field_type)); + field_types.emplace_back(std::move(field_type)); + field_names.emplace_back(child->name()); + } + *doris_type = make_nullable(std::make_shared<DataTypeStruct>(field_types, field_names)); + return Status::OK(); + } + default: + return Status::NotSupported("unsupported Lance Arrow type: {}", arrow_type->ToString()); + } +} + +// Widen little-endian Lance BFloat16 values to Arrow Float32 without precision loss. +Status convert_bfloat16_array(const std::shared_ptr<arrow::Array>& array, + arrow::MemoryPool* memory_pool, + std::shared_ptr<arrow::Array>* normalized) { + DORIS_CHECK(array != nullptr); + DORIS_CHECK(memory_pool != nullptr); + DORIS_CHECK(normalized != nullptr); + const auto fixed_binary = std::dynamic_pointer_cast<arrow::FixedSizeBinaryArray>(array); + if (fixed_binary == nullptr || fixed_binary->byte_width() != 2) { + return Status::InvalidArgument("invalid Lance BFloat16 array storage: {}", + array->type()->ToString()); + } + if (config::enable_arrow_input_validation) { + check_arrow_fixed_width_buffer(*fixed_binary, sizeof(uint16_t)); + } + + arrow::FloatBuilder builder(memory_pool); + auto arrow_status = builder.Reserve(fixed_binary->length()); + if (!arrow_status.ok()) { + return Status::InternalError("reserve Lance BFloat16 output failed: {}", + arrow_status.message()); + } + for (int64_t row = 0; row < fixed_binary->length(); ++row) { + if (fixed_binary->IsNull(row)) { + arrow_status = builder.AppendNull(); + } else { + const auto bits = LittleEndian::Load16(fixed_binary->GetValue(row)); + arrow_status = builder.Append(std::bit_cast<float>(static_cast<uint32_t>(bits) << 16)); + } + if (!arrow_status.ok()) { + return Status::InternalError("append Lance BFloat16 value failed: {}", + arrow_status.message()); + } + } + std::shared_ptr<arrow::FloatArray> result; + arrow_status = builder.Finish(&result); + if (!arrow_status.ok()) { + return Status::InternalError("finish Lance BFloat16 conversion failed: {}", + arrow_status.message()); + } + *normalized = std::move(result); + return Status::OK(); +} + +// Narrow a Decimal256 array whose declared precision fits Decimal128. The safe cast keeps every +// in-precision value and its scale, and rejects a value that exceeds the declared precision +// instead of truncating it: unscaled 10^18 stored in a decimal256(18, 2) field fails with +// "Decimal value does not fit in precision 18". +Status narrow_decimal256_array(std::string_view field_name, + const std::shared_ptr<arrow::Array>& array, + const std::shared_ptr<arrow::DataType>& decimal128_type, + arrow::MemoryPool* memory_pool, + std::shared_ptr<arrow::Array>* normalized) { + DORIS_CHECK(array != nullptr); + DORIS_CHECK(memory_pool != nullptr); + DORIS_CHECK(normalized != nullptr); + arrow::compute::ExecContext exec_context(memory_pool); + auto narrowed = arrow::compute::Cast(*array, decimal128_type, + arrow::compute::CastOptions::Safe(), &exec_context); + if (!narrowed.ok()) { + return Status::InvalidArgument("narrow Lance Decimal256 field '{}' to {} failed: {}", + field_name, decimal128_type->ToString(), + narrowed.status().message()); + } + *normalized = std::move(narrowed).ValueUnsafe(); + return Status::OK(); +} + +// Rebuild a nested Arrow type after one or more child arrays changed physical type. +Status set_lance_nested_type(std::string_view field_name, + const std::shared_ptr<arrow::DataType>& source_type, + const arrow::FieldVector& child_fields, + std::shared_ptr<arrow::ArrayData>* data) { + switch (source_type->id()) { + case arrow::Type::LIST: + (*data)->type = arrow::list(child_fields[0]); + break; + case arrow::Type::LARGE_LIST: + (*data)->type = arrow::large_list(child_fields[0]); + break; + case arrow::Type::FIXED_SIZE_LIST: + (*data)->type = arrow::fixed_size_list( + child_fields[0], + std::static_pointer_cast<arrow::FixedSizeListType>(source_type)->list_size()); + break; + case arrow::Type::STRUCT: + (*data)->type = arrow::struct_(child_fields); + break; + case arrow::Type::MAP: { + const auto map_type = std::static_pointer_cast<arrow::MapType>(source_type); + auto normalized_type = arrow::MapType::Make(child_fields[0], map_type->keys_sorted()); + if (!normalized_type.ok()) { + return Status::InvalidArgument("normalize Lance map field '{}' failed: {}", field_name, + normalized_type.status().message()); + } + (*data)->type = std::move(normalized_type).ValueUnsafe(); + break; + } + default: + return Status::InvalidArgument("Lance field '{}' has unexpected child-bearing type {}", + field_name, source_type->ToString()); + } + return Status::OK(); +} + +// Materialize the visible range into an offset-zero Arrow array for Doris SerDes. +Status compact_lance_array(const std::shared_ptr<arrow::Array>& array, + arrow::MemoryPool* memory_pool, + std::shared_ptr<arrow::Array>* compacted) { + DORIS_CHECK(memory_pool != nullptr); + const auto validation = array->ValidateFull(); + if (!validation.ok()) { + return Status::InvalidArgument("validate sliced Lance array failed: {}", + validation.message()); + } + auto builder_result = arrow::MakeBuilder(array->type(), memory_pool); + if (!builder_result.ok()) { + return Status::InternalError("create sliced Lance array builder failed: {}", + builder_result.status().message()); + } + auto builder = std::move(builder_result).ValueUnsafe(); + auto arrow_status = builder->Reserve(array->length()); + if (!arrow_status.ok()) { + return Status::InternalError("reserve sliced Lance array builder failed: {}", + arrow_status.message()); + } + arrow_status = builder->AppendArraySlice(*array->data(), 0, array->length()); + if (!arrow_status.ok()) { + return Status::InternalError("copy sliced Lance array failed: {}", arrow_status.message()); + } + arrow_status = builder->Finish(compacted); + if (!arrow_status.ok()) { + return Status::InternalError("finish sliced Lance array copy failed: {}", + arrow_status.message()); + } + if ((*compacted)->offset() != 0) { + return Status::InternalError("compacted Lance array retained offset {}", + (*compacted)->offset()); + } + return Status::OK(); +} + +// Compact a sliced variable-offset parent and all of its visible descendants. +template <typename ArrayType> +Status compact_lance_offset_array(const std::shared_ptr<arrow::Array>& array, + arrow::MemoryPool* memory_pool, + std::shared_ptr<arrow::Array>* compacted) { + const auto offset_array = std::dynamic_pointer_cast<ArrayType>(array); + if (offset_array == nullptr) { + return Status::InvalidArgument("invalid sliced Lance offset array: {}", + array->type()->ToString()); + } + const auto child_begin = static_cast<int64_t>(offset_array->value_offset(0)); + const auto child_end = static_cast<int64_t>(offset_array->value_offset(offset_array->length())); + const auto& values = offset_array->values(); + if (child_begin < 0 || child_end < child_begin || child_end > values->length()) { + return Status::InvalidArgument("invalid sliced Lance offsets [{}, {}) for child length {}", + child_begin, child_end, values->length()); + } + if (array->offset() == 0 && child_begin == 0 && child_end == values->length()) { + *compacted = array; + return Status::OK(); + } + return compact_lance_array(array, memory_pool, compacted); +} + +// Compact sliced arrays and nested children only when Doris cannot consume their current layout. +Status compact_lance_array_if_needed(const std::shared_ptr<arrow::Array>& array, + arrow::MemoryPool* memory_pool, + std::shared_ptr<arrow::Array>* compacted) { + switch (array->type_id()) { + case arrow::Type::LIST: + return compact_lance_offset_array<arrow::ListArray>(array, memory_pool, compacted); + case arrow::Type::LARGE_LIST: + return compact_lance_offset_array<arrow::LargeListArray>(array, memory_pool, compacted); + case arrow::Type::MAP: + return compact_lance_offset_array<arrow::MapArray>(array, memory_pool, compacted); + case arrow::Type::FIXED_SIZE_LIST: { + const auto list = std::dynamic_pointer_cast<arrow::FixedSizeListArray>(array); + if (list == nullptr) { + return Status::InvalidArgument("invalid sliced Lance fixed-size list array: {}", + array->type()->ToString()); + } + const auto child_begin = list->value_offset(0); + const auto child_length = list->length() * list->value_length(); + const auto& values = list->values(); + if (child_begin < 0 || child_length < 0 || child_begin > values->length() - child_length) { + return Status::InvalidArgument( + "invalid sliced Lance fixed-size list range [{}, {}) for child length {}", + child_begin, child_begin + child_length, values->length()); + } + if (array->offset() == 0 && child_begin == 0 && child_length == values->length()) { + *compacted = array; + return Status::OK(); + } + return compact_lance_array(array, memory_pool, compacted); + } + case arrow::Type::STRUCT: { + const auto struct_array = std::dynamic_pointer_cast<arrow::StructArray>(array); + if (struct_array == nullptr) { + return Status::InvalidArgument("invalid sliced Lance struct array: {}", + array->type()->ToString()); + } + bool requires_compaction = array->offset() != 0; + for (const auto& child : array->data()->child_data) { + requires_compaction |= child->length != array->length(); Review Comment: Confirmed. The pinned v1 plain decoder returns a Boolean range with bit offset `start % 8`, and `read_struct_array` / `read_list_array` wrap that child unchanged, so a struct or list that is tight at its own level can reach the SerDes with an offset descendant, which `check_arrow_array_range()` rejects under the default Arrow input validation. The shortcut missed more than direct struct children: list, map and fixed-size-list values and deeper descendants were not checked either, and `LIST<BOOLEAN>` gets an offset child whenever a writer batch is larger than `batch_size`. Fixed in ff6fbfe9989: an array is passed through without a copy only when it and every descendant start at offset 0 and reference exactly their child ranges; otherwise it is compacted. The new unit tests build that layout directly (STRUCT, LIST, MAP, STRUCT<STRUCT> and STRUCT<LIST> over a Boolean child at offset 3) and read it through the record batch converter with validation on, rather than writing a legacy dat aset; arrays that are offset-free at every level keep the zero-copy path. branch-4.1 and branch-4.2 carry the same normalizer. ########## be/src/format_v2/lance/lance_record_batch_converter.cpp: ########## @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_record_batch_converter.h" + +#include <arrow/array.h> +#include <arrow/memory_pool.h> +#include <arrow/record_batch.h> +#include <arrow/type.h> + +#include <limits> + +#include "common/consts.h" +#include "common/exception.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_nullable.h" +#include "format_v2/lance/lance_reader_helper.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "storage/utils.h" + +namespace doris::format::lance { +namespace { + +constexpr size_t IGNORED_OUTPUT_INDEX = std::numeric_limits<size_t>::max(); + +} // namespace + +arrow::MemoryPool* LanceRecordBatchConverter::_query_arrow_memory_pool( + RuntimeState* runtime_state) { + DORIS_CHECK(runtime_state != nullptr); + if (runtime_state->exec_env() != nullptr) { + if (auto* memory_pool = runtime_state->exec_env()->arrow_memory_pool(); + memory_pool != nullptr) { + return memory_pool; + } + } + return arrow::default_memory_pool(); +} + +Status LanceRecordBatchConverter::init(RuntimeState* runtime_state, + const std::vector<ColumnDefinition>& projected_columns, + SearchKind search_kind) { + DORIS_CHECK(runtime_state != nullptr); + _search_kind = search_kind; + _timezone = runtime_state->timezone_obj(); + _memory_pool = _query_arrow_memory_pool(runtime_state); + _output_columns.clear(); + _output_columns.reserve(projected_columns.size()); + _output_index_by_input_name.clear(); + _output_index_by_input_name.reserve(projected_columns.size()); + _global_rowid_output_index.reset(); + reset_schema(); + + for (const auto& column : projected_columns) { + if (column.type == nullptr) { + return Status::InvalidArgument("Lance projected column '{}' has no type", column.name); + } + if (column.name.starts_with(BeConsts::GLOBAL_ROWID_COL)) { + if (_search_kind == SearchKind::NORMAL) { + return Status::NotSupported( + "Lance global row id is currently supported only for external search"); + } + if (_global_rowid_output_index.has_value()) { + return Status::InvalidArgument("duplicate Lance global row id projected column: {}", + column.name); + } + if (remove_nullable(column.type)->get_primitive_type() != TYPE_STRING) { + return Status::InvalidArgument( + "Lance global row id column '{}' must have Doris STRING type, but was {}", + column.name, column.type->get_name()); + } + _global_rowid_output_index = _output_columns.size(); + // Lance returns the native `_rowid`, while Doris exposes it through a generated + // global-rowid column. One input column cannot also satisfy a regular `_rowid` + // projection, so reject an existing binding instead of silently replacing it. + if (!_output_index_by_input_name + .emplace(LANCE_ROW_ID_COLUMN, *_global_rowid_output_index) + .second) { + return Status::InvalidArgument( + "Lance global row id cannot be combined with projected column '{}'", + LANCE_ROW_ID_COLUMN); + } + _output_columns.push_back({column.name, nullptr}); + continue; + } + const auto output_index = _output_columns.size(); + if (!_output_index_by_input_name.emplace(column.name, output_index).second) { + return Status::InvalidArgument("duplicate Lance projected column: {}", column.name); + } + if (_search_kind == SearchKind::VECTOR && column.name == LANCE_DISTANCE_COLUMN && + remove_nullable(column.type)->get_primitive_type() != TYPE_FLOAT) { + return Status::InvalidArgument( + "Lance vector search column '{}' must have Doris FLOAT type, but was {}", + LANCE_DISTANCE_COLUMN, column.type->get_name()); + } + if (_search_kind == SearchKind::FULL_TEXT && column.name == LANCE_SCORE_COLUMN && + remove_nullable(column.type)->get_primitive_type() != TYPE_FLOAT) { + return Status::InvalidArgument( + "Lance full-text search column '{}' must have Doris FLOAT type, but was {}", + LANCE_SCORE_COLUMN, column.type->get_name()); + } + _output_columns.push_back({column.name, column.type->get_serde()}); + } + return Status::OK(); +} + +Status LanceRecordBatchConverter::_bind_schema(const std::shared_ptr<arrow::Schema>& schema) { + DORIS_CHECK(schema != nullptr); + if (_bound_schema == schema || + (_bound_schema != nullptr && _bound_schema->Equals(*schema, true))) { + return Status::OK(); + } + + std::vector<size_t> arrow_column_to_output(static_cast<size_t>(schema->num_fields()), + IGNORED_OUTPUT_INDEX); + std::vector<LanceArrowArrayNormalizer> array_normalizer_for_arrow_column( + static_cast<size_t>(schema->num_fields())); + std::vector<bool> output_has_source_column(_output_columns.size(), false); + for (int arrow_index = 0; arrow_index < schema->num_fields(); ++arrow_index) { + const auto& field = schema->field(arrow_index); + const auto output = _output_index_by_input_name.find(field->name()); + if (output == _output_index_by_input_name.end()) { + const bool is_omitted_search_result = + (_search_kind == SearchKind::VECTOR && + field->name() == LANCE_DISTANCE_COLUMN) || + (_search_kind == SearchKind::FULL_TEXT && field->name() == LANCE_SCORE_COLUMN); + // Lance may return this generated result even when Doris did not project it. + if (is_omitted_search_result) { + continue; + } + return Status::InternalError("Lance returned unknown column '{}'", field->name()); + } + const auto output_index = output->second; + if (output_has_source_column[output_index]) { + return Status::InternalError("Lance returned duplicate column '{}'", field->name()); + } + output_has_source_column[output_index] = true; + arrow_column_to_output[arrow_index] = output_index; + RETURN_IF_ERROR(LanceArrowArrayNormalizer::create( + field, &array_normalizer_for_arrow_column[static_cast<size_t>(arrow_index)])); + } + for (size_t output_index = 0; output_index < _output_columns.size(); ++output_index) { + if (!output_has_source_column[output_index]) { + return Status::InternalError("Lance did not return requested column '{}'", + _output_columns[output_index].name); + } + } + _bound_schema = schema; + _arrow_column_to_output_index = std::move(arrow_column_to_output); + _array_normalizer_for_arrow_column = std::move(array_normalizer_for_arrow_column); + return Status::OK(); +} + +Status LanceRecordBatchConverter::convert_record_batch_to_block( + const std::shared_ptr<arrow::RecordBatch>& record_batch, Block* block, + const std::optional<GlobalRowIdContext>& global_rowid_context, size_t* rows) { + DORIS_CHECK(record_batch != nullptr); + DORIS_CHECK(block != nullptr); + DORIS_CHECK(rows != nullptr); + RETURN_IF_ERROR(_bind_schema(record_batch->schema())); + + const auto row_count = static_cast<size_t>(record_batch->num_rows()); + auto columns_guard = block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + for (int arrow_index = 0; arrow_index < record_batch->num_columns(); ++arrow_index) { + const auto output_index = _arrow_column_to_output_index[static_cast<size_t>(arrow_index)]; + if (output_index == IGNORED_OUTPUT_INDEX) { + continue; + } + const auto& output = _output_columns[output_index]; + const auto& arrow_column = record_batch->column(arrow_index); + if (is_global_rowid_output(output_index)) { + if (!global_rowid_context.has_value()) { + return Status::InvalidArgument( + "Lance global row id requested without global row id context"); + } + RETURN_IF_ERROR(_append_global_row_ids(arrow_column, columns[output_index], + *global_rowid_context)); + continue; + } + const auto& field = record_batch->schema()->field(arrow_index); + try { + if (arrow_column->type_id() == arrow::Type::NA) { + columns[output_index]->insert_many_defaults(row_count); + continue; + } + std::shared_ptr<arrow::Array> normalized_column; + RETURN_IF_ERROR( + _array_normalizer_for_arrow_column[static_cast<size_t>(arrow_index)] + .normalize_for_doris(arrow_column, _memory_pool, &normalized_column)); + RETURN_IF_ERROR(output.serde->read_column_from_arrow( Review Comment: Confirmed, and it is not specific to Lance: `DataTypeDateTimeV2SerDe::read_column_from_arrow` packs the civil year from `from_unixtime()` into `unchecked_set_time(uint16_t year, ...)` without a range check, so ADBC, Arrow stream load, paimon-rust and remote Doris reads return the same `0001-01-01` for 2005980768000000000 us, and years 10000..65535 come back as non-null values outside the DATETIME range. The TIMESTAMPTZ Arrow path and the UTC-adjusted Parquet paths call `is_valid_date()` only after packing, which a wrapped year passes. branch-4.1 and branch-4.2 have the same code. Unlike the DATE32 check in this PR, which ports a fix branch-4.1 / 4.2 already have (#68649), this one is missing on every branch, and the change belongs in the shared SerDe used by all of those readers. It goes in a separate PR that checks the civil year before narrowing, as the ORC path does since #68516, for the Arrow and Parquet timestamp paths, with first-out-of-range, year-wrap, pre-0000 and nested cases. This PR does not touch that SerDe. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
