eldenmoon commented on code in PR #65561: URL: https://github.com/apache/doris/pull/65561#discussion_r3575584248
########## be/src/core/column/column_variant_v2.cpp: ########## @@ -0,0 +1,1651 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/column/column_variant_v2.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <cstdint> +#include <limits> +#include <string_view> +#include <typeinfo> +#include <utility> + +#include "common/check.h" +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/column/column_const.h" +#include "core/column/column_decimal.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_variant_v2_typed_scalar.h" +#include "core/column/column_vector.h" +#include "core/column/columns_common.h" +#include "core/custom_allocator.h" +#include "util/hash_util.hpp" +#include "util/variant/variant_canonical.h" +#include "util/variant/variant_encoded_block.h" +#include "util/variant/variant_scalar_encoding.h" + +namespace doris { +namespace { + +using MetaIdsColumn = ColumnVector<TYPE_UINT32>; +using column_variant_v2_internal::TypedFallbackKind; +using column_variant_v2_internal::dispatch_typed_column; +using column_variant_v2_internal::exact_typed_identity; +using column_variant_v2_internal::is_supported_typed_identity; +using column_variant_v2_internal::validate_typed_decimal_scale; +using column_variant_v2_internal::visit_typed_canonical_rows; +using column_variant_v2_internal::visit_typed_rows; +constexpr uint32_t UNMAPPED_METADATA_ID = std::numeric_limits<uint32_t>::max(); +constexpr size_t CANONICAL_SIZE_PREFIX = sizeof(uint32_t); +constexpr std::array<char, 3> EMPTY_OBJECT_METADATA {static_cast<char>(0x11), 0, 0}; +constexpr std::array<char, 3> EMPTY_OBJECT_VALUE {static_cast<char>(0x02), 0, 0}; + +uint32_t read_canonical_payload_size(const char* pos) { + DCHECK(pos != nullptr); + uint32_t payload_size = 0; + for (uint8_t byte = 0; byte < CANONICAL_SIZE_PREFIX; ++byte) { + payload_size |= static_cast<uint32_t>(static_cast<uint8_t>(pos[byte])) << (byte * 8); + } + return payload_size; +} + +size_t trusted_canonical_cell_size(const char* pos) { + return CANONICAL_SIZE_PREFIX + read_canonical_payload_size(pos); +} + +void check_hash_range(const ColumnVariantV2& column, size_t start, size_t end) { + DCHECK_LE(start, end); + DCHECK_LE(end, column.size()); +} + +template <typename Sink, typename Hash> +void update_canonical_hash_range(const ColumnVariantV2& column, size_t start, size_t end, + Hash& hash, const uint8_t* __restrict null_data) { + check_hash_range(column, start, end); + Sink sink(hash); + for (size_t row = start; row < end; ++row) { + if (null_data == nullptr || null_data[row] == 0) { + canonical_hash(column.get_value_ref(row), sink); + } + } + hash = sink.digest(); +} + +template <typename Sink, typename Hash> +void update_canonical_hashes(const ColumnVariantV2& column, Hash* __restrict hashes, + const uint8_t* __restrict null_data) { + for (size_t row = 0; row < column.size(); ++row) { + if (null_data == nullptr || null_data[row] == 0) { + Sink sink(hashes[row]); + canonical_hash(column.get_value_ref(row), sink); + hashes[row] = sink.digest(); + } + } +} + +void validate_offsets(StringRef bytes, std::span<const uint32_t> offsets, + std::string_view description) { + DORIS_CHECK(bytes.data != nullptr || bytes.size == 0) + << description << " bytes have a null pointer"; + DORIS_CHECK(!offsets.empty()) << description << " offsets cannot be empty"; + DORIS_CHECK_EQ(offsets.front(), 0) << description << " offsets must start at zero"; + DORIS_CHECK_EQ(static_cast<size_t>(offsets.back()), bytes.size) + << description << " offsets must end at the byte size"; + for (size_t index = 1; index < offsets.size(); ++index) { + DORIS_CHECK_LT(offsets[index - 1], offsets[index]) + << description << " offsets must be strictly increasing"; + } +} + +void require_exclusive(const IColumn::WrappedPtr& column, std::string_view description) { + const auto& immutable = static_cast<const IColumn::Ptr&>(column); + DORIS_CHECK(immutable->is_exclusive()) + << "ColumnVariantV2 " << description << " must be COW-detached before mutation"; +} + +void reserve_rows(ColumnString& values, MetaIdsColumn& metadata_ids, size_t value_bytes, + size_t rows) { + const size_t final_value_bytes = values.get_chars().size() + value_bytes; + ColumnString::check_chars_length(final_value_bytes, values.size() + rows, values.size()); + values.get_chars().reserve(final_value_bytes); + values.get_offsets().reserve(values.size() + rows); + metadata_ids.get_data().reserve(metadata_ids.size() + rows); +} + +void append_metadata_ids(MetaIdsColumn& destination, const DorisVector<uint32_t>& ids) { + auto& data = destination.get_data(); + const size_t old_size = data.size(); + data.resize(old_size + ids.size()); + std::ranges::copy(ids, data.begin() + old_size); +} + +struct SelectedRows { + DorisVector<uint32_t> metadata_ids; + size_t value_bytes = 0; +}; + +size_t validate_selected_indices(const uint32_t* indices_begin, const uint32_t* indices_end, + size_t source_rows) { + if (indices_begin == indices_end) { + return 0; + } + DORIS_CHECK(indices_begin != nullptr && indices_end != nullptr) + << "non-empty source indices cannot be null"; + DORIS_CHECK_LT(indices_begin, indices_end) << "source index range is reversed"; + const size_t rows = indices_end - indices_begin; + DORIS_CHECK_LT(*std::max_element(indices_begin, indices_end), source_rows) + << "source index is out of range"; + return rows; +} + +DorisVector<uint32_t> collect_range_metadata_ids( + const MetaIdsColumn::Container& source_metadata_ids, size_t source_metadata_count, + size_t start, size_t length) { + DorisVector<uint32_t> selected(length); + for (size_t row = 0; row < length; ++row) { + const uint32_t metadata_id = source_metadata_ids[start + row]; + DCHECK_LT(metadata_id, source_metadata_count) << "source metadata id is out of range"; + selected[row] = metadata_id; + } + return selected; +} + +SelectedRows collect_selected_rows(const MetaIdsColumn::Container& source_metadata_ids, + const ColumnString& source_metadatas, + const ColumnString& source_values, const uint32_t* indices_begin, + size_t rows) { + SelectedRows selected; + selected.metadata_ids.resize(rows); + for (size_t row = 0; row < rows; ++row) { + const uint32_t source_row = indices_begin[row]; + const uint32_t metadata_id = source_metadata_ids[source_row]; + DCHECK_LT(metadata_id, source_metadatas.size()) << "source metadata id is out of range"; + selected.metadata_ids[row] = metadata_id; + const size_t row_bytes = source_values.get_data_at(source_row).size; + DORIS_CHECK_LE(row_bytes, std::numeric_limits<size_t>::max() - selected.value_bytes) + << "selected Variant value bytes overflow size_t"; + selected.value_bytes += row_bytes; + } + return selected; +} + +template <typename Sink, typename Hash> +void update_typed_hashes(const ColumnNullable& nullable, PrimitiveType type, uint32_t scale, + Hash* __restrict hashes, const uint8_t* __restrict null_data) { + dispatch_typed_column(nullable, type, [&]<PrimitiveType Type>(const auto& column) { + visit_typed_canonical_rows<Type>(nullable, column, scale, 0, nullable.size(), + [&](size_t row, auto&& canonical_factory) { + if (null_data == nullptr || null_data[row] == 0) { + VariantCanonicalScalarRef canonical = + canonical_factory(); + Sink sink(hashes[row]); + canonical_hash(canonical, sink); + hashes[row] = sink.digest(); + } + }); + }); +} + +template <typename Sink, typename Hash> +void update_typed_hash_range(const ColumnNullable& nullable, PrimitiveType type, uint32_t scale, + size_t start, size_t end, Hash& hash, + const uint8_t* __restrict null_data) { + Sink sink(hash); + dispatch_typed_column(nullable, type, [&]<PrimitiveType Type>(const auto& column) { + visit_typed_canonical_rows<Type>( + nullable, column, scale, start, end, [&](size_t row, auto&& canonical_factory) { + if (null_data == nullptr || null_data[row] == 0) { + VariantCanonicalScalarRef canonical = canonical_factory(); + canonical_hash(canonical, sink); + } + }); + }); + hash = sink.digest(); +} + +struct TypedEncodingResult { + ColumnString::MutablePtr metadatas; + MetaIdsColumn::MutablePtr metadata_ids; + ColumnString::MutablePtr values; + ColumnVariantV2::TypedEncodingStats stats; +}; + +void count_fallback(TypedFallbackKind fallback, ColumnVariantV2::TypedEncodingStats& stats) { + switch (fallback) { + case TypedFallbackKind::NONE: + return; + case TypedFallbackKind::LARGEINT: + ++stats.largeint_string_fallback_rows; + return; + case TypedFallbackKind::DECIMAL256: + ++stats.decimal256_string_fallback_rows; + return; + case TypedFallbackKind::IP: + ++stats.ip_string_fallback_rows; + return; + } +} + +template <PrimitiveType Type, typename Column> +TypedEncodingResult encode_typed_column(const ColumnNullable& nullable, const Column& column, + uint32_t scale) { + size_t value_bytes = 0; + TypedEncodingResult result; + visit_typed_rows<Type>( + nullable, column, scale, 0, nullable.size(), + [&](size_t, auto&& physical_factory, auto&&, TypedFallbackKind fallback) { + const VariantScalarEncodingPlan plan = physical_factory(); + if (plan.size() > std::numeric_limits<size_t>::max() - value_bytes) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Typed Variant values exceed addressable size"); + } + value_bytes += plan.size(); + count_fallback(fallback, result.stats); + }); + ColumnString::check_chars_length(value_bytes, nullable.size(), 0); + + result.metadatas = ColumnString::create(); + result.metadata_ids = MetaIdsColumn::create(); + result.values = ColumnString::create(); + if (!nullable.empty()) { + result.metadatas->insert_data(EMPTY_OBJECT_METADATA.data(), EMPTY_OBJECT_METADATA.size()); + } + result.metadata_ids->get_data().resize(nullable.size()); + std::fill(result.metadata_ids->get_data().begin(), result.metadata_ids->get_data().end(), 0); + auto& chars = result.values->get_chars(); + auto& offsets = result.values->get_offsets(); + chars.resize(value_bytes); + offsets.resize(nullable.size()); + + size_t offset = 0; + visit_typed_rows<Type>(nullable, column, scale, 0, nullable.size(), + [&](size_t row, auto&& physical_factory, auto&&, TypedFallbackKind) { + const VariantScalarEncodingPlan plan = physical_factory(); + plan.write(reinterpret_cast<char*>(chars.data()) + offset, + plan.size()); + offset += plan.size(); + offsets[row] = static_cast<ColumnString::Offset>(offset); + }); + DCHECK_EQ(offset, value_bytes); + return result; +} + +struct ValidatedTypedInput { + MutableColumnPtr column; + DataTypePtr type; +}; + +ValidatedTypedInput validate_typed_input(ColumnPtr column, DataTypePtr scalar_type) { + DORIS_CHECK(static_cast<bool>(column)) << "typed ColumnVariantV2 column must not be null"; + DORIS_CHECK(scalar_type != nullptr) << "typed ColumnVariantV2 type must not be null"; + DORIS_CHECK(!scalar_type->is_nullable()) + << "typed ColumnVariantV2 requires a non-nullable scalar type"; + DORIS_CHECK(check_and_get_column<ColumnConst>(column.get()) == nullptr) + << "typed ColumnVariantV2 input must not be ColumnConst"; + const IColumn* input_column = column.get(); + DORIS_CHECK(typeid(*input_column) == typeid(ColumnNullable)) + << "typed ColumnVariantV2 input must be an exact ColumnNullable"; + + const auto& nullable = assert_cast<const ColumnNullable&>(*column); + const IColumn& nested = nullable.get_nested_column(); + DORIS_CHECK_EQ(nested.size(), nullable.get_null_map_column().size()) + << "typed ColumnVariantV2 null map size does not match nested column size"; + const PrimitiveType type = scalar_type->get_primitive_type(); + DORIS_CHECK(is_supported_typed_identity(type)) + << "unsupported typed identity " << scalar_type->get_name(); + MutableColumnPtr expected = scalar_type->create_column(); + const IColumn* expected_column = expected.get(); + DORIS_CHECK(typeid(nested) == typeid(*expected_column)) + << "typed nested column " << nested.get_name() << " does not match data type " + << scalar_type->get_name(); + validate_typed_decimal_scale(nested, type, scalar_type->get_scale()); + + return {.column = IColumn::mutate(std::move(column)), .type = std::move(scalar_type)}; +} + +[[noreturn]] void throw_deferred(std::string_view method, std::string_view task) { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, + "ColumnVariantV2::{} is deferred until {} is complete", method, task); +} + +[[noreturn]] void throw_unsupported(std::string_view method) { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, + "ColumnVariantV2::{} is intentionally unsupported for Variant values", method); +} + +} // namespace + +struct ColumnVariantV2::MetaDictIndex { + using Candidates = DorisVector<uint32_t>; + using Buckets = DorisMap<uint64_t, Candidates>; + + size_t allocated_bytes() const { + // std::map does not expose node allocation sizes. Count the logical node payload and a + // conservative four-pointer tree overhead; the actual allocations are independently + // charged through CustomStdAllocator. + size_t bytes = + sizeof(*this) + buckets.size() * (sizeof(Buckets::value_type) + 4 * sizeof(void*)); + for (const auto& entry : buckets) { + bytes += entry.second.capacity() * sizeof(uint32_t); + } + return bytes; + } + + Buckets buckets; +}; + +#ifdef BE_TEST +void ColumnVariantV2::TestAccess::reset_metadata_index(ColumnVariantV2& column) { + column._meta_index.reset(); +} + +void ColumnVariantV2::TestAccess::replace_encoded_subcolumn(ColumnVariantV2& column, size_t index, + ColumnPtr replacement) { + DORIS_CHECK(!column._typed); + switch (index) { + case 0: + static_cast<IColumn::Ptr&>(column._metadatas) = std::move(replacement); + break; + case 1: + static_cast<IColumn::Ptr&>(column._meta_ids) = std::move(replacement); + break; + case 2: + static_cast<IColumn::Ptr&>(column._values) = std::move(replacement); + break; + default: + DORIS_CHECK(false) << "ColumnVariantV2 test subcolumn index is out of range"; + } + column._meta_index.reset(); +} +#endif + +ColumnVariantV2::ColumnVariantV2() + : _metadatas(ColumnString::create()), + _meta_ids(MetaIdsColumn::create()), + _values(ColumnString::create()) { + _check_invariants(); +} + +ColumnVariantV2::ColumnVariantV2(const ColumnVariantV2& other) + : _metadatas(other._metadatas), + _meta_ids(other._meta_ids), + _values(other._values), + _typed(other._typed), + _typed_type(other._typed_type) {} + +ColumnVariantV2::~ColumnVariantV2() = default; + +ColumnVariantV2::MutablePtr ColumnVariantV2::create_typed_from_scan(ColumnPtr column, + DataTypePtr scalar_type) { + ValidatedTypedInput input = validate_typed_input(std::move(column), std::move(scalar_type)); + auto result = ColumnVariantV2::create(); + static_cast<IColumn::Ptr&>(result->_typed) = std::move(input.column); + result->_typed_type = std::move(input.type); + result->_check_invariants(); + return result; +} + +ColumnVariantV2::MutablePtr ColumnVariantV2::create_typed_from_element_at(ColumnPtr column, + DataTypePtr scalar_type) { + ValidatedTypedInput input = validate_typed_input(std::move(column), std::move(scalar_type)); + auto result = ColumnVariantV2::create(); + static_cast<IColumn::Ptr&>(result->_typed) = std::move(input.column); + result->_typed_type = std::move(input.type); + result->_check_invariants(); + return result; +} + +ColumnVariantV2::MutablePtr ColumnVariantV2::create_typed_from_cast(ColumnPtr column, + DataTypePtr scalar_type) { + ValidatedTypedInput input = validate_typed_input(std::move(column), std::move(scalar_type)); + auto result = ColumnVariantV2::create(); + static_cast<IColumn::Ptr&>(result->_typed) = std::move(input.column); + result->_typed_type = std::move(input.type); + result->_check_invariants(); + return result; +} + +ColumnVariantV2::MutablePtr ColumnVariantV2::create_typed_from_exchange(ColumnPtr column, + DataTypePtr scalar_type) { + ValidatedTypedInput input = validate_typed_input(std::move(column), std::move(scalar_type)); + auto result = ColumnVariantV2::create(); + static_cast<IColumn::Ptr&>(result->_typed) = std::move(input.column); + result->_typed_type = std::move(input.type); + result->_check_invariants(); + return result; +} + +const IColumn& ColumnVariantV2::typed_column() const { + DORIS_CHECK(_typed != nullptr) << "typed_column requires ColumnVariantV2 typed state"; + return *_typed; +} + +const DataTypePtr& ColumnVariantV2::typed_type() const { + DORIS_CHECK(_typed_type != nullptr) << "typed_type requires ColumnVariantV2 typed state"; + return _typed_type; +} + +ColumnVariantV2::TypedEncodingStats ColumnVariantV2::ensure_encoded() { + if (!_typed) { + DCHECK(_typed_type == nullptr); + return {}; + } + + const auto& typed = static_cast<const IColumn::Ptr&>(_typed); + const auto& nullable = assert_cast<const ColumnNullable&>(*typed); + const PrimitiveType type = _typed_type->get_primitive_type(); + const uint32_t scale = _typed_type->get_scale(); + TypedEncodingResult encoded; + dispatch_typed_column(nullable, type, [&]<PrimitiveType Type>(const auto& column) { + encoded = encode_typed_column<Type>(nullable, column, scale); + }); + + static_cast<IColumn::Ptr&>(_metadatas) = std::move(encoded.metadatas); + static_cast<IColumn::Ptr&>(_meta_ids) = std::move(encoded.metadata_ids); + static_cast<IColumn::Ptr&>(_values) = std::move(encoded.values); + static_cast<IColumn::Ptr&>(_typed).reset(); + _typed_type.reset(); + _meta_index.reset(); + _check_invariants(); + return encoded.stats; +} + +std::string ColumnVariantV2::get_name() const { + if (_typed) { + DORIS_CHECK(_typed_type != nullptr); + return "variant_v2(typed=" + _typed_type->get_name() + ")"; + } + DCHECK(_typed_type == nullptr); + return "variant_v2"; +} + +size_t ColumnVariantV2::size() const { + if (_typed) { + DCHECK(_typed_type != nullptr); + DCHECK(_metadatas->empty()); + DCHECK(_meta_ids->empty()); + DCHECK(_values->empty()); + return _typed->size(); + } + + DCHECK(_typed_type == nullptr); + DCHECK_EQ(_meta_ids->size(), _values->size()); + return _meta_ids->size(); +} + +size_t ColumnVariantV2::byte_size() const { + if (_typed) { + DCHECK(_metadatas->empty()); + DCHECK(_meta_ids->empty()); + DCHECK(_values->empty()); + return _typed->byte_size(); + } + DCHECK_EQ(_meta_ids->size(), _values->size()); + return _metadatas->byte_size() + _meta_ids->byte_size() + _values->byte_size(); +} + +size_t ColumnVariantV2::allocated_bytes() const { + if (_typed) { + DCHECK(_metadatas->empty()); + DCHECK(_meta_ids->empty()); + DCHECK(_values->empty()); + return _typed->allocated_bytes(); + } + DCHECK_EQ(_meta_ids->size(), _values->size()); + const size_t index_bytes = _meta_index == nullptr ? 0 : _meta_index->allocated_bytes(); + return _metadatas->allocated_bytes() + _meta_ids->allocated_bytes() + + _values->allocated_bytes() + index_bytes; +} + +bool ColumnVariantV2::has_enough_capacity(const IColumn& src) const { + const auto& source = assert_cast<const ColumnVariantV2&>(src); + if (static_cast<bool>(_typed) != static_cast<bool>(source._typed)) { + return false; + } + if (_typed) { + if (!exact_typed_identity(_typed_type, source._typed_type)) { + return false; + } + return _typed->has_enough_capacity(*source._typed); + } + return _metadatas->has_enough_capacity(*source._metadatas) && + _meta_ids->has_enough_capacity(*source._meta_ids) && + _values->has_enough_capacity(*source._values); +} + +bool ColumnVariantV2::structure_equals(const IColumn& rhs) const { + return typeid(rhs) == typeid(ColumnVariantV2); +} + +void ColumnVariantV2::sanity_check() const { + if (_typed) { + _typed->sanity_check(); + } else { + _metadatas->sanity_check(); + _meta_ids->sanity_check(); + _values->sanity_check(); + } + _check_invariants(); +} + +void ColumnVariantV2::for_each_subcolumn(ColumnCallback callback) const { + if (_typed) { + callback(*static_cast<const IColumn::Ptr&>(_typed)); + } else { + callback(*static_cast<const IColumn::Ptr&>(_metadatas)); + callback(*static_cast<const IColumn::Ptr&>(_meta_ids)); + callback(*static_cast<const IColumn::Ptr&>(_values)); + } +} + +void ColumnVariantV2::mutate_subcolumns() { + _meta_index.reset(); + if (_typed) { + mutate_subcolumn(_typed); + } else { + mutate_subcolumn(_metadatas); + mutate_subcolumn(_meta_ids); + mutate_subcolumn(_values); + } +} + +void ColumnVariantV2::clear() { + if (_typed) { + require_exclusive(_typed, "typed column"); + static_cast<IColumn::Ptr&>(_typed) = _typed->clone_empty(); + } else { + auto& metadata_ptr = static_cast<IColumn::Ptr&>(_metadatas); + if (metadata_ptr->is_exclusive()) { + _metadatas->clear(); + } else { + metadata_ptr = ColumnString::create(); + } + require_exclusive(_meta_ids, "metadata ids"); + require_exclusive(_values, "values"); + _meta_ids->clear(); + _values->clear(); + } + _meta_index.reset(); + _check_invariants(); +} + +void ColumnVariantV2::insert_variant_field(const VariantField& field) { + const VariantValueRef value = field.ref(); + if (value.metadata.size > std::numeric_limits<uint32_t>::max() || + value.size > std::numeric_limits<uint32_t>::max()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Variant row exceeds the ColumnString uint32 byte limit"); + } + const std::array<uint32_t, 2> metadata_offsets {0, static_cast<uint32_t>(value.metadata.size)}; + const std::array<uint32_t, 2> value_offsets {0, static_cast<uint32_t>(value.size)}; + insert_encoded_rows({.metadata_bytes = {value.metadata.data, value.metadata.size}, + .metadata_offsets = metadata_offsets, + .meta_ids = {}, + .value_bytes = {value.data, value.size}, + .value_offsets = value_offsets}); +} + +// Encoded insertion is a transactional state transition that keeps metadata, ids, and values in +// sync; splitting it would obscure the shared rollback boundary. +void ColumnVariantV2::insert_encoded_rows( // NOLINT(readability-function-size) + const EncodedDataView& data) { + if (_typed) { + require_exclusive(_typed, "typed column"); + auto replacement = ColumnVariantV2::create(); + static_cast<IColumn::Ptr&>(replacement->_typed) = _typed->clone_resized(size()); + replacement->_typed_type = _typed_type; + replacement->ensure_encoded(); + replacement->insert_encoded_rows(data); + _adopt_state_from(*replacement); + return; + } + DORIS_CHECK(_typed_type == nullptr) << "encoded state cannot retain a typed data type"; + validate_offsets(data.metadata_bytes, data.metadata_offsets, "metadata"); + validate_offsets(data.value_bytes, data.value_offsets, "value"); + + const size_t metadata_count = data.metadata_offsets.size() - 1; + const size_t rows = data.value_offsets.size() - 1; + DORIS_CHECK_LE(metadata_count, std::numeric_limits<uint32_t>::max()) + << "metadata count exceeds the uint32 id domain"; + if (rows == 0) { + DORIS_CHECK(data.meta_ids.empty()) << "empty encoded batch cannot contain metadata ids"; + return; + } + DORIS_CHECK_NE(metadata_count, 0) << "encoded rows require at least one metadata blob"; + if (data.meta_ids.empty()) { + DORIS_CHECK_EQ(metadata_count, 1) + << "omitted metadata ids require exactly one metadata blob"; + } else { + DORIS_CHECK_EQ(data.meta_ids.size(), rows) + << "metadata id count must match the encoded row count"; + for (uint32_t metadata_id : data.meta_ids) { + DORIS_CHECK_LT(metadata_id, metadata_count) + << "encoded row metadata id is out of range"; + } + } + + DorisVector<uint32_t> remap(metadata_count, UNMAPPED_METADATA_ID); + DorisVector<uint32_t> appended_ids(rows); + require_exclusive(_meta_ids, "metadata ids"); + require_exclusive(_values, "values"); + auto& values = assert_cast<ColumnString&>(*_values); + auto& metadata_ids = assert_cast<MetaIdsColumn&>(*_meta_ids); + reserve_rows(values, metadata_ids, data.value_bytes.size, rows); + + const auto& metadata_ptr = static_cast<const IColumn::Ptr&>(_metadatas); + const size_t old_metadata_count = metadata_ptr->size(); + IColumn::Ptr original_metadata; + if (!metadata_ptr->is_exclusive()) { + original_metadata = metadata_ptr; + } + const size_t old_rows = size(); Review Comment: add comment for the complex code ########## be/src/core/column/column_variant_v2.cpp: ########## @@ -0,0 +1,1651 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/column/column_variant_v2.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <cstdint> +#include <limits> +#include <string_view> +#include <typeinfo> +#include <utility> + +#include "common/check.h" +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/column/column_const.h" +#include "core/column/column_decimal.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_variant_v2_typed_scalar.h" +#include "core/column/column_vector.h" +#include "core/column/columns_common.h" +#include "core/custom_allocator.h" +#include "util/hash_util.hpp" +#include "util/variant/variant_canonical.h" +#include "util/variant/variant_encoded_block.h" +#include "util/variant/variant_scalar_encoding.h" + +namespace doris { +namespace { + +using MetaIdsColumn = ColumnVector<TYPE_UINT32>; +using column_variant_v2_internal::TypedFallbackKind; +using column_variant_v2_internal::dispatch_typed_column; +using column_variant_v2_internal::exact_typed_identity; +using column_variant_v2_internal::is_supported_typed_identity; +using column_variant_v2_internal::validate_typed_decimal_scale; +using column_variant_v2_internal::visit_typed_canonical_rows; +using column_variant_v2_internal::visit_typed_rows; +constexpr uint32_t UNMAPPED_METADATA_ID = std::numeric_limits<uint32_t>::max(); +constexpr size_t CANONICAL_SIZE_PREFIX = sizeof(uint32_t); +constexpr std::array<char, 3> EMPTY_OBJECT_METADATA {static_cast<char>(0x11), 0, 0}; +constexpr std::array<char, 3> EMPTY_OBJECT_VALUE {static_cast<char>(0x02), 0, 0}; + +uint32_t read_canonical_payload_size(const char* pos) { + DCHECK(pos != nullptr); + uint32_t payload_size = 0; + for (uint8_t byte = 0; byte < CANONICAL_SIZE_PREFIX; ++byte) { + payload_size |= static_cast<uint32_t>(static_cast<uint8_t>(pos[byte])) << (byte * 8); + } + return payload_size; +} + +size_t trusted_canonical_cell_size(const char* pos) { + return CANONICAL_SIZE_PREFIX + read_canonical_payload_size(pos); +} + +void check_hash_range(const ColumnVariantV2& column, size_t start, size_t end) { + DCHECK_LE(start, end); + DCHECK_LE(end, column.size()); +} + +template <typename Sink, typename Hash> +void update_canonical_hash_range(const ColumnVariantV2& column, size_t start, size_t end, + Hash& hash, const uint8_t* __restrict null_data) { + check_hash_range(column, start, end); + Sink sink(hash); + for (size_t row = start; row < end; ++row) { + if (null_data == nullptr || null_data[row] == 0) { + canonical_hash(column.get_value_ref(row), sink); + } + } + hash = sink.digest(); +} + +template <typename Sink, typename Hash> +void update_canonical_hashes(const ColumnVariantV2& column, Hash* __restrict hashes, + const uint8_t* __restrict null_data) { + for (size_t row = 0; row < column.size(); ++row) { + if (null_data == nullptr || null_data[row] == 0) { + Sink sink(hashes[row]); + canonical_hash(column.get_value_ref(row), sink); + hashes[row] = sink.digest(); + } + } +} + +void validate_offsets(StringRef bytes, std::span<const uint32_t> offsets, + std::string_view description) { + DORIS_CHECK(bytes.data != nullptr || bytes.size == 0) + << description << " bytes have a null pointer"; + DORIS_CHECK(!offsets.empty()) << description << " offsets cannot be empty"; + DORIS_CHECK_EQ(offsets.front(), 0) << description << " offsets must start at zero"; + DORIS_CHECK_EQ(static_cast<size_t>(offsets.back()), bytes.size) + << description << " offsets must end at the byte size"; + for (size_t index = 1; index < offsets.size(); ++index) { + DORIS_CHECK_LT(offsets[index - 1], offsets[index]) + << description << " offsets must be strictly increasing"; + } +} + +void require_exclusive(const IColumn::WrappedPtr& column, std::string_view description) { + const auto& immutable = static_cast<const IColumn::Ptr&>(column); + DORIS_CHECK(immutable->is_exclusive()) + << "ColumnVariantV2 " << description << " must be COW-detached before mutation"; +} + +void reserve_rows(ColumnString& values, MetaIdsColumn& metadata_ids, size_t value_bytes, + size_t rows) { + const size_t final_value_bytes = values.get_chars().size() + value_bytes; + ColumnString::check_chars_length(final_value_bytes, values.size() + rows, values.size()); + values.get_chars().reserve(final_value_bytes); + values.get_offsets().reserve(values.size() + rows); + metadata_ids.get_data().reserve(metadata_ids.size() + rows); +} + +void append_metadata_ids(MetaIdsColumn& destination, const DorisVector<uint32_t>& ids) { + auto& data = destination.get_data(); + const size_t old_size = data.size(); + data.resize(old_size + ids.size()); + std::ranges::copy(ids, data.begin() + old_size); +} + +struct SelectedRows { + DorisVector<uint32_t> metadata_ids; + size_t value_bytes = 0; +}; + +size_t validate_selected_indices(const uint32_t* indices_begin, const uint32_t* indices_end, + size_t source_rows) { + if (indices_begin == indices_end) { + return 0; + } + DORIS_CHECK(indices_begin != nullptr && indices_end != nullptr) + << "non-empty source indices cannot be null"; + DORIS_CHECK_LT(indices_begin, indices_end) << "source index range is reversed"; + const size_t rows = indices_end - indices_begin; + DORIS_CHECK_LT(*std::max_element(indices_begin, indices_end), source_rows) + << "source index is out of range"; + return rows; +} + +DorisVector<uint32_t> collect_range_metadata_ids( + const MetaIdsColumn::Container& source_metadata_ids, size_t source_metadata_count, + size_t start, size_t length) { + DorisVector<uint32_t> selected(length); + for (size_t row = 0; row < length; ++row) { + const uint32_t metadata_id = source_metadata_ids[start + row]; + DCHECK_LT(metadata_id, source_metadata_count) << "source metadata id is out of range"; + selected[row] = metadata_id; + } + return selected; +} + +SelectedRows collect_selected_rows(const MetaIdsColumn::Container& source_metadata_ids, + const ColumnString& source_metadatas, + const ColumnString& source_values, const uint32_t* indices_begin, + size_t rows) { + SelectedRows selected; + selected.metadata_ids.resize(rows); + for (size_t row = 0; row < rows; ++row) { + const uint32_t source_row = indices_begin[row]; + const uint32_t metadata_id = source_metadata_ids[source_row]; + DCHECK_LT(metadata_id, source_metadatas.size()) << "source metadata id is out of range"; + selected.metadata_ids[row] = metadata_id; + const size_t row_bytes = source_values.get_data_at(source_row).size; + DORIS_CHECK_LE(row_bytes, std::numeric_limits<size_t>::max() - selected.value_bytes) + << "selected Variant value bytes overflow size_t"; + selected.value_bytes += row_bytes; + } + return selected; +} + +template <typename Sink, typename Hash> +void update_typed_hashes(const ColumnNullable& nullable, PrimitiveType type, uint32_t scale, + Hash* __restrict hashes, const uint8_t* __restrict null_data) { + dispatch_typed_column(nullable, type, [&]<PrimitiveType Type>(const auto& column) { + visit_typed_canonical_rows<Type>(nullable, column, scale, 0, nullable.size(), + [&](size_t row, auto&& canonical_factory) { + if (null_data == nullptr || null_data[row] == 0) { + VariantCanonicalScalarRef canonical = + canonical_factory(); + Sink sink(hashes[row]); + canonical_hash(canonical, sink); + hashes[row] = sink.digest(); + } + }); + }); +} + +template <typename Sink, typename Hash> +void update_typed_hash_range(const ColumnNullable& nullable, PrimitiveType type, uint32_t scale, + size_t start, size_t end, Hash& hash, + const uint8_t* __restrict null_data) { + Sink sink(hash); + dispatch_typed_column(nullable, type, [&]<PrimitiveType Type>(const auto& column) { + visit_typed_canonical_rows<Type>( + nullable, column, scale, start, end, [&](size_t row, auto&& canonical_factory) { + if (null_data == nullptr || null_data[row] == 0) { + VariantCanonicalScalarRef canonical = canonical_factory(); + canonical_hash(canonical, sink); + } + }); + }); + hash = sink.digest(); +} + +struct TypedEncodingResult { + ColumnString::MutablePtr metadatas; + MetaIdsColumn::MutablePtr metadata_ids; + ColumnString::MutablePtr values; + ColumnVariantV2::TypedEncodingStats stats; +}; + +void count_fallback(TypedFallbackKind fallback, ColumnVariantV2::TypedEncodingStats& stats) { + switch (fallback) { + case TypedFallbackKind::NONE: + return; + case TypedFallbackKind::LARGEINT: + ++stats.largeint_string_fallback_rows; + return; + case TypedFallbackKind::DECIMAL256: + ++stats.decimal256_string_fallback_rows; + return; + case TypedFallbackKind::IP: + ++stats.ip_string_fallback_rows; + return; + } +} + +template <PrimitiveType Type, typename Column> +TypedEncodingResult encode_typed_column(const ColumnNullable& nullable, const Column& column, + uint32_t scale) { + size_t value_bytes = 0; + TypedEncodingResult result; + visit_typed_rows<Type>( + nullable, column, scale, 0, nullable.size(), + [&](size_t, auto&& physical_factory, auto&&, TypedFallbackKind fallback) { + const VariantScalarEncodingPlan plan = physical_factory(); + if (plan.size() > std::numeric_limits<size_t>::max() - value_bytes) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Typed Variant values exceed addressable size"); + } + value_bytes += plan.size(); + count_fallback(fallback, result.stats); + }); + ColumnString::check_chars_length(value_bytes, nullable.size(), 0); + + result.metadatas = ColumnString::create(); + result.metadata_ids = MetaIdsColumn::create(); + result.values = ColumnString::create(); + if (!nullable.empty()) { + result.metadatas->insert_data(EMPTY_OBJECT_METADATA.data(), EMPTY_OBJECT_METADATA.size()); + } + result.metadata_ids->get_data().resize(nullable.size()); + std::fill(result.metadata_ids->get_data().begin(), result.metadata_ids->get_data().end(), 0); + auto& chars = result.values->get_chars(); + auto& offsets = result.values->get_offsets(); + chars.resize(value_bytes); + offsets.resize(nullable.size()); + + size_t offset = 0; + visit_typed_rows<Type>(nullable, column, scale, 0, nullable.size(), + [&](size_t row, auto&& physical_factory, auto&&, TypedFallbackKind) { + const VariantScalarEncodingPlan plan = physical_factory(); + plan.write(reinterpret_cast<char*>(chars.data()) + offset, + plan.size()); + offset += plan.size(); + offsets[row] = static_cast<ColumnString::Offset>(offset); + }); + DCHECK_EQ(offset, value_bytes); + return result; +} + +struct ValidatedTypedInput { + MutableColumnPtr column; + DataTypePtr type; +}; + +ValidatedTypedInput validate_typed_input(ColumnPtr column, DataTypePtr scalar_type) { + DORIS_CHECK(static_cast<bool>(column)) << "typed ColumnVariantV2 column must not be null"; + DORIS_CHECK(scalar_type != nullptr) << "typed ColumnVariantV2 type must not be null"; + DORIS_CHECK(!scalar_type->is_nullable()) + << "typed ColumnVariantV2 requires a non-nullable scalar type"; + DORIS_CHECK(check_and_get_column<ColumnConst>(column.get()) == nullptr) + << "typed ColumnVariantV2 input must not be ColumnConst"; + const IColumn* input_column = column.get(); + DORIS_CHECK(typeid(*input_column) == typeid(ColumnNullable)) + << "typed ColumnVariantV2 input must be an exact ColumnNullable"; + + const auto& nullable = assert_cast<const ColumnNullable&>(*column); + const IColumn& nested = nullable.get_nested_column(); + DORIS_CHECK_EQ(nested.size(), nullable.get_null_map_column().size()) + << "typed ColumnVariantV2 null map size does not match nested column size"; + const PrimitiveType type = scalar_type->get_primitive_type(); + DORIS_CHECK(is_supported_typed_identity(type)) + << "unsupported typed identity " << scalar_type->get_name(); + MutableColumnPtr expected = scalar_type->create_column(); + const IColumn* expected_column = expected.get(); + DORIS_CHECK(typeid(nested) == typeid(*expected_column)) + << "typed nested column " << nested.get_name() << " does not match data type " + << scalar_type->get_name(); + validate_typed_decimal_scale(nested, type, scalar_type->get_scale()); + + return {.column = IColumn::mutate(std::move(column)), .type = std::move(scalar_type)}; +} + +[[noreturn]] void throw_deferred(std::string_view method, std::string_view task) { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, + "ColumnVariantV2::{} is deferred until {} is complete", method, task); +} + +[[noreturn]] void throw_unsupported(std::string_view method) { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, + "ColumnVariantV2::{} is intentionally unsupported for Variant values", method); +} + +} // namespace + +struct ColumnVariantV2::MetaDictIndex { + using Candidates = DorisVector<uint32_t>; + using Buckets = DorisMap<uint64_t, Candidates>; + + size_t allocated_bytes() const { + // std::map does not expose node allocation sizes. Count the logical node payload and a + // conservative four-pointer tree overhead; the actual allocations are independently + // charged through CustomStdAllocator. + size_t bytes = + sizeof(*this) + buckets.size() * (sizeof(Buckets::value_type) + 4 * sizeof(void*)); + for (const auto& entry : buckets) { + bytes += entry.second.capacity() * sizeof(uint32_t); + } + return bytes; + } + + Buckets buckets; +}; + +#ifdef BE_TEST +void ColumnVariantV2::TestAccess::reset_metadata_index(ColumnVariantV2& column) { + column._meta_index.reset(); +} + +void ColumnVariantV2::TestAccess::replace_encoded_subcolumn(ColumnVariantV2& column, size_t index, + ColumnPtr replacement) { + DORIS_CHECK(!column._typed); + switch (index) { + case 0: + static_cast<IColumn::Ptr&>(column._metadatas) = std::move(replacement); + break; + case 1: + static_cast<IColumn::Ptr&>(column._meta_ids) = std::move(replacement); + break; + case 2: + static_cast<IColumn::Ptr&>(column._values) = std::move(replacement); + break; + default: + DORIS_CHECK(false) << "ColumnVariantV2 test subcolumn index is out of range"; + } + column._meta_index.reset(); +} +#endif + +ColumnVariantV2::ColumnVariantV2() + : _metadatas(ColumnString::create()), + _meta_ids(MetaIdsColumn::create()), + _values(ColumnString::create()) { + _check_invariants(); +} + +ColumnVariantV2::ColumnVariantV2(const ColumnVariantV2& other) + : _metadatas(other._metadatas), + _meta_ids(other._meta_ids), + _values(other._values), + _typed(other._typed), + _typed_type(other._typed_type) {} + +ColumnVariantV2::~ColumnVariantV2() = default; + +ColumnVariantV2::MutablePtr ColumnVariantV2::create_typed_from_scan(ColumnPtr column, + DataTypePtr scalar_type) { + ValidatedTypedInput input = validate_typed_input(std::move(column), std::move(scalar_type)); + auto result = ColumnVariantV2::create(); + static_cast<IColumn::Ptr&>(result->_typed) = std::move(input.column); + result->_typed_type = std::move(input.type); + result->_check_invariants(); + return result; +} + +ColumnVariantV2::MutablePtr ColumnVariantV2::create_typed_from_element_at(ColumnPtr column, + DataTypePtr scalar_type) { + ValidatedTypedInput input = validate_typed_input(std::move(column), std::move(scalar_type)); + auto result = ColumnVariantV2::create(); + static_cast<IColumn::Ptr&>(result->_typed) = std::move(input.column); + result->_typed_type = std::move(input.type); + result->_check_invariants(); + return result; +} + +ColumnVariantV2::MutablePtr ColumnVariantV2::create_typed_from_cast(ColumnPtr column, + DataTypePtr scalar_type) { + ValidatedTypedInput input = validate_typed_input(std::move(column), std::move(scalar_type)); + auto result = ColumnVariantV2::create(); + static_cast<IColumn::Ptr&>(result->_typed) = std::move(input.column); + result->_typed_type = std::move(input.type); + result->_check_invariants(); + return result; +} + +ColumnVariantV2::MutablePtr ColumnVariantV2::create_typed_from_exchange(ColumnPtr column, + DataTypePtr scalar_type) { + ValidatedTypedInput input = validate_typed_input(std::move(column), std::move(scalar_type)); + auto result = ColumnVariantV2::create(); + static_cast<IColumn::Ptr&>(result->_typed) = std::move(input.column); + result->_typed_type = std::move(input.type); + result->_check_invariants(); + return result; +} + +const IColumn& ColumnVariantV2::typed_column() const { + DORIS_CHECK(_typed != nullptr) << "typed_column requires ColumnVariantV2 typed state"; + return *_typed; +} + +const DataTypePtr& ColumnVariantV2::typed_type() const { + DORIS_CHECK(_typed_type != nullptr) << "typed_type requires ColumnVariantV2 typed state"; + return _typed_type; +} + +ColumnVariantV2::TypedEncodingStats ColumnVariantV2::ensure_encoded() { + if (!_typed) { + DCHECK(_typed_type == nullptr); + return {}; + } + + const auto& typed = static_cast<const IColumn::Ptr&>(_typed); + const auto& nullable = assert_cast<const ColumnNullable&>(*typed); + const PrimitiveType type = _typed_type->get_primitive_type(); + const uint32_t scale = _typed_type->get_scale(); + TypedEncodingResult encoded; + dispatch_typed_column(nullable, type, [&]<PrimitiveType Type>(const auto& column) { + encoded = encode_typed_column<Type>(nullable, column, scale); + }); + + static_cast<IColumn::Ptr&>(_metadatas) = std::move(encoded.metadatas); + static_cast<IColumn::Ptr&>(_meta_ids) = std::move(encoded.metadata_ids); + static_cast<IColumn::Ptr&>(_values) = std::move(encoded.values); + static_cast<IColumn::Ptr&>(_typed).reset(); + _typed_type.reset(); + _meta_index.reset(); + _check_invariants(); + return encoded.stats; +} + +std::string ColumnVariantV2::get_name() const { + if (_typed) { + DORIS_CHECK(_typed_type != nullptr); + return "variant_v2(typed=" + _typed_type->get_name() + ")"; + } + DCHECK(_typed_type == nullptr); + return "variant_v2"; +} + +size_t ColumnVariantV2::size() const { + if (_typed) { + DCHECK(_typed_type != nullptr); + DCHECK(_metadatas->empty()); + DCHECK(_meta_ids->empty()); + DCHECK(_values->empty()); + return _typed->size(); + } + + DCHECK(_typed_type == nullptr); + DCHECK_EQ(_meta_ids->size(), _values->size()); + return _meta_ids->size(); +} + +size_t ColumnVariantV2::byte_size() const { + if (_typed) { + DCHECK(_metadatas->empty()); + DCHECK(_meta_ids->empty()); + DCHECK(_values->empty()); + return _typed->byte_size(); + } + DCHECK_EQ(_meta_ids->size(), _values->size()); + return _metadatas->byte_size() + _meta_ids->byte_size() + _values->byte_size(); +} + +size_t ColumnVariantV2::allocated_bytes() const { + if (_typed) { + DCHECK(_metadatas->empty()); + DCHECK(_meta_ids->empty()); + DCHECK(_values->empty()); + return _typed->allocated_bytes(); + } + DCHECK_EQ(_meta_ids->size(), _values->size()); + const size_t index_bytes = _meta_index == nullptr ? 0 : _meta_index->allocated_bytes(); + return _metadatas->allocated_bytes() + _meta_ids->allocated_bytes() + + _values->allocated_bytes() + index_bytes; +} + +bool ColumnVariantV2::has_enough_capacity(const IColumn& src) const { + const auto& source = assert_cast<const ColumnVariantV2&>(src); + if (static_cast<bool>(_typed) != static_cast<bool>(source._typed)) { + return false; + } + if (_typed) { + if (!exact_typed_identity(_typed_type, source._typed_type)) { + return false; + } + return _typed->has_enough_capacity(*source._typed); + } + return _metadatas->has_enough_capacity(*source._metadatas) && + _meta_ids->has_enough_capacity(*source._meta_ids) && + _values->has_enough_capacity(*source._values); +} + +bool ColumnVariantV2::structure_equals(const IColumn& rhs) const { + return typeid(rhs) == typeid(ColumnVariantV2); +} + +void ColumnVariantV2::sanity_check() const { + if (_typed) { + _typed->sanity_check(); + } else { + _metadatas->sanity_check(); + _meta_ids->sanity_check(); + _values->sanity_check(); + } + _check_invariants(); +} + +void ColumnVariantV2::for_each_subcolumn(ColumnCallback callback) const { + if (_typed) { + callback(*static_cast<const IColumn::Ptr&>(_typed)); + } else { + callback(*static_cast<const IColumn::Ptr&>(_metadatas)); + callback(*static_cast<const IColumn::Ptr&>(_meta_ids)); + callback(*static_cast<const IColumn::Ptr&>(_values)); + } +} + +void ColumnVariantV2::mutate_subcolumns() { + _meta_index.reset(); + if (_typed) { + mutate_subcolumn(_typed); + } else { + mutate_subcolumn(_metadatas); + mutate_subcolumn(_meta_ids); + mutate_subcolumn(_values); + } +} + +void ColumnVariantV2::clear() { + if (_typed) { + require_exclusive(_typed, "typed column"); + static_cast<IColumn::Ptr&>(_typed) = _typed->clone_empty(); + } else { + auto& metadata_ptr = static_cast<IColumn::Ptr&>(_metadatas); + if (metadata_ptr->is_exclusive()) { + _metadatas->clear(); + } else { + metadata_ptr = ColumnString::create(); + } + require_exclusive(_meta_ids, "metadata ids"); + require_exclusive(_values, "values"); + _meta_ids->clear(); + _values->clear(); + } + _meta_index.reset(); + _check_invariants(); +} + +void ColumnVariantV2::insert_variant_field(const VariantField& field) { + const VariantValueRef value = field.ref(); + if (value.metadata.size > std::numeric_limits<uint32_t>::max() || + value.size > std::numeric_limits<uint32_t>::max()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Variant row exceeds the ColumnString uint32 byte limit"); + } + const std::array<uint32_t, 2> metadata_offsets {0, static_cast<uint32_t>(value.metadata.size)}; + const std::array<uint32_t, 2> value_offsets {0, static_cast<uint32_t>(value.size)}; + insert_encoded_rows({.metadata_bytes = {value.metadata.data, value.metadata.size}, + .metadata_offsets = metadata_offsets, + .meta_ids = {}, + .value_bytes = {value.data, value.size}, + .value_offsets = value_offsets}); +} + +// Encoded insertion is a transactional state transition that keeps metadata, ids, and values in +// sync; splitting it would obscure the shared rollback boundary. +void ColumnVariantV2::insert_encoded_rows( // NOLINT(readability-function-size) + const EncodedDataView& data) { + if (_typed) { + require_exclusive(_typed, "typed column"); + auto replacement = ColumnVariantV2::create(); + static_cast<IColumn::Ptr&>(replacement->_typed) = _typed->clone_resized(size()); + replacement->_typed_type = _typed_type; + replacement->ensure_encoded(); + replacement->insert_encoded_rows(data); + _adopt_state_from(*replacement); + return; + } + DORIS_CHECK(_typed_type == nullptr) << "encoded state cannot retain a typed data type"; + validate_offsets(data.metadata_bytes, data.metadata_offsets, "metadata"); + validate_offsets(data.value_bytes, data.value_offsets, "value"); + + const size_t metadata_count = data.metadata_offsets.size() - 1; + const size_t rows = data.value_offsets.size() - 1; + DORIS_CHECK_LE(metadata_count, std::numeric_limits<uint32_t>::max()) + << "metadata count exceeds the uint32 id domain"; + if (rows == 0) { + DORIS_CHECK(data.meta_ids.empty()) << "empty encoded batch cannot contain metadata ids"; + return; + } + DORIS_CHECK_NE(metadata_count, 0) << "encoded rows require at least one metadata blob"; + if (data.meta_ids.empty()) { + DORIS_CHECK_EQ(metadata_count, 1) + << "omitted metadata ids require exactly one metadata blob"; + } else { + DORIS_CHECK_EQ(data.meta_ids.size(), rows) + << "metadata id count must match the encoded row count"; + for (uint32_t metadata_id : data.meta_ids) { + DORIS_CHECK_LT(metadata_id, metadata_count) + << "encoded row metadata id is out of range"; + } + } + + DorisVector<uint32_t> remap(metadata_count, UNMAPPED_METADATA_ID); + DorisVector<uint32_t> appended_ids(rows); + require_exclusive(_meta_ids, "metadata ids"); + require_exclusive(_values, "values"); + auto& values = assert_cast<ColumnString&>(*_values); + auto& metadata_ids = assert_cast<MetaIdsColumn&>(*_meta_ids); + reserve_rows(values, metadata_ids, data.value_bytes.size, rows); + + const auto& metadata_ptr = static_cast<const IColumn::Ptr&>(_metadatas); + const size_t old_metadata_count = metadata_ptr->size(); + IColumn::Ptr original_metadata; + if (!metadata_ptr->is_exclusive()) { + original_metadata = metadata_ptr; + } + const size_t old_rows = size(); Review Comment: add comment for the complex code bellow -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
