This is an automated email from the ASF dual-hosted git repository.
csun5285 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3d2d9aac541 [refactor](be) Read stored values through StorageLayout
(#69609)
3d2d9aac541 is described below
commit 3d2d9aac54148056529c13ef4b9fbdbcb65fab77
Author: Chenyang Sun <[email protected]>
AuthorDate: Tue Oct 27 18:30:46 2026 +0800
[refactor](be) Read stored values through StorageLayout (#69609)
Read stored values through StorageLayout
---
be/src/core/column/column.h | 6 -
be/src/core/column/column_decimal.cpp | 19 --
be/src/core/column/column_decimal.h | 2 -
be/src/core/column/column_nullable.h | 5 -
be/src/core/column/column_vector.h | 40 ----
be/src/core/data_type/primitive_type.h | 43 -----
.../data_type_serde/data_type_decimal_serde.cpp | 1 +
be/src/core/field.h | 23 ---
.../aggregate/aggregate_function_window_funnel.h | 1 -
be/src/storage/field_type.h | 2 +-
.../bloom_filter/bloom_filter_index_writer.cpp | 2 +-
.../index/inverted/clucene/inverted_index_writer.h | 2 +-
be/src/storage/index/zone_map/zone_map_index.cpp | 13 +-
be/src/storage/index/zone_map/zone_map_index.h | 7 +-
be/src/storage/key_coder.h | 33 ++--
be/src/storage/segment/binary_dict_page.cpp | 2 +-
be/src/storage/segment/bitshuffle_page.h | 7 +-
be/src/storage/segment/column_reader.cpp | 49 ++++-
be/src/storage/segment/encoding_info.cpp | 20 +-
be/src/storage/segment/encoding_info.h | 6 +-
be/src/storage/segment/plain_page.h | 7 +-
be/src/storage/storage_layout.cpp | 42 ++++-
be/src/storage/storage_layout.h | 54 ++++--
be/src/storage/types.h | 208 ++++++++++-----------
be/test/core/column/column_array_test.cpp | 9 -
be/test/core/column/column_decimal_test.cpp | 6 -
be/test/core/column/column_vector_test.cpp | 24 ---
be/test/core/column/common_column_test.h | 49 -----
.../writer/iceberg/partition_transformers_test.cpp | 16 +-
be/test/storage/index/date_bloom_filter_test.cpp | 24 +--
be/test/storage/index/index_builder_test.cpp | 2 +-
.../inverted/snii/bkd/bkd_differential_test.cpp | 4 +-
.../index/inverted/snii/bkd/bkd_property_test.cpp | 10 +-
.../inverted/snii/snii_bkd_adapter_types_test.cpp | 4 +-
be/test/storage/key_coder_test.cpp | 35 ++--
be/test/storage/olap_type_test.cpp | 37 ++--
be/test/storage/row_cursor_test.cpp | 14 +-
be/test/storage/segment/bitshuffle_page_test.cpp | 82 ++++++++
.../bloom_filter_index_reader_writer_test.cpp | 2 +-
.../storage/segment/inverted_index_reader_test.cpp | 21 +--
.../storage/segment/page_builder_column_test.cpp | 12 +-
be/test/storage/segment/page_format_test.cpp | 124 ++++++++----
be/test/storage/segment/plain_page_test.cpp | 68 +++++++
be/test/storage/storage_layout_test.cpp | 116 +++++++++++-
.../storage_layout_test_util.h} | 30 ++-
45 files changed, 736 insertions(+), 547 deletions(-)
diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h
index bde5b4786c1..ad2b6cba3e1 100644
--- a/be/src/core/column/column.h
+++ b/be/src/core/column/column.h
@@ -239,12 +239,6 @@ public:
"Method insert_data is not supported for " +
get_name());
}
- virtual void insert_many_fix_len_data(const char* pos, size_t num) {
- throw doris::Exception(
- ErrorCode::NOT_IMPLEMENTED_ERROR,
- "Method insert_many_fix_len_data is not supported for " +
get_name());
- }
-
// todo(zeno) Use dict_args temp object to cover all arguments
virtual void insert_many_dict_data(const int32_t* data_array, size_t
start_index,
const StringRef* dict, size_t data_num,
diff --git a/be/src/core/column/column_decimal.cpp
b/be/src/core/column/column_decimal.cpp
index 314bccd5d54..cbc21186c46 100644
--- a/be/src/core/column/column_decimal.cpp
+++ b/be/src/core/column/column_decimal.cpp
@@ -30,7 +30,6 @@
#include "core/column/columns_common.h"
#include "core/data_type/data_type.h"
#include "core/data_type/data_type_decimal.h"
-#include "core/decimal12.h"
#include "core/value/decimalv2_value.h"
#include "exec/common/int_exp.h"
#include "exec/common/sip_hash.h"
@@ -390,24 +389,6 @@ void ColumnDecimal<T>::insert_data(const char* src, size_t
/*length*/) {
data.emplace_back(tmp);
}
-template <PrimitiveType T>
-void ColumnDecimal<T>::insert_many_fix_len_data(const char* data_ptr, size_t
num) {
- size_t old_size = data.size();
- data.resize(old_size + num);
-
- if constexpr (T == TYPE_DECIMALV2) {
- DecimalV2Value* target = (DecimalV2Value*)(data.data() + old_size);
- for (int i = 0; i < num; i++) {
- const char* cur_ptr = data_ptr + sizeof(decimal12_t) * i;
- auto int_value = unaligned_load<int64_t>(cur_ptr);
- auto frac_value = unaligned_load<int32_t>(cur_ptr +
sizeof(int64_t));
- target[i].from_olap_decimal(int_value, frac_value);
- }
- } else {
- memcpy(data.data() + old_size, data_ptr, num * sizeof(value_type));
- }
-}
-
template <PrimitiveType T>
void ColumnDecimal<T>::insert_many_from(const IColumn& src, size_t position,
size_t length) {
auto old_size = data.size();
diff --git a/be/src/core/column/column_decimal.h
b/be/src/core/column/column_decimal.h
index 8cd7be991a7..23ae05b4a85 100644
--- a/be/src/core/column/column_decimal.h
+++ b/be/src/core/column/column_decimal.h
@@ -128,8 +128,6 @@ public:
data.data() + origin_size, indices_begin, indices_end);
}
- void insert_many_fix_len_data(const char* data_ptr, size_t num) override;
-
void insert_many_raw_data(const char* pos, size_t num) override {
DCHECK(pos);
size_t old_size = data.size();
diff --git a/be/src/core/column/column_nullable.h
b/be/src/core/column/column_nullable.h
index c90f72d4400..382ce7c4fc4 100644
--- a/be/src/core/column/column_nullable.h
+++ b/be/src/core/column/column_nullable.h
@@ -161,11 +161,6 @@ public:
_null_map->insert_from(src_concrete.get_null_map_column(), n);
}
- void insert_many_fix_len_data(const char* pos, size_t num) override {
- push_false_to_nullmap(num);
- get_nested_column().insert_many_fix_len_data(pos, num);
- }
-
void insert_many_raw_data(const char* pos, size_t num) override {
DCHECK(pos);
push_false_to_nullmap(num);
diff --git a/be/src/core/column/column_vector.h
b/be/src/core/column/column_vector.h
index 5f27d18566b..b6d79ae9d26 100644
--- a/be/src/core/column/column_vector.h
+++ b/be/src/core/column/column_vector.h
@@ -47,7 +47,6 @@
#include "core/pod_array_fwd.h"
#include "core/string_ref.h"
#include "core/types.h"
-#include "core/uint24.h"
#include "core/value/timestamp_ns_value.h"
#include "core/value/vdatetime_value.h"
#include "util/unaligned.h"
@@ -133,45 +132,6 @@ public:
}
}
- void insert_date_column(const char* data_ptr, size_t num) {
- data.reserve(data.size() + num);
- constexpr size_t input_value_size = sizeof(uint24_t);
-
- for (int i = 0; i < num; i++) {
- uint24_t val = 0;
- memcpy((char*)(&val), data_ptr, input_value_size);
- data_ptr += input_value_size;
-
- VecDateTimeValue date;
- date.set_olap_date(val);
- data.push_back_without_reserve(date);
- }
- }
-
- void insert_datetime_column(const char* data_ptr, size_t num) {
- data.reserve(data.size() + num);
- size_t value_size = sizeof(uint64_t);
- for (int i = 0; i < num; i++) {
- const char* cur_ptr = data_ptr + value_size * i;
- auto value = unaligned_load<uint64_t>(cur_ptr);
- VecDateTimeValue datetime =
VecDateTimeValue::create_from_olap_datetime(value);
- this->insert_data(reinterpret_cast<char*>(&datetime), 0);
- }
- }
-
- /*
- use by date, datetime, basic type
- */
- void insert_many_fix_len_data(const char* data_ptr, size_t num) override {
- if (T == TYPE_DATE) {
- insert_date_column(data_ptr, num);
- } else if (T == TYPE_DATETIME) {
- insert_datetime_column(data_ptr, num);
- } else {
- insert_many_raw_data(data_ptr, num);
- }
- }
-
void insert_many_raw_data(const char* data_ptr, size_t num) override {
DCHECK(data_ptr);
auto old_size = data.size();
diff --git a/be/src/core/data_type/primitive_type.h
b/be/src/core/data_type/primitive_type.h
index 71591884ce5..41506fe5953 100644
--- a/be/src/core/data_type/primitive_type.h
+++ b/be/src/core/data_type/primitive_type.h
@@ -27,10 +27,8 @@
#include "common/cast_set.h"
#include "core/data_type/define_primitive_type.h"
-#include "core/decimal12.h"
#include "core/string_view.h"
#include "core/types.h"
-#include "core/uint24.h"
#include "core/value/timestamptz_value.h"
#include "core/value/vdatetime_value.h"
#include "exec/common/template_helpers.hpp"
@@ -284,113 +282,96 @@ struct PrimitiveTypeTraits;
/**
* CppType: Doris type in execution engine
- * StorageFieldType: Doris type in storage engine
* DataType: DataType which is mapping to this PrimitiveType
* ColumnType: ColumnType which is mapping to this PrimitiveType
*/
template <>
struct PrimitiveTypeTraits<TYPE_BOOLEAN> {
using CppType = UInt8;
- using StorageFieldType = CppType;
using DataType = DataTypeBool;
using ColumnType = ColumnUInt8;
};
template <>
struct PrimitiveTypeTraits<TYPE_TINYINT> {
using CppType = int8_t;
- using StorageFieldType = CppType;
using DataType = DataTypeInt8;
using ColumnType = ColumnInt8;
};
template <>
struct PrimitiveTypeTraits<TYPE_SMALLINT> {
using CppType = int16_t;
- using StorageFieldType = CppType;
using DataType = DataTypeInt16;
using ColumnType = ColumnInt16;
};
template <>
struct PrimitiveTypeTraits<TYPE_INT> {
using CppType = int32_t;
- using StorageFieldType = CppType;
using DataType = DataTypeInt32;
using ColumnType = ColumnInt32;
};
template <>
struct PrimitiveTypeTraits<TYPE_BIGINT> {
using CppType = int64_t;
- using StorageFieldType = CppType;
using DataType = DataTypeInt64;
using ColumnType = ColumnInt64;
};
template <>
struct PrimitiveTypeTraits<TYPE_LARGEINT> {
using CppType = __int128_t;
- using StorageFieldType = CppType;
using DataType = DataTypeInt128;
using ColumnType = ColumnInt128;
};
template <>
struct PrimitiveTypeTraits<TYPE_NULL> {
using CppType = Null;
- using StorageFieldType = CppType;
using DataType = DataTypeNothing;
using ColumnType = IColumnDummy;
};
template <>
struct PrimitiveTypeTraits<TYPE_FLOAT> {
using CppType = float;
- using StorageFieldType = CppType;
using DataType = DataTypeFloat32;
using ColumnType = ColumnFloat32;
};
template <>
struct PrimitiveTypeTraits<TYPE_DOUBLE> {
using CppType = double;
- using StorageFieldType = CppType;
using DataType = DataTypeFloat64;
using ColumnType = ColumnFloat64;
};
template <>
struct PrimitiveTypeTraits<TYPE_TIMEV2> {
using CppType = Float64;
- using StorageFieldType = CppType;
using DataType = DataTypeTimeV2;
using ColumnType = ColumnTimeV2;
};
template <>
struct PrimitiveTypeTraits<TYPE_DATE> {
using CppType = doris::VecDateTimeValue;
- /// Different with compute layer, the DateV1 was stored as uint24_t(3
bytes).
- using StorageFieldType = uint24_t;
using DataType = DataTypeDate;
using ColumnType = ColumnDate;
};
template <>
struct PrimitiveTypeTraits<TYPE_DATETIME> {
using CppType = doris::VecDateTimeValue;
- using StorageFieldType = uint64_t;
using DataType = DataTypeDateTime;
using ColumnType = ColumnDateTime;
};
template <>
struct PrimitiveTypeTraits<TYPE_DATETIMEV2> {
using CppType = DateV2Value<DateTimeV2ValueType>;
- using StorageFieldType = uint64_t;
using DataType = DataTypeDateTimeV2;
using ColumnType = ColumnDateTimeV2;
};
template <>
struct PrimitiveTypeTraits<TYPE_TIMESTAMP_NS> {
using CppType = TimeStampNsValue;
- using StorageFieldType = int64_t;
using DataType = DataTypeTimeStampNs;
using ColumnType = ColumnTimeStampNs;
};
template <>
struct PrimitiveTypeTraits<TYPE_DATEV2> {
using CppType = DateV2Value<DateV2ValueType>;
- using StorageFieldType = uint32_t;
using DataType = DataTypeDateV2;
using ColumnType = ColumnDateV2;
};
@@ -398,7 +379,6 @@ struct PrimitiveTypeTraits<TYPE_DATEV2> {
template <>
struct PrimitiveTypeTraits<TYPE_TIMESTAMPTZ> {
using CppType = TimestampTzValue;
- using StorageFieldType = uint64_t;
using DataType = DataTypeTimeStampTz;
using ColumnType = ColumnTimeStampTz;
};
@@ -406,155 +386,132 @@ struct PrimitiveTypeTraits<TYPE_TIMESTAMPTZ> {
template <>
struct PrimitiveTypeTraits<TYPE_DECIMALV2> {
using CppType = DecimalV2Value;
- /// Different with compute layer, the DecimalV1 was stored as
decimal12_t(12 bytes).
- using StorageFieldType = decimal12_t;
using DataType = DataTypeDecimalV2;
using ColumnType = ColumnDecimal128V2;
};
template <>
struct PrimitiveTypeTraits<TYPE_DECIMAL32> {
using CppType = Decimal32;
- using StorageFieldType = Int32;
using DataType = DataTypeDecimal32;
using ColumnType = ColumnDecimal32;
};
template <>
struct PrimitiveTypeTraits<TYPE_DECIMAL64> {
using CppType = Decimal64;
- using StorageFieldType = Int64;
using DataType = DataTypeDecimal64;
using ColumnType = ColumnDecimal64;
};
template <>
struct PrimitiveTypeTraits<TYPE_DECIMAL128I> {
using CppType = Decimal128V3;
- using StorageFieldType = Int128;
using DataType = DataTypeDecimal128;
using ColumnType = ColumnDecimal128V3;
};
template <>
struct PrimitiveTypeTraits<TYPE_DECIMAL256> {
using CppType = Decimal256;
- using StorageFieldType = wide::Int256;
using DataType = DataTypeDecimal256;
using ColumnType = ColumnDecimal256;
};
template <>
struct PrimitiveTypeTraits<TYPE_IPV4> {
using CppType = IPv4;
- using StorageFieldType = CppType;
using DataType = DataTypeIPv4;
using ColumnType = ColumnIPv4;
};
template <>
struct PrimitiveTypeTraits<TYPE_IPV6> {
using CppType = IPv6;
- using StorageFieldType = CppType;
using DataType = DataTypeIPv6;
using ColumnType = ColumnIPv6;
};
template <>
struct PrimitiveTypeTraits<TYPE_UUID> {
using CppType = UUIDValueType;
- using StorageFieldType = CppType;
using DataType = DataTypeUUID;
using ColumnType = ColumnUUID;
};
template <>
struct PrimitiveTypeTraits<TYPE_CHAR> {
using CppType = String;
- using StorageFieldType = CppType;
using DataType = DataTypeString;
using ColumnType = ColumnString;
};
template <>
struct PrimitiveTypeTraits<TYPE_VARCHAR> {
using CppType = String;
- using StorageFieldType = CppType;
using DataType = DataTypeString;
using ColumnType = ColumnString;
};
template <>
struct PrimitiveTypeTraits<TYPE_STRING> {
using CppType = String;
- using StorageFieldType = CppType;
using DataType = DataTypeString;
using ColumnType = ColumnString;
};
template <>
struct PrimitiveTypeTraits<TYPE_VARBINARY> {
using CppType = doris::StringView;
- using StorageFieldType = CppType;
using DataType = DataTypeVarbinary;
using ColumnType = ColumnVarbinary;
};
template <>
struct PrimitiveTypeTraits<TYPE_HLL> {
using CppType = HyperLogLog;
- using StorageFieldType = CppType;
using DataType = DataTypeHLL;
using ColumnType = ColumnHLL;
};
template <>
struct PrimitiveTypeTraits<TYPE_JSONB> {
using CppType = JsonbField;
- using StorageFieldType = CppType;
using DataType = DataTypeJsonb;
using ColumnType = ColumnString;
};
template <>
struct PrimitiveTypeTraits<TYPE_ARRAY> {
using CppType = Array;
- using StorageFieldType = CppType;
using DataType = DataTypeArray;
using ColumnType = ColumnArray;
};
template <>
struct PrimitiveTypeTraits<TYPE_MAP> {
using CppType = Map;
- using StorageFieldType = CppType;
using DataType = DataTypeMap;
using ColumnType = ColumnMap;
};
template <>
struct PrimitiveTypeTraits<TYPE_STRUCT> {
using CppType = Struct;
- using StorageFieldType = CppType;
using DataType = DataTypeStruct;
using ColumnType = ColumnStruct;
};
template <>
struct PrimitiveTypeTraits<TYPE_VARIANT> {
using CppType = VariantField;
- using StorageFieldType = CppType;
using DataType = DataTypeVariant;
using ColumnType = ColumnVariantV2;
};
template <>
struct PrimitiveTypeTraits<TYPE_BITMAP> {
using CppType = BitmapValue;
- using StorageFieldType = CppType;
using DataType = DataTypeBitMap;
using ColumnType = ColumnBitmap;
};
template <>
struct PrimitiveTypeTraits<TYPE_QUANTILE_STATE> {
using CppType = QuantileState;
- using StorageFieldType = CppType;
using DataType = DataTypeQuantileState;
using ColumnType = ColumnQuantileState;
};
template <>
struct PrimitiveTypeTraits<TYPE_UINT32> {
using CppType = UInt32;
- using StorageFieldType = CppType;
using DataType = DataTypeUInt32;
using ColumnType = ColumnOffset32;
};
template <>
struct PrimitiveTypeTraits<TYPE_UINT64> {
using CppType = UInt64;
- using StorageFieldType = CppType;
using DataType = DataTypeUInt64;
using ColumnType = ColumnOffset64;
};
diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp
b/be/src/core/data_type_serde/data_type_decimal_serde.cpp
index bb136460743..7c8e92a5788 100644
--- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp
@@ -39,6 +39,7 @@
#include "core/data_type_serde/decoded_column_view.h"
#include "core/data_type_serde/orc_serde_utils.h"
#include "core/data_type_serde/parquet_decode_source.h"
+#include "core/decimal12.h"
#include "core/types.h"
#include "exec/common/arithmetic_overflow.h"
#include "exec/common/endian.h"
diff --git a/be/src/core/field.h b/be/src/core/field.h
index 155278fa337..60b391bc211 100644
--- a/be/src/core/field.h
+++ b/be/src/core/field.h
@@ -204,29 +204,6 @@ public:
return f;
}
- template <PrimitiveType PType, typename ValType = std::conditional_t<
- doris::is_string_type(PType),
StringRef,
- typename
PrimitiveTypeTraits<PType>::StorageFieldType>>
- static Field create_field_from_olap_value(const ValType& data) {
- auto f = Field(PType);
- typename PrimitiveTypeTraits<PType>::CppType cpp_value;
- if constexpr (is_string_type(PType)) {
- cpp_value = String(data.data, data.size);
- } else if constexpr (is_date_or_datetime(PType)) {
- if constexpr (PType == TYPE_DATE) {
- cpp_value.from_olap_date(data);
- } else {
- cpp_value.from_olap_datetime(data);
- }
- } else if constexpr (is_decimalv2(PType)) {
- cpp_value = DecimalV2Value(data.integer, data.fraction);
- } else {
- cpp_value = typename PrimitiveTypeTraits<PType>::CppType(data);
- }
- f.template create_concrete<PType>(std::move(cpp_value));
- return f;
- }
-
/** Despite the presence of a template constructor, this constructor is
still needed,
* since, in its absence, the compiler will still generate the default
constructor.
*/
diff --git a/be/src/exprs/aggregate/aggregate_function_window_funnel.h
b/be/src/exprs/aggregate/aggregate_function_window_funnel.h
index 7069cf9ca00..d7686fb5e9e 100644
--- a/be/src/exprs/aggregate/aggregate_function_window_funnel.h
+++ b/be/src/exprs/aggregate/aggregate_function_window_funnel.h
@@ -102,7 +102,6 @@ struct DataValue {
template <PrimitiveType T>
struct WindowFunnelState {
static constexpr PrimitiveType PType = T;
- using NativeType = typename PrimitiveTypeTraits<T>::StorageFieldType;
using DateValueType = typename PrimitiveTypeTraits<T>::CppType;
int event_count = 0;
int64_t window;
diff --git a/be/src/storage/field_type.h b/be/src/storage/field_type.h
index f27df867645..965dd5a835c 100644
--- a/be/src/storage/field_type.h
+++ b/be/src/storage/field_type.h
@@ -20,7 +20,7 @@
namespace doris {
// Storage-engine cell types, used by TabletColumn / KeyCoder and the
-// data_type traits chain. When adding a new value, also extend CppTypeTraits,
+// data_type traits chain. When adding a new value, also extend
StorageTypeTraits,
// FieldTypeTraits and the field_type_size() switch in storage/types.h. Decide
how it maps to
// PrimitiveType and explicitly define its behavior in
primitive_type_to_storage_field_type() and
// storage_field_type_to_primitive_type(), either by providing a mapping or by
throwing.
diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp
b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp
index 06da761a099..b1f7aa42565 100644
--- a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp
+++ b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp
@@ -63,7 +63,7 @@ struct BloomFilterTraits<int128_t> {
template <FieldType field_type>
class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter {
public:
- using CppType = typename CppTypeTraits<field_type>::CppType;
+ using CppType = typename StorageTypeTraits<field_type>::StorageType;
using ValueDict = typename BloomFilterTraits<CppType>::ValueDict;
explicit BloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options)
diff --git a/be/src/storage/index/inverted/clucene/inverted_index_writer.h
b/be/src/storage/index/inverted/clucene/inverted_index_writer.h
index ef6d21aa8a5..51ff4546d44 100644
--- a/be/src/storage/index/inverted/clucene/inverted_index_writer.h
+++ b/be/src/storage/index/inverted/clucene/inverted_index_writer.h
@@ -43,7 +43,7 @@ using namespace doris::segment_v2::inverted_index;
template <FieldType field_type>
class InvertedIndexColumnWriter : public IndexColumnWriter {
public:
- using CppType = typename CppTypeTraits<field_type>::CppType;
+ using CppType = typename StorageTypeTraits<field_type>::StorageType;
InvertedIndexColumnWriter(const std::string& field_name, IndexFileWriter*
index_file_writer,
const TabletIndex* index_meta, const bool
single_field = true);
diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp
b/be/src/storage/index/zone_map/zone_map_index.cpp
index d2b086cc51b..5e96d91a7c2 100644
--- a/be/src/storage/index/zone_map/zone_map_index.cpp
+++ b/be/src/storage/index/zone_map/zone_map_index.cpp
@@ -36,6 +36,7 @@
#include "storage/index/indexed_column_writer.h"
#include "storage/olap_common.h"
#include "storage/segment/encoding_info.h"
+#include "storage/storage_layout.h"
#include "storage/tablet/tablet_schema.h"
#include "storage/types.h"
#include "storage/utils.h"
@@ -205,8 +206,16 @@ void
TypedZoneMapIndexWriter<Type>::_materialize_page_minmax() {
if (!_page_zone_map.has_not_null) {
return;
}
- _page_zone_map.min_value =
doris::Field::create_field_from_olap_value<Type>(_page_min);
- _page_zone_map.max_value =
doris::Field::create_field_from_olap_value<Type>(_page_max);
+ if constexpr (is_string_type(Type)) {
+ _page_zone_map.min_value =
+ Field::create_field<Type>(String(_page_min.data,
_page_min.size));
+ _page_zone_map.max_value =
+ Field::create_field<Type>(String(_page_max.data,
_page_max.size));
+ } else {
+ using Layout =
StorageLayout<primitive_type_to_storage_field_type(Type)>;
+ _page_zone_map.min_value =
Field::create_field<Type>(Layout::to_primitive(_page_min));
+ _page_zone_map.max_value =
Field::create_field<Type>(Layout::to_primitive(_page_max));
+ }
}
template <PrimitiveType Type>
diff --git a/be/src/storage/index/zone_map/zone_map_index.h
b/be/src/storage/index/zone_map/zone_map_index.h
index 7360fb9f797..9b44badbcae 100644
--- a/be/src/storage/index/zone_map/zone_map_index.h
+++ b/be/src/storage/index/zone_map/zone_map_index.h
@@ -29,10 +29,12 @@
#include "common/status.h"
#include "core/data_type/data_type.h"
#include "core/data_type/define_primitive_type.h"
+#include "core/data_type/storage_field_type.h"
#include "core/string_ref.h"
#include "io/fs/file_reader_writer_fwd.h"
#include "storage/metadata_adder.h"
#include "storage/tablet/tablet_schema.h"
+#include "storage/types.h"
#include "util/once.h"
namespace doris {
@@ -118,8 +120,9 @@ public:
template <PrimitiveType Type>
class TypedZoneMapIndexWriter final : public ZoneMapIndexWriter {
public:
- using ValType = std::conditional_t<is_string_type(Type), StringRef,
- typename
PrimitiveTypeTraits<Type>::StorageFieldType>;
+ using ValType = std::conditional_t<
+ is_string_type(Type), StringRef,
+ typename
StorageTypeTraits<primitive_type_to_storage_field_type(Type)>::StorageType>;
explicit TypedZoneMapIndexWriter(DataTypePtr&& data_type);
void add_values(const uint8_t* values, size_t count) override;
diff --git a/be/src/storage/key_coder.h b/be/src/storage/key_coder.h
index 8076e940174..c9d5bbeb8c5 100644
--- a/be/src/storage/key_coder.h
+++ b/be/src/storage/key_coder.h
@@ -84,13 +84,14 @@ template <FieldType field_type, typename Enable = void>
class KeyCoderTraits {};
template <FieldType field_type>
-class KeyCoderTraits<field_type,
- typename std::enable_if<
- IsIntegral<typename
CppTypeTraits<field_type>::CppType>::value ||
- IsDecimalNumber<typename
CppTypeTraits<field_type>::CppType>>::type> {
+class KeyCoderTraits<
+ field_type,
+ typename std::enable_if<
+ IsIntegral<typename
StorageTypeTraits<field_type>::StorageType>::value ||
+ IsDecimalNumber<typename
StorageTypeTraits<field_type>::StorageType>>::type> {
public:
- using CppType = typename CppTypeTraits<field_type>::CppType;
- using UnsignedCppType = typename
CppTypeTraits<field_type>::UnsignedCppType;
+ using CppType = typename StorageTypeTraits<field_type>::StorageType;
+ using UnsignedCppType = typename
StorageTypeTraits<field_type>::UnsignedStorageType;
static void full_encode_ascending(const void* value, std::string* buf) {
UnsignedCppType unsigned_val;
@@ -133,9 +134,9 @@ public:
template <>
class KeyCoderTraits<FieldType::OLAP_FIELD_TYPE_DATE> {
public:
- using CppType = typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::CppType;
+ using CppType = typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::StorageType;
using UnsignedCppType =
- typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::UnsignedCppType;
+ typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::UnsignedStorageType;
public:
static void full_encode_ascending(const void* value, std::string* buf) {
@@ -167,9 +168,9 @@ public:
template <>
class KeyCoderTraits<FieldType::OLAP_FIELD_TYPE_DATEV2> {
public:
- using CppType = typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2>::CppType;
+ using CppType = typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2>::StorageType;
using UnsignedCppType =
- typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2>::UnsignedCppType;
+ typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2>::UnsignedStorageType;
public:
static void full_encode_ascending(const void* value, std::string* buf) {
@@ -202,9 +203,9 @@ public:
template <>
class KeyCoderTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2> {
public:
- using CppType = typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>::CppType;
+ using CppType = typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>::StorageType;
using UnsignedCppType =
- typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>::UnsignedCppType;
+ typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>::UnsignedStorageType;
public:
static void full_encode_ascending(const void* value, std::string* buf) {
@@ -237,9 +238,9 @@ public:
template <>
class KeyCoderTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ> {
public:
- using CppType = typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>::CppType;
+ using CppType = typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>::StorageType;
using UnsignedCppType =
- typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>::UnsignedCppType;
+ typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>::UnsignedStorageType;
public:
static void full_encode_ascending(const void* value, std::string* buf) {
@@ -361,8 +362,8 @@ public:
template <FieldType field_type>
class KeyCoderTraitsForFloat {
public:
- using CppType = typename CppTypeTraits<field_type>::CppType;
- using UnsignedCppType = typename
CppTypeTraits<field_type>::UnsignedCppType;
+ using CppType = typename StorageTypeTraits<field_type>::StorageType;
+ using UnsignedCppType = typename
StorageTypeTraits<field_type>::UnsignedStorageType;
static UnsignedCppType encode_float(CppType value) {
return sortable_float_bits(float_to_int_bits(value));
diff --git a/be/src/storage/segment/binary_dict_page.cpp
b/be/src/storage/segment/binary_dict_page.cpp
index fb85020bb15..97f1bb4a657 100644
--- a/be/src/storage/segment/binary_dict_page.cpp
+++ b/be/src/storage/segment/binary_dict_page.cpp
@@ -288,7 +288,7 @@ Status BinaryDictPageDecoder::next_batch(size_t* n,
MutableColumnPtr& dst) {
// without copying actual char data. This allows length() to work.
// ColumnDictI32 does not implement insert_offsets_from_lengths, so
convert
// it to a predicate column (ColumnString) first. This is a no-op for
- // non-dictionary columns and for ColumnNullable it converts the
nested column.
+ // non-dictionary columns; a Nullable never reaches a page decoder.
dst = dst->convert_to_predicate_column_if_dictionary();
const auto* data_array = reinterpret_cast<const
int32_t*>(_bit_shuffle_ptr->get_data(0));
size_t start_index = _bit_shuffle_ptr->_cur_index;
diff --git a/be/src/storage/segment/bitshuffle_page.h
b/be/src/storage/segment/bitshuffle_page.h
index 0d58fdb17d5..b480de9426a 100644
--- a/be/src/storage/segment/bitshuffle_page.h
+++ b/be/src/storage/segment/bitshuffle_page.h
@@ -38,6 +38,7 @@
#include "storage/segment/options.h"
#include "storage/segment/page_builder.h"
#include "storage/segment/page_decoder.h"
+#include "storage/storage_layout.h"
#include "storage/types.h"
#include "util/alignment.h"
#include "util/coding.h"
@@ -393,7 +394,8 @@ public:
size_t max_fetch = std::min(*n, _num_elements - _cur_index);
- dst->insert_many_fix_len_data(get_data(_cur_index), max_fetch);
+ StorageLayout<Type>::storage_to_column(
+ reinterpret_cast<const uint8_t*>(get_data(_cur_index)),
max_fetch, *dst);
*n = max_fetch;
if constexpr (forward_index) {
_cur_index += max_fetch;
@@ -425,7 +427,8 @@ public:
}
if (LIKELY(read_count > 0)) {
- dst->insert_many_fix_len_data((char*)_buffer.data(), read_count);
+ StorageLayout<Type>::storage_to_column(reinterpret_cast<const
uint8_t*>(_buffer.data()),
+ read_count, *dst);
}
*n = read_count;
diff --git a/be/src/storage/segment/column_reader.cpp
b/be/src/storage/segment/column_reader.cpp
index 30d123a4d8b..ea660a07187 100644
--- a/be/src/storage/segment/column_reader.cpp
+++ b/be/src/storage/segment/column_reader.cpp
@@ -2924,6 +2924,34 @@ Status
FileColumnIterator::next_batch_of_zone_map(size_t* n, MutableColumnPtr& d
return _reader->next_batch_of_zone_map(n, dst);
}
+namespace {
+
+// Runs `decode`, a page decoder call that appends values to the column it is
given. The decoders
+// append to the column of the type, as ScalarColumnWriter::append peels the
Nullable off before
+// the values reach a page, so a Nullable dst lends them its nested column and
the rows they
+// append are marked not NULL here. A dictionary page decoder may replace the
column it appends to
+// (a ColumnDictI32 becomes a ColumnString when a page falls back to plain
encoding), and the
+// replacement goes back into dst.
+template <typename Decode>
+Status decode_values(MutableColumnPtr& dst, const Decode& decode) {
+ if (!is_column_nullable(*dst)) {
+ return decode(dst);
+ }
+ auto& nullable = static_cast<ColumnNullable&>(*dst);
+ MutableColumnPtr nested = nullable.get_nested_column_ptr();
+ const IColumn* const lent = nested.get();
+ const size_t rows = lent->size();
+ Status st = decode(nested);
+ nullable.push_false_to_nullmap(nested->size() - rows);
+ if (nested.get() != lent) {
+ ColumnPtr replacement = std::move(nested);
+ nullable.change_nested_column(replacement);
+ }
+ return st;
+}
+
+} // namespace
+
Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool*
has_null) {
if (!need_to_read()) {
DLOG(INFO) << "File column iterator column " << _column_name << " skip
reading.";
@@ -3004,7 +3032,9 @@ Status FileColumnIterator::next_batch(size_t* n,
MutableColumnPtr& dst, bool* ha
// we use num_rows only for CHECK
size_t num_rows = this_run;
if (!is_null) {
- RETURN_IF_ERROR(_page.data_decoder->next_batch(&num_rows,
dst));
+ RETURN_IF_ERROR(decode_values(dst, [&](MutableColumnPtr&
values) {
+ return _page.data_decoder->next_batch(&num_rows,
values);
+ }));
DCHECK_EQ(this_run, num_rows);
} else {
*has_null = true;
@@ -3021,7 +3051,9 @@ Status FileColumnIterator::next_batch(size_t* n,
MutableColumnPtr& dst, bool* ha
_current_ordinal += this_run;
}
} else {
- RETURN_IF_ERROR(_page.data_decoder->next_batch(&nrows_to_read,
dst));
+ RETURN_IF_ERROR(decode_values(dst, [&](MutableColumnPtr& values) {
+ return _page.data_decoder->next_batch(&nrows_to_read, values);
+ }));
DCHECK_EQ(nrows_to_read, nrows_in_page);
_page.offset_in_page += nrows_to_read;
@@ -3168,8 +3200,11 @@ Status FileColumnIterator::read_by_rowids(const rowid_t*
rowids, const size_t co
// so we need use `page_start_off_in_decoder` to
calculate the actual offset in `data_decoder`
size_t page_start_off_in_decoder =
_page.first_ordinal + _page.offset_in_page -
origin_index;
- RETURN_IF_ERROR(_page.data_decoder->read_by_rowids(
- &rowids[offset], page_start_off_in_decoder,
&read_count, dst));
+ RETURN_IF_ERROR(decode_values(dst,
[&](MutableColumnPtr& values) {
+ return
_page.data_decoder->read_by_rowids(&rowids[offset],
+
page_start_off_in_decoder,
+
&read_count, values);
+ }));
DCHECK_EQ(read_count, this_read_count);
}
}
@@ -3188,8 +3223,10 @@ Status FileColumnIterator::read_by_rowids(const rowid_t*
rowids, const size_t co
total_read_count += nrows_to_read;
remaining -= nrows_to_read;
} else {
- RETURN_IF_ERROR(_page.data_decoder->read_by_rowids(
- &rowids[total_read_count], _page.first_ordinal,
&nrows_to_read, dst));
+ RETURN_IF_ERROR(decode_values(dst, [&](MutableColumnPtr& values) {
+ return _page.data_decoder->read_by_rowids(
+ &rowids[total_read_count], _page.first_ordinal,
&nrows_to_read, values);
+ }));
total_read_count += nrows_to_read;
remaining -= nrows_to_read;
}
diff --git a/be/src/storage/segment/encoding_info.cpp
b/be/src/storage/segment/encoding_info.cpp
index 7490a8fe4cf..dc8655c248e 100644
--- a/be/src/storage/segment/encoding_info.cpp
+++ b/be/src/storage/segment/encoding_info.cpp
@@ -151,8 +151,9 @@ struct TypeEncodingTraits<type, DICT_ENCODING, Slice> {
};
template <>
-struct TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_DATE, FOR_ENCODING,
- typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::CppType> {
+struct TypeEncodingTraits<
+ FieldType::OLAP_FIELD_TYPE_DATE, FOR_ENCODING,
+ typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::StorageType> {
static Status create_page_builder(const PageBuilderOptions& opts,
PageBuilder** builder) {
return
FrameOfReferencePageBuilder<FieldType::OLAP_FIELD_TYPE_DATE>::create(builder,
opts);
}
@@ -164,8 +165,9 @@ struct TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_DATE,
FOR_ENCODING,
};
template <>
-struct TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING,
- typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2>::CppType> {
+struct TypeEncodingTraits<
+ FieldType::OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING,
+ typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2>::StorageType> {
static Status create_page_builder(const PageBuilderOptions& opts,
PageBuilder** builder) {
return
FrameOfReferencePageBuilder<FieldType::OLAP_FIELD_TYPE_DATEV2>::create(builder,
opts);
@@ -178,8 +180,9 @@ struct
TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING,
};
template <>
-struct TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, FOR_ENCODING,
- typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>::CppType> {
+struct TypeEncodingTraits<
+ FieldType::OLAP_FIELD_TYPE_DATETIMEV2, FOR_ENCODING,
+ typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>::StorageType> {
static Status create_page_builder(const PageBuilderOptions& opts,
PageBuilder** builder) {
return
FrameOfReferencePageBuilder<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>::create(builder,
opts);
@@ -193,8 +196,9 @@ struct
TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, FOR_ENCODING,
};
template <>
-struct TypeEncodingTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FOR_ENCODING,
- typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>::CppType> {
+struct TypeEncodingTraits<
+ FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FOR_ENCODING,
+ typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>::StorageType> {
static Status create_page_builder(const PageBuilderOptions& opts,
PageBuilder** builder) {
return
FrameOfReferencePageBuilder<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>::create(builder,
opts);
diff --git a/be/src/storage/segment/encoding_info.h
b/be/src/storage/segment/encoding_info.h
index dfbf7b1d7cf..c5ce7eed27a 100644
--- a/be/src/storage/segment/encoding_info.h
+++ b/be/src/storage/segment/encoding_info.h
@@ -27,7 +27,7 @@
#include "common/status.h"
#include "storage/cache/page_cache.h"
#include "storage/segment/options.h"
-#include "storage/types.h" // CppTypeTraits, used by the EncodingTraits base
below
+#include "storage/types.h" // StorageTypeTraits, used by the EncodingTraits
base below
#include "util/slice.h"
namespace doris {
@@ -172,8 +172,8 @@ struct TypeEncodingTraits {};
template <FieldType field_type, EncodingTypePB encoding_type>
struct EncodingTraits : TypeEncodingTraits<field_type, encoding_type,
- typename
CppTypeTraits<field_type>::CppType> {
- using CppType = typename CppTypeTraits<field_type>::CppType;
+ typename
StorageTypeTraits<field_type>::StorageType> {
+ using CppType = typename StorageTypeTraits<field_type>::StorageType;
static const FieldType type = field_type;
static const EncodingTypePB encoding = encoding_type;
};
diff --git a/be/src/storage/segment/plain_page.h
b/be/src/storage/segment/plain_page.h
index cfc5e428dc8..76b72a999bc 100644
--- a/be/src/storage/segment/plain_page.h
+++ b/be/src/storage/segment/plain_page.h
@@ -22,6 +22,7 @@
#include "storage/segment/options.h"
#include "storage/segment/page_builder.h"
#include "storage/segment/page_decoder.h"
+#include "storage/storage_layout.h"
#include "storage/types.h"
#include "util/coding.h"
#include "util/faststring.h"
@@ -191,7 +192,8 @@ public:
size_t max_fetch = std::min(*n, static_cast<size_t>(_num_elems -
_cur_idx));
const void* src_data = &_data[PLAIN_PAGE_HEADER_SIZE + _cur_idx *
SIZE_OF_TYPE];
- dst->insert_many_fix_len_data((const char*)src_data, max_fetch);
+ StorageLayout<Type>::storage_to_column(static_cast<const
uint8_t*>(src_data), max_fetch,
+ *dst);
*n = max_fetch;
_cur_idx += max_fetch;
@@ -220,7 +222,8 @@ public:
}
if (LIKELY(read_count > 0)) {
- dst->insert_many_fix_len_data((char*)_buffer.data(), read_count);
+ StorageLayout<Type>::storage_to_column(reinterpret_cast<const
uint8_t*>(_buffer.data()),
+ read_count, *dst);
}
*n = read_count;
diff --git a/be/src/storage/storage_layout.cpp
b/be/src/storage/storage_layout.cpp
index 2e10e040475..6bd8107d436 100644
--- a/be/src/storage/storage_layout.cpp
+++ b/be/src/storage/storage_layout.cpp
@@ -47,9 +47,14 @@ const uint8_t* BitCast<FT>::column_to_storage(const IColumn&
column, size_t row_
row_pos);
}
+template <FieldType FT>
+void BitCast<FT>::storage_to_column(const uint8_t* values, size_t n, IColumn&
dst) {
+ dst.insert_many_raw_data(reinterpret_cast<const char*>(values), n);
+}
+
template <FieldType FT>
void repack_from_column(const IColumn& column, size_t row_pos, size_t n,
uint8_t* dst) {
- using StorageValue = typename CppTypeTraits<FT>::CppType;
+ using StorageValue = typename StorageTypeTraits<FT>::StorageType;
using Column =
typename
PrimitiveTypeTraits<storage_field_type_to_primitive_type(FT)>::ColumnType;
const auto& data = assert_cast<const Column&>(column).get_data();
@@ -59,6 +64,21 @@ void repack_from_column(const IColumn& column, size_t
row_pos, size_t n, uint8_t
}
}
+// Appends n stored values of a repacked layout, each turned back into its
PrimitiveValue.
+template <FieldType FT>
+void repack_to_column(const uint8_t* values, size_t n, IColumn& dst) {
+ using StorageValue = typename StorageTypeTraits<FT>::StorageType;
+ using Column =
+ typename
PrimitiveTypeTraits<storage_field_type_to_primitive_type(FT)>::ColumnType;
+ auto& data = assert_cast<Column&>(dst).get_data();
+ data.reserve(data.size() + n);
+ for (size_t i = 0; i < n; ++i) {
+ StorageValue value;
+ memcpy(&value, values + i * sizeof(StorageValue),
sizeof(StorageValue));
+ data.push_back_without_reserve(StorageLayout<FT>::to_primitive(value));
+ }
+}
+
template <FieldType FT>
const uint8_t* FloatingPoint<FT>::column_to_storage(const IColumn& column,
size_t row_pos, size_t n,
StorageBuffers& buffers) {
@@ -81,6 +101,11 @@ const uint8_t* FloatingPoint<FT>::column_to_storage(const
IColumn& column, size_
return bytes.data();
}
+template <FieldType FT>
+void FloatingPoint<FT>::storage_to_column(const uint8_t* values, size_t n,
IColumn& dst) {
+ dst.insert_many_raw_data(reinterpret_cast<const char*>(values), n);
+}
+
const uint8_t* Bytes::column_to_storage(const IColumn& column, size_t row_pos,
size_t n,
StorageBuffers& buffers) {
const auto& strings = assert_cast<const ColumnString&>(column);
@@ -167,6 +192,21 @@ const uint8_t*
StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL>::column_to_stor
return buffers.bytes.data();
}
+void StorageLayout<FieldType::OLAP_FIELD_TYPE_DATE>::storage_to_column(const
uint8_t* values,
+ size_t
n, IColumn& dst) {
+
storage_layout_detail::repack_to_column<FieldType::OLAP_FIELD_TYPE_DATE>(values,
n, dst);
+}
+
+void
StorageLayout<FieldType::OLAP_FIELD_TYPE_DATETIME>::storage_to_column(const
uint8_t* values,
+
size_t n, IColumn& dst) {
+
storage_layout_detail::repack_to_column<FieldType::OLAP_FIELD_TYPE_DATETIME>(values,
n, dst);
+}
+
+void
StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL>::storage_to_column(const
uint8_t* values,
+
size_t n, IColumn& dst) {
+
storage_layout_detail::repack_to_column<FieldType::OLAP_FIELD_TYPE_DECIMAL>(values,
n, dst);
+}
+
const uint8_t*
StorageLayout<FieldType::OLAP_FIELD_TYPE_CHAR>::column_to_storage(
const IColumn& column, size_t row_pos, size_t n, size_t length, const
uint8_t* null_map,
StorageBuffers& buffers) {
diff --git a/be/src/storage/storage_layout.h b/be/src/storage/storage_layout.h
index 17cdc6d404f..2831f3cf16a 100644
--- a/be/src/storage/storage_layout.h
+++ b/be/src/storage/storage_layout.h
@@ -55,7 +55,7 @@ class IColumn;
// 5. Serialised: BITMAP, HLL, QUANTILE_STATE.
// 6. AGG_STATE: a string, a bitmap or a fixed-length object, whichever its
aggregate
// function serialises to; the column holding the states tells which.
-// Compute-layer types: PrimitiveTypeTraits<PT>; storage-layer types:
CppTypeTraits<FT>.
+// Compute-layer types: PrimitiveTypeTraits<PT>; storage-layer types:
StorageTypeTraits<FT>.
//
// Every layout converts a batch the same way: column_to_storage(column,
row_pos, n, buffers)
// returns the stored cells of rows [row_pos, row_pos + n) back to back, n
fixed-width values
@@ -65,6 +65,12 @@ class IColumn;
// NULL row holds converts without harm, and no caller uses a NULL row's cell.
CHAR padding is
// the one exception: it needs a range without NULL rows, see
StorageLayout<CHAR>. to_storage
// retains a typed result for single-value key and predicate conversions.
+//
+// The fixed-width layouts also read back. to_primitive is the inverse of
to_storage for one
+// value, and storage_to_column(values, n, dst) appends the n stored values at
`values`, which
+// need not be aligned, to `dst`, the column of the PrimitiveType:
FileColumnIterator peels a
+// Nullable off first, as ScalarColumnWriter::append does. The page decoders
read through it,
+// and the zone map turns its stored bounds back with to_primitive.
template <FieldType FT>
struct StorageLayout;
@@ -109,19 +115,22 @@ namespace storage_layout_detail {
// stored values.
template <FieldType FT>
struct BitCast {
-private:
- using StorageValue = typename CppTypeTraits<FT>::CppType;
+ using StorageValue = typename StorageTypeTraits<FT>::StorageType;
using PrimitiveValue =
typename
PrimitiveTypeTraits<storage_field_type_to_primitive_type(FT)>::CppType;
static_assert(sizeof(StorageValue) == sizeof(PrimitiveValue));
-public:
static StorageValue to_storage(const PrimitiveValue& value) {
return std::bit_cast<StorageValue>(value);
}
+ static PrimitiveValue to_primitive(const StorageValue& value) {
+ return std::bit_cast<PrimitiveValue>(value);
+ }
// Always borrows the column's bytes.
static const uint8_t* column_to_storage(const IColumn& column, size_t
row_pos, size_t n,
StorageBuffers& buffers);
+ // One copy: the stored values are the column's values.
+ static void storage_to_column(const uint8_t* values, size_t n, IColumn&
dst);
};
// Every NaN is stored as the quiet NaN, so equal values have equal stored
bytes
@@ -129,23 +138,25 @@ public:
// treat all NaNs as one value.
template <FieldType FT>
struct FloatingPoint {
-private:
- using StorageValue = typename CppTypeTraits<FT>::CppType;
+ using StorageValue = typename StorageTypeTraits<FT>::StorageType;
using PrimitiveValue =
typename
PrimitiveTypeTraits<storage_field_type_to_primitive_type(FT)>::CppType;
static_assert(std::is_floating_point_v<StorageValue>);
static_assert(std::is_same_v<StorageValue, PrimitiveValue>);
-public:
static StorageValue canonicalize_nan(StorageValue value) {
return std::isnan(value) ?
std::numeric_limits<StorageValue>::quiet_NaN() : value;
}
static StorageValue to_storage(PrimitiveValue value) { return
canonicalize_nan(value); }
+ // A stored value is read back as it is: only writing canonicalises.
+ static PrimitiveValue to_primitive(StorageValue value) { return value; }
// Borrows the column's bytes when [row_pos, row_pos + n) has no NaN,
otherwise
// converts the range into buffers.bytes. NaNs outside the range do not
require a
// copy; the source column is never modified.
static const uint8_t* column_to_storage(const IColumn& column, size_t
row_pos, size_t n,
StorageBuffers& buffers);
+ // One copy: the stored values are the column's values.
+ static void storage_to_column(const uint8_t* values, size_t n, IColumn&
dst);
};
// A string is stored as its own bytes, whatever their length: the cells are
Slice
@@ -216,49 +227,60 @@ extern template struct
storage_layout_detail::FloatingPoint<FieldType::OLAP_FIEL
// V1 DATE: three bytes of (year << 9) | (month << 5) | day.
template <>
struct StorageLayout<FieldType::OLAP_FIELD_TYPE_DATE> {
-private:
- using StorageValue = typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::CppType;
+ using StorageValue = typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::StorageType;
using PrimitiveValue = typename PrimitiveTypeTraits<TYPE_DATE>::CppType;
static_assert(std::is_same_v<StorageValue, uint24_t>);
-public:
static StorageValue to_storage(const PrimitiveValue& value) {
return StorageValue(cast_set<uint32_t>(value.to_olap_date()));
}
+ // The fields are taken as stored: a date was checked when it was written.
+ static PrimitiveValue to_primitive(const StorageValue& value) {
+ PrimitiveValue primitive;
+ primitive.set_olap_date(value);
+ return primitive;
+ }
static const uint8_t* column_to_storage(const IColumn& column, size_t
row_pos, size_t n,
StorageBuffers& buffers);
+ static void storage_to_column(const uint8_t* values, size_t n, IColumn&
dst);
};
// V1 DATETIME: the decimal digits YYYYMMDDhhmmss packed into one integer.
template <>
struct StorageLayout<FieldType::OLAP_FIELD_TYPE_DATETIME> {
-private:
- using StorageValue = typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIME>::CppType;
+ using StorageValue =
+ typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIME>::StorageType;
using PrimitiveValue = typename
PrimitiveTypeTraits<TYPE_DATETIME>::CppType;
static_assert(std::is_same_v<StorageValue, int64_t>);
-public:
static StorageValue to_storage(const PrimitiveValue& value) {
return static_cast<StorageValue>(value.to_olap_datetime());
}
+ static PrimitiveValue to_primitive(const StorageValue& value) {
+ return
PrimitiveValue::create_from_olap_datetime(static_cast<uint64_t>(value));
+ }
static const uint8_t* column_to_storage(const IColumn& column, size_t
row_pos, size_t n,
StorageBuffers& buffers);
+ static void storage_to_column(const uint8_t* values, size_t n, IColumn&
dst);
};
// V1 DECIMAL: the integer and the nine-digit fraction stored side by side.
template <>
struct StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL> {
-private:
- using StorageValue = typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL>::CppType;
+ using StorageValue =
+ typename
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL>::StorageType;
using PrimitiveValue = typename
PrimitiveTypeTraits<TYPE_DECIMALV2>::CppType;
static_assert(std::is_same_v<StorageValue, decimal12_t>);
-public:
static StorageValue to_storage(const PrimitiveValue& value) {
return {value.int_value(), value.frac_value()};
}
+ static PrimitiveValue to_primitive(const StorageValue& value) {
+ return PrimitiveValue(value.integer, value.fraction);
+ }
static const uint8_t* column_to_storage(const IColumn& column, size_t
row_pos, size_t n,
StorageBuffers& buffers);
+ static void storage_to_column(const uint8_t* values, size_t n, IColumn&
dst);
};
template <>
diff --git a/be/src/storage/types.h b/be/src/storage/types.h
index fc3602f0df6..d7e806ebf2e 100644
--- a/be/src/storage/types.h
+++ b/be/src/storage/types.h
@@ -59,189 +59,189 @@ static const std::vector<std::string> DATE_FORMATS {
// ARRAY / MAP / STRUCT are intentionally NOT specialized here: they are
// containers of other types, so only their element types have a storage-layer
// cell representation. The primary template below uses a deferred
-// static_assert so that instantiating `CppTypeTraits<X>` for any
+// static_assert so that instantiating `StorageTypeTraits<X>` for any
// unspecialized FieldType — most importantly ARRAY/MAP/STRUCT — fails at
// build time with a clear message, rather than at runtime.
namespace detail {
template <FieldType>
-inline constexpr bool cpp_type_traits_unspecialized = false;
+inline constexpr bool storage_type_traits_unspecialized = false;
} // namespace detail
template <FieldType field_type>
-struct CppTypeTraits {
- static_assert(detail::cpp_type_traits_unspecialized<field_type>,
- "CppTypeTraits not specialized for this FieldType. "
+struct StorageTypeTraits {
+ static_assert(detail::storage_type_traits_unspecialized<field_type>,
+ "StorageTypeTraits not specialized for this FieldType. "
"ARRAY / MAP / STRUCT and similar container types have no "
"storage-layer cell representation — operate on the element "
"type instead.");
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_BOOL> {
- using CppType = uint8_t;
- using UnsignedCppType = uint8_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_BOOL> {
+ using StorageType = uint8_t;
+ using UnsignedStorageType = uint8_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_TINYINT> {
- using CppType = int8_t;
- using UnsignedCppType = uint8_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_TINYINT> {
+ using StorageType = int8_t;
+ using UnsignedStorageType = uint8_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_SMALLINT> {
- using CppType = int16_t;
- using UnsignedCppType = uint16_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_SMALLINT> {
+ using StorageType = int16_t;
+ using UnsignedStorageType = uint16_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_INT> {
- using CppType = int32_t;
- using UnsignedCppType = uint32_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_INT> {
+ using StorageType = int32_t;
+ using UnsignedStorageType = uint32_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT> {
- using CppType = uint32_t;
- using UnsignedCppType = uint32_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT> {
+ using StorageType = uint32_t;
+ using UnsignedStorageType = uint32_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_BIGINT> {
- using CppType = int64_t;
- using UnsignedCppType = uint64_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_BIGINT> {
+ using StorageType = int64_t;
+ using UnsignedStorageType = uint64_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT> {
- using CppType = uint64_t;
- using UnsignedCppType = uint64_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT> {
+ using StorageType = uint64_t;
+ using UnsignedStorageType = uint64_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_LARGEINT> {
- using CppType = int128_t;
- using UnsignedCppType = uint128_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_LARGEINT> {
+ using StorageType = int128_t;
+ using UnsignedStorageType = uint128_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_FLOAT> {
- using CppType = float;
- using UnsignedCppType = uint32_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_FLOAT> {
+ using StorageType = float;
+ using UnsignedStorageType = uint32_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DOUBLE> {
- using CppType = double;
- using UnsignedCppType = uint64_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DOUBLE> {
+ using StorageType = double;
+ using UnsignedStorageType = uint64_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL> {
- using CppType = decimal12_t;
- using UnsignedCppType = decimal12_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL> {
+ using StorageType = decimal12_t;
+ using UnsignedStorageType = decimal12_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL32> {
- using CppType = int32_t;
- using UnsignedCppType = uint32_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL32> {
+ using StorageType = int32_t;
+ using UnsignedStorageType = uint32_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL64> {
- using CppType = int64_t;
- using UnsignedCppType = uint64_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL64> {
+ using StorageType = int64_t;
+ using UnsignedStorageType = uint64_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL128I> {
- using CppType = int128_t;
- using UnsignedCppType = uint128_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL128I> {
+ using StorageType = int128_t;
+ using UnsignedStorageType = uint128_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL256> {
- using CppType = wide::Int256;
- using UnsignedCppType = wide::UInt256;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL256> {
+ using StorageType = wide::Int256;
+ using UnsignedStorageType = wide::UInt256;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE> {
- using CppType = uint24_t;
- using UnsignedCppType = uint24_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE> {
+ using StorageType = uint24_t;
+ using UnsignedStorageType = uint24_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2> {
- using CppType = uint32_t;
- using UnsignedCppType = uint32_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2> {
+ using StorageType = uint32_t;
+ using UnsignedStorageType = uint32_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMEV2> {
- using CppType = uint64_t;
- using UnsignedCppType = uint64_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMEV2> {
+ using StorageType = uint64_t;
+ using UnsignedStorageType = uint64_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2> {
- using CppType = uint64_t;
- using UnsignedCppType = uint64_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2> {
+ using StorageType = uint64_t;
+ using UnsignedStorageType = uint64_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS> {
- using CppType = int64_t;
- using UnsignedCppType = uint64_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS> {
+ using StorageType = int64_t;
+ using UnsignedStorageType = uint64_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ> {
- using CppType = uint64_t;
- using UnsignedCppType = uint64_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ> {
+ using StorageType = uint64_t;
+ using UnsignedStorageType = uint64_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIME> {
- using CppType = int64_t;
- using UnsignedCppType = uint64_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIME> {
+ using StorageType = int64_t;
+ using UnsignedStorageType = uint64_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_IPV4> {
- using CppType = uint32_t;
- using UnsignedCppType = uint32_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_IPV4> {
+ using StorageType = uint32_t;
+ using UnsignedStorageType = uint32_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_IPV6> {
- using CppType = uint128_t;
- using UnsignedCppType = uint128_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_IPV6> {
+ using StorageType = uint128_t;
+ using UnsignedStorageType = uint128_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_UUID> {
- using CppType = uint128_t;
- using UnsignedCppType = uint128_t;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_UUID> {
+ using StorageType = uint128_t;
+ using UnsignedStorageType = uint128_t;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_CHAR> {
- using CppType = Slice;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_CHAR> {
+ using StorageType = Slice;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_VARCHAR> {
- using CppType = Slice;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_VARCHAR> {
+ using StorageType = Slice;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_STRING> {
- using CppType = Slice;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_STRING> {
+ using StorageType = Slice;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_JSONB> {
- using CppType = Slice;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_JSONB> {
+ using StorageType = Slice;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_VARIANT> {
- using CppType = Slice;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_VARIANT> {
+ using StorageType = Slice;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_HLL> {
- using CppType = Slice;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_HLL> {
+ using StorageType = Slice;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_BITMAP> {
- using CppType = Slice;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_BITMAP> {
+ using StorageType = Slice;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE> {
- using CppType = Slice;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE> {
+ using StorageType = Slice;
};
template <>
-struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_AGG_STATE> {
- using CppType = Slice;
+struct StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_AGG_STATE> {
+ using StorageType = Slice;
};
template <FieldType field_type>
-struct BaseFieldTypeTraits : public CppTypeTraits<field_type> {
- using CppType = typename CppTypeTraits<field_type>::CppType;
+struct BaseFieldTypeTraits : public StorageTypeTraits<field_type> {
+ using CppType = typename StorageTypeTraits<field_type>::StorageType;
static inline CppType get_cpp_type_value(const void* address) {
if constexpr (field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT) {
@@ -271,7 +271,7 @@ struct BaseFieldTypeTraits : public
CppTypeTraits<field_type> {
// FieldType::OLAP_FIELD_TYPE_DOUBLE, to reduce redundant code
template <FieldType fieldType, bool isArithmetic>
struct NumericFieldTypeTraits : public BaseFieldTypeTraits<fieldType> {
- using CppType = typename CppTypeTraits<fieldType>::CppType;
+ using CppType = typename StorageTypeTraits<fieldType>::StorageType;
};
template <FieldType fieldType>
@@ -386,14 +386,14 @@ struct
FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_AGG_STATE>
// Instantiate this template to get static access to the type traits.
template <FieldType field_type>
struct TypeTraits : public FieldTypeTraits<field_type> {
- using CppType = typename CppTypeTraits<field_type>::CppType;
+ using CppType = typename StorageTypeTraits<field_type>::StorageType;
static const FieldType type = field_type;
static const int32_t size = sizeof(CppType);
};
// In-memory storage cell footprint for one value of `field_type`,
-// i.e. sizeof(CppTypeTraits<field_type>::CppType).
+// i.e. sizeof(StorageTypeTraits<field_type>::StorageType).
//
// This is NOT the schema-declared length:
// - CHAR(N) / VARCHAR(N) / STRING / JSONB / VARIANT / HLL / BITMAP /
@@ -417,7 +417,7 @@ inline size_t field_type_size(FieldType field_type) {
switch (field_type) {
#define DORIS_FIELD_TYPE_SIZE_CASE(ft) \
case FieldType::ft: \
- return sizeof(typename CppTypeTraits<FieldType::ft>::CppType);
+ return sizeof(typename StorageTypeTraits<FieldType::ft>::StorageType);
DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_BOOL)
DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TINYINT)
DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_SMALLINT)
diff --git a/be/test/core/column/column_array_test.cpp
b/be/test/core/column/column_array_test.cpp
index e0dd2c6febe..13d532536d4 100644
--- a/be/test/core/column/column_array_test.cpp
+++ b/be/test/core/column/column_array_test.cpp
@@ -365,15 +365,6 @@ protected:
};
//////////////////////// basic function from column.h ////////////////////////
-TEST_F(ColumnArrayTest, InsertManyFixLengthDataTest) {
- auto callback = [&](MutableColumns& load_cols, DataTypeSerDeSPtrs serders)
{
- for (auto& col : array_columns) {
- EXPECT_ANY_THROW(col->insert_many_fix_len_data(nullptr, 0));
- }
- };
- assert_insert_many_fix_len_data(array_columns, serdes, callback);
-}
-
TEST_F(ColumnArrayTest, InsertManyDictDataTest) {
auto callback = [&](MutableColumns& load_cols, DataTypeSerDeSPtrs serders)
{
for (auto& col : array_columns) {
diff --git a/be/test/core/column/column_decimal_test.cpp
b/be/test/core/column/column_decimal_test.cpp
index 01beb71e360..ca4cffa0cab 100644
--- a/be/test/core/column/column_decimal_test.cpp
+++ b/be/test/core/column/column_decimal_test.cpp
@@ -238,12 +238,6 @@ TEST_F(ColumnDecimalTest, insert_indices_from) {
_column_decimal_common_test_with_type(assert_column_vector_insert_indices_from_callback,
false);
}
-// decimal, vector, nullable, PredicateColumnType
-TEST_F(ColumnDecimalTest, insert_many_fix_len_data) {
-
_column_decimal_common_test_with_type(assert_column_vector_insert_many_fix_len_data_callback,
- true);
-}
-
TEST_F(ColumnDecimalTest, insert_many_raw_data) {
_column_decimal_common_test_with_type(assert_column_vector_insert_many_raw_data_callback,
false);
diff --git a/be/test/core/column/column_vector_test.cpp
b/be/test/core/column/column_vector_test.cpp
index 1517a2df370..0552a7e7271 100644
--- a/be/test/core/column/column_vector_test.cpp
+++ b/be/test/core/column/column_vector_test.cpp
@@ -249,30 +249,6 @@ TEST_F(ColumnVectorTest, insert_range_of_integer) {
assert_column_vector_insert_range_of_integer_callback<TYPE_DOUBLE>((Float64)0,
column_float64->get_ptr());
}
-// void insert_date_column(const char* data_ptr, size_t num) {
-// decimal, vector, nullable, PredicateColumnType
-TEST_F(ColumnVectorTest, insert_many_fix_len_data) {
-
assert_column_vector_insert_many_fix_len_data_callback<TYPE_TINYINT>((Int8)0,
-
column_int8->get_ptr());
-
assert_column_vector_insert_many_fix_len_data_callback<TYPE_SMALLINT>((Int16)0,
-
column_int16->get_ptr());
- assert_column_vector_insert_many_fix_len_data_callback<TYPE_INT>((Int32)0,
-
column_int32->get_ptr());
-
assert_column_vector_insert_many_fix_len_data_callback<TYPE_BIGINT>((Int64)0,
-
column_int64->get_ptr());
-
assert_column_vector_insert_many_fix_len_data_callback<TYPE_LARGEINT>((Int128)0,
-
column_int128->get_ptr());
-
assert_column_vector_insert_many_fix_len_data_callback<TYPE_BOOLEAN>((UInt8)0,
-
column_uint8->get_ptr());
- assert_column_vector_insert_many_fix_len_data_callback<TYPE_DATETIMEV2>(
- (UInt64)0, column_datetime_v2_0->get_ptr());
- assert_column_vector_insert_many_fix_len_data_callback<TYPE_DATETIMEV2>(
- (UInt64)0, column_datetime_v2_5->get_ptr());
- assert_column_vector_insert_many_fix_len_data_callback<TYPE_DATETIMEV2>(
- (UInt64)0, column_datetime_v2_6->get_ptr());
-
assert_column_vector_insert_many_fix_len_data_callback<TYPE_DATEV2>((UInt32)0,
-
column_date_v2->get_ptr());
-}
TEST_F(ColumnVectorTest, insert_many_raw_data) {
_column_vector_common_test(assert_column_vector_insert_many_raw_data_callback);
}
diff --git a/be/test/core/column/common_column_test.h
b/be/test/core/column/common_column_test.h
index 8a97ae0f10d..6320dd52aa3 100644
--- a/be/test/core/column/common_column_test.h
+++ b/be/test/core/column/common_column_test.h
@@ -264,17 +264,8 @@ public:
////////// =================== column data insert interface assert(16)
=================== //////////
// In storage layer such as segment_iterator to call these function
- // insert_many_fix_len_data (const char *pos, size_t num);
// insert_many_dict_data (const int32_t *data_array, size_t start_index,
const StringRef *dict, size_t data_num, uint32_t dict_num=0)
// insert_many_continuous_binary_data (const char *data, const uint32_t
*offsets, const size_t num)
- static void assert_insert_many_fix_len_data(
- MutableColumns& load_cols, DataTypeSerDeSPtrs serders,
- std::function<void(MutableColumns& load_cols, DataTypeSerDeSPtrs
serders)>
- assert_callback) {
- // Create a column to verify `insert_many_fix_len_data` functionality
- assert_callback(load_cols, serders);
- }
-
static void assert_insert_many_dict_data(
MutableColumns& load_cols, DataTypeSerDeSPtrs serders,
std::function<void(MutableColumns& load_cols, DataTypeSerDeSPtrs
serders)>
@@ -2836,46 +2827,6 @@ auto
assert_column_vector_insert_range_of_integer_callback = [](auto x, const Mu
}
};
template <PrimitiveType PType>
-auto assert_column_vector_insert_many_fix_len_data_callback = [](auto x, const
MutableColumnPtr&
-
source_column) {
- using T = decltype(x);
- using ColumnVecType =
- std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<PType>,
ColumnVector<PType>>;
- std::vector<size_t> insert_vals_count = {0, 10, 1000};
- auto* col_vec_src = assert_cast<ColumnVecType*>(source_column.get());
- auto src_size = source_column->size();
- std::vector<size_t> src_data_indices = {0, src_size - 1, (src_size + 1) >>
1};
-
- auto test_func = [&](size_t clone_count) {
- size_t actual_clone_count = std::min(clone_count, src_size);
- auto target_column = source_column->clone_resized(actual_clone_count);
- auto* col_vec_target =
assert_cast<ColumnVecType*>(target_column.get());
- for (auto pos = src_data_indices.begin(); pos <
src_data_indices.end(); ++pos) {
- if (*pos >= src_size) {
- continue;
- }
- for (auto n : insert_vals_count) {
- col_vec_target->resize(actual_clone_count);
- size_t actual_insert_count = std::min(n, src_size - *pos);
-
col_vec_target->insert_many_fix_len_data(source_column->get_data_at(*pos).data,
- actual_insert_count);
- auto target_size = col_vec_target->size();
- EXPECT_EQ(target_size, actual_clone_count +
actual_insert_count);
- size_t i = 0;
- for (; i < actual_clone_count; ++i) {
- EXPECT_EQ(col_vec_target->get_element(i),
col_vec_src->get_element(i));
- }
- for (size_t j = *pos; i < target_size; ++i, ++j) {
- EXPECT_EQ(col_vec_target->get_element(i),
col_vec_src->get_element(j))
- << col_vec_src->get_name() << ' ' <<
col_vec_target->get_name();
- }
- }
- }
- };
- test_func(0);
- test_func(10);
-};
-template <PrimitiveType PType>
auto assert_column_vector_insert_many_raw_data_callback = [](auto x, const
MutableColumnPtr&
source_column) {
using T = decltype(x);
diff --git a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp
b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp
index d73b7dd7344..673e41dac64 100644
--- a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp
+++ b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp
@@ -218,7 +218,7 @@ TEST_F(PartitionTransformersTest,
date_partitions_before_epoch_use_calendar_ordi
TEST_F(PartitionTransformersTest, test_integer_truncate_transform) {
const std::vector<int32_t> values({1, -1});
auto column = ColumnInt32::create();
- column->insert_many_fix_len_data(reinterpret_cast<const
char*>(values.data()), values.size());
+ column->insert_many_raw_data(reinterpret_cast<const char*>(values.data()),
values.size());
ColumnWithTypeAndName test_int(column->get_ptr(),
std::make_shared<DataTypeInt32>(),
"test_int");
@@ -239,7 +239,7 @@ TEST_F(PartitionTransformersTest,
test_integer_truncate_transform) {
TEST_F(PartitionTransformersTest, test_bigint_truncate_transform) {
const std::vector<int64_t> values({1, -1});
auto column = ColumnInt64::create();
- column->insert_many_fix_len_data(reinterpret_cast<const
char*>(values.data()), values.size());
+ column->insert_many_raw_data(reinterpret_cast<const char*>(values.data()),
values.size());
ColumnWithTypeAndName test_bigint(column->get_ptr(),
std::make_shared<DataTypeInt64>(),
"test_bigint");
@@ -261,7 +261,7 @@ TEST_F(PartitionTransformersTest,
test_bigint_truncate_transform) {
TEST_F(PartitionTransformersTest, test_decimal32_truncate_transform) {
const std::vector<int32_t> values({1065});
auto column = ColumnDecimal32::create(0, 2);
- column->insert_many_fix_len_data(reinterpret_cast<const
char*>(values.data()), values.size());
+ column->insert_many_raw_data(reinterpret_cast<const char*>(values.data()),
values.size());
ColumnWithTypeAndName test_decimal32(
column->get_ptr(), std::make_shared<DataTypeDecimal32>(4, 2),
"test_decimal32");
@@ -326,7 +326,7 @@ TEST_F(PartitionTransformersTest,
test_floating_point_special_partition_value) {
TEST_F(PartitionTransformersTest, test_integer_bucket_transform) {
const std::vector<int32_t> values({34, -123}); // 2017239379, -471378254
auto column = ColumnInt32::create();
- column->insert_many_fix_len_data(reinterpret_cast<const
char*>(values.data()), values.size());
+ column->insert_many_raw_data(reinterpret_cast<const char*>(values.data()),
values.size());
ColumnWithTypeAndName test_int(column->get_ptr(),
std::make_shared<DataTypeInt32>(),
"test_int");
@@ -347,7 +347,7 @@ TEST_F(PartitionTransformersTest,
test_integer_bucket_transform) {
TEST_F(PartitionTransformersTest, test_bigint_bucket_transform) {
const std::vector<int64_t> values({34, -123}); // 2017239379, -471378254
auto column = ColumnInt64::create();
- column->insert_many_fix_len_data(reinterpret_cast<const
char*>(values.data()), values.size());
+ column->insert_many_raw_data(reinterpret_cast<const char*>(values.data()),
values.size());
ColumnWithTypeAndName test_bigint(column->get_ptr(),
std::make_shared<DataTypeInt64>(),
"test_bigint");
@@ -369,7 +369,7 @@ TEST_F(PartitionTransformersTest,
test_bigint_bucket_transform) {
TEST_F(PartitionTransformersTest, test_decimal32_bucket_transform) {
const std::vector<int32_t> values({1420}); // -500754589
auto column = ColumnDecimal32::create(0, 2);
- column->insert_many_fix_len_data(reinterpret_cast<const
char*>(values.data()), values.size());
+ column->insert_many_raw_data(reinterpret_cast<const char*>(values.data()),
values.size());
ColumnWithTypeAndName test_decimal32(
column->get_ptr(), std::make_shared<DataTypeDecimal32>(4, 2),
"test_decimal32");
@@ -649,7 +649,7 @@ TEST_F(PartitionTransformersTest,
test_timestamp_hour_transform) {
TEST_F(PartitionTransformersTest, test_void_transform) {
const std::vector<int32_t> values({1, -1});
auto column = ColumnInt32::create();
- column->insert_many_fix_len_data(reinterpret_cast<const
char*>(values.data()), values.size());
+ column->insert_many_raw_data(reinterpret_cast<const char*>(values.data()),
values.size());
ColumnWithTypeAndName test_int(column->get_ptr(),
std::make_shared<DataTypeInt32>(),
"test_int");
@@ -671,7 +671,7 @@ TEST_F(PartitionTransformersTest,
test_nullable_column_integer_truncate_transfor
const std::vector<int32_t> values({1, -1});
auto column = ColumnNullable::create(ColumnInt32::create(),
ColumnUInt8::create());
column->insert_data(nullptr, 0);
- column->insert_many_fix_len_data(reinterpret_cast<const
char*>(values.data()), values.size());
+ column->insert_many_raw_data(reinterpret_cast<const char*>(values.data()),
values.size());
ColumnWithTypeAndName test_int(
column->get_ptr(),
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>()),
"test_int");
diff --git a/be/test/storage/index/date_bloom_filter_test.cpp
b/be/test/storage/index/date_bloom_filter_test.cpp
index 00703648056..e5473e52a7a 100644
--- a/be/test/storage/index/date_bloom_filter_test.cpp
+++ b/be/test/storage/index/date_bloom_filter_test.cpp
@@ -145,17 +145,13 @@ TEST_F(DateBloomFilterTest, query_index_test) {
auto date = timestamp_from_date("2024-11-08");
auto datetime = timestamp_from_datetime("2024-11-08 09:00:00");
- uint24_t olap_date_value(date.to_olap_date());
- uint64_t olap_datetime_value(datetime.to_olap_datetime());
- columns[0]->insert_many_fix_len_data(reinterpret_cast<const
char*>(&olap_date_value), 1);
- columns[1]->insert_many_fix_len_data(reinterpret_cast<const
char*>(&olap_datetime_value), 1);
+ columns[0]->insert_data(reinterpret_cast<const char*>(&date),
sizeof(date));
+ columns[1]->insert_data(reinterpret_cast<const char*>(&datetime),
sizeof(datetime));
date = timestamp_from_date("2024-11-09");
datetime = timestamp_from_datetime("2024-11-09 09:00:00");
- olap_date_value = date.to_olap_date();
- olap_datetime_value = datetime.to_olap_datetime();
- columns[0]->insert_many_fix_len_data(reinterpret_cast<const
char*>(&olap_date_value), 1);
- columns[1]->insert_many_fix_len_data(reinterpret_cast<const
char*>(&olap_datetime_value), 1);
+ columns[0]->insert_data(reinterpret_cast<const char*>(&date),
sizeof(date));
+ columns[1]->insert_data(reinterpret_cast<const char*>(&datetime),
sizeof(datetime));
block.set_columns(std::move(columns));
Status st;
@@ -242,17 +238,13 @@ TEST_F(DateBloomFilterTest, in_list_predicate_test) {
// Insert test data
auto date = timestamp_from_date("2024-11-08");
auto datetime = timestamp_from_datetime("2024-11-08 09:00:00");
- uint24_t olap_date_value(date.to_olap_date());
- uint64_t olap_datetime_value(datetime.to_olap_datetime());
- columns[0]->insert_many_fix_len_data(reinterpret_cast<const
char*>(&olap_date_value), 1);
- columns[1]->insert_many_fix_len_data(reinterpret_cast<const
char*>(&olap_datetime_value), 1);
+ columns[0]->insert_data(reinterpret_cast<const char*>(&date),
sizeof(date));
+ columns[1]->insert_data(reinterpret_cast<const char*>(&datetime),
sizeof(datetime));
date = timestamp_from_date("2024-11-09");
datetime = timestamp_from_datetime("2024-11-09 09:00:00");
- olap_date_value = date.to_olap_date();
- olap_datetime_value = datetime.to_olap_datetime();
- columns[0]->insert_many_fix_len_data(reinterpret_cast<const
char*>(&olap_date_value), 1);
- columns[1]->insert_many_fix_len_data(reinterpret_cast<const
char*>(&olap_datetime_value), 1);
+ columns[0]->insert_data(reinterpret_cast<const char*>(&date),
sizeof(date));
+ columns[1]->insert_data(reinterpret_cast<const char*>(&datetime),
sizeof(datetime));
block.set_columns(std::move(columns));
EXPECT_TRUE(rowset_writer->add_block(&block).ok());
diff --git a/be/test/storage/index/index_builder_test.cpp
b/be/test/storage/index/index_builder_test.cpp
index 746d93536e3..40e70cc1dd7 100644
--- a/be/test/storage/index/index_builder_test.cpp
+++ b/be/test/storage/index/index_builder_test.cpp
@@ -3648,7 +3648,7 @@ TEST_F(IndexBuilderTest,
BuildIndexReusesStringDescriptorsAcrossNullRuns) {
TEST_F(IndexBuilderTest,
FloatingPointBuildIndexBorrowsCleanRunsAndPreservesSource) {
auto check = [&]<FieldType FT>() {
- using Value = typename CppTypeTraits<FT>::CppType;
+ using Value = typename StorageTypeTraits<FT>::StorageType;
using UInt = std::conditional_t<sizeof(Value) == sizeof(uint32_t),
uint32_t, uint64_t>;
using Column =
typename
PrimitiveTypeTraits<storage_field_type_to_primitive_type(FT)>::ColumnType;
diff --git a/be/test/storage/index/inverted/snii/bkd/bkd_differential_test.cpp
b/be/test/storage/index/inverted/snii/bkd/bkd_differential_test.cpp
index 359b9121401..f315d003eed 100644
--- a/be/test/storage/index/inverted/snii/bkd/bkd_differential_test.cpp
+++ b/be/test/storage/index/inverted/snii/bkd/bkd_differential_test.cpp
@@ -101,7 +101,7 @@ struct Dataset {
// everything downstream is type-agnostic bytes.
template <FieldType FT>
struct ValueFactory {
- using CppType = typename CppTypeTraits<FT>::CppType;
+ using CppType = typename StorageTypeTraits<FT>::StorageType;
// A deterministic mapping from a rank to a value, injective wherever the
// domain allows it. Narrow types (BOOL, TINYINT) wrap, which is a
legitimate
@@ -158,7 +158,7 @@ struct ValueFactory {
// construction, and that is asserted rather than assumed: a KeyCoder whose
// output is not sizeof(CppType) would silently break INV-2 for every case
below.
template <FieldType FT>
-Bytes encode(const typename CppTypeTraits<FT>::CppType& value) {
+Bytes encode(const typename StorageTypeTraits<FT>::StorageType& value) {
Bytes buf;
get_key_coder(FT)->full_encode_ascending(&value, &buf);
EXPECT_EQ(buf.size(), field_type_size(FT));
diff --git a/be/test/storage/index/inverted/snii/bkd/bkd_property_test.cpp
b/be/test/storage/index/inverted/snii/bkd/bkd_property_test.cpp
index c0ae37dda20..2ef27ced5e7 100644
--- a/be/test/storage/index/inverted/snii/bkd/bkd_property_test.cpp
+++ b/be/test/storage/index/inverted/snii/bkd/bkd_property_test.cpp
@@ -107,7 +107,7 @@ constexpr uint32_t kLeafCapacities[] = {1, 2, 5, 16, 1024};
// construction, and that is asserted rather than assumed: a KeyCoder emitting
// something other than sizeof(CppType) would silently break INV-2.
template <FieldType FT>
-Bytes encode(const typename CppTypeTraits<FT>::CppType& value) {
+Bytes encode(const typename StorageTypeTraits<FT>::StorageType& value) {
Bytes buf;
get_key_coder(FT)->full_encode_ascending(&value, &buf);
EXPECT_EQ(buf.size(), field_type_size(FT));
@@ -119,7 +119,7 @@ Bytes encode(const typename CppTypeTraits<FT>::CppType&
value) {
// repeats across thousands of points and dozens of leaves.
template <FieldType FT>
Bytes encode_rank(int64_t rank) {
- using CppType = typename CppTypeTraits<FT>::CppType;
+ using CppType = typename StorageTypeTraits<FT>::StorageType;
return encode<FT>(static_cast<CppType>(rank));
}
@@ -672,8 +672,8 @@ TYPED_TEST(BkdPropertyTest,
BuildIsDeterministicUnderPerDocValueOrder) {
// Check encoded ranges against native value order across signed, unsigned,
and floating-point types.
template <FieldType FT>
-std::vector<typename CppTypeTraits<FT>::CppType> native_spread() {
- using CppType = typename CppTypeTraits<FT>::CppType;
+std::vector<typename StorageTypeTraits<FT>::StorageType> native_spread() {
+ using CppType = typename StorageTypeTraits<FT>::StorageType;
std::vector<CppType> values;
if constexpr (std::is_floating_point_v<CppType>) {
// Negative zero is deliberate: it must not sort strictly below +0.0
nor
@@ -685,7 +685,7 @@ std::vector<typename CppTypeTraits<FT>::CppType>
native_spread() {
values.push_back(std::numeric_limits<CppType>::lowest());
values.push_back(std::numeric_limits<CppType>::max());
} else {
- using UnsignedType = typename CppTypeTraits<FT>::UnsignedCppType;
+ using UnsignedType = typename
StorageTypeTraits<FT>::UnsignedStorageType;
constexpr size_t kBits = sizeof(CppType) * 8;
constexpr bool kIsSigned = std::is_signed_v<CppType>;
const CppType lo = kIsSigned ? static_cast<CppType>(UnsignedType(1) <<
(kBits - 1))
diff --git
a/be/test/storage/index/inverted/snii/snii_bkd_adapter_types_test.cpp
b/be/test/storage/index/inverted/snii/snii_bkd_adapter_types_test.cpp
index 57797bb151f..44cea68bbb5 100644
--- a/be/test/storage/index/inverted/snii/snii_bkd_adapter_types_test.cpp
+++ b/be/test/storage/index/inverted/snii/snii_bkd_adapter_types_test.cpp
@@ -64,7 +64,7 @@ struct FieldTag {
// correct way rather than reinvented here.
template <FieldType FT>
struct ValueOf {
- using CppType = typename CppTypeTraits<FT>::CppType;
+ using CppType = typename StorageTypeTraits<FT>::StorageType;
static CppType from_rank(int64_t rank) {
if constexpr (std::is_same_v<CppType, decimal12_t>) {
decimal12_t value;
@@ -125,7 +125,7 @@ TYPED_TEST_SUITE(SniiBkdAdapterTypesTest, FieldTypeTags);
TYPED_TEST(SniiBkdAdapterTypesTest, AdapterPreservesValueToRowAssociation) {
constexpr FieldType FT = TypeParam::kFieldType;
- using CppType = typename CppTypeTraits<FT>::CppType;
+ using CppType = typename StorageTypeTraits<FT>::StorageType;
// 240 rows over 60 distinct ranks, so most values own several rows and a
// shifted-by-one stride cannot coincidentally reproduce the mapping.
diff --git a/be/test/storage/key_coder_test.cpp
b/be/test/storage/key_coder_test.cpp
index ee14881ca42..43d56d2de72 100644
--- a/be/test/storage/key_coder_test.cpp
+++ b/be/test/storage/key_coder_test.cpp
@@ -45,7 +45,7 @@ public:
template <FieldType type>
void test_integer_encode() {
- using CppType = typename CppTypeTraits<type>::CppType;
+ using CppType = typename StorageTypeTraits<type>::StorageType;
auto key_coder = get_key_coder(type);
@@ -111,9 +111,9 @@ void test_integer_encode() {
}
template <FieldType field_type>
-typename CppTypeTraits<field_type>::CppType decode_float(const std::string&
encoded) {
+typename StorageTypeTraits<field_type>::StorageType decode_float(const
std::string& encoded) {
Slice encoded_key(encoded);
- typename CppTypeTraits<field_type>::CppType result;
+ typename StorageTypeTraits<field_type>::StorageType result;
uint8_t* ptr = reinterpret_cast<uint8_t*>(&result);
Status status = KeyCoderTraits<field_type>::decode_ascending(&encoded_key,
sizeof(result), ptr);
EXPECT_TRUE(status.ok());
@@ -121,22 +121,23 @@ typename CppTypeTraits<field_type>::CppType
decode_float(const std::string& enco
}
template <FieldType field_type>
-std::string encode_float(typename CppTypeTraits<field_type>::CppType value) {
+std::string encode_float(typename StorageTypeTraits<field_type>::StorageType
value) {
std::string buf;
KeyCoderTraits<field_type>::full_encode_ascending(&value, &buf);
return buf;
}
template <FieldType field_type>
-void test_encode_decode(typename CppTypeTraits<field_type>::CppType value) {
+void test_encode_decode(typename StorageTypeTraits<field_type>::StorageType
value) {
std::string encoded = encode_float<field_type>(value);
- typename CppTypeTraits<field_type>::CppType decoded =
decode_float<field_type>(encoded);
+ typename StorageTypeTraits<field_type>::StorageType decoded =
decode_float<field_type>(encoded);
EXPECT_EQ(value, decoded);
}
template <FieldType field_type>
-void test_ordering(typename CppTypeTraits<field_type>::CppType a,
- typename CppTypeTraits<field_type>::CppType b) {
+// NOLINTNEXTLINE(readability-function-cognitive-complexity) -- GTest
assertion macros inflate it.
+void test_ordering(typename StorageTypeTraits<field_type>::StorageType a,
+ typename StorageTypeTraits<field_type>::StorageType b) {
std::string encoded_a = encode_float<field_type>(a);
std::string encoded_b = encode_float<field_type>(b);
@@ -907,7 +908,7 @@ void WriteStringToFile(std::ofstream& out, const
std::string& value, FieldType f
// Helper function to read from file and decode
template <FieldType field_type>
void ReadAndDecodeNumber(std::ifstream& in) {
- using T = typename CppTypeTraits<field_type>::CppType;
+ using T = typename StorageTypeTraits<field_type>::StorageType;
// 2. read original_value
size_t value_size = 0;
@@ -974,7 +975,7 @@ void ReadAndDecodeString(std::ifstream& in) {
template <FieldType field_type>
void WriteAllBoundaryValues(std::ofstream& out) {
- using T = typename CppTypeTraits<field_type>::CppType;
+ using T = typename StorageTypeTraits<field_type>::StorageType;
WriteNumberToFile<T>(out, std::numeric_limits<T>::min(), field_type);
WriteNumberToFile<T>(out, T(0), field_type);
WriteNumberToFile<T>(out, std::numeric_limits<T>::max(), field_type);
@@ -994,7 +995,7 @@ void WriteStringsValues(std::ofstream& out, const
std::string& str) {
template <FieldType field_type>
void WriteFloatValues(std::ofstream& out) {
- using T = typename CppTypeTraits<field_type>::CppType;
+ using T = typename StorageTypeTraits<field_type>::StorageType;
WriteNumberToFile<T>(out, +0.0, field_type);
WriteNumberToFile<T>(out, -0.0, field_type);
WriteNumberToFile<T>(out, 1.0, field_type);
@@ -1209,7 +1210,7 @@ void WriteSingleValueToComplexFile(std::ofstream& out) {
out.write(reinterpret_cast<const char*>(&value), value_size);
} else if constexpr (field_is_numeric_type(field_type) ||
field_type ==
FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT) {
- using T = typename CppTypeTraits<field_type>::CppType;
+ using T = typename StorageTypeTraits<field_type>::StorageType;
T value = std::numeric_limits<T>::min();
size_t value_size = sizeof(value);
out.write(reinterpret_cast<const char*>(&value_size),
sizeof(value_size));
@@ -1248,7 +1249,7 @@ void WriteSingleEncodedValueToComplexFile(std::ofstream&
out) {
coder->full_encode_ascending(&value, &encoded_buf);
} else if constexpr (field_is_numeric_type(field_type) ||
field_type ==
FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT) {
- using T = typename CppTypeTraits<field_type>::CppType;
+ using T = typename StorageTypeTraits<field_type>::StorageType;
T value = std::numeric_limits<T>::min();
coder->full_encode_ascending(&value, &encoded_buf);
} else if constexpr (field_is_slice_type(field_type)) {
@@ -1392,7 +1393,7 @@ void ReadAndDecodeFromComplexFile(std::ifstream& in) {
std::string value_i(value_size_i, '\0');
in.read(value_i.data(), value_size_i);
- using T = typename CppTypeTraits<field_type_j>::CppType;
+ using T = typename StorageTypeTraits<field_type_j>::StorageType;
size_t value_size = 0;
in.read(reinterpret_cast<char*>(&value_size), sizeof(value_size));
@@ -1420,7 +1421,7 @@ void ReadAndDecodeFromComplexFile(std::ifstream& in) {
EXPECT_TRUE(status.ok());
EXPECT_EQ(decoded_value, original_value);
} else if constexpr (!field_is_slice_type(field_type_i) &&
field_is_slice_type(field_type_j)) {
- using T = typename CppTypeTraits<field_type_i>::CppType;
+ using T = typename StorageTypeTraits<field_type_i>::StorageType;
size_t value_size = 0;
in.read(reinterpret_cast<char*>(&value_size), sizeof(value_size));
@@ -1456,8 +1457,8 @@ void ReadAndDecodeFromComplexFile(std::ifstream& in) {
EXPECT_TRUE(status.ok());
EXPECT_EQ(decoded_value, original_value);
} else {
- using T = typename CppTypeTraits<field_type_i>::CppType;
- using U = typename CppTypeTraits<field_type_j>::CppType;
+ using T = typename StorageTypeTraits<field_type_i>::StorageType;
+ using U = typename StorageTypeTraits<field_type_j>::StorageType;
size_t value_size_i;
in.read(reinterpret_cast<char*>(&value_size_i), sizeof(value_size_i));
T original_value_i;
diff --git a/be/test/storage/olap_type_test.cpp
b/be/test/storage/olap_type_test.cpp
index c32ec5b7b7e..020f82f1add 100644
--- a/be/test/storage/olap_type_test.cpp
+++ b/be/test/storage/olap_type_test.cpp
@@ -33,6 +33,7 @@
#include "exprs/function/cast/cast_to_string.h"
#include "gtest/gtest_pred_impl.h"
#include "storage/olap_common.h"
+#include "storage/storage_layout_test_util.h"
namespace doris {
@@ -1119,8 +1120,7 @@ TEST_F(OlapTypeTest, char_type_with_padding) {
expected.append(15, '\0');
std::string expected_serde = expected;
- auto field = Field::create_field_from_olap_value<TYPE_CHAR>(
- StringRef(olap_value.data, olap_value.size));
+ auto field = Field::create_field<TYPE_CHAR>(String(olap_value.data,
olap_value.size));
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(expected_serde, serde_str) << "serde mismatch for CHAR(20)
'hello'"
@@ -1136,8 +1136,7 @@ TEST_F(OlapTypeTest, char_type_with_padding) {
std::string expected(20, 'x');
std::string expected_serde = expected;
- auto field = Field::create_field_from_olap_value<TYPE_CHAR>(
- StringRef(olap_value.data, olap_value.size));
+ auto field = Field::create_field<TYPE_CHAR>(String(olap_value.data,
olap_value.size));
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(expected_serde, serde_str) << "serde mismatch for CHAR(20)
filled 'x'";
@@ -1151,8 +1150,7 @@ TEST_F(OlapTypeTest, char_type_with_padding) {
std::string expected(20, '\0');
std::string expected_serde = expected;
- auto field = Field::create_field_from_olap_value<TYPE_CHAR>(
- StringRef(olap_value.data, olap_value.size));
+ auto field = Field::create_field<TYPE_CHAR>(String(olap_value.data,
olap_value.size));
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(expected_serde, serde_str) << "serde mismatch for CHAR(20)
empty"
@@ -1179,8 +1177,7 @@ TEST_F(OlapTypeTest, varchar_type) {
for (auto& tc : test_cases) {
Slice olap_value(tc.input.data(), tc.input.size());
- auto field = Field::create_field_from_olap_value<TYPE_VARCHAR>(
- StringRef(olap_value.data, olap_value.size));
+ auto field = Field::create_field<TYPE_VARCHAR>(String(olap_value.data,
olap_value.size));
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected_serde, serde_str)
@@ -1211,7 +1208,7 @@ TEST_F(OlapTypeTest, date_v1_type) {
for (auto& tc : test_cases) {
uint24_t olap_value = make_olap_date(tc.year, tc.month, tc.day);
- auto field =
Field::create_field_from_olap_value<TYPE_DATE>(olap_value);
+ auto field = field_of_cell<TYPE_DATE>(olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected_serde, serde_str) << "serde mismatch for DATE "
<< tc.expected;
@@ -1235,7 +1232,7 @@ TEST_F(OlapTypeTest, datetime_v1_type) {
};
for (auto& tc : test_cases) {
- auto field =
Field::create_field_from_olap_value<TYPE_DATETIME>((uint64_t)tc.olap_value);
+ auto field = field_of_cell<TYPE_DATETIME>((uint64_t)tc.olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected_serde, serde_str) << "serde mismatch for
DATETIME " << tc.expected;
@@ -1265,7 +1262,7 @@ TEST_F(OlapTypeTest, datev2_type) {
for (auto& tc : test_cases) {
uint32_t olap_value = make_datev2(tc.year, tc.month, tc.day);
- auto field =
Field::create_field_from_olap_value<TYPE_DATEV2>(olap_value);
+ auto field = field_of_cell<TYPE_DATEV2>(olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected_serde, serde_str) << "serde mismatch for DATEV2
" << tc.expected;
@@ -1307,7 +1304,7 @@ TEST_F(OlapTypeTest, datetimev2_type) {
auto serde = data_type->get_serde();
for (auto& tc : test_cases) {
- auto field =
Field::create_field_from_olap_value<TYPE_DATETIMEV2>(tc.olap_value);
+ auto field = field_of_cell<TYPE_DATETIMEV2>(tc.olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected, serde_str)
@@ -1325,7 +1322,7 @@ TEST_F(OlapTypeTest,
datetime_v1_vs_v2_precision_difference) {
int64_t olap_value = 20230615123456L;
std::string expected = "2023-06-15 12:34:56";
std::string expected_serde = expected;
- auto field =
Field::create_field_from_olap_value<TYPE_DATETIME>((uint64_t)olap_value);
+ auto field = field_of_cell<TYPE_DATETIME>((uint64_t)olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(expected_serde, serde_str) << "serde mismatch for DATETIME
V1";
@@ -1347,7 +1344,7 @@ TEST_F(OlapTypeTest,
datetime_v1_vs_v2_precision_difference) {
uint64_t olap_value = make_datetimev2(2023, 6, 15, 12, 34, 56, 123456);
std::string expected = "2023-06-15 12:34:56.123456";
std::string expected_serde = expected;
- auto field =
Field::create_field_from_olap_value<TYPE_DATETIMEV2>(olap_value);
+ auto field = field_of_cell<TYPE_DATETIMEV2>(olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(expected_serde, serde_str) << "serde mismatch for
DATETIMEV2";
@@ -1394,7 +1391,7 @@ TEST_F(OlapTypeTest, decimalv2_type) {
olap_value.integer = tc.integer;
olap_value.fraction = tc.fraction;
- auto field =
Field::create_field_from_olap_value<TYPE_DECIMALV2>(olap_value);
+ auto field = field_of_cell<TYPE_DECIMALV2>(olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected_serde, serde_str)
@@ -1428,7 +1425,7 @@ TEST_F(OlapTypeTest, decimal32_type) {
auto serde = data_type->get_serde();
int32_t olap_value = tc.value;
- auto field =
Field::create_field_from_olap_value<TYPE_DECIMAL32>(olap_value);
+ auto field = field_of_cell<TYPE_DECIMAL32>(olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected_serde, serde_str)
@@ -1460,7 +1457,7 @@ TEST_F(OlapTypeTest, decimal64_type) {
auto serde = data_type->get_serde();
int64_t olap_value = tc.value;
- auto field =
Field::create_field_from_olap_value<TYPE_DECIMAL64>(olap_value);
+ auto field = field_of_cell<TYPE_DECIMAL64>(olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected_serde, serde_str)
@@ -1492,7 +1489,7 @@ TEST_F(OlapTypeTest, decimal128i_type) {
auto serde = data_type->get_serde();
int128_t olap_value = tc.value;
- auto field =
Field::create_field_from_olap_value<TYPE_DECIMAL128I>(olap_value);
+ auto field = field_of_cell<TYPE_DECIMAL128I>(olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected_serde, serde_str)
@@ -1521,7 +1518,7 @@ TEST_F(OlapTypeTest, decimal256_type) {
auto serde = data_type->get_serde();
wide::Int256 olap_value = tc.value;
- auto field =
Field::create_field_from_olap_value<TYPE_DECIMAL256>(olap_value);
+ auto field = field_of_cell<TYPE_DECIMAL256>(olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected_serde, serde_str)
@@ -1859,7 +1856,7 @@ TEST_F(OlapTypeTest, timestamptz_type) {
};
for (auto& tc : test_cases) {
- auto field =
Field::create_field_from_olap_value<TYPE_TIMESTAMPTZ>(tc.olap_value);
+ auto field = field_of_cell<TYPE_TIMESTAMPTZ>(tc.olap_value);
std::string serde_str = serde->to_olap_string(field);
EXPECT_EQ(tc.expected_serde, serde_str)
diff --git a/be/test/storage/row_cursor_test.cpp
b/be/test/storage/row_cursor_test.cpp
index 31e88ab2b4f..333912a2246 100644
--- a/be/test/storage/row_cursor_test.cpp
+++ b/be/test/storage/row_cursor_test.cpp
@@ -31,6 +31,7 @@
#include "core/uint24.h"
#include "core/wide_integer.h"
#include "storage/schema.h"
+#include "storage/storage_layout_test_util.h"
#include "storage/tablet/tablet_schema.h"
#include "storage/tablet/tablet_schema_helper.h"
#include "util/debug_util.h"
@@ -433,8 +434,7 @@ TEST_F(TestRowCursor, encode_key_int_date_varchar) {
row.mutable_field(0) = Field::create_field<TYPE_INT>(int32_t(12345));
// DATE(2020-01-01): raw uint24_t = 2020*512 + 1*32 + 1 = 1034273
- row.mutable_field(1) =
- Field::create_field_from_olap_value<TYPE_DATE>(uint24_t(2020 * 512
+ 1 * 32 + 1));
+ row.mutable_field(1) = field_of_cell<TYPE_DATE>(uint24_t(2020 * 512 + 1 *
32 + 1));
row.mutable_field(2) = Field::create_field<TYPE_STRING>(String("hello"));
// encode_key: VARCHAR('hello') truncated to 4 bytes
@@ -469,7 +469,7 @@ TEST_F(TestRowCursor, encode_key_double_nan_decimal_string)
{
Field::create_field<TYPE_DOUBLE>(std::numeric_limits<double>::quiet_NaN());
// DECIMAL(123.456000000): stored as decimal12_t{123, 456000000}
row.mutable_field(1) =
- Field::create_field_from_olap_value<TYPE_DECIMALV2>(decimal12_t
{123, 456000000});
+ field_of_cell<TYPE_DECIMALV2>(decimal12_t {.integer = 123,
.fraction = 456000000});
row.mutable_field(2) = Field::create_field<TYPE_STRING>(String("hello
world"));
// encode_key: STRING truncated to 4 bytes
@@ -541,8 +541,7 @@ TEST_F(TestRowCursor,
encode_key_float_inf_datetime_varchar) {
row.mutable_field(0) =
Field::create_field<TYPE_FLOAT>(std::numeric_limits<float>::infinity());
// DATETIME(2020-01-01 12:00:00) = 20200101120000 in olap datetime format
- row.mutable_field(1) =
-
Field::create_field_from_olap_value<TYPE_DATETIME>(uint64_t(20200101120000ULL));
+ row.mutable_field(1) =
field_of_cell<TYPE_DATETIME>(int64_t(20200101120000));
row.mutable_field(2) = Field::create_field<TYPE_STRING>(String("ab"));
// encode_key: VARCHAR('ab') only 2 bytes (< index_size=4, not truncated)
@@ -610,7 +609,7 @@ TEST_F(TestRowCursor, encode_key_negative_values) {
row.mutable_field(1) = Field::create_field<TYPE_DOUBLE>(-1.0);
// DECIMAL(-123, -456000000) representing -123.456000000
row.mutable_field(2) =
- Field::create_field_from_olap_value<TYPE_DECIMALV2>(decimal12_t
{-123, -456000000});
+ field_of_cell<TYPE_DECIMALV2>(decimal12_t {.integer = -123,
.fraction = -456000000});
{
std::string buf;
@@ -706,8 +705,7 @@ TEST_F(TestRowCursor, encode_key_datetime_v1) {
{
RowCursor row;
init_row_cursor_with_nulls(row, tablet_schema, 1);
- row.mutable_field(0) =
-
Field::create_field_from_olap_value<TYPE_DATETIME>(uint64_t(20241231123045ULL));
+ row.mutable_field(0) =
field_of_cell<TYPE_DATETIME>(int64_t(20241231123045));
std::string buf;
row.encode_key(&buf, 1);
diff --git a/be/test/storage/segment/bitshuffle_page_test.cpp
b/be/test/storage/segment/bitshuffle_page_test.cpp
index 761b7a03bfd..b182279faa4 100644
--- a/be/test/storage/segment/bitshuffle_page_test.cpp
+++ b/be/test/storage/segment/bitshuffle_page_test.cpp
@@ -21,11 +21,16 @@
#include <memory>
+#include "core/assert_cast.h"
+#include "core/column/column_vector.h"
#include "core/data_type/data_type_factory.hpp"
+#include "core/uint24.h"
+#include "core/value/vdatetime_value.h"
#include "storage/segment/bitshuffle_page_pre_decoder.h"
#include "storage/segment/options.h"
#include "storage/segment/page_builder.h"
#include "storage/segment/page_decoder.h"
+#include "storage/storage_layout.h"
using doris::segment_v2::PageBuilderOptions;
using doris::OlapReaderStatistics;
@@ -162,6 +167,39 @@ public:
status = page_decoder.seek_at_or_after_value(bigger_than_biggest,
&exact_match);
EXPECT_EQ(status.code(), ENTRY_NOT_FOUND);
}
+
+ // Eight consecutive days from 2020-01-01 in one bitshuffle page of V1
DATE cells, each date
+ // stored as a uint24, unshuffled into `decoded_page` as the page cache
holds it. `page` is
+ // left pointing at the unshuffled page.
+ void build_date_v1_page(std::vector<VecDateTimeValue>* days,
+ std::unique_ptr<DataPage>* decoded_page, Slice*
page) {
+ using Layout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DATE>;
+ constexpr size_t kRows = 8;
+ days->resize(kRows);
+ std::vector<uint24_t> cells(kRows);
+ for (size_t i = 0; i < kRows; ++i) {
+
ASSERT_TRUE((*days)[i].from_date_int64(static_cast<int64_t>(20200101 + i)));
+ cells[i] = Layout::to_storage((*days)[i]);
+ }
+ PageBuilderOptions options;
+ options.data_page_size = 256 * 1024;
+ segment_v2::PageBuilder* builder = nullptr;
+
ASSERT_TRUE(segment_v2::BitshufflePageBuilder<FieldType::OLAP_FIELD_TYPE_DATE>::create(
+ &builder, options)
+ .ok());
+ std::unique_ptr<segment_v2::PageBuilder> page_builder(builder);
+ size_t added = kRows;
+ ASSERT_TRUE(page_builder->add(reinterpret_cast<const
uint8_t*>(cells.data()), &added).ok());
+ ASSERT_EQ(added, kRows);
+ OwnedSlice encoded;
+ ASSERT_TRUE(page_builder->finish(&encoded).ok());
+ segment_v2::BitShufflePagePreDecoder pre_decoder;
+ *page = encoded.slice();
+ ASSERT_TRUE(
+ pre_decoder
+ .decode(decoded_page, page, 0, false,
segment_v2::PageTypePB::DATA_PAGE, "")
+ .ok());
+ }
};
// Test for bitshuffle block, for INT32, INT64, FLOAT, DOUBLE
@@ -363,4 +401,48 @@ TEST_F(BitShufflePageTest,
TestBitShuffleDecimal12BlockEncoderSeekValue) {
decimals.get(), size, &small_than_smallest, &bigger_than_biggest);
}
+// A V1 DATE page holds each date as a uint24 cell; a batch decodes the cells
into the dates
+// of a ColumnDate.
+TEST_F(BitShufflePageTest, TestBitShuffleDateV1DecodesIntoDates) {
+ std::vector<VecDateTimeValue> days;
+ std::unique_ptr<DataPage> decoded_page;
+ Slice page;
+ ASSERT_NO_FATAL_FAILURE(build_date_v1_page(&days, &decoded_page, &page));
+ segment_v2::PageDecoderOptions decoder_options;
+ segment_v2::BitShufflePageDecoder<FieldType::OLAP_FIELD_TYPE_DATE>
decoder(page,
+
decoder_options);
+ ASSERT_TRUE(decoder.init().ok());
+
+ MutableColumnPtr dates = ColumnDate::create();
+ size_t n = days.size();
+ ASSERT_TRUE(decoder.next_batch(&n, dates).ok());
+ ASSERT_EQ(n, days.size());
+ const auto& decoded = assert_cast<const ColumnDate&>(*dates).get_data();
+ for (size_t i = 0; i < days.size(); ++i) {
+ EXPECT_EQ(decoded[i], days[i]) << "row " << i;
+ }
+}
+
+// Rows read by id into a ColumnDate come back as their dates.
+TEST_F(BitShufflePageTest, TestBitShuffleDateV1ReadsRowIdsIntoDates) {
+ std::vector<VecDateTimeValue> days;
+ std::unique_ptr<DataPage> decoded_page;
+ Slice page;
+ ASSERT_NO_FATAL_FAILURE(build_date_v1_page(&days, &decoded_page, &page));
+ segment_v2::PageDecoderOptions decoder_options;
+ segment_v2::BitShufflePageDecoder<FieldType::OLAP_FIELD_TYPE_DATE>
decoder(page,
+
decoder_options);
+ ASSERT_TRUE(decoder.init().ok());
+
+ const segment_v2::rowid_t rowids[] = {1, 3, 6};
+ MutableColumnPtr picked = ColumnDate::create();
+ size_t n = 3;
+ ASSERT_TRUE(decoder.read_by_rowids(rowids, 0, &n, picked).ok());
+ ASSERT_EQ(n, 3);
+ const auto& picked_dates = assert_cast<const
ColumnDate&>(*picked).get_data();
+ for (size_t i = 0; i < 3; ++i) {
+ EXPECT_EQ(picked_dates[i], days[rowids[i]]) << "row id " << rowids[i];
+ }
+}
+
} // namespace doris
diff --git a/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp
b/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp
index a661eca4225..314281dfd47 100644
--- a/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp
+++ b/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp
@@ -67,7 +67,7 @@ Status write_bloom_filter_index_file(const std::string&
file_name, const void* v
size_t value_count, size_t null_count,
ColumnIndexMetaPB* index_meta,
bool use_primary_key_bloom_filter =
false, double fpp = 0.05) {
- using CppType = typename CppTypeTraits<type>::CppType;
+ using CppType = typename StorageTypeTraits<type>::StorageType;
std::string fname = dname + "/" + file_name;
auto fs = io::global_local_filesystem();
{
diff --git a/be/test/storage/segment/inverted_index_reader_test.cpp
b/be/test/storage/segment/inverted_index_reader_test.cpp
index d35f6fe3e13..11b560aabc2 100644
--- a/be/test/storage/segment/inverted_index_reader_test.cpp
+++ b/be/test/storage/segment/inverted_index_reader_test.cpp
@@ -52,6 +52,7 @@
#include "storage/index/inverted/query/scoring/collection_similarity.h"
#include "storage/index/inverted/reader/collection_statistics.h"
#include "storage/key_coder.h"
+#include "storage/storage_layout_test_util.h"
#include "storage/tablet/tablet_schema.h"
#include "storage/tablet/tablet_schema_helper.h"
#include "util/slice.h"
@@ -3718,8 +3719,9 @@ public:
EXPECT_NE(bkd_reader, nullptr);
auto run_query = [&](InvertedIndexQueryType qt, T thr) {
- using raw_t = typename PrimitiveTypeTraits<PT>::StorageFieldType;
- Field qp =
Field::create_field_from_olap_value<PT>(static_cast<raw_t>(thr));
+ using raw_t =
+ typename
StorageLayout<primitive_type_to_storage_field_type(PT)>::StorageValue;
+ Field qp = field_of_cell<PT>(static_cast<raw_t>(thr));
auto bitmap = std::make_shared<roaring::Roaring>();
auto status = bkd_reader->query(context, column_name, qp, qt,
bitmap);
EXPECT_TRUE(status.ok()) << column_name << ": " << status;
@@ -4331,8 +4333,7 @@ public:
for (auto& test_case : test_cases) {
std::shared_ptr<roaring::Roaring> bitmap =
std::make_shared<roaring::Roaring>();
- Field qp_3262 =
-
Field::create_field_from_olap_value<TYPE_TIMESTAMPTZ>(test_case.second);
+ Field qp_3262 =
field_of_cell<TYPE_TIMESTAMPTZ>(test_case.second);
auto status = bkd_reader->query(context, "c_timestamptz",
qp_3262, test_case.first,
bitmap);
EXPECT_TRUE(status.ok()) << "Query type: " <<
static_cast<int>(test_case.first);
@@ -4345,8 +4346,7 @@ public:
for (auto& test_case : test_cases) {
size_t count = 0;
- Field qp_3274 =
-
Field::create_field_from_olap_value<TYPE_TIMESTAMPTZ>(test_case.second);
+ Field qp_3274 =
field_of_cell<TYPE_TIMESTAMPTZ>(test_case.second);
auto status = bkd_reader->try_query(context, "c_timestamptz",
qp_3274,
test_case.first, &count);
EXPECT_TRUE(status.ok()) << "Try query type: " <<
static_cast<int>(test_case.first);
@@ -4444,8 +4444,8 @@ public:
uint32_t query_value = 20240102;
std::shared_ptr<roaring::Roaring> bitmap =
std::make_shared<roaring::Roaring>();
// TYPE_DATE storage is uint24_t — narrow from the test's uint32_t.
- typename PrimitiveTypeTraits<TYPE_DATE>::StorageFieldType
date_storage(query_value);
- Field qp_3366 =
Field::create_field_from_olap_value<TYPE_DATE>(date_storage);
+ const uint24_t date_storage(query_value);
+ Field qp_3366 = field_of_cell<TYPE_DATE>(date_storage);
auto status = bkd_reader->query(context, "c_date", qp_3366,
InvertedIndexQueryType::EQUAL_QUERY, bitmap);
EXPECT_TRUE(status.ok());
@@ -4471,8 +4471,7 @@ public:
int64_t query_value = 20240101130000LL;
std::shared_ptr<roaring::Roaring> bitmap =
std::make_shared<roaring::Roaring>();
- Field qp_3391 = Field::create_field_from_olap_value<TYPE_DATETIME>(
- static_cast<uint64_t>(query_value));
+ Field qp_3391 = field_of_cell<TYPE_DATETIME>(query_value);
auto status = bkd_reader->query(context, "c_datetime", qp_3391,
InvertedIndexQueryType::EQUAL_QUERY, bitmap);
EXPECT_TRUE(status.ok());
@@ -4651,7 +4650,7 @@ public:
uint64_t query_value = 20240201130000ULL;
std::shared_ptr<roaring::Roaring> bitmap =
std::make_shared<roaring::Roaring>();
- Field qp_3561 =
Field::create_field_from_olap_value<TYPE_TIMESTAMPTZ>(query_value);
+ Field qp_3561 = field_of_cell<TYPE_TIMESTAMPTZ>(query_value);
auto status = bkd_reader->query(context, "c_timestamptz", qp_3561,
InvertedIndexQueryType::EQUAL_QUERY, bitmap);
EXPECT_TRUE(status.ok());
diff --git a/be/test/storage/segment/page_builder_column_test.cpp
b/be/test/storage/segment/page_builder_column_test.cpp
index 8035fb3311e..026b3c2450d 100644
--- a/be/test/storage/segment/page_builder_column_test.cpp
+++ b/be/test/storage/segment/page_builder_column_test.cpp
@@ -142,7 +142,8 @@ Pages drain(PageBuilder& builder, size_t num_rows, AddFn
add, size_t append_rows
// for a fixed-width FieldType, one Slice per row otherwise; CHAR padded to
char_length.
template <FieldType FT>
struct ReferenceCells {
- static constexpr bool kSlices = std::is_same_v<typename
CppTypeTraits<FT>::CppType, Slice>;
+ static constexpr bool kSlices =
+ std::is_same_v<typename StorageTypeTraits<FT>::StorageType, Slice>;
std::vector<uint8_t> bytes;
std::vector<Slice> slices;
@@ -168,7 +169,7 @@ struct ReferenceCells {
using Column = typename
PrimitiveTypeTraits<storage_field_type_to_primitive_type(
FT)>::ColumnType;
const auto& data = assert_cast<const Column&>(column).get_data();
- bytes.resize(n * sizeof(typename CppTypeTraits<FT>::CppType));
+ bytes.resize(n * sizeof(typename
StorageTypeTraits<FT>::StorageType));
for (size_t i = 0; i < n; ++i) {
const auto value = StorageLayout<FT>::to_storage(data[i]);
memcpy(bytes.data() + i * sizeof(value), &value,
sizeof(value));
@@ -182,7 +183,8 @@ struct ReferenceCells {
if constexpr (kSlices) {
return builder.add(reinterpret_cast<const uint8_t*>(slices.data()
+ pos), n);
} else {
- return builder.add(bytes.data() + pos * sizeof(typename
CppTypeTraits<FT>::CppType), n);
+ return builder.add(
+ bytes.data() + pos * sizeof(typename
StorageTypeTraits<FT>::StorageType), n);
}
}
};
@@ -210,7 +212,7 @@ constexpr EncodingTypePB builder_encoding() {
template <FieldType FT>
void expect_page_inputs(const RecordingPageBuilder& builder, const IColumn&
column) {
- if constexpr (std::is_floating_point_v<typename
CppTypeTraits<FT>::CppType>) {
+ if constexpr (std::is_floating_point_v<typename
StorageTypeTraits<FT>::StorageType>) {
using Column =
typename
PrimitiveTypeTraits<storage_field_type_to_primitive_type(FT)>::ColumnType;
const auto& values = assert_cast<const Column&>(column).get_data();
@@ -260,7 +262,7 @@ void expect_same_pages(const IColumn& source,
PageBuilderOptions options, size_t
meta.set_column_id(0);
meta.set_unique_id(0);
meta.set_type(int(FT));
- meta.set_length(sizeof(typename CppTypeTraits<FT>::CppType));
+ meta.set_length(sizeof(typename StorageTypeTraits<FT>::StorageType));
meta.set_encoding(builder_encoding<Builder, FT>());
meta.set_compression(NO_COMPRESSION);
meta.set_is_nullable(false);
diff --git a/be/test/storage/segment/page_format_test.cpp
b/be/test/storage/segment/page_format_test.cpp
index b4e4a17ae6d..1b0087d53ba 100644
--- a/be/test/storage/segment/page_format_test.cpp
+++ b/be/test/storage/segment/page_format_test.cpp
@@ -33,6 +33,7 @@
#include <limits>
#include <map>
#include <memory>
+#include <numeric>
#include <set>
#include <string>
#include <string_view>
@@ -45,6 +46,7 @@
#include "core/assert_cast.h"
#include "core/column/column_complex.h"
#include "core/column/column_decimal.h"
+#include "core/column/column_dictionary.h"
#include "core/column/column_nullable.h"
#include "core/column/column_string.h"
#include "core/column/column_vector.h"
@@ -848,42 +850,30 @@ std::vector<std::string> storage_cells(FieldType type,
const IColumn& data) {
return cells;
}
-// Reads the column back the way a Segment reads it and checks that it holds
the rows written.
-Status verify_read_back(const CaseInput& input, const WrittenColumn& written,
- const std::string& path) {
- io::FileReaderSPtr file_reader;
- RETURN_IF_ERROR(io::global_local_filesystem()->open_file(path,
&file_reader));
- ColumnReaderOptions reader_options;
- reader_options.be_exec_version =
BeExecVersionManager::get_newest_version();
- std::shared_ptr<ColumnReader> reader;
- RETURN_IF_ERROR(ColumnReader::create(reader_options, written.meta,
written.meta.num_rows(),
- file_reader, &reader));
- ColumnIteratorUPtr iterator;
- RETURN_IF_ERROR(reader->new_iterator(&iterator, &input.column));
- OlapReaderStatistics stats;
- ColumnIteratorOptions iterator_options;
- iterator_options.stats = &stats;
- iterator_options.file_reader = file_reader.get();
- RETURN_IF_ERROR(iterator->init(iterator_options));
- RETURN_IF_ERROR(iterator->seek_to_ordinal(0));
- MutableColumnPtr read = input.data.column->clone_empty();
- size_t rows = written.meta.num_rows();
- bool has_null = false;
- RETURN_IF_ERROR(iterator->next_batch(&rows, read, &has_null));
- if (read->size() != input.rows()) {
- return Status::InternalError("read {} rows back, wrote {}",
read->size(), input.rows());
+// Checks that `read` holds the rows written at `rowids`.
+Status check_rows_read(const CaseInput& input, const WrittenColumn& written,
+ const std::vector<rowid_t>& rowids, const IColumn&
read) {
+ if (read.size() != rowids.size()) {
+ return Status::InternalError("read {} rows back, wanted {}",
read.size(), rowids.size());
}
-
if (input.column.is_nullable()) {
- const auto& read_null_map = assert_cast<const
ColumnNullable&>(*read).get_null_map_data();
+ const auto& read_null_map = assert_cast<const
ColumnNullable&>(read).get_null_map_data();
const auto& null_map =
assert_cast<const
ColumnNullable&>(*input.data.column).get_null_map_data();
- if (!std::equal(read_null_map.begin(), read_null_map.end(),
null_map.begin())) {
- return Status::InternalError("the NULL rows read back differ from
the ones written");
+ for (size_t i = 0; i < rowids.size(); ++i) {
+ if (read_null_map[i] != null_map[rowids[i]]) {
+ return Status::InternalError("row {} reads back as {}NULL",
rowids[i],
+ read_null_map[i] != 0 ? "" : "not
");
+ }
}
}
- auto expected = storage_cells(input.column.type(), *input.data.column);
- auto actual = storage_cells(input.column.type(), *read);
+ const auto written_cells = storage_cells(input.column.type(),
*input.data.column);
+ std::vector<std::string> expected;
+ expected.reserve(rowids.size());
+ for (const auto row : rowids) {
+ expected.push_back(written_cells[row]);
+ }
+ auto actual = storage_cells(input.column.type(), read);
// Reading turns a single-value bitmap into a one-value set, which is
written differently, so
// bitmaps are compared by value.
if
(check_and_get_column<ColumnBitmap>(*remove_nullable(input.data.column)) !=
nullptr) {
@@ -902,9 +892,9 @@ Status verify_read_back(const CaseInput& input, const
WrittenColumn& written,
written.meta.encoding() == PREFIX_ENCODING) {
const auto& null_map =
assert_cast<const
ColumnNullable&>(*input.data.column).get_null_map_data();
- for (size_t row = 0; row < expected.size(); ++row) {
- if (null_map[row] == 0) {
- expected[row].resize(input.column.length(), '\0');
+ for (size_t i = 0; i < rowids.size(); ++i) {
+ if (null_map[rowids[i]] == 0) {
+ expected[i].resize(input.column.length(), '\0');
}
}
}
@@ -919,12 +909,66 @@ Status verify_read_back(const CaseInput& input, const
WrittenColumn& written,
return text;
};
return Status::InternalError("row {} reads back as {} instead of {}",
- std::distance(actual.begin(),
actual_cell), hex(*actual_cell),
- hex(*expected_cell));
+ rowids[std::distance(actual.begin(),
actual_cell)],
+ hex(*actual_cell), hex(*expected_cell));
}
return Status::OK();
}
+// Reads the column back the way a Segment reads it, into columns like
`prototype`: every row by
+// batch, then two rows of every three by row id, so that the reads start and
stop inside NULL
+// runs. Checks that both hold the rows written.
+Status verify_read_back(const CaseInput& input, const WrittenColumn& written,
+ const std::string& path, const IColumn& prototype) {
+ io::FileReaderSPtr file_reader;
+ RETURN_IF_ERROR(io::global_local_filesystem()->open_file(path,
&file_reader));
+ ColumnReaderOptions reader_options;
+ reader_options.be_exec_version =
BeExecVersionManager::get_newest_version();
+ std::shared_ptr<ColumnReader> reader;
+ RETURN_IF_ERROR(ColumnReader::create(reader_options, written.meta,
written.meta.num_rows(),
+ file_reader, &reader));
+ ColumnIteratorUPtr iterator;
+ RETURN_IF_ERROR(reader->new_iterator(&iterator, &input.column));
+ OlapReaderStatistics stats;
+ ColumnIteratorOptions iterator_options;
+ iterator_options.stats = &stats;
+ iterator_options.file_reader = file_reader.get();
+ RETURN_IF_ERROR(iterator->init(iterator_options));
+ RETURN_IF_ERROR(iterator->seek_to_ordinal(0));
+ std::vector<rowid_t> rowids(input.rows());
+ std::iota(rowids.begin(), rowids.end(), rowid_t {0});
+ MutableColumnPtr read = prototype.clone_empty();
+ size_t rows = written.meta.num_rows();
+ bool has_null = false;
+ RETURN_IF_ERROR(iterator->next_batch(&rows, read, &has_null));
+ // A ColumnDictI32 holds codes: compare the strings they stand for.
+ read = read->convert_to_predicate_column_if_dictionary();
+ RETURN_IF_ERROR(check_rows_read(input, written, rowids, *read));
+
+ // BinaryPrefixPageDecoder does not read by row id.
+ if (written.meta.encoding() == PREFIX_ENCODING) {
+ return Status::OK();
+ }
+ std::erase_if(rowids, [](rowid_t row) { return row % 3 == 1; });
+ MutableColumnPtr picked = prototype.clone_empty();
+ // A ColumnDictI32 appends codes only into room reserved beforehand, as
SegmentIterator
+ // reserves the columns it reads into.
+ picked->reserve(rowids.size());
+ RETURN_IF_ERROR(iterator->read_by_rowids(rowids.data(), rowids.size(),
picked));
+ picked = picked->convert_to_predicate_column_if_dictionary();
+ return check_rows_read(input, written, rowids, *picked);
+}
+
+// The column a query that filters a string column reads it into, as
+// ReadSchema::get_predicate_column_ptr makes it.
+MutableColumnPtr dictionary_column(const CaseInput& input) {
+ MutableColumnPtr column = ColumnDictI32::create();
+ if (input.column.is_nullable()) {
+ column = ColumnNullable::create(std::move(column),
ColumnUInt8::create());
+ }
+ return column;
+}
+
// A page ends with its PageFooterPB, the footer length and the checksum.
uint32_t page_rows(const std::string& page) {
const size_t footer_size =
@@ -1143,8 +1187,16 @@ TEST_F(PageFormatTest,
RegisteredEncodingsKeepTheirPageBytes) {
EXPECT_TRUE(written.value().data_pages ==
rewritten.value().data_pages &&
written.value().dict_page ==
rewritten.value().dict_page)
<< "page bytes are not deterministic";
- const auto read_back = verify_read_back(input.value(),
written.value(), path);
+ const auto read_back = verify_read_back(input.value(),
written.value(), path,
+
*input.value().data.column);
EXPECT_TRUE(read_back.ok()) << read_back;
+ if (page_case.encoding == DICT_ENCODING) {
+ // The pages that fall back to plain encoding make the
dictionary page decoder
+ // replace the ColumnDictI32 it appends to with a ColumnString.
+ const auto dictionary_read_back = verify_read_back(
+ input.value(), written.value(), path,
*dictionary_column(input.value()));
+ EXPECT_TRUE(dictionary_read_back.ok()) << dictionary_read_back;
+ }
add_goldens(&goldens, page_case, written.value());
}
}
diff --git a/be/test/storage/segment/plain_page_test.cpp
b/be/test/storage/segment/plain_page_test.cpp
index 9c8ed90d9e8..f2975ccb617 100644
--- a/be/test/storage/segment/plain_page_test.cpp
+++ b/be/test/storage/segment/plain_page_test.cpp
@@ -22,10 +22,15 @@
#include <iostream>
#include "common/logging.h"
+#include "core/assert_cast.h"
+#include "core/column/column_vector.h"
#include "core/data_type/data_type_factory.hpp"
+#include "core/uint24.h"
+#include "core/value/vdatetime_value.h"
#include "storage/olap_common.h"
#include "storage/segment/page_builder.h"
#include "storage/segment/page_decoder.h"
+#include "storage/storage_layout.h"
#include "storage/types.h"
namespace doris {
@@ -154,6 +159,29 @@ public:
EXPECT_EQ(status.code(), ENTRY_NOT_FOUND);
}
}
+
+ // Eight consecutive days from 2020-01-01 in one plain page of V1 DATE
cells, each date
+ // stored as a uint24.
+ void build_date_v1_page(std::vector<VecDateTimeValue>* days, OwnedSlice*
page) {
+ using Layout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DATE>;
+ constexpr size_t kRows = 8;
+ days->resize(kRows);
+ std::vector<uint24_t> cells(kRows);
+ for (size_t i = 0; i < kRows; ++i) {
+
ASSERT_TRUE((*days)[i].from_date_int64(static_cast<int64_t>(20200101 + i)));
+ cells[i] = Layout::to_storage((*days)[i]);
+ }
+ PageBuilderOptions options;
+ options.data_page_size = 256 * 1024;
+ PageBuilder* builder = nullptr;
+ ASSERT_TRUE(
+
PlainPageBuilder<FieldType::OLAP_FIELD_TYPE_DATE>::create(&builder,
options).ok());
+ std::unique_ptr<PageBuilder> page_builder(builder);
+ size_t added = kRows;
+ ASSERT_TRUE(page_builder->add(reinterpret_cast<const
uint8_t*>(cells.data()), &added).ok());
+ ASSERT_EQ(added, kRows);
+ ASSERT_TRUE(page_builder->finish(page).ok());
+ }
};
TEST_F(PlainPageTest, TestInt32PlainPageRandom) {
@@ -266,5 +294,45 @@ TEST_F(PlainPageTest, TestBoolPlainPageSeekValue) {
nullptr);
}
+// A V1 DATE page holds each date as a uint24 cell; a batch decodes the cells
into the dates
+// of a ColumnDate.
+TEST_F(PlainPageTest, TestDateV1PlainPageDecodesIntoDates) {
+ std::vector<VecDateTimeValue> days;
+ OwnedSlice page;
+ ASSERT_NO_FATAL_FAILURE(build_date_v1_page(&days, &page));
+ PageDecoderOptions decoder_options;
+ PlainPageDecoder<FieldType::OLAP_FIELD_TYPE_DATE> decoder(page.slice(),
decoder_options);
+ ASSERT_TRUE(decoder.init().ok());
+
+ MutableColumnPtr dates = ColumnDate::create();
+ size_t n = days.size();
+ ASSERT_TRUE(decoder.next_batch(&n, dates).ok());
+ ASSERT_EQ(n, days.size());
+ const auto& decoded = assert_cast<const ColumnDate&>(*dates).get_data();
+ for (size_t i = 0; i < days.size(); ++i) {
+ EXPECT_EQ(decoded[i], days[i]) << "row " << i;
+ }
+}
+
+// Rows read by id into a ColumnDate come back as their dates.
+TEST_F(PlainPageTest, TestDateV1PlainPageReadsRowIdsIntoDates) {
+ std::vector<VecDateTimeValue> days;
+ OwnedSlice page;
+ ASSERT_NO_FATAL_FAILURE(build_date_v1_page(&days, &page));
+ PageDecoderOptions decoder_options;
+ PlainPageDecoder<FieldType::OLAP_FIELD_TYPE_DATE> decoder(page.slice(),
decoder_options);
+ ASSERT_TRUE(decoder.init().ok());
+
+ const rowid_t rowids[] = {1, 3, 6};
+ MutableColumnPtr picked = ColumnDate::create();
+ size_t n = 3;
+ ASSERT_TRUE(decoder.read_by_rowids(rowids, 0, &n, picked).ok());
+ ASSERT_EQ(n, 3);
+ const auto& picked_dates = assert_cast<const
ColumnDate&>(*picked).get_data();
+ for (size_t i = 0; i < 3; ++i) {
+ EXPECT_EQ(picked_dates[i], days[rowids[i]]) << "row id " << rowids[i];
+ }
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/test/storage/storage_layout_test.cpp
b/be/test/storage/storage_layout_test.cpp
index 3e3583d5c1e..b2185ba26ae 100644
--- a/be/test/storage/storage_layout_test.cpp
+++ b/be/test/storage/storage_layout_test.cpp
@@ -46,8 +46,10 @@
namespace doris {
TEST(StorageLayoutTest, StorageValueSizeIsFieldTypeSize) {
-#define CHECK_SIZE(FT) \
- EXPECT_EQ(sizeof(CppTypeTraits<FieldType::FT>::CppType),
field_type_size(FieldType::FT)) << #FT;
+#define CHECK_SIZE(FT) \
+ EXPECT_EQ(sizeof(StorageTypeTraits<FieldType::FT>::StorageType), \
+ field_type_size(FieldType::FT)) \
+ << #FT;
CHECK_SIZE(OLAP_FIELD_TYPE_BOOL)
CHECK_SIZE(OLAP_FIELD_TYPE_TINYINT)
CHECK_SIZE(OLAP_FIELD_TYPE_SMALLINT)
@@ -214,7 +216,7 @@ TEST(StorageLayoutTest, ColumnToStoragePreservesSource) {
dates->insert_value(day);
dates->insert_value(day);
using DateLayout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DATE>;
- using StoredDate = CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::CppType;
+ using StoredDate =
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::StorageType;
StorageBuffers date_buffers;
const auto* date_out = DateLayout::column_to_storage(*dates, 0, 2,
date_buffers);
for (size_t i = 0; i < 2; ++i) {
@@ -228,7 +230,7 @@ namespace {
template <FieldType FT>
void expect_floating_point_range_views() {
- using Value = typename CppTypeTraits<FT>::CppType;
+ using Value = typename StorageTypeTraits<FT>::StorageType;
using UInt = std::conditional_t<sizeof(Value) == sizeof(uint32_t),
uint32_t, uint64_t>;
using Column =
typename
PrimitiveTypeTraits<storage_field_type_to_primitive_type(FT)>::ColumnType;
@@ -393,7 +395,7 @@ TEST(StorageLayoutTest,
RuntimeTypeConvertsThroughItsLayout) {
VecDateTimeValue day;
ASSERT_TRUE(day.from_date_int64(20200101));
dates->insert_value(day);
- using StoredDate = CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::CppType;
+ using StoredDate =
StorageTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::StorageType;
StoredDate stored;
memcpy(&stored, column_to_storage(FieldType::OLAP_FIELD_TYPE_DATE, *dates,
0, 1, buffers),
sizeof(stored));
@@ -409,6 +411,110 @@ TEST(StorageLayoutTest,
RuntimeTypeConvertsThroughItsLayout) {
Exception);
}
+TEST(StorageLayoutTest, ToPrimitiveInvertsToStorage) {
+ using DateLayout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DATE>;
+ VecDateTimeValue day;
+ ASSERT_TRUE(day.from_date_int64(20200229));
+ const VecDateTimeValue read_day =
DateLayout::to_primitive(DateLayout::to_storage(day));
+ EXPECT_EQ(read_day, day);
+ EXPECT_EQ(read_day.to_olap_date(), day.to_olap_date());
+ VecDateTimeValue last_day;
+ ASSERT_TRUE(last_day.from_date_int64(99991231));
+ EXPECT_EQ(DateLayout::to_primitive(DateLayout::to_storage(last_day)),
last_day);
+
+ using DateTimeLayout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DATETIME>;
+ VecDateTimeValue moment;
+ ASSERT_TRUE(moment.from_date_int64(20200101123045));
+
EXPECT_EQ(DateTimeLayout::to_primitive(DateTimeLayout::to_storage(moment)),
moment);
+
+ using DecimalLayout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL>;
+ const DecimalV2Value amount(123, 456000000);
+ EXPECT_EQ(DecimalLayout::to_primitive(DecimalLayout::to_storage(amount)),
amount);
+ const DecimalV2Value debt(-123, -456000000);
+ EXPECT_EQ(DecimalLayout::to_primitive(DecimalLayout::to_storage(debt)),
debt);
+ const DecimalV2Value read_debt =
+ DecimalLayout::to_primitive(decimal12_t {.integer = -123,
.fraction = -456000000});
+ EXPECT_EQ(read_debt.int_value(), -123);
+ EXPECT_EQ(read_debt.frac_value(), -456000000);
+
+ // Reading does not canonicalise: a stored NaN keeps its payload.
+ using FloatLayout = StorageLayout<FieldType::OLAP_FIELD_TYPE_FLOAT>;
+ const auto payload_nan = std::bit_cast<float>(0x7FC12345U);
+ EXPECT_EQ(std::bit_cast<uint32_t>(FloatLayout::to_primitive(payload_nan)),
0x7FC12345U);
+ EXPECT_EQ(FloatLayout::to_primitive(FloatLayout::to_storage(1.5F)), 1.5F);
+
+ DateV2Value<DateV2ValueType> date;
+ ASSERT_TRUE(date.from_date_int64(20200101));
+ EXPECT_EQ(
+
StorageLayout<FieldType::OLAP_FIELD_TYPE_DATEV2>::to_primitive(date.to_date_int_val()),
+ date);
+
EXPECT_EQ(StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL32>::to_primitive(-12345),
+ Decimal32(-12345));
+
EXPECT_EQ(StorageLayout<FieldType::OLAP_FIELD_TYPE_BOOL>::to_primitive(uint8_t(1)),
UInt8(1));
+}
+
+TEST(StorageLayoutTest, StorageToColumnRepacksTheV1LayoutsFromUnalignedBytes) {
+ using DateLayout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DATE>;
+ using DateTimeLayout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DATETIME>;
+ using DecimalLayout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL>;
+
+ VecDateTimeValue day1;
+ VecDateTimeValue day2;
+ ASSERT_TRUE(day1.from_date_int64(20200101));
+ ASSERT_TRUE(day2.from_date_int64(20211231));
+ VecDateTimeValue moment;
+ ASSERT_TRUE(moment.from_date_int64(20200101123045));
+ const DecimalV2Value amount(-123, -456000000);
+
+ // A page holds the stored values after a header, so they are not aligned.
+ std::string page(1, 'h');
+ auto put = [&](const auto& value) {
+ page.append(reinterpret_cast<const char*>(&value), sizeof(value));
+ };
+ put(DateLayout::to_storage(day1));
+ put(DateLayout::to_storage(day2));
+ put(DateTimeLayout::to_storage(moment));
+ put(DecimalLayout::to_storage(amount));
+ const auto* stored = reinterpret_cast<const uint8_t*>(page.data()) + 1;
+
+ auto dates = ColumnDate::create();
+ dates->insert_value(day2);
+ DateLayout::storage_to_column(stored, 2, *dates);
+ ASSERT_EQ(dates->size(), 3);
+ EXPECT_EQ(dates->get_data()[1], day1);
+ EXPECT_EQ(dates->get_data()[2], day2);
+ stored += 2 * sizeof(uint24_t);
+
+ auto moments = ColumnDateTime::create();
+ DateTimeLayout::storage_to_column(stored, 1, *moments);
+ ASSERT_EQ(moments->size(), 1);
+ EXPECT_EQ(moments->get_data()[0], moment);
+ stored += sizeof(int64_t);
+
+ auto amounts = ColumnDecimal128V2::create(0, 9);
+ DecimalLayout::storage_to_column(stored, 1, *amounts);
+ ASSERT_EQ(amounts->size(), 1);
+ EXPECT_EQ(amounts->get_data()[0], amount);
+}
+
+TEST(StorageLayoutTest, StorageToColumnCopiesStoredValuesAsTheyAre) {
+ const float stored_floats[] = {1.5F, std::bit_cast<float>(0x7FC12345U)};
+ auto floats = ColumnFloat32::create();
+ StorageLayout<FieldType::OLAP_FIELD_TYPE_FLOAT>::storage_to_column(
+ reinterpret_cast<const uint8_t*>(stored_floats), 2, *floats);
+ ASSERT_EQ(floats->size(), 2);
+ EXPECT_EQ(floats->get_data()[0], 1.5F);
+ EXPECT_EQ(std::bit_cast<uint32_t>(floats->get_data()[1]), 0x7FC12345U);
+
+ const int32_t stored_decimals[] = {-12345, 67890};
+ auto decimals = ColumnDecimal32::create(0, 3);
+ StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL32>::storage_to_column(
+ reinterpret_cast<const uint8_t*>(stored_decimals), 2, *decimals);
+ ASSERT_EQ(decimals->size(), 2);
+ EXPECT_EQ(decimals->get_data()[0], Decimal32(-12345));
+ EXPECT_EQ(decimals->get_data()[1], Decimal32(67890));
+}
+
TEST(StorageLayoutTest, ObjectBatchSerialisesEveryValue) {
auto bitmaps = ColumnBitmap::create();
for (size_t i = 0; i < 3; ++i) {
diff --git a/be/test/core/block/column_vector_test.cpp
b/be/test/storage/storage_layout_test_util.h
similarity index 59%
rename from be/test/core/block/column_vector_test.cpp
rename to be/test/storage/storage_layout_test_util.h
index f1d233f6115..32bb4c4493e 100644
--- a/be/test/core/block/column_vector_test.cpp
+++ b/be/test/storage/storage_layout_test_util.h
@@ -1,4 +1,3 @@
-
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -16,26 +15,23 @@
// specific language governing permissions and limitations
// under the License.
-#include "core/column/column_vector.h"
-
-#include <gtest/gtest-message.h>
-#include <gtest/gtest-test-part.h>
+#pragma once
-#include <string>
-
-#include "gtest/gtest_pred_impl.h"
+#include "core/data_type/primitive_type.h"
+#include "core/data_type/storage_field_type.h"
+#include "core/field.h"
+#include "storage/storage_layout.h"
namespace doris {
-TEST(VColumnVectorTest, insert_date_column) {
- auto column = ColumnInt64::create();
-
- size_t rows = 4096;
- int64_t val = 0;
- for (size_t i = 0; i < rows; ++i) {
- column->insert_date_column(reinterpret_cast<char*>(&val), 1);
- }
- ASSERT_EQ(column->size(), rows);
+// The Field a stored cell of PT's FieldType reads back as, for tests that
state their
+// expectations in stored bytes.
+template <PrimitiveType PT>
+Field field_of_cell(
+ const typename
StorageLayout<primitive_type_to_storage_field_type(PT)>::StorageValue&
+ value) {
+ return Field::create_field<PT>(
+
StorageLayout<primitive_type_to_storage_field_type(PT)>::to_primitive(value));
}
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
