This is an automated email from the ASF dual-hosted git repository.
csun5285 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fdfc63b0207 [refactor](be) Replace PrimitiveTypeConvertor with
StorageLayout keyed by FieldType (#68664)
fdfc63b0207 is described below
commit fdfc63b0207a76373f94229a72267bf2ee3dee6c
Author: Chenyang Sun <[email protected]>
AuthorDate: Fri Oct 9 20:07:38 2026 +0800
[refactor](be) Replace PrimitiveTypeConvertor with StorageLayout keyed by
FieldType (#68664)
### What problem does this PR solve?
Issue Number: None
Related PR: #68608, #68609
Problem Summary: How a compute-layer value becomes the value a segment
stores is written down in several places. PrimitiveTypeConvertor<PT>
gives the V1 DATE, DATETIME and DECIMAL layouts to the key encodings of
RowCursor and the BKD index; KeyCoderTraitsForFloat canonicalises NaN on
its own; and the comparison and IN-list predicates repeat the V1
conversions by hand to probe a bloom filter. The key-encodable types are
listed as (FieldType, PrimitiveType) pairs so that the key helpers can
be keyed by PrimitiveType.
Add storage/storage_layout.h. For every fixed-width FieldType,
StorageLayout<FT> gives the StorageValue a segment stores, the
PrimitiveValue the compute layer holds, and to_storage from one to the
other: a bit cast, the quiet NaN for every NaN of FLOAT/DOUBLE, and the
V1 packing of DATE, DATETIME and DECIMAL. Make the PrimitiveType <->
FieldType mapping in storage_field_type.h constexpr, so that a template
picks its layout with
StorageLayout<primitive_type_to_storage_field_type(PT)>; data_type.h now
declares FieldType itself instead of including that header.
Move the users onto it: full_encode_field_as_key and so the RowCursor
and BKD query key encodings, the BKD +/- infinity sentinels, the NaN
handling of KeyCoderTraitsForFloat, and the bloom filter probes of the
comparison and IN-list predicates. The key-encodable types are listed by
FieldType alone. Delete PrimitiveTypeConvertor, whose last user was a
test helper.
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
---
be/src/core/data_type/data_type.h | 2 +-
be/src/core/data_type/primitive_type.h | 44 -----
be/src/core/data_type/storage_field_type.cpp | 210 ---------------------
be/src/core/data_type/storage_field_type.h | 193 ++++++++++++++++++-
be/src/storage/index/bkd_field_encoding.cpp | 32 ++--
.../index/inverted/inverted_index_reader.cpp | 31 +--
be/src/storage/key_coder.h | 83 ++++----
be/src/storage/predicate/comparison_predicate.h | 23 +--
be/src/storage/predicate/in_list_predicate.h | 31 +--
be/src/storage/row_cursor.cpp | 6 +-
be/src/storage/row_cursor.h | 4 +-
be/src/storage/storage_layout.h | 159 ++++++++++++++++
be/test/storage/key_coder_test.cpp | 114 +++++------
.../storage/segment/inverted_index_reader_test.cpp | 4 +-
be/test/storage/segment/test_segment_writer.h | 3 +-
be/test/storage/storage_layout_test.cpp | 119 ++++++++++++
16 files changed, 604 insertions(+), 454 deletions(-)
diff --git a/be/src/core/data_type/data_type.h
b/be/src/core/data_type/data_type.h
index f0c6ad80596..8948b6072bc 100644
--- a/be/src/core/data_type/data_type.h
+++ b/be/src/core/data_type/data_type.h
@@ -38,11 +38,11 @@
#include "core/cow.h"
#include "core/data_type/define_primitive_type.h"
#include "core/data_type/primitive_type.h"
-#include "core/data_type/storage_field_type.h"
#include "core/data_type_serde/data_type_serde.h"
#include "core/types.h"
namespace doris {
+enum class FieldType;
class PColumnMeta;
class PTypeDesc;
class PTypeNode;
diff --git a/be/src/core/data_type/primitive_type.h
b/be/src/core/data_type/primitive_type.h
index 9b4ec11a244..6169076a035 100644
--- a/be/src/core/data_type/primitive_type.h
+++ b/be/src/core/data_type/primitive_type.h
@@ -555,50 +555,6 @@ struct PrimitiveTypeTraits<TYPE_UINT64> {
using ColumnType = ColumnOffset64;
};
-template <PrimitiveType PT>
-struct PrimitiveTypeConvertor {
- using CppType = typename PrimitiveTypeTraits<PT>::CppType;
- using StorageFieldType = typename
PrimitiveTypeTraits<PT>::StorageFieldType;
-
- static inline StorageFieldType&& to_storage_field_type(CppType&& value) {
- return static_cast<StorageFieldType&&>(std::forward<CppType>(value));
- }
-
- static inline const StorageFieldType& to_storage_field_type(const CppType&
value) {
- return *reinterpret_cast<const StorageFieldType*>(&value);
- }
-};
-
-template <>
-struct PrimitiveTypeConvertor<TYPE_DATE> {
- using CppType = typename PrimitiveTypeTraits<TYPE_DATE>::CppType;
- using StorageFieldType = typename
PrimitiveTypeTraits<TYPE_DATE>::StorageFieldType;
-
- static inline StorageFieldType to_storage_field_type(const CppType& value)
{
- return StorageFieldType(cast_set<uint32_t>(value.to_olap_date()));
- }
-};
-
-template <>
-struct PrimitiveTypeConvertor<TYPE_DATETIME> {
- using CppType = typename PrimitiveTypeTraits<TYPE_DATETIME>::CppType;
- using StorageFieldType = typename
PrimitiveTypeTraits<TYPE_DATETIME>::StorageFieldType;
-
- static inline StorageFieldType to_storage_field_type(const CppType& value)
{
- return value.to_olap_datetime();
- }
-};
-
-template <>
-struct PrimitiveTypeConvertor<TYPE_DECIMALV2> {
- using CppType = typename PrimitiveTypeTraits<TYPE_DECIMALV2>::CppType;
- using StorageFieldType = typename
PrimitiveTypeTraits<TYPE_DECIMALV2>::StorageFieldType;
-
- static inline StorageFieldType to_storage_field_type(const CppType& value)
{
- return {value.int_value(), value.frac_value()};
- }
-};
-
inline TTypeDesc create_type_desc(PrimitiveType type, int precision = 0, int
scale = 0) {
TTypeDesc type_desc;
std::vector<TTypeNode> node_type;
diff --git a/be/src/core/data_type/storage_field_type.cpp
b/be/src/core/data_type/storage_field_type.cpp
deleted file mode 100644
index 7eb96d83029..00000000000
--- a/be/src/core/data_type/storage_field_type.cpp
+++ /dev/null
@@ -1,210 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "core/data_type/storage_field_type.h"
-
-#include "common/exception.h"
-#include "storage/olap_common.h"
-
-namespace doris {
-
-// NOLINTNEXTLINE(readability-function-size): keep the exhaustive mapping
together for auditability.
-FieldType primitive_type_to_storage_field_type(PrimitiveType type) {
- switch (type) {
- case PrimitiveType::INVALID_TYPE:
- return FieldType::OLAP_FIELD_TYPE_UNKNOWN;
- case PrimitiveType::TYPE_NULL:
- return FieldType::OLAP_FIELD_TYPE_NONE;
- case PrimitiveType::TYPE_BOOLEAN:
- return FieldType::OLAP_FIELD_TYPE_BOOL;
- case PrimitiveType::TYPE_TINYINT:
- return FieldType::OLAP_FIELD_TYPE_TINYINT;
- case PrimitiveType::TYPE_SMALLINT:
- return FieldType::OLAP_FIELD_TYPE_SMALLINT;
- case PrimitiveType::TYPE_INT:
- return FieldType::OLAP_FIELD_TYPE_INT;
- case PrimitiveType::TYPE_BIGINT:
- return FieldType::OLAP_FIELD_TYPE_BIGINT;
- case PrimitiveType::TYPE_LARGEINT:
- return FieldType::OLAP_FIELD_TYPE_LARGEINT;
- case PrimitiveType::TYPE_FLOAT:
- return FieldType::OLAP_FIELD_TYPE_FLOAT;
- case PrimitiveType::TYPE_DOUBLE:
- return FieldType::OLAP_FIELD_TYPE_DOUBLE;
- case PrimitiveType::TYPE_VARCHAR:
- return FieldType::OLAP_FIELD_TYPE_VARCHAR;
- case PrimitiveType::TYPE_DATE:
- return FieldType::OLAP_FIELD_TYPE_DATE;
- case PrimitiveType::TYPE_DATETIME:
- return FieldType::OLAP_FIELD_TYPE_DATETIME;
- case PrimitiveType::TYPE_CHAR:
- return FieldType::OLAP_FIELD_TYPE_CHAR;
- case PrimitiveType::TYPE_STRUCT:
- return FieldType::OLAP_FIELD_TYPE_STRUCT;
- case PrimitiveType::TYPE_ARRAY:
- return FieldType::OLAP_FIELD_TYPE_ARRAY;
- case PrimitiveType::TYPE_MAP:
- return FieldType::OLAP_FIELD_TYPE_MAP;
- case PrimitiveType::TYPE_HLL:
- return FieldType::OLAP_FIELD_TYPE_HLL;
- case PrimitiveType::TYPE_DECIMALV2:
- return FieldType::OLAP_FIELD_TYPE_DECIMAL;
- case PrimitiveType::TYPE_BITMAP:
- return FieldType::OLAP_FIELD_TYPE_BITMAP;
- case PrimitiveType::TYPE_STRING:
- return FieldType::OLAP_FIELD_TYPE_STRING;
- case PrimitiveType::TYPE_QUANTILE_STATE:
- return FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE;
- case PrimitiveType::TYPE_DATEV2:
- return FieldType::OLAP_FIELD_TYPE_DATEV2;
- case PrimitiveType::TYPE_DATETIMEV2:
- return FieldType::OLAP_FIELD_TYPE_DATETIMEV2;
- case PrimitiveType::TYPE_TIMESTAMP_NS:
- return FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS;
- case PrimitiveType::TYPE_TIMEV2:
- return FieldType::OLAP_FIELD_TYPE_TIMEV2;
- case PrimitiveType::TYPE_DECIMAL32:
- return FieldType::OLAP_FIELD_TYPE_DECIMAL32;
- case PrimitiveType::TYPE_DECIMAL64:
- return FieldType::OLAP_FIELD_TYPE_DECIMAL64;
- case PrimitiveType::TYPE_DECIMAL128I:
- return FieldType::OLAP_FIELD_TYPE_DECIMAL128I;
- case PrimitiveType::TYPE_JSONB:
- return FieldType::OLAP_FIELD_TYPE_JSONB;
- case PrimitiveType::TYPE_VARIANT:
- return FieldType::OLAP_FIELD_TYPE_VARIANT;
- case PrimitiveType::TYPE_AGG_STATE:
- return FieldType::OLAP_FIELD_TYPE_AGG_STATE;
- case PrimitiveType::TYPE_DECIMAL256:
- return FieldType::OLAP_FIELD_TYPE_DECIMAL256;
- case PrimitiveType::TYPE_IPV4:
- return FieldType::OLAP_FIELD_TYPE_IPV4;
- case PrimitiveType::TYPE_IPV6:
- return FieldType::OLAP_FIELD_TYPE_IPV6;
- case PrimitiveType::TYPE_UUID:
- return FieldType::OLAP_FIELD_TYPE_UUID;
- case PrimitiveType::TYPE_UINT32:
- return FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT;
- case PrimitiveType::TYPE_UINT64:
- return FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT;
- case PrimitiveType::TYPE_TIMESTAMPTZ:
- return FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ;
- case PrimitiveType::TYPE_BINARY:
- case static_cast<PrimitiveType>(14): // TYPE_DECIMAL (deprecated)
- case static_cast<PrimitiveType>(21): // TYPE_TIME (deprecated)
- case static_cast<PrimitiveType>(33): // TYPE_LAMBDA_FUNCTION (deprecated)
- case PrimitiveType::TYPE_FIXED_LENGTH_OBJECT:
- case PrimitiveType::TYPE_VARBINARY:
- break;
- }
-
- throw Exception(ErrorCode::INTERNAL_ERROR, "Cannot convert PrimitiveType
{} to FieldType",
- static_cast<int>(type));
-}
-
-// NOLINTNEXTLINE(readability-function-size): keep the exhaustive mapping
together for auditability.
-PrimitiveType storage_field_type_to_primitive_type(FieldType type) {
- switch (type) {
- case FieldType::OLAP_FIELD_TYPE_TINYINT:
- return PrimitiveType::TYPE_TINYINT;
- case FieldType::OLAP_FIELD_TYPE_SMALLINT:
- return PrimitiveType::TYPE_SMALLINT;
- case FieldType::OLAP_FIELD_TYPE_INT:
- return PrimitiveType::TYPE_INT;
- case FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT:
- return PrimitiveType::TYPE_UINT32;
- case FieldType::OLAP_FIELD_TYPE_BIGINT:
- return PrimitiveType::TYPE_BIGINT;
- case FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT:
- return PrimitiveType::TYPE_UINT64;
- case FieldType::OLAP_FIELD_TYPE_LARGEINT:
- return PrimitiveType::TYPE_LARGEINT;
- case FieldType::OLAP_FIELD_TYPE_FLOAT:
- return PrimitiveType::TYPE_FLOAT;
- case FieldType::OLAP_FIELD_TYPE_DOUBLE:
- return PrimitiveType::TYPE_DOUBLE;
- case FieldType::OLAP_FIELD_TYPE_CHAR:
- return PrimitiveType::TYPE_CHAR;
- case FieldType::OLAP_FIELD_TYPE_DATE:
- return PrimitiveType::TYPE_DATE;
- case FieldType::OLAP_FIELD_TYPE_DATETIME:
- return PrimitiveType::TYPE_DATETIME;
- case FieldType::OLAP_FIELD_TYPE_DECIMAL:
- return PrimitiveType::TYPE_DECIMALV2;
- case FieldType::OLAP_FIELD_TYPE_VARCHAR:
- return PrimitiveType::TYPE_VARCHAR;
- case FieldType::OLAP_FIELD_TYPE_STRUCT:
- return PrimitiveType::TYPE_STRUCT;
- case FieldType::OLAP_FIELD_TYPE_ARRAY:
- return PrimitiveType::TYPE_ARRAY;
- case FieldType::OLAP_FIELD_TYPE_MAP:
- return PrimitiveType::TYPE_MAP;
- case FieldType::OLAP_FIELD_TYPE_UNKNOWN:
- return PrimitiveType::INVALID_TYPE;
- case FieldType::OLAP_FIELD_TYPE_NONE:
- return PrimitiveType::TYPE_NULL;
- case FieldType::OLAP_FIELD_TYPE_HLL:
- return PrimitiveType::TYPE_HLL;
- case FieldType::OLAP_FIELD_TYPE_BOOL:
- return PrimitiveType::TYPE_BOOLEAN;
- case FieldType::OLAP_FIELD_TYPE_BITMAP:
- return PrimitiveType::TYPE_BITMAP;
- case FieldType::OLAP_FIELD_TYPE_STRING:
- return PrimitiveType::TYPE_STRING;
- case FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE:
- return PrimitiveType::TYPE_QUANTILE_STATE;
- case FieldType::OLAP_FIELD_TYPE_DATEV2:
- return PrimitiveType::TYPE_DATEV2;
- case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
- return PrimitiveType::TYPE_DATETIMEV2;
- case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
- return PrimitiveType::TYPE_TIMESTAMP_NS;
- case FieldType::OLAP_FIELD_TYPE_TIMEV2:
- return PrimitiveType::TYPE_TIMEV2;
- case FieldType::OLAP_FIELD_TYPE_DECIMAL32:
- return PrimitiveType::TYPE_DECIMAL32;
- case FieldType::OLAP_FIELD_TYPE_DECIMAL64:
- return PrimitiveType::TYPE_DECIMAL64;
- case FieldType::OLAP_FIELD_TYPE_DECIMAL128I:
- return PrimitiveType::TYPE_DECIMAL128I;
- case FieldType::OLAP_FIELD_TYPE_JSONB:
- return PrimitiveType::TYPE_JSONB;
- case FieldType::OLAP_FIELD_TYPE_VARIANT:
- return PrimitiveType::TYPE_VARIANT;
- case FieldType::OLAP_FIELD_TYPE_AGG_STATE:
- return PrimitiveType::TYPE_AGG_STATE;
- case FieldType::OLAP_FIELD_TYPE_DECIMAL256:
- return PrimitiveType::TYPE_DECIMAL256;
- case FieldType::OLAP_FIELD_TYPE_IPV4:
- return PrimitiveType::TYPE_IPV4;
- case FieldType::OLAP_FIELD_TYPE_IPV6:
- return PrimitiveType::TYPE_IPV6;
- case FieldType::OLAP_FIELD_TYPE_UUID:
- return PrimitiveType::TYPE_UUID;
- case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ:
- return PrimitiveType::TYPE_TIMESTAMPTZ;
- case FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT:
- case FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT:
- case FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE:
- break;
- }
-
- throw Exception(ErrorCode::INTERNAL_ERROR, "Cannot convert FieldType {} to
PrimitiveType",
- static_cast<int>(type));
-}
-
-} // namespace doris
diff --git a/be/src/core/data_type/storage_field_type.h
b/be/src/core/data_type/storage_field_type.h
index a1d48e53eea..8145f205f8a 100644
--- a/be/src/core/data_type/storage_field_type.h
+++ b/be/src/core/data_type/storage_field_type.h
@@ -17,16 +17,199 @@
#pragma once
+#include "common/exception.h"
#include "core/data_type/define_primitive_type.h"
+#include "storage/field_type.h"
namespace doris {
-enum class FieldType;
+// The FieldType a segment stores a PrimitiveType's values as, and back. Both
+// directions are constexpr, so a template can pick the storage traits of a
+// compute-layer type; at run time an unmapped type throws.
-// Throws Exception when the primitive type has no storage field type mapping.
-[[nodiscard]] FieldType primitive_type_to_storage_field_type(PrimitiveType
type);
+// NOLINTNEXTLINE(readability-function-size): keep the exhaustive mapping
together for auditability.
+constexpr FieldType primitive_type_to_storage_field_type(PrimitiveType type) {
+ switch (type) {
+ case PrimitiveType::INVALID_TYPE:
+ return FieldType::OLAP_FIELD_TYPE_UNKNOWN;
+ case PrimitiveType::TYPE_NULL:
+ return FieldType::OLAP_FIELD_TYPE_NONE;
+ case PrimitiveType::TYPE_BOOLEAN:
+ return FieldType::OLAP_FIELD_TYPE_BOOL;
+ case PrimitiveType::TYPE_TINYINT:
+ return FieldType::OLAP_FIELD_TYPE_TINYINT;
+ case PrimitiveType::TYPE_SMALLINT:
+ return FieldType::OLAP_FIELD_TYPE_SMALLINT;
+ case PrimitiveType::TYPE_INT:
+ return FieldType::OLAP_FIELD_TYPE_INT;
+ case PrimitiveType::TYPE_BIGINT:
+ return FieldType::OLAP_FIELD_TYPE_BIGINT;
+ case PrimitiveType::TYPE_LARGEINT:
+ return FieldType::OLAP_FIELD_TYPE_LARGEINT;
+ case PrimitiveType::TYPE_FLOAT:
+ return FieldType::OLAP_FIELD_TYPE_FLOAT;
+ case PrimitiveType::TYPE_DOUBLE:
+ return FieldType::OLAP_FIELD_TYPE_DOUBLE;
+ case PrimitiveType::TYPE_VARCHAR:
+ return FieldType::OLAP_FIELD_TYPE_VARCHAR;
+ case PrimitiveType::TYPE_DATE:
+ return FieldType::OLAP_FIELD_TYPE_DATE;
+ case PrimitiveType::TYPE_DATETIME:
+ return FieldType::OLAP_FIELD_TYPE_DATETIME;
+ case PrimitiveType::TYPE_CHAR:
+ return FieldType::OLAP_FIELD_TYPE_CHAR;
+ case PrimitiveType::TYPE_STRUCT:
+ return FieldType::OLAP_FIELD_TYPE_STRUCT;
+ case PrimitiveType::TYPE_ARRAY:
+ return FieldType::OLAP_FIELD_TYPE_ARRAY;
+ case PrimitiveType::TYPE_MAP:
+ return FieldType::OLAP_FIELD_TYPE_MAP;
+ case PrimitiveType::TYPE_HLL:
+ return FieldType::OLAP_FIELD_TYPE_HLL;
+ case PrimitiveType::TYPE_DECIMALV2:
+ return FieldType::OLAP_FIELD_TYPE_DECIMAL;
+ case PrimitiveType::TYPE_BITMAP:
+ return FieldType::OLAP_FIELD_TYPE_BITMAP;
+ case PrimitiveType::TYPE_STRING:
+ return FieldType::OLAP_FIELD_TYPE_STRING;
+ case PrimitiveType::TYPE_QUANTILE_STATE:
+ return FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE;
+ case PrimitiveType::TYPE_DATEV2:
+ return FieldType::OLAP_FIELD_TYPE_DATEV2;
+ case PrimitiveType::TYPE_DATETIMEV2:
+ return FieldType::OLAP_FIELD_TYPE_DATETIMEV2;
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ return FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS;
+ case PrimitiveType::TYPE_TIMEV2:
+ return FieldType::OLAP_FIELD_TYPE_TIMEV2;
+ case PrimitiveType::TYPE_DECIMAL32:
+ return FieldType::OLAP_FIELD_TYPE_DECIMAL32;
+ case PrimitiveType::TYPE_DECIMAL64:
+ return FieldType::OLAP_FIELD_TYPE_DECIMAL64;
+ case PrimitiveType::TYPE_DECIMAL128I:
+ return FieldType::OLAP_FIELD_TYPE_DECIMAL128I;
+ case PrimitiveType::TYPE_JSONB:
+ return FieldType::OLAP_FIELD_TYPE_JSONB;
+ case PrimitiveType::TYPE_VARIANT:
+ return FieldType::OLAP_FIELD_TYPE_VARIANT;
+ case PrimitiveType::TYPE_AGG_STATE:
+ return FieldType::OLAP_FIELD_TYPE_AGG_STATE;
+ case PrimitiveType::TYPE_DECIMAL256:
+ return FieldType::OLAP_FIELD_TYPE_DECIMAL256;
+ case PrimitiveType::TYPE_IPV4:
+ return FieldType::OLAP_FIELD_TYPE_IPV4;
+ case PrimitiveType::TYPE_IPV6:
+ return FieldType::OLAP_FIELD_TYPE_IPV6;
+ case PrimitiveType::TYPE_UUID:
+ return FieldType::OLAP_FIELD_TYPE_UUID;
+ case PrimitiveType::TYPE_UINT32:
+ return FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT;
+ case PrimitiveType::TYPE_UINT64:
+ return FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT;
+ case PrimitiveType::TYPE_TIMESTAMPTZ:
+ return FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ;
+ case PrimitiveType::TYPE_BINARY:
+ case static_cast<PrimitiveType>(14): // TYPE_DECIMAL (deprecated)
+ case static_cast<PrimitiveType>(21): // TYPE_TIME (deprecated)
+ case static_cast<PrimitiveType>(33): // TYPE_LAMBDA_FUNCTION (deprecated)
+ case PrimitiveType::TYPE_FIXED_LENGTH_OBJECT:
+ case PrimitiveType::TYPE_VARBINARY:
+ break;
+ }
-// Throws Exception when the storage field type has no primitive type mapping.
-[[nodiscard]] PrimitiveType storage_field_type_to_primitive_type(FieldType
type);
+ throw Exception(ErrorCode::INTERNAL_ERROR, "Cannot convert PrimitiveType
{} to FieldType",
+ static_cast<int>(type));
+}
+
+// NOLINTNEXTLINE(readability-function-size): keep the exhaustive mapping
together for auditability.
+constexpr PrimitiveType storage_field_type_to_primitive_type(FieldType type) {
+ switch (type) {
+ case FieldType::OLAP_FIELD_TYPE_TINYINT:
+ return PrimitiveType::TYPE_TINYINT;
+ case FieldType::OLAP_FIELD_TYPE_SMALLINT:
+ return PrimitiveType::TYPE_SMALLINT;
+ case FieldType::OLAP_FIELD_TYPE_INT:
+ return PrimitiveType::TYPE_INT;
+ case FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT:
+ return PrimitiveType::TYPE_UINT32;
+ case FieldType::OLAP_FIELD_TYPE_BIGINT:
+ return PrimitiveType::TYPE_BIGINT;
+ case FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT:
+ return PrimitiveType::TYPE_UINT64;
+ case FieldType::OLAP_FIELD_TYPE_LARGEINT:
+ return PrimitiveType::TYPE_LARGEINT;
+ case FieldType::OLAP_FIELD_TYPE_FLOAT:
+ return PrimitiveType::TYPE_FLOAT;
+ case FieldType::OLAP_FIELD_TYPE_DOUBLE:
+ return PrimitiveType::TYPE_DOUBLE;
+ case FieldType::OLAP_FIELD_TYPE_CHAR:
+ return PrimitiveType::TYPE_CHAR;
+ case FieldType::OLAP_FIELD_TYPE_DATE:
+ return PrimitiveType::TYPE_DATE;
+ case FieldType::OLAP_FIELD_TYPE_DATETIME:
+ return PrimitiveType::TYPE_DATETIME;
+ case FieldType::OLAP_FIELD_TYPE_DECIMAL:
+ return PrimitiveType::TYPE_DECIMALV2;
+ case FieldType::OLAP_FIELD_TYPE_VARCHAR:
+ return PrimitiveType::TYPE_VARCHAR;
+ case FieldType::OLAP_FIELD_TYPE_STRUCT:
+ return PrimitiveType::TYPE_STRUCT;
+ case FieldType::OLAP_FIELD_TYPE_ARRAY:
+ return PrimitiveType::TYPE_ARRAY;
+ case FieldType::OLAP_FIELD_TYPE_MAP:
+ return PrimitiveType::TYPE_MAP;
+ case FieldType::OLAP_FIELD_TYPE_UNKNOWN:
+ return PrimitiveType::INVALID_TYPE;
+ case FieldType::OLAP_FIELD_TYPE_NONE:
+ return PrimitiveType::TYPE_NULL;
+ case FieldType::OLAP_FIELD_TYPE_HLL:
+ return PrimitiveType::TYPE_HLL;
+ case FieldType::OLAP_FIELD_TYPE_BOOL:
+ return PrimitiveType::TYPE_BOOLEAN;
+ case FieldType::OLAP_FIELD_TYPE_BITMAP:
+ return PrimitiveType::TYPE_BITMAP;
+ case FieldType::OLAP_FIELD_TYPE_STRING:
+ return PrimitiveType::TYPE_STRING;
+ case FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE:
+ return PrimitiveType::TYPE_QUANTILE_STATE;
+ case FieldType::OLAP_FIELD_TYPE_DATEV2:
+ return PrimitiveType::TYPE_DATEV2;
+ case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
+ return PrimitiveType::TYPE_DATETIMEV2;
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
+ return PrimitiveType::TYPE_TIMESTAMP_NS;
+ case FieldType::OLAP_FIELD_TYPE_TIMEV2:
+ return PrimitiveType::TYPE_TIMEV2;
+ case FieldType::OLAP_FIELD_TYPE_DECIMAL32:
+ return PrimitiveType::TYPE_DECIMAL32;
+ case FieldType::OLAP_FIELD_TYPE_DECIMAL64:
+ return PrimitiveType::TYPE_DECIMAL64;
+ case FieldType::OLAP_FIELD_TYPE_DECIMAL128I:
+ return PrimitiveType::TYPE_DECIMAL128I;
+ case FieldType::OLAP_FIELD_TYPE_JSONB:
+ return PrimitiveType::TYPE_JSONB;
+ case FieldType::OLAP_FIELD_TYPE_VARIANT:
+ return PrimitiveType::TYPE_VARIANT;
+ case FieldType::OLAP_FIELD_TYPE_AGG_STATE:
+ return PrimitiveType::TYPE_AGG_STATE;
+ case FieldType::OLAP_FIELD_TYPE_DECIMAL256:
+ return PrimitiveType::TYPE_DECIMAL256;
+ case FieldType::OLAP_FIELD_TYPE_IPV4:
+ return PrimitiveType::TYPE_IPV4;
+ case FieldType::OLAP_FIELD_TYPE_IPV6:
+ return PrimitiveType::TYPE_IPV6;
+ case FieldType::OLAP_FIELD_TYPE_UUID:
+ return PrimitiveType::TYPE_UUID;
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ:
+ return PrimitiveType::TYPE_TIMESTAMPTZ;
+ case FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT:
+ case FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT:
+ case FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE:
+ break;
+ }
+
+ throw Exception(ErrorCode::INTERNAL_ERROR, "Cannot convert FieldType {} to
PrimitiveType",
+ static_cast<int>(type));
+}
} // namespace doris
diff --git a/be/src/storage/index/bkd_field_encoding.cpp
b/be/src/storage/index/bkd_field_encoding.cpp
index ba50871897e..b737670bbd4 100644
--- a/be/src/storage/index/bkd_field_encoding.cpp
+++ b/be/src/storage/index/bkd_field_encoding.cpp
@@ -18,30 +18,32 @@
#include "storage/index/bkd_field_encoding.h"
#include "core/data_type/primitive_type.h"
+#include "core/data_type/storage_field_type.h"
namespace doris {
Status encode_bkd_field_ascending(FieldType ft, const Field& field, const
KeyCoder* coder,
std::string* out) {
- // `actual` is the primitive type of the query Field from the caller;
`PrimitiveType::PT` is the
- // scalar type the BKD index stores (e.g. INT for an INT column or
ARRAY<INT> index).
+ // `actual` is the primitive type of the query Field from the caller; `pt`
is the scalar
+ // type the BKD index stores (e.g. INT for an INT column or ARRAY<INT>
index).
// Normally they match: `int_col = 1` -> both INT;
`array_contains(int_arr, 2)` -> both INT.
// Mismatch happens when the query Field carries a non-scalar while BKD
records the inner scalar:
// `arr = []` reaches here via `FunctionComparison<EqualsOp>` with the
entire const ARRAY literal
- // as the query Field, so `actual = TYPE_ARRAY` while PT is the inner
scalar -- the predicate
+ // as the query Field, so `actual = TYPE_ARRAY` while `pt` is the inner
scalar -- the predicate
// cannot be answered by BKD. Return INVERTED_INDEX_EVALUATE_SKIPPED so
`_apply_index_expr`
- // downgrades to scalar evaluation instead of crashing on
`Field::get<PT>()` DCHECK below.
-#define CASE(FT, PT)
\
- case FieldType::FT: {
\
- const auto actual = field.get_type();
\
- if (actual != PrimitiveType::PT && actual != PrimitiveType::TYPE_NULL
&& \
- !(is_string_type(actual) && is_string_type(PrimitiveType::PT))) {
\
- return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED,
false>( \
- "BKD query value type {} does not match index type {}",
\
- static_cast<int>(actual), static_cast<int>(ft));
\
- }
\
- full_encode_field_as_key<PrimitiveType::PT>(field, coder, out);
\
- return Status::OK();
\
+ // downgrades to scalar evaluation instead of crashing on
`Field::get<pt>()` DCHECK below.
+#define CASE(FT)
\
+ case FieldType::FT: {
\
+ constexpr PrimitiveType pt =
storage_field_type_to_primitive_type(FieldType::FT); \
+ const auto actual = field.get_type();
\
+ if (actual != pt && actual != PrimitiveType::TYPE_NULL &&
\
+ !(is_string_type(actual) && is_string_type(pt))) {
\
+ return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED,
false>( \
+ "BKD query value type {} does not match index type {}",
\
+ static_cast<int>(actual), static_cast<int>(ft));
\
+ }
\
+ full_encode_field_as_key<FieldType::FT>(field, coder, out);
\
+ return Status::OK();
\
}
switch (ft) {
DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES(CASE)
diff --git a/be/src/storage/index/inverted/inverted_index_reader.cpp
b/be/src/storage/index/inverted/inverted_index_reader.cpp
index 84e2f110b66..c244872f140 100644
--- a/be/src/storage/index/inverted/inverted_index_reader.cpp
+++ b/be/src/storage/index/inverted/inverted_index_reader.cpp
@@ -58,31 +58,36 @@
#include "storage/index/inverted/query/query_factory.h"
#include "storage/key_coder.h"
#include "storage/olap_common.h"
+#include "storage/storage_layout.h"
#include "storage/types.h"
#include "util/faststring.h"
namespace {
// Sentinel values are sourced from the compute-layer `type_limit<CppType>` and
-// then projected onto the storage-layer POD via `PrimitiveTypeConvertor<PT>`.
+// then projected onto the storage-layer POD via
`StorageLayout<FT>::to_storage`.
// Routing through the compute layer keeps the +/- infinity constants
// single-sourced (e.g. DecimalV2 max lives only in
DecimalV2Value::get_max_decimal,
// DATE bounds only in VecDateTimeValue::datetime_min/max_value), so types like
// decimal12_t and uint24_t — which have no std::numeric_limits specialisation
—
// no longer need their own type_limit<> entries.
-template <doris::PrimitiveType PT>
+template <doris::FieldType FT>
static void bkd_encode_min(const doris::KeyCoder* coder, std::string* out) {
- using compute_t = typename doris::PrimitiveTypeTraits<PT>::CppType;
+ using compute_t =
+ typename
doris::PrimitiveTypeTraits<doris::storage_field_type_to_primitive_type(
+ FT)>::CppType;
auto compute_v = doris::type_limit<compute_t>::min();
- auto v =
doris::PrimitiveTypeConvertor<PT>::to_storage_field_type(compute_v);
+ auto v = doris::StorageLayout<FT>::to_storage(compute_v);
coder->full_encode_ascending(&v, out);
}
-template <doris::PrimitiveType PT>
+template <doris::FieldType FT>
static void bkd_encode_max(const doris::KeyCoder* coder, std::string* out) {
- using compute_t = typename doris::PrimitiveTypeTraits<PT>::CppType;
+ using compute_t =
+ typename
doris::PrimitiveTypeTraits<doris::storage_field_type_to_primitive_type(
+ FT)>::CppType;
auto compute_v = doris::type_limit<compute_t>::max();
- auto v =
doris::PrimitiveTypeConvertor<PT>::to_storage_field_type(compute_v);
+ auto v = doris::StorageLayout<FT>::to_storage(compute_v);
coder->full_encode_ascending(&v, out);
}
@@ -94,9 +99,9 @@ using doris::encode_bkd_field_ascending;
static doris::Status encode_bkd_min_ascending(doris::FieldType ft, const
doris::KeyCoder* coder,
std::string* out) {
-#define CASE(FT, PT) \
- case doris::FieldType::FT: \
- bkd_encode_min<doris::PrimitiveType::PT>(coder, out); \
+#define CASE(FT) \
+ case doris::FieldType::FT: \
+ bkd_encode_min<doris::FieldType::FT>(coder, out); \
return doris::Status::OK();
switch (ft) {
DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES(CASE)
@@ -109,9 +114,9 @@ static doris::Status
encode_bkd_min_ascending(doris::FieldType ft, const doris::
static doris::Status encode_bkd_max_ascending(doris::FieldType ft, const
doris::KeyCoder* coder,
std::string* out) {
-#define CASE(FT, PT) \
- case doris::FieldType::FT: \
- bkd_encode_max<doris::PrimitiveType::PT>(coder, out); \
+#define CASE(FT) \
+ case doris::FieldType::FT: \
+ bkd_encode_max<doris::FieldType::FT>(coder, out); \
return doris::Status::OK();
switch (ft) {
DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES(CASE)
diff --git a/be/src/storage/key_coder.h b/be/src/storage/key_coder.h
index f8d8a1e009b..8076e940174 100644
--- a/be/src/storage/key_coder.h
+++ b/be/src/storage/key_coder.h
@@ -30,12 +30,14 @@
#include "absl/strings/substitute.h"
#include "common/status.h"
#include "core/data_type/primitive_type.h"
+#include "core/data_type/storage_field_type.h"
#include "core/decimal12.h"
#include "core/extended_types.h"
#include "core/field.h"
#include "core/types.h"
#include "exec/common/endian.h"
#include "storage/olap_common.h"
+#include "storage/storage_layout.h"
#include "storage/types.h"
#include "util/slice.h"
@@ -406,16 +408,9 @@ public:
private:
static UnsignedCppType float_to_int_bits(CppType value) {
- if (std::isnan(value)) {
- if constexpr (std::is_same_v<CppType, float>) {
- return 0x7FC00000U;
- } else {
- return 0x7FF8000000000000ULL;
- }
- }
-
+ const CppType canonical =
StorageLayout<field_type>::canonicalize_nan(value);
UnsignedCppType result;
- std::memcpy(&result, &value, sizeof(CppType));
+ std::memcpy(&result, &canonical, sizeof(CppType));
return result;
}
@@ -448,43 +443,43 @@ template <>
class KeyCoderTraits<FieldType::OLAP_FIELD_TYPE_DOUBLE>
: public KeyCoderTraitsForFloat<FieldType::OLAP_FIELD_TYPE_DOUBLE> {};
-// X-macro listing every (FieldType, PrimitiveType) pair that goes through
KeyCoder
-// as a non-string scalar key. Strings are handled separately because they need
-// length / padding logic outside KeyCoder. Each entry: M(FT_suffix,
PT_suffix).
+// Every FieldType that goes through KeyCoder as a non-string scalar key.
+// Strings are handled separately because they need length / padding logic
+// outside KeyCoder.
#define DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES(M) \
- M(OLAP_FIELD_TYPE_BOOL, TYPE_BOOLEAN) \
- M(OLAP_FIELD_TYPE_TINYINT, TYPE_TINYINT) \
- M(OLAP_FIELD_TYPE_SMALLINT, TYPE_SMALLINT) \
- M(OLAP_FIELD_TYPE_INT, TYPE_INT) \
- M(OLAP_FIELD_TYPE_BIGINT, TYPE_BIGINT) \
- M(OLAP_FIELD_TYPE_LARGEINT, TYPE_LARGEINT) \
- M(OLAP_FIELD_TYPE_FLOAT, TYPE_FLOAT) \
- M(OLAP_FIELD_TYPE_DOUBLE, TYPE_DOUBLE) \
- M(OLAP_FIELD_TYPE_DECIMAL, TYPE_DECIMALV2) \
- M(OLAP_FIELD_TYPE_DECIMAL32, TYPE_DECIMAL32) \
- M(OLAP_FIELD_TYPE_DECIMAL64, TYPE_DECIMAL64) \
- M(OLAP_FIELD_TYPE_DECIMAL128I, TYPE_DECIMAL128I) \
- M(OLAP_FIELD_TYPE_DECIMAL256, TYPE_DECIMAL256) \
- M(OLAP_FIELD_TYPE_DATE, TYPE_DATE) \
- M(OLAP_FIELD_TYPE_DATETIME, TYPE_DATETIME) \
- M(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2) \
- M(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2) \
- M(OLAP_FIELD_TYPE_TIMESTAMP_NS, TYPE_TIMESTAMP_NS) \
- M(OLAP_FIELD_TYPE_TIMESTAMPTZ, TYPE_TIMESTAMPTZ) \
- M(OLAP_FIELD_TYPE_IPV4, TYPE_IPV4) \
- M(OLAP_FIELD_TYPE_IPV6, TYPE_IPV6) \
- M(OLAP_FIELD_TYPE_UUID, TYPE_UUID)
-
-// True for exactly the PrimitiveTypes listed in
+ M(OLAP_FIELD_TYPE_BOOL) \
+ M(OLAP_FIELD_TYPE_TINYINT) \
+ M(OLAP_FIELD_TYPE_SMALLINT) \
+ M(OLAP_FIELD_TYPE_INT) \
+ M(OLAP_FIELD_TYPE_BIGINT) \
+ M(OLAP_FIELD_TYPE_LARGEINT) \
+ M(OLAP_FIELD_TYPE_FLOAT) \
+ M(OLAP_FIELD_TYPE_DOUBLE) \
+ M(OLAP_FIELD_TYPE_DECIMAL) \
+ M(OLAP_FIELD_TYPE_DECIMAL32) \
+ M(OLAP_FIELD_TYPE_DECIMAL64) \
+ M(OLAP_FIELD_TYPE_DECIMAL128I) \
+ M(OLAP_FIELD_TYPE_DECIMAL256) \
+ M(OLAP_FIELD_TYPE_DATE) \
+ M(OLAP_FIELD_TYPE_DATETIME) \
+ M(OLAP_FIELD_TYPE_DATEV2) \
+ M(OLAP_FIELD_TYPE_DATETIMEV2) \
+ M(OLAP_FIELD_TYPE_TIMESTAMP_NS) \
+ M(OLAP_FIELD_TYPE_TIMESTAMPTZ) \
+ M(OLAP_FIELD_TYPE_IPV4) \
+ M(OLAP_FIELD_TYPE_IPV6) \
+ M(OLAP_FIELD_TYPE_UUID)
+
+// True for exactly the FieldTypes listed in
// DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES. Strings
(CHAR/VARCHAR/STRING/
// VARBINARY) have their own short-key code path in row_cursor.cpp that calls
// storage_field->full_encode_ascending directly, and nested/aggregate types
// (ARRAY/MAP/STRUCT/VARIANT/HLL/BITMAP/JSONB/QUANTILE_STATE/AGG_STATE) are not
// key-encodable at all -- both groups must never reach the helpers below.
-constexpr bool is_key_encodable_non_string_type(PrimitiveType pt) {
- switch (pt) {
-#define DORIS_KEY_ENCODABLE_CASE(FT, PT) \
- case PrimitiveType::PT: \
+constexpr bool is_key_encodable_non_string_type(FieldType ft) {
+ switch (ft) {
+#define DORIS_KEY_ENCODABLE_CASE(FT) \
+ case FieldType::FT: \
return true;
DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES(DORIS_KEY_ENCODABLE_CASE)
#undef DORIS_KEY_ENCODABLE_CASE
@@ -493,16 +488,16 @@ constexpr bool
is_key_encodable_non_string_type(PrimitiveType pt) {
}
}
-// Convert a Field value to its storage representation (via
PrimitiveTypeConvertor)
+// Convert a Field value to its StorageValue (via StorageLayout)
// and full-encode it as a byte-comparable ascending key via KeyCoder.
-template <PrimitiveType PT>
+template <FieldType FT>
inline void full_encode_field_as_key(const Field& f, const KeyCoder* coder,
std::string* buf) {
- static_assert(is_key_encodable_non_string_type(PT),
+ static_assert(is_key_encodable_non_string_type(FT),
"full_encode_field_as_key is for non-string scalar keys
only; "
"strings have their own path in RowCursor that calls "
"storage_field->full_encode_ascending directly, and nested /
"
"aggregate types are not key-encodable");
- auto v = PrimitiveTypeConvertor<PT>::to_storage_field_type(f.get<PT>());
+ const auto v =
StorageLayout<FT>::to_storage(f.get<storage_field_type_to_primitive_type(FT)>());
coder->full_encode_ascending(&v, buf);
}
diff --git a/be/src/storage/predicate/comparison_predicate.h
b/be/src/storage/predicate/comparison_predicate.h
index d420e550ff5..fb43fb6a119 100644
--- a/be/src/storage/predicate/comparison_predicate.h
+++ b/be/src/storage/predicate/comparison_predicate.h
@@ -29,6 +29,7 @@
#include "storage/index/inverted/inverted_index_cache.h" // IWYU pragma: keep
#include "storage/index/inverted/inverted_index_reader.h"
#include "storage/predicate/column_predicate.h"
+#include "storage/storage_layout.h"
namespace doris {
template <PrimitiveType Type, PredicateType PT>
@@ -290,24 +291,10 @@ public:
} else if constexpr (is_string_type(Type)) {
return bf->test_bytes(_value.data(), _value.size());
} else {
- // DecimalV2 using decimal12_t in bloom filter, should convert
value to decimal12_t
- if constexpr (Type == PrimitiveType::TYPE_DECIMALV2) {
- decimal12_t decimal12_t_val(_value.int_value(),
_value.frac_value());
- return bf->test_bytes(reinterpret_cast<const
char*>(&decimal12_t_val),
- sizeof(decimal12_t));
- // Datev1 using uint24_t in bloom filter
- } else if constexpr (Type == PrimitiveType::TYPE_DATE) {
- uint24_t date_value(uint32_t(_value.to_olap_date()));
- return bf->test_bytes(reinterpret_cast<const
char*>(&date_value),
- sizeof(uint24_t));
- // DatetimeV1 using int64_t in bloom filter
- } else if constexpr (Type == PrimitiveType::TYPE_DATETIME) {
- int64_t datetime_value(_value.to_olap_datetime());
- return bf->test_bytes(reinterpret_cast<const
char*>(&datetime_value),
- sizeof(int64_t));
- } else {
- return bf->test_bytes(reinterpret_cast<const
char*>(&_value), sizeof(T));
- }
+ const auto stored =
+
StorageLayout<primitive_type_to_storage_field_type(Type)>::to_storage(
+ _value);
+ return bf->test_bytes(reinterpret_cast<const char*>(&stored),
sizeof(stored));
}
} else {
LOG(FATAL) << "Bloom filter is not supported by predicate type.";
diff --git a/be/src/storage/predicate/in_list_predicate.h
b/be/src/storage/predicate/in_list_predicate.h
index a0de781415b..b1ee283c253 100644
--- a/be/src/storage/predicate/in_list_predicate.h
+++ b/be/src/storage/predicate/in_list_predicate.h
@@ -40,6 +40,7 @@
#include "storage/index/inverted/inverted_index_reader.h"
#include "storage/olap_common.h"
#include "storage/predicate/column_predicate.h"
+#include "storage/storage_layout.h"
// for uint24_t
template <>
@@ -358,34 +359,12 @@ public:
if (bf->test_bytes(value->data, value->size)) {
return true;
}
- } else if constexpr (Type == PrimitiveType::TYPE_DECIMALV2) {
- // DecimalV2 using decimal12_t in bloom filter in storage
layer,
- // should convert value to decimal12_t
- // Datev1/DatetimeV1 using VecDatetimeValue in bloom
filter, NO need to convert.
- const T* value = (const T*)(iter->get_value());
- decimal12_t decimal12_t_val(value->int_value(),
value->frac_value());
- if (bf->test_bytes(reinterpret_cast<const
char*>(&decimal12_t_val),
- sizeof(decimal12_t))) {
- return true;
- }
- } else if constexpr (Type == PrimitiveType::TYPE_DATE) {
- const T* value = (const T*)(iter->get_value());
- uint24_t date_value(uint32_t(value->to_olap_date()));
- if (bf->test_bytes(reinterpret_cast<const
char*>(&date_value),
- sizeof(uint24_t))) {
- return true;
- }
- // DatetimeV1 using int64_t in bloom filter
- } else if constexpr (Type == PrimitiveType::TYPE_DATETIME) {
- const T* value = (const T*)(iter->get_value());
- int64_t datetime_value(value->to_olap_datetime());
- if (bf->test_bytes(reinterpret_cast<const
char*>(&datetime_value),
- sizeof(int64_t))) {
- return true;
- }
} else {
const T* value = (const T*)(iter->get_value());
- if (bf->test_bytes(reinterpret_cast<const char*>(value),
sizeof(*value))) {
+ const auto stored =
+
StorageLayout<primitive_type_to_storage_field_type(Type)>::to_storage(
+ *value);
+ if (bf->test_bytes(reinterpret_cast<const char*>(&stored),
sizeof(stored))) {
return true;
}
}
diff --git a/be/src/storage/row_cursor.cpp b/be/src/storage/row_cursor.cpp
index b2cabb70fa1..4cf22dfac37 100644
--- a/be/src/storage/row_cursor.cpp
+++ b/be/src/storage/row_cursor.cpp
@@ -147,9 +147,9 @@ void RowCursor::_encode_column_value(const TabletColumn*
column, const Field& va
// ignores `index_size` and delegates to full_encode_ascending, so the
// `full_encode` flag here is a no-op and we always call the full helper.
switch (ft) {
-#define CASE(FT, PT) \
- case FieldType::FT: \
- full_encode_field_as_key<PrimitiveType::PT>(value, coder, buf); \
+#define CASE(FT) \
+ case FieldType::FT: \
+ full_encode_field_as_key<FieldType::FT>(value, coder, buf); \
break;
DORIS_APPLY_FOR_KEY_ENCODABLE_NON_STRING_TYPES(CASE)
#undef CASE
diff --git a/be/src/storage/row_cursor.h b/be/src/storage/row_cursor.h
index 94d4eec63cc..3b505434408 100644
--- a/be/src/storage/row_cursor.h
+++ b/be/src/storage/row_cursor.h
@@ -68,8 +68,8 @@ public:
std::string to_string() const;
// Encode one row into binary according given num_keys.
- // Internally converts each core::Field to its storage representation via
- // PrimitiveTypeConvertor before passing to KeyCoder.
+ // Internally converts each core::Field to its StorageValue via
StorageLayout
+ // before passing to KeyCoder.
// CHAR fields are zero-padded to column.length() for encoding.
template <bool is_mow = false>
void encode_key_with_padding(std::string* buf, size_t num_keys, bool
padding_minimal) const;
diff --git a/be/src/storage/storage_layout.h b/be/src/storage/storage_layout.h
new file mode 100644
index 00000000000..6034529a71e
--- /dev/null
+++ b/be/src/storage/storage_layout.h
@@ -0,0 +1,159 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <limits>
+#include <type_traits>
+
+#include "common/cast_set.h"
+#include "core/data_type/primitive_type.h"
+#include "core/data_type/storage_field_type.h"
+#include "core/decimal12.h"
+#include "core/uint24.h"
+#include "core/value/decimalv2_value.h"
+#include "core/value/vdatetime_value.h"
+#include "storage/field_type.h"
+#include "storage/types.h"
+
+namespace doris {
+
+// How a compute-layer value becomes what a segment stores for FieldType FT:
its own
+// bytes, except for the V1 DATE, DATETIME and DECIMAL layouts and NaN, stored
as the
+// quiet NaN. The two types are PrimitiveTypeTraits<PT>::CppType and
+// CppTypeTraits<FT>::CppType; the aliases inside are private shorthand.
Declared only,
+// so a FieldType without a specialisation is a compile error, not a silent
byte copy.
+template <FieldType FT>
+struct StorageLayout;
+
+namespace storage_layout_detail {
+
+// A StorageValue is the PrimitiveValue's own bytes.
+template <FieldType FT>
+struct BitCast {
+private:
+ using StorageValue = typename CppTypeTraits<FT>::CppType;
+ using PrimitiveValue =
+ typename
PrimitiveTypeTraits<storage_field_type_to_primitive_type(FT)>::CppType;
+ static_assert(sizeof(StorageValue) == sizeof(PrimitiveValue));
+
+public:
+ static StorageValue to_storage(const PrimitiveValue& value) {
+ return std::bit_cast<StorageValue>(value);
+ }
+};
+
+// Every NaN is stored as the quiet NaN, so equal values have equal stored
bytes
+// and the bloom filter / dictionary / key encodings that hash or compare bytes
+// treat all NaNs as one value.
+template <FieldType FT>
+struct FloatingPoint {
+private:
+ using StorageValue = typename CppTypeTraits<FT>::CppType;
+ using PrimitiveValue =
+ typename
PrimitiveTypeTraits<storage_field_type_to_primitive_type(FT)>::CppType;
+ static_assert(std::is_floating_point_v<StorageValue>);
+ static_assert(std::is_same_v<StorageValue, PrimitiveValue>);
+
+public:
+ static StorageValue canonicalize_nan(StorageValue value) {
+ return std::isnan(value) ?
std::numeric_limits<StorageValue>::quiet_NaN() : value;
+ }
+ static StorageValue to_storage(PrimitiveValue value) { return
canonicalize_nan(value); }
+};
+
+} // namespace storage_layout_detail
+
+#define DORIS_STORAGE_LAYOUT_BIT_CAST(FT) \
+ template <> \
+ struct StorageLayout<FieldType::FT> :
storage_layout_detail::BitCast<FieldType::FT> {};
+
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_BOOL)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_TINYINT)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_SMALLINT)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_INT)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_BIGINT)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_LARGEINT)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_UNSIGNED_INT)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_UNSIGNED_BIGINT)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_DECIMAL32)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_DECIMAL64)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_DECIMAL128I)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_DECIMAL256)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_DATEV2)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_DATETIMEV2)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_TIMESTAMP_NS)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_TIMESTAMPTZ)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_IPV4)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_IPV6)
+DORIS_STORAGE_LAYOUT_BIT_CAST(OLAP_FIELD_TYPE_UUID)
+#undef DORIS_STORAGE_LAYOUT_BIT_CAST
+
+template <>
+struct StorageLayout<FieldType::OLAP_FIELD_TYPE_FLOAT>
+ :
storage_layout_detail::FloatingPoint<FieldType::OLAP_FIELD_TYPE_FLOAT> {};
+
+template <>
+struct StorageLayout<FieldType::OLAP_FIELD_TYPE_DOUBLE>
+ :
storage_layout_detail::FloatingPoint<FieldType::OLAP_FIELD_TYPE_DOUBLE> {};
+
+// V1 DATE: three bytes of (year << 9) | (month << 5) | day.
+template <>
+struct StorageLayout<FieldType::OLAP_FIELD_TYPE_DATE> {
+private:
+ using StorageValue = typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATE>::CppType;
+ using PrimitiveValue = typename PrimitiveTypeTraits<TYPE_DATE>::CppType;
+ static_assert(std::is_same_v<StorageValue, uint24_t>);
+
+public:
+ static StorageValue to_storage(const PrimitiveValue& value) {
+ return StorageValue(cast_set<uint32_t>(value.to_olap_date()));
+ }
+};
+
+// V1 DATETIME: the decimal digits YYYYMMDDhhmmss packed into one integer.
+template <>
+struct StorageLayout<FieldType::OLAP_FIELD_TYPE_DATETIME> {
+private:
+ using StorageValue = typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIME>::CppType;
+ using PrimitiveValue = typename
PrimitiveTypeTraits<TYPE_DATETIME>::CppType;
+ static_assert(std::is_same_v<StorageValue, int64_t>);
+
+public:
+ static StorageValue to_storage(const PrimitiveValue& value) {
+ return static_cast<StorageValue>(value.to_olap_datetime());
+ }
+};
+
+// V1 DECIMAL: the integer and the nine-digit fraction stored side by side.
+template <>
+struct StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL> {
+private:
+ using StorageValue = typename
CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DECIMAL>::CppType;
+ using PrimitiveValue = typename
PrimitiveTypeTraits<TYPE_DECIMALV2>::CppType;
+ static_assert(std::is_same_v<StorageValue, decimal12_t>);
+
+public:
+ static StorageValue to_storage(const PrimitiveValue& value) {
+ return {value.int_value(), value.frac_value()};
+ }
+};
+
+} // namespace doris
diff --git a/be/test/storage/key_coder_test.cpp
b/be/test/storage/key_coder_test.cpp
index 07d3f5021eb..ee14881ca42 100644
--- a/be/test/storage/key_coder_test.cpp
+++ b/be/test/storage/key_coder_test.cpp
@@ -285,32 +285,34 @@ TEST_F(KeyCoderTest, test_decimal) {
}
// Encode an ascending sequence of compute-layer values via
-// full_encode_field_as_key<PT> -- the same helper RowCursor::encode_key and
+// full_encode_field_as_key<FT> -- the same helper RowCursor::encode_key and
// the BKD inverted-index reader use -- and assert the byte order matches.
//
-// Locks in the contract that PrimitiveTypeConvertor<PT> + KeyCoder together
-// preserve the compute-layer ordering for every (PrimitiveType, FieldType)
-// pair used as a sortable key.
+// Locks in the contract that StorageLayout<FT>::to_storage + KeyCoder together
+// preserve the compute-layer ordering for every FieldType used as a sortable
+// key.
//
// Why no `scale` or `frac` parameter? The contract under test is exactly
// "the encode path does not read scale/frac". They live on TabletColumn
// metadata (`_precision`, `_frac`) one layer above and never reach
-// `Field::create_field<PT>`, `PrimitiveTypeConvertor<PT>`, or `KeyCoder`.
+// `Field::create_field<PT>`, `StorageLayout<FT>`, or `KeyCoder`.
// So each subgroup below picks raw ints that *would* arise from a column at
// some hypothetical scale, labels the subgroup `scale=N` to make the human
// interpretation explicit, and asserts ordering -- the encode result is by
// construction identical regardless of which scale the column declared.
-template <PrimitiveType PT, typename ComputeT>
-static void check_full_encode_preserves_order(FieldType ft, const
std::vector<ComputeT>& ascending,
- const char* label) {
- const KeyCoder* coder = get_key_coder(ft);
+template <FieldType FT>
+static void check_full_encode_preserves_order(
+ const std::vector<typename PrimitiveTypeTraits<
+ storage_field_type_to_primitive_type(FT)>::CppType>& ascending,
+ const char* label) {
+ const KeyCoder* coder = get_key_coder(FT);
ASSERT_NE(coder, nullptr) << label;
std::vector<std::string> encoded;
encoded.reserve(ascending.size());
for (const auto& v : ascending) {
- Field f = Field::create_field<PT>(v);
+ Field f =
Field::create_field<storage_field_type_to_primitive_type(FT)>(v);
std::string buf;
- full_encode_field_as_key<PT>(f, coder, &buf);
+ full_encode_field_as_key<FT>(f, coder, &buf);
encoded.push_back(std::move(buf));
}
for (size_t i = 0; i + 1 < encoded.size(); ++i) {
@@ -347,8 +349,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// -------- DECIMAL32 (compute=Decimal32, storage=Int32) --------
// scale=0 (DECIMAL(9,0)): whole int32 range
- check_full_encode_preserves_order<TYPE_DECIMAL32, Decimal32>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL32,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL32>(
{Decimal32(std::numeric_limits<int32_t>::min()), // INT32_MIN
Decimal32(int32_t(-99999)), // -99999
Decimal32(int32_t(-1)), // -1
@@ -360,8 +361,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// scale=2 (DECIMAL(9,2)): raw = whole * 100 + frac
{
const int32_t s = pow10_i32(2);
- check_full_encode_preserves_order<TYPE_DECIMAL32, Decimal32>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL32,
+
check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL32>(
{Decimal32(-123456 * s), // -123456.00
Decimal32(-12 * s - 34), // -12.34
Decimal32(-1), // -0.01
@@ -374,8 +374,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// scale=9 (DECIMAL(9,9)): -0.999999999 .. +0.999999999 (whole always 0)
{
const int32_t s = pow10_i32(9);
- check_full_encode_preserves_order<TYPE_DECIMAL32, Decimal32>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL32,
+
check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL32>(
{Decimal32(-(s - 1)), // -0.999999999
Decimal32(-1), // -0.000000001
Decimal32(0), // 0
@@ -386,8 +385,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// -------- DECIMAL64 (compute=Decimal64, storage=Int64) --------
// scale=0: whole int64 range
- check_full_encode_preserves_order<TYPE_DECIMAL64, Decimal64>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL64,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL64>(
{Decimal64(std::numeric_limits<int64_t>::min()), // INT64_MIN
Decimal64(int64_t(-1'000'000'000LL)), // -1 000 000
000
Decimal64(int64_t(-1)), Decimal64(int64_t(0)),
Decimal64(int64_t(1)),
@@ -397,8 +395,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// scale=4 (DECIMAL(18,4))
{
const int64_t s = pow10_i64(4);
- check_full_encode_preserves_order<TYPE_DECIMAL64, Decimal64>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL64,
+
check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL64>(
{Decimal64(-123456 * s - 7890), //
-123456.7890
Decimal64(-int64_t(1)), //
-0.0001
Decimal64(int64_t(0)), //
0.0000
@@ -409,8 +406,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// scale=18 (DECIMAL(18,18)): whole always 0, full fractional range
{
const int64_t s = pow10_i64(18);
- check_full_encode_preserves_order<TYPE_DECIMAL64, Decimal64>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL64,
+
check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL64>(
{Decimal64(-(s - 1)), // -0.999999999999999999
Decimal64(int64_t(-1)), Decimal64(int64_t(0)),
Decimal64(int64_t(1)),
Decimal64(s - 1)}, // 0.999999999999999999
@@ -419,8 +415,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// -------- DECIMAL128I (compute=Decimal128V3, storage=Int128) --------
// scale=0: span ±2^100 to exercise both halves of int128.
- check_full_encode_preserves_order<TYPE_DECIMAL128I, Decimal128V3>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL128I,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL128I>(
{Decimal128V3(-(static_cast<int128_t>(1) << 100)), // -2^100
Decimal128V3(int128_t(-1)), Decimal128V3(int128_t(0)),
Decimal128V3(int128_t(1)),
Decimal128V3(static_cast<int128_t>(1) << 100)}, // 2^100
@@ -428,8 +423,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// scale=10 (DECIMAL(38,10))
{
const int128_t s = pow10_i128(10);
- check_full_encode_preserves_order<TYPE_DECIMAL128I, Decimal128V3>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL128I,
+
check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL128I>(
{Decimal128V3(-int128_t(100) * s), // -100.0000000000
Decimal128V3(int128_t(-1)), // -0.0000000001
Decimal128V3(int128_t(0)),
@@ -442,8 +436,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// -------- DECIMAL256 (compute=Decimal256, storage=wide::Int256) --------
// scale=0: span beyond int128 to exercise the upper halves of int256.
- check_full_encode_preserves_order<TYPE_DECIMAL256, Decimal256>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL256,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL256>(
{Decimal256(wide::Int256(-1'000'000'000'000LL)), // -10^12
Decimal256(wide::Int256(-1)), Decimal256(wide::Int256(0)),
Decimal256(wide::Int256(1)),
Decimal256(wide::Int256(1'000'000'000'000'000'000LL))}, // 10^18
@@ -452,8 +445,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
{
const wide::Int256 s = pow10_i256(20);
const wide::Int256 big = pow10_i256(36); // far beyond int128
- check_full_encode_preserves_order<TYPE_DECIMAL256, Decimal256>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL256,
+
check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL256>(
{Decimal256(-big), // -10^36
Decimal256(-s), // -1.0
Decimal256(wide::Int256(-1)), // -1e-20
@@ -469,8 +461,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// splits the int128 into {int_part, frac_part} where frac is the lower
// 9 digits (DecimalV2Value's `frac_value()` * 10^-9). Caller passes the
// two parts with matching sign.
- check_full_encode_preserves_order<TYPE_DECIMALV2, DecimalV2Value>(
- FieldType::OLAP_FIELD_TYPE_DECIMAL,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DECIMAL>(
{DecimalV2Value::get_min_decimal(), //
-999999999999999999.999999999
DecimalV2Value(int64_t(-100), int64_t(0)), //
-100.000000000
DecimalV2Value(int64_t(-1), int64_t(-500'000'000)), //
-1.500000000
@@ -482,8 +473,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// -------- DATEV2 (compute=DateV2Value<DateV2ValueType>, storage=uint32)
--------
auto pack_d = [](int y, int m, int d) -> uint32_t { return uint32_t((y <<
9) | (m << 5) | d); };
- check_full_encode_preserves_order<TYPE_DATEV2,
DateV2Value<DateV2ValueType>>(
- FieldType::OLAP_FIELD_TYPE_DATEV2,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DATEV2>(
{DateV2Value<DateV2ValueType>(pack_d(0001, 1, 1)),
DateV2Value<DateV2ValueType>(pack_d(1970, 1, 1)),
DateV2Value<DateV2ValueType>(pack_d(2024, 12, 31)),
@@ -503,14 +493,12 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
};
using DTV2 = DateV2Value<DateTimeV2ValueType>;
// scale=0: microsecond always zero
- check_full_encode_preserves_order<TYPE_DATETIMEV2, DTV2>(
- FieldType::OLAP_FIELD_TYPE_DATETIMEV2,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>(
{DTV2(pack_dt(2020, 1, 1, 12, 0, 0)), DTV2(pack_dt(2024, 3, 10, 9,
30, 0)),
DTV2(pack_dt(2024, 12, 31, 23, 59, 59))},
"DATETIMEV2 scale=0");
// scale=3: microsecond multiples of 1000
- check_full_encode_preserves_order<TYPE_DATETIMEV2, DTV2>(
- FieldType::OLAP_FIELD_TYPE_DATETIMEV2,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>(
{DTV2(pack_dt(2024, 3, 10, 9, 30, 0, 1'000)),
DTV2(pack_dt(2024, 3, 10, 9, 30, 0, 123'000)),
DTV2(pack_dt(2024, 3, 10, 9, 30, 0, 999'000)),
@@ -518,8 +506,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
"DATETIMEV2 scale=3");
// scale=6: full microsecond resolution; verifies ordering is byte-stable
// even at the boundary between us=999999 and the next-second carry.
- check_full_encode_preserves_order<TYPE_DATETIMEV2, DTV2>(
- FieldType::OLAP_FIELD_TYPE_DATETIMEV2,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>(
{DTV2(pack_dt(2024, 3, 10, 9, 30, 0, 0)), DTV2(pack_dt(2024, 3,
10, 9, 30, 0, 1)),
DTV2(pack_dt(2024, 3, 10, 9, 30, 0, 999'998)),
DTV2(pack_dt(2024, 3, 10, 9, 30, 0, 999'999)),
@@ -527,40 +514,34 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
"DATETIMEV2 scale=6");
// -------- TIMESTAMPTZ (same packing as DATETIMEV2) --------
- check_full_encode_preserves_order<TYPE_TIMESTAMPTZ, TimestampTzValue>(
- FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>(
{TimestampTzValue(pack_dt(2020, 1, 1, 12, 0, 0)),
TimestampTzValue(pack_dt(2024, 3, 10, 9, 30, 0, 123'456)),
TimestampTzValue(pack_dt(2024, 12, 31, 23, 59, 59, 999'999))},
"TIMESTAMPTZ");
// -------- BOOLEAN (compute=storage=UInt8, only {0, 1}) --------
- check_full_encode_preserves_order<TYPE_BOOLEAN,
UInt8>(FieldType::OLAP_FIELD_TYPE_BOOL,
- {UInt8(0),
UInt8(1)}, "BOOLEAN");
+
check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_BOOL>({UInt8(0),
UInt8(1)},
+
"BOOLEAN");
// -------- Plain integer keys (compute=storage) --------
- check_full_encode_preserves_order<TYPE_TINYINT, int8_t>(
- FieldType::OLAP_FIELD_TYPE_TINYINT,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_TINYINT>(
{std::numeric_limits<int8_t>::min(), int8_t(-1), int8_t(0),
int8_t(1),
std::numeric_limits<int8_t>::max()},
"TINYINT");
- check_full_encode_preserves_order<TYPE_SMALLINT, int16_t>(
- FieldType::OLAP_FIELD_TYPE_SMALLINT,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_SMALLINT>(
{std::numeric_limits<int16_t>::min(), int16_t(-1), int16_t(0),
int16_t(1),
std::numeric_limits<int16_t>::max()},
"SMALLINT");
- check_full_encode_preserves_order<TYPE_INT, int32_t>(
- FieldType::OLAP_FIELD_TYPE_INT,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_INT>(
{std::numeric_limits<int32_t>::min(), int32_t(-1), int32_t(0),
int32_t(1),
std::numeric_limits<int32_t>::max()},
"INT");
- check_full_encode_preserves_order<TYPE_BIGINT, int64_t>(
- FieldType::OLAP_FIELD_TYPE_BIGINT,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_BIGINT>(
{std::numeric_limits<int64_t>::min(), int64_t(-1), int64_t(0),
int64_t(1),
std::numeric_limits<int64_t>::max()},
"BIGINT");
- check_full_encode_preserves_order<TYPE_LARGEINT, int128_t>(
- FieldType::OLAP_FIELD_TYPE_LARGEINT,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_LARGEINT>(
{-(static_cast<int128_t>(1) << 100), int128_t(-1), int128_t(0),
int128_t(1),
static_cast<int128_t>(1) << 100},
"LARGEINT");
@@ -569,23 +550,20 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// KeyCoderTraitsForFloat byte-encodes finite values byte-comparably; NaN
and
// signed-zero ambiguity have their own dedicated tests above
(FloatOrdering /
// FloatComprehensiveOrdering), so here we just exercise the typical path.
- check_full_encode_preserves_order<TYPE_FLOAT, float>(
- FieldType::OLAP_FIELD_TYPE_FLOAT,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_FLOAT>(
{-std::numeric_limits<float>::infinity(), -1e10f, -1.0f, -1e-10f,
0.0f, 1e-10f, 1.0f,
1e10f, std::numeric_limits<float>::infinity()},
"FLOAT");
- check_full_encode_preserves_order<TYPE_DOUBLE, double>(
- FieldType::OLAP_FIELD_TYPE_DOUBLE,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DOUBLE>(
{-std::numeric_limits<double>::infinity(), -1e100, -1.0, -1e-100,
0.0, 1e-100, 1.0,
1e100, std::numeric_limits<double>::infinity()},
"DOUBLE");
// -------- DATE V1 (compute=VecDateTimeValue, storage=uint24_t packed)
--------
// Storage = (year << 9) | (month << 5) | day, same packing as DATEV2 just
- // narrower. The PrimitiveTypeConvertor specialisation runs to_olap_date()
+ // narrower. StorageLayout<DATE>::to_storage runs to_olap_date()
// to project compute -> storage.
- check_full_encode_preserves_order<TYPE_DATE, VecDateTimeValue>(
- FieldType::OLAP_FIELD_TYPE_DATE,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DATE>(
{VecDateTimeValue::create_from_olap_date(pack_d(1900, 1, 1)),
VecDateTimeValue::create_from_olap_date(pack_d(1970, 1, 1)),
VecDateTimeValue::create_from_olap_date(pack_d(2024, 12, 31)),
@@ -595,8 +573,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
// -------- DATETIME V1 (compute=VecDateTimeValue, storage=int64
decimal-packed) --------
// Storage = YYYYMMDDhhmmss interpreted as int64 (sparse compared to V2 bit
// packing but still chronologically ordered as integers).
- check_full_encode_preserves_order<TYPE_DATETIME, VecDateTimeValue>(
- FieldType::OLAP_FIELD_TYPE_DATETIME,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_DATETIME>(
{VecDateTimeValue::create_from_olap_datetime(uint64_t(19700101000000ULL)),
VecDateTimeValue::create_from_olap_datetime(uint64_t(20240310093000ULL)),
VecDateTimeValue::create_from_olap_datetime(uint64_t(20241231235959ULL)),
@@ -607,8 +584,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
auto ip4 = [](uint8_t a, uint8_t b, uint8_t c, uint8_t d) -> uint32_t {
return (uint32_t(a) << 24) | (uint32_t(b) << 16) | (uint32_t(c) << 8)
| uint32_t(d);
};
- check_full_encode_preserves_order<TYPE_IPV4, uint32_t>(
- FieldType::OLAP_FIELD_TYPE_IPV4,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_IPV4>(
{ip4(0, 0, 0, 1), // 0.0.0.1
ip4(10, 0, 0, 1), // 10.0.0.1
ip4(127, 0, 0, 1), // 127.0.0.1
@@ -620,8 +596,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
auto ip6 = [](uint64_t hi, uint64_t lo) -> uint128_t {
return (static_cast<uint128_t>(hi) << 64) | lo;
};
- check_full_encode_preserves_order<TYPE_IPV6, uint128_t>(
- FieldType::OLAP_FIELD_TYPE_IPV6,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_IPV6>(
{ip6(0, 1), // ::1
ip6(0, 0x0000FFFF7F000001ULL), // ::ffff:127.0.0.1
ip6(0x20010DB800000000ULL, 1), // 2001:db8::1
@@ -631,8 +606,7 @@ TEST_F(KeyCoderTest,
full_encode_field_as_key_preserves_compute_layer_ordering)
"IPV6");
// -------- UUID (uint128_t, canonical unsigned order) --------
- check_full_encode_preserves_order<TYPE_UUID, uint128_t>(
- FieldType::OLAP_FIELD_TYPE_UUID,
+ check_full_encode_preserves_order<FieldType::OLAP_FIELD_TYPE_UUID>(
{ip6(0, 0), ip6(0, 1), ip6(0, 0xFFFFFFFFFFFFFFFFULL), ip6(1, 0),
ip6(0x8000000000000000ULL, 0), ip6(0xFFFFFFFFFFFFFFFFULL,
0xFFFFFFFFFFFFFFFFULL)},
"UUID");
diff --git a/be/test/storage/segment/inverted_index_reader_test.cpp
b/be/test/storage/segment/inverted_index_reader_test.cpp
index a2ca5d29ac9..d53277f43ac 100644
--- a/be/test/storage/segment/inverted_index_reader_test.cpp
+++ b/be/test/storage/segment/inverted_index_reader_test.cpp
@@ -3312,7 +3312,7 @@ public:
// Locks in:
// * the typed-param interface (TypedInvertedIndexQueryParam<PT>)
// * the +/-infinity sentinels routed through type_limit<compute_t> +
- // PrimitiveTypeConvertor<PT>
+ // StorageLayout<FT>::to_storage
// * BKD's writer/reader/visitor agreement on KeyCoder-encoded bytes
template <PrimitiveType PT, typename T>
void verify_bkd_range_queries(int col_id, std::string_view rowset_id,
@@ -4398,7 +4398,7 @@ TEST_F(InvertedIndexReaderTest, BkdIndexRead) {
// cardinalities derived from the values via std::count_if.
//
// Locks in the typed-param interface, the +/-infinity sentinels routed
-// through type_limit<compute_t> + PrimitiveTypeConvertor<PT>, and BKD
+// through type_limit<compute_t> + StorageLayout<FT>::to_storage, and BKD
// writer/reader/visitor agreement.
TEST_F(InvertedIndexReaderTest, BkdRangeIntRangeQuery) {
test_bkd_range_int();
diff --git a/be/test/storage/segment/test_segment_writer.h
b/be/test/storage/segment/test_segment_writer.h
index f1f398f8581..9448205681a 100644
--- a/be/test/storage/segment/test_segment_writer.h
+++ b/be/test/storage/segment/test_segment_writer.h
@@ -29,6 +29,7 @@
#include "storage/row_cursor.h"
#include "storage/segment/column_writer.h"
#include "storage/segment/vertical_segment_writer.h"
+#include "storage/storage_layout.h"
#include "util/slice.h"
namespace doris::segment_v2 {
@@ -73,7 +74,7 @@ public:
break;
}
case FieldType::OLAP_FIELD_TYPE_DECIMAL: {
- auto v =
PrimitiveTypeConvertor<TYPE_DECIMALV2>::to_storage_field_type(
+ auto v =
StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL>::to_storage(
f.get<TYPE_DECIMALV2>());
memcpy(buf, &v, sizeof(v));
ptr = buf;
diff --git a/be/test/storage/storage_layout_test.cpp
b/be/test/storage/storage_layout_test.cpp
new file mode 100644
index 00000000000..35936aeb032
--- /dev/null
+++ b/be/test/storage/storage_layout_test.cpp
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/storage_layout.h"
+
+#include <gtest/gtest.h>
+
+#include <bit>
+#include <cstdint>
+#include <limits>
+
+#include "core/types.h"
+#include "core/value/decimalv2_value.h"
+#include "core/value/vdatetime_value.h"
+#include "storage/types.h"
+
+namespace doris {
+
+TEST(StorageLayoutTest, StorageValueSizeIsFieldTypeSize) {
+#define CHECK_SIZE(FT) \
+ EXPECT_EQ(sizeof(CppTypeTraits<FieldType::FT>::CppType),
field_type_size(FieldType::FT)) << #FT;
+ CHECK_SIZE(OLAP_FIELD_TYPE_BOOL)
+ CHECK_SIZE(OLAP_FIELD_TYPE_TINYINT)
+ CHECK_SIZE(OLAP_FIELD_TYPE_SMALLINT)
+ CHECK_SIZE(OLAP_FIELD_TYPE_INT)
+ CHECK_SIZE(OLAP_FIELD_TYPE_BIGINT)
+ CHECK_SIZE(OLAP_FIELD_TYPE_LARGEINT)
+ CHECK_SIZE(OLAP_FIELD_TYPE_UNSIGNED_INT)
+ CHECK_SIZE(OLAP_FIELD_TYPE_UNSIGNED_BIGINT)
+ CHECK_SIZE(OLAP_FIELD_TYPE_FLOAT)
+ CHECK_SIZE(OLAP_FIELD_TYPE_DOUBLE)
+ CHECK_SIZE(OLAP_FIELD_TYPE_DECIMAL)
+ CHECK_SIZE(OLAP_FIELD_TYPE_DECIMAL32)
+ CHECK_SIZE(OLAP_FIELD_TYPE_DECIMAL64)
+ CHECK_SIZE(OLAP_FIELD_TYPE_DECIMAL128I)
+ CHECK_SIZE(OLAP_FIELD_TYPE_DECIMAL256)
+ CHECK_SIZE(OLAP_FIELD_TYPE_DATE)
+ CHECK_SIZE(OLAP_FIELD_TYPE_DATETIME)
+ CHECK_SIZE(OLAP_FIELD_TYPE_DATEV2)
+ CHECK_SIZE(OLAP_FIELD_TYPE_DATETIMEV2)
+ CHECK_SIZE(OLAP_FIELD_TYPE_TIMESTAMP_NS)
+ CHECK_SIZE(OLAP_FIELD_TYPE_TIMESTAMPTZ)
+ CHECK_SIZE(OLAP_FIELD_TYPE_IPV4)
+ CHECK_SIZE(OLAP_FIELD_TYPE_IPV6)
+ CHECK_SIZE(OLAP_FIELD_TYPE_UUID)
+#undef CHECK_SIZE
+}
+
+TEST(StorageLayoutTest, DateV1PacksYearMonthDayIntoThreeBytes) {
+ VecDateTimeValue value;
+ ASSERT_TRUE(value.from_date_int64(20200101));
+ const uint24_t stored =
StorageLayout<FieldType::OLAP_FIELD_TYPE_DATE>::to_storage(value);
+ EXPECT_EQ(static_cast<uint32_t>(stored), (2020U << 9) | (1U << 5) | 1U);
+}
+
+TEST(StorageLayoutTest, DateTimeV1PacksDecimalDigits) {
+ VecDateTimeValue value;
+ ASSERT_TRUE(value.from_date_int64(20200101123045));
+
EXPECT_EQ(StorageLayout<FieldType::OLAP_FIELD_TYPE_DATETIME>::to_storage(value),
+ 20200101123045);
+}
+
+TEST(StorageLayoutTest, DecimalV1SplitsIntegerAndFraction) {
+ using Layout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL>;
+ decimal12_t stored = Layout::to_storage(DecimalV2Value(123, 456000000));
+ EXPECT_EQ(stored.integer, 123);
+ EXPECT_EQ(stored.fraction, 456000000);
+
+ stored = Layout::to_storage(DecimalV2Value(-123, -456000000));
+ EXPECT_EQ(stored.integer, -123);
+ EXPECT_EQ(stored.fraction, -456000000);
+}
+
+TEST(StorageLayoutTest, FloatingPointCanonicalisesEveryNaN) {
+ using FloatLayout = StorageLayout<FieldType::OLAP_FIELD_TYPE_FLOAT>;
+ using DoubleLayout = StorageLayout<FieldType::OLAP_FIELD_TYPE_DOUBLE>;
+
+ const auto payload_nan = std::bit_cast<float>(0x7FC12345U);
+
EXPECT_EQ(std::bit_cast<uint32_t>(FloatLayout::canonicalize_nan(payload_nan)),
0x7FC00000U);
+
EXPECT_EQ(std::bit_cast<uint32_t>(FloatLayout::canonicalize_nan(-payload_nan)),
0x7FC00000U);
+ EXPECT_EQ(std::bit_cast<uint32_t>(FloatLayout::canonicalize_nan(-0.0F)),
0x80000000U);
+ EXPECT_EQ(FloatLayout::canonicalize_nan(1.5F), 1.5F);
+ EXPECT_EQ(std::bit_cast<uint32_t>(FloatLayout::to_storage(-payload_nan)),
0x7FC00000U);
+
+ const auto payload_nan64 = std::bit_cast<double>(0xFFF8000000000001ULL);
+
EXPECT_EQ(std::bit_cast<uint64_t>(DoubleLayout::canonicalize_nan(payload_nan64)),
+ 0x7FF8000000000000ULL);
+ EXPECT_EQ(DoubleLayout::canonicalize_nan(-2.25), -2.25);
+
EXPECT_EQ(DoubleLayout::canonicalize_nan(std::numeric_limits<double>::infinity()),
+ std::numeric_limits<double>::infinity());
+ EXPECT_EQ(std::bit_cast<uint64_t>(DoubleLayout::to_storage(payload_nan64)),
+ 0x7FF8000000000000ULL);
+}
+
+TEST(StorageLayoutTest, BitCastStoresThePrimitiveBytes) {
+ DateV2Value<DateV2ValueType> date;
+ ASSERT_TRUE(date.from_date_int64(20200101));
+
EXPECT_EQ(StorageLayout<FieldType::OLAP_FIELD_TYPE_DATEV2>::to_storage(date),
+ date.to_date_int_val());
+
+
EXPECT_EQ(StorageLayout<FieldType::OLAP_FIELD_TYPE_DECIMAL32>::to_storage(Decimal32(-12345)),
+ -12345);
+}
+
+} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
