This is an automated email from the ASF dual-hosted git repository.
caiconghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new e285d09 [Enhancement](load) speed up stream load for duplicate table,
use template for faster get_type_info. (#8500)
e285d09 is described below
commit e285d091571251c69dc3f09334297454c25cbf7d
Author: zbtzbtzbt <[email protected]>
AuthorDate: Fri Mar 25 15:18:43 2022 +0800
[Enhancement](load) speed up stream load for duplicate table, use template
for faster get_type_info. (#8500)
---
be/src/olap/aggregate_func.h | 3 ++-
be/src/olap/memtable.cpp | 5 ++---
be/src/olap/olap_common.h | 9 ++++----
be/src/olap/rowset/segment_v2/column_reader.cpp | 2 +-
be/src/olap/types.cpp | 24 +++-----------------
be/src/olap/types.h | 29 ++++++++++++++++++++++---
6 files changed, 38 insertions(+), 34 deletions(-)
diff --git a/be/src/olap/aggregate_func.h b/be/src/olap/aggregate_func.h
index 15302dc..99fd284 100644
--- a/be/src/olap/aggregate_func.h
+++ b/be/src/olap/aggregate_func.h
@@ -102,7 +102,8 @@ struct BaseAggregateFuncs {
auto _type_info = get_collection_type_info(sub_type);
_type_info->deep_copy(dst->mutable_cell_ptr(), src, mem_pool);
} else {
- auto _type_info = get_type_info(field_type);
+ // get type at compile time for performance
+ auto _type_info = get_scalar_type_info<field_type>();
_type_info->deep_copy(dst->mutable_cell_ptr(), src, mem_pool);
}
}
diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index 8c56e38..2ab6ab1 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -108,9 +108,8 @@ void MemTable::_tuple_to_row(const Tuple* tuple,
ContiguousRow* row, MemPool* me
const SlotDescriptor* slot = (*_slot_descs)[i];
bool is_null = tuple->is_null(slot->null_indicator_offset());
- const void* value = tuple->get_slot(slot->tuple_offset());
- _schema->column(i)->consume(&cell, (const char*)value, is_null,
mem_pool,
- &_agg_buffer_pool);
+ const auto* value = (const char*)tuple->get_slot(slot->tuple_offset());
+ _schema->column(i)->consume(&cell, value, is_null, mem_pool,
&_agg_buffer_pool);
}
}
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 56c23fb..b06334c 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -55,10 +55,10 @@ struct DataDirInfo {
FilePathDesc path_desc;
size_t path_hash = 0;
int64_t disk_capacity = 1; // actual disk capacity
- int64_t available = 0; // 可用空间，单位字节
+ int64_t available = 0; // available space, in bytes unit
int64_t data_used_capacity = 0;
- bool is_used = false; // 是否可用标识
- TStorageMedium::type storage_medium = TStorageMedium::HDD; //
存储介质类型：SSD|HDD
+ bool is_used = false; // whether
available mark
+ TStorageMedium::type storage_medium = TStorageMedium::HDD; // Storage
medium type: SSD|HDD
};
// Sort DataDirInfo by available space.
@@ -114,8 +114,7 @@ enum DelCondSatisfied {
DEL_NOT_SATISFIED = 1, //not satisfy delete condition
DEL_PARTIAL_SATISFIED = 2, //partially satisfy delete condition
};
-
-// 定义Field支持的所有数据类型
+// Define all data types supported by Field.
enum FieldType {
OLAP_FIELD_TYPE_TINYINT = 1, // MYSQL_TYPE_TINY
OLAP_FIELD_TYPE_UNSIGNED_TINYINT = 2,
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 058bb79..e7d4ac4 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -393,7 +393,7 @@ Status ArrayFileColumnIterator::init(const
ColumnIteratorOptions& opts) {
if (_array_reader->is_nullable()) {
RETURN_IF_ERROR(_null_iterator->init(opts));
}
- auto offset_type_info =
get_scalar_type_info(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT);
+ auto offset_type_info = get_scalar_type_info(OLAP_FIELD_TYPE_UNSIGNED_INT);
RETURN_IF_ERROR(
ColumnVectorBatch::create(1024, false, offset_type_info, nullptr,
&_length_batch));
return Status::OK();
diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp
index d3f1f3b..4a1d327 100644
--- a/be/src/olap/types.cpp
+++ b/be/src/olap/types.cpp
@@ -23,24 +23,6 @@ namespace doris {
void (*FieldTypeTraits<OLAP_FIELD_TYPE_CHAR>::set_to_max)(void*) = nullptr;
-template <typename TypeTraitsClass>
-ScalarTypeInfo::ScalarTypeInfo(TypeTraitsClass t)
- : _equal(TypeTraitsClass::equal),
- _cmp(TypeTraitsClass::cmp),
- _shallow_copy(TypeTraitsClass::shallow_copy),
- _deep_copy(TypeTraitsClass::deep_copy),
- _copy_object(TypeTraitsClass::copy_object),
- _direct_copy(TypeTraitsClass::direct_copy),
- _direct_copy_may_cut(TypeTraitsClass::direct_copy_may_cut),
- _convert_from(TypeTraitsClass::convert_from),
- _from_string(TypeTraitsClass::from_string),
- _to_string(TypeTraitsClass::to_string),
- _set_to_max(TypeTraitsClass::set_to_max),
- _set_to_min(TypeTraitsClass::set_to_min),
- _hash_code(TypeTraitsClass::hash_code),
- _size(TypeTraitsClass::size),
- _field_type(TypeTraitsClass::type) {}
-
class ScalarTypeInfoResolver {
DECLARE_SINGLETON(ScalarTypeInfoResolver);
@@ -150,10 +132,10 @@ public:
}
private:
- template <FieldType item_type>
+ template <FieldType field_type>
void add_mapping() {
- _type_mapping.emplace(item_type, std::shared_ptr<const TypeInfo>(new
ArrayTypeInfo(
-
get_scalar_type_info(item_type))));
+ _type_mapping.emplace(field_type, std::shared_ptr<const TypeInfo>(new
ArrayTypeInfo(
+
get_scalar_type_info(field_type))));
}
// item_type_info -> list_type_info
diff --git a/be/src/olap/types.h b/be/src/olap/types.h
index 6be2dec..e4ec59e 100644
--- a/be/src/olap/types.h
+++ b/be/src/olap/types.h
@@ -49,7 +49,6 @@ extern bool is_olap_string_type(FieldType field_type);
class TypeInfo {
public:
virtual ~TypeInfo() = default;
- ;
virtual bool equal(const void* left, const void* right) const = 0;
virtual int cmp(const void* left, const void* right) const = 0;
@@ -133,6 +132,24 @@ public:
inline FieldType type() const override { return _field_type; }
+ template <typename TypeTraitsClass>
+ ScalarTypeInfo(TypeTraitsClass t)
+ : _equal(TypeTraitsClass::equal),
+ _cmp(TypeTraitsClass::cmp),
+ _shallow_copy(TypeTraitsClass::shallow_copy),
+ _deep_copy(TypeTraitsClass::deep_copy),
+ _copy_object(TypeTraitsClass::copy_object),
+ _direct_copy(TypeTraitsClass::direct_copy),
+ _direct_copy_may_cut(TypeTraitsClass::direct_copy_may_cut),
+ _convert_from(TypeTraitsClass::convert_from),
+ _from_string(TypeTraitsClass::from_string),
+ _to_string(TypeTraitsClass::to_string),
+ _set_to_max(TypeTraitsClass::set_to_max),
+ _set_to_min(TypeTraitsClass::set_to_min),
+ _hash_code(TypeTraitsClass::hash_code),
+ _size(TypeTraitsClass::size),
+ _field_type(TypeTraitsClass::type) {}
+
private:
bool (*_equal)(const void* left, const void* right);
int (*_cmp)(const void* left, const void* right);
@@ -157,8 +174,6 @@ private:
const FieldType _field_type;
friend class ScalarTypeInfoResolver;
- template <typename TypeTraitsClass>
- ScalarTypeInfo(TypeTraitsClass t);
};
class ArrayTypeInfo : public TypeInfo {
@@ -1251,6 +1266,14 @@ struct TypeTraits : public FieldTypeTraits<field_type> {
static const int32_t size = sizeof(CppType);
};
+// get ScalarTypeInfo at compile time for performance
+template <FieldType field_type>
+inline TypeInfo* get_scalar_type_info() {
+ static constexpr TypeTraits<field_type> traits;
+ static auto _scala_type_info = ScalarTypeInfo(traits);
+ return dynamic_cast<TypeInfo*>(&_scala_type_info);
+}
+
} // namespace doris
#endif // DORIS_BE_SRC_OLAP_TYPES_H
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
