This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 4d45072760 [Improvement](dict) compute hash only if needed (#18058)
(#18083)
4d45072760 is described below
commit 4d450727609eba88f6d70679f386590219117063
Author: Gabriel <[email protected]>
AuthorDate: Fri Mar 24 15:49:29 2023 +0800
[Improvement](dict) compute hash only if needed (#18058) (#18083)
pick #18058
---
be/src/olap/block_column_predicate.cpp | 2 +-
be/src/olap/olap_common.h | 3 ++
be/src/olap/reader.h | 2 +-
be/src/olap/rowset/beta_rowset_reader.h | 2 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 8 +--
be/src/vec/columns/column.h | 2 +-
be/src/vec/columns/column_dictionary.h | 62 +++++++++++++---------
be/src/vec/columns/column_nullable.h | 4 +-
be/src/vec/exec/scan/new_olap_scan_node.cpp | 9 +++-
be/src/vec/exec/scan/new_olap_scan_node.h | 5 +-
be/src/vec/exec/scan/new_olap_scanner.cpp | 7 ++-
11 files changed, 69 insertions(+), 37 deletions(-)
diff --git a/be/src/olap/block_column_predicate.cpp
b/be/src/olap/block_column_predicate.cpp
index c35eff0153..0548f88713 100644
--- a/be/src/olap/block_column_predicate.cpp
+++ b/be/src/olap/block_column_predicate.cpp
@@ -81,7 +81,7 @@ void
SingleColumnBlockPredicate::evaluate_vec(vectorized::MutableColumns& block,
if (PredicateTypeTraits::is_range(_predicate->type())) {
column->convert_dict_codes_if_necessary();
} else if (PredicateTypeTraits::is_bloom_filter(_predicate->type())) {
- column->generate_hash_values_for_runtime_filter();
+ column->initialize_hash_values_for_runtime_filter();
}
_predicate->evaluate_vec(*column, size, flags);
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 41f9c2132a..5c6647ef60 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -306,6 +306,9 @@ struct OlapReaderStatistics {
int64_t raw_rows_read = 0;
int64_t rows_vec_cond_filtered = 0;
+ int64_t rows_short_circuit_cond_filtered = 0;
+ int64_t vec_cond_input_rows = 0;
+ int64_t short_circuit_cond_input_rows = 0;
int64_t rows_vec_del_cond_filtered = 0;
int64_t vec_cond_ns = 0;
int64_t short_cond_ns = 0;
diff --git a/be/src/olap/reader.h b/be/src/olap/reader.h
index fbb4a3018d..2ceeb8c5bc 100644
--- a/be/src/olap/reader.h
+++ b/be/src/olap/reader.h
@@ -142,7 +142,7 @@ public:
uint64_t filtered_rows() const {
return _stats.rows_del_filtered + _stats.rows_del_by_bitmap +
_stats.rows_conditions_filtered +
_stats.rows_vec_del_cond_filtered +
- _stats.rows_vec_cond_filtered;
+ _stats.rows_vec_cond_filtered +
_stats.rows_short_circuit_cond_filtered;
}
void set_batch_size(int batch_size) { _batch_size = batch_size; }
diff --git a/be/src/olap/rowset/beta_rowset_reader.h
b/be/src/olap/rowset/beta_rowset_reader.h
index d09cf5445f..3ca18b17f9 100644
--- a/be/src/olap/rowset/beta_rowset_reader.h
+++ b/be/src/olap/rowset/beta_rowset_reader.h
@@ -59,7 +59,7 @@ public:
int64_t filtered_rows() override {
return _stats->rows_del_filtered + _stats->rows_del_by_bitmap +
_stats->rows_conditions_filtered +
_stats->rows_vec_del_cond_filtered +
- _stats->rows_vec_cond_filtered;
+ _stats->rows_vec_cond_filtered +
_stats->rows_short_circuit_cond_filtered;
}
RowsetTypePB type() const override { return RowsetTypePB::BETA_ROWSET; }
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index ea21e6a50b..d9cac145c8 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1090,6 +1090,7 @@ uint16_t
SegmentIterator::_evaluate_vectorization_predicate(uint16_t* sel_rowid_
}
}
+ _opts.stats->vec_cond_input_rows += original_size;
_opts.stats->rows_vec_cond_filtered += original_size - new_size;
return new_size;
}
@@ -1107,7 +1108,8 @@ uint16_t
SegmentIterator::_evaluate_short_circuit_predicate(uint16_t* vec_sel_ro
auto& short_cir_column = _current_return_columns[column_id];
selected_size = predicate->evaluate(*short_cir_column,
vec_sel_rowid_idx, selected_size);
}
- _opts.stats->rows_vec_cond_filtered += original_size - selected_size;
+ _opts.stats->short_circuit_cond_input_rows += original_size;
+ _opts.stats->rows_short_circuit_cond_filtered += original_size -
selected_size;
// evaluate delete condition
original_size = selected_size;
@@ -1295,7 +1297,7 @@ void
SegmentIterator::_convert_dict_code_for_predicate_if_necessary() {
}
for (auto column_id : _delete_bloom_filter_column_ids) {
-
_current_return_columns[column_id].get()->generate_hash_values_for_runtime_filter();
+
_current_return_columns[column_id].get()->initialize_hash_values_for_runtime_filter();
}
}
@@ -1307,7 +1309,7 @@ void
SegmentIterator::_convert_dict_code_for_predicate_if_necessary_impl(
if (PredicateTypeTraits::is_range(predicate->type())) {
col_ptr->convert_dict_codes_if_necessary();
} else if (PredicateTypeTraits::is_bloom_filter(predicate->type())) {
- col_ptr->generate_hash_values_for_runtime_filter();
+ col_ptr->initialize_hash_values_for_runtime_filter();
}
}
diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index 6bd5ac7855..0a2c66183e 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -111,7 +111,7 @@ public:
virtual void convert_dict_codes_if_necessary() {}
/// If column is ColumnDictionary, and is a bloom filter predicate,
generate_hash_values
- virtual void generate_hash_values_for_runtime_filter() {}
+ virtual void initialize_hash_values_for_runtime_filter() {}
/// Creates empty column with the same type.
virtual MutablePtr clone_empty() const { return clone_resized(0); }
diff --git a/be/src/vec/columns/column_dictionary.h
b/be/src/vec/columns/column_dictionary.h
index 2812c07103..ca79748ce2 100644
--- a/be/src/vec/columns/column_dictionary.h
+++ b/be/src/vec/columns/column_dictionary.h
@@ -265,11 +265,11 @@ public:
return _dict.find_code_by_bound(value, greater, eq);
}
- void generate_hash_values_for_runtime_filter() override {
- _dict.generate_hash_values_for_runtime_filter(_type);
+ void initialize_hash_values_for_runtime_filter() override {
+ _dict.initialize_hash_values_for_runtime_filter();
}
- uint32_t get_hash_value(uint32_t idx) const { return
_dict.get_hash_value(_codes[idx]); }
+ uint32_t get_hash_value(uint32_t idx) const { return
_dict.get_hash_value(_codes[idx], _type); }
void find_codes(const phmap::flat_hash_set<StringValue>& values,
std::vector<vectorized::UInt8>& selected) const {
@@ -345,31 +345,38 @@ public:
inline const StringValue& get_value(T code) const { return
(*_dict_data)[code]; }
// The function is only used in the runtime filter feature
- inline void generate_hash_values_for_runtime_filter(FieldType type) {
+ inline void initialize_hash_values_for_runtime_filter() {
if (_hash_values.empty()) {
_hash_values.resize(_dict_data->size());
- for (size_t i = 0; i < _dict_data->size(); i++) {
- auto& sv = (*_dict_data)[i];
- // The char data is stored in the disk with the schema
length,
- // and zeros are filled if the length is insufficient
-
- // When reading data, use
shrink_char_type_column_suffix_zero(_char_type_idx)
- // Remove the suffix 0
- // When writing data, use the CharField::consume function
to fill in the trailing 0.
-
- // For dictionary data of char type, sv.len is the schema
length,
- // so use strnlen to remove the 0 at the end to get the
actual length.
- int32_t len = sv.len;
- if (type == OLAP_FIELD_TYPE_CHAR) {
- len = strnlen(sv.ptr, sv.len);
- }
- uint32_t hash_val = HashUtil::murmur_hash3_32(sv.ptr, len,
0);
- _hash_values[i] = hash_val;
- }
+ _compute_hash_value_flags.resize(_dict_data->size());
+ _compute_hash_value_flags.assign(_dict_data->size(), 0);
}
}
- inline uint32_t get_hash_value(T code) const { return
_hash_values[code]; }
+ inline uint32_t get_hash_value(T code, FieldType type) const {
+ if (_compute_hash_value_flags[code]) {
+ return _hash_values[code];
+ } else {
+ auto& sv = (*_dict_data)[code];
+ // The char data is stored in the disk with the schema length,
+ // and zeros are filled if the length is insufficient
+
+ // When reading data, use
shrink_char_type_column_suffix_zero(_char_type_idx)
+ // Remove the suffix 0
+ // When writing data, use the CharField::consume function to
fill in the trailing 0.
+
+ // For dictionary data of char type, sv.size is the schema
length,
+ // so use strnlen to remove the 0 at the end to get the actual
length.
+ int32_t len = sv.len;
+ if (type == OLAP_FIELD_TYPE_CHAR) {
+ len = strnlen(sv.ptr, sv.len);
+ }
+ uint32_t hash_val = HashUtil::murmur_hash3_32(sv.ptr, len, 0);
+ _hash_values[code] = hash_val;
+ _compute_hash_value_flags[code] = 1;
+ return _hash_values[code];
+ }
+ }
// For > , code takes upper_bound - 1; For >= , code takes upper_bound
// For < , code takes upper_bound; For <=, code takes upper_bound - 1
@@ -414,9 +421,13 @@ public:
_dict_data->clear();
_code_convert_table.clear();
_hash_values.clear();
+ _compute_hash_value_flags.clear();
}
- void clear_hash_values() { _hash_values.clear(); }
+ void clear_hash_values() {
+ _hash_values.clear();
+ _compute_hash_value_flags.clear();
+ }
void sort() {
size_t dict_size = _dict_data->size();
@@ -486,7 +497,8 @@ public:
// But in TPC-DS 1GB q60,we see no significant improvement.
// This may because the magnitude of the data is not large enough(in
q60, only about 80k rows data is filtered for largest table)
// So we may need more test here.
- HashValueContainer _hash_values;
+ mutable HashValueContainer _hash_values;
+ mutable std::vector<uint8> _compute_hash_value_flags;
IColumn::Permutation _perm;
size_t _total_str_len;
};
diff --git a/be/src/vec/columns/column_nullable.h
b/be/src/vec/columns/column_nullable.h
index 2b053ef939..fe34724954 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -314,8 +314,8 @@ public:
get_nested_column().convert_dict_codes_if_necessary();
}
- void generate_hash_values_for_runtime_filter() override {
- get_nested_column().generate_hash_values_for_runtime_filter();
+ void initialize_hash_values_for_runtime_filter() override {
+ get_nested_column().initialize_hash_values_for_runtime_filter();
}
void sort_column(const ColumnSorter* sorter, EqualFlags& flags,
IColumn::Permutation& perms,
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.cpp
b/be/src/vec/exec/scan/new_olap_scan_node.cpp
index f0a484d84e..076b9f89ea 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.cpp
+++ b/be/src/vec/exec/scan/new_olap_scan_node.cpp
@@ -75,7 +75,14 @@ Status NewOlapScanNode::_init_profile() {
_block_init_seek_counter = ADD_COUNTER(_segment_profile,
"BlockInitSeekCount", TUnit::UNIT);
_block_conditions_filtered_timer = ADD_TIMER(_segment_profile,
"BlockConditionsFilteredTime");
- _rows_vec_cond_counter = ADD_COUNTER(_segment_profile,
"RowsVectorPredFiltered", TUnit::UNIT);
+ _rows_vec_cond_filtered_counter =
+ ADD_COUNTER(_segment_profile, "RowsVectorPredFiltered",
TUnit::UNIT);
+ _rows_short_circuit_cond_filtered_counter =
+ ADD_COUNTER(_segment_profile, "RowsShortCircuitPredFiltered",
TUnit::UNIT);
+ _rows_vec_cond_input_counter =
+ ADD_COUNTER(_segment_profile, "RowsVectorPredInput", TUnit::UNIT);
+ _rows_short_circuit_cond_input_counter =
+ ADD_COUNTER(_segment_profile, "RowsShortCircuitPredInput",
TUnit::UNIT);
_vec_cond_timer = ADD_TIMER(_segment_profile, "VectorPredEvalTime");
_short_cond_timer = ADD_TIMER(_segment_profile, "ShortPredEvalTime");
_first_read_timer = ADD_TIMER(_segment_profile, "FirstReadTime");
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.h
b/be/src/vec/exec/scan/new_olap_scan_node.h
index 4305faf0a8..4e2869c945 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.h
+++ b/be/src/vec/exec/scan/new_olap_scan_node.h
@@ -78,7 +78,10 @@ private:
RuntimeProfile::Counter* _read_uncompressed_counter = nullptr;
RuntimeProfile::Counter* _raw_rows_counter = nullptr;
- RuntimeProfile::Counter* _rows_vec_cond_counter = nullptr;
+ RuntimeProfile::Counter* _rows_vec_cond_filtered_counter = nullptr;
+ RuntimeProfile::Counter* _rows_short_circuit_cond_filtered_counter =
nullptr;
+ RuntimeProfile::Counter* _rows_vec_cond_input_counter = nullptr;
+ RuntimeProfile::Counter* _rows_short_circuit_cond_input_counter = nullptr;
RuntimeProfile::Counter* _vec_cond_timer = nullptr;
RuntimeProfile::Counter* _short_cond_timer = nullptr;
RuntimeProfile::Counter* _output_col_timer = nullptr;
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index 7c78a92b27..08da003a5b 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -394,7 +394,12 @@ void NewOlapScanner::_update_counters_before_close() {
COUNTER_UPDATE(olap_parent->_lazy_read_seek_timer,
stats.block_lazy_read_seek_ns);
COUNTER_UPDATE(olap_parent->_lazy_read_seek_counter,
stats.block_lazy_read_seek_num);
COUNTER_UPDATE(olap_parent->_output_col_timer, stats.output_col_ns);
- COUNTER_UPDATE(olap_parent->_rows_vec_cond_counter,
stats.rows_vec_cond_filtered);
+ COUNTER_UPDATE(olap_parent->_rows_vec_cond_filtered_counter,
stats.rows_vec_cond_filtered);
+ COUNTER_UPDATE(olap_parent->_rows_short_circuit_cond_filtered_counter,
+ stats.rows_short_circuit_cond_filtered);
+ COUNTER_UPDATE(olap_parent->_rows_vec_cond_input_counter,
stats.vec_cond_input_rows);
+ COUNTER_UPDATE(olap_parent->_rows_short_circuit_cond_input_counter,
+ stats.short_circuit_cond_input_rows);
COUNTER_UPDATE(olap_parent->_stats_filtered_counter,
stats.rows_stats_filtered);
COUNTER_UPDATE(olap_parent->_bf_filtered_counter, stats.rows_bf_filtered);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
