This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4e783af [feature] add Generic debug timer for debugging or profiling
(#7923)
4e783af is described below
commit 4e783afa7a27a751034004531dcd620fbbfe30e4
Author: zuochunwei <[email protected]>
AuthorDate: Mon Jan 31 22:15:43 2022 +0800
[feature] add Generic debug timer for debugging or profiling (#7923)
add a group of debug-timer for the purpose of profiling or testing
you can use these timers for custom meaning purpose unlike the specific
named timer
---
be/src/exec/olap_scan_node.cpp | 7 +++++++
be/src/exec/olap_scan_node.h | 3 +++
be/src/exec/olap_scanner.cpp | 5 +++++
be/src/olap/olap_common.h | 13 +++++++++++++
be/src/olap/rowset/segment_v2/binary_plain_page.h | 2 --
be/src/olap/rowset/segment_v2/column_reader.cpp | 3 ++-
be/src/olap/rowset/segment_v2/indexed_column_reader.cpp | 2 +-
be/src/olap/rowset/segment_v2/options.h | 5 ++++-
be/src/olap/rowset/segment_v2/parsed_page.h | 5 +++--
9 files changed, 38 insertions(+), 7 deletions(-)
diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp
index dfc9a29..48f0ab0 100644
--- a/be/src/exec/olap_scan_node.cpp
+++ b/be/src/exec/olap_scan_node.cpp
@@ -159,6 +159,13 @@ void OlapScanNode::_init_counter(RuntimeState* state) {
// time of node to wait for batch/block queue
_olap_wait_batch_queue_timer = ADD_TIMER(_runtime_profile,
"BatchQueueWaitTime");
+
+ // for the purpose of debugging or profiling
+ for (int i = 0; i <
sizeof(_general_debug_timer)/sizeof(*_general_debug_timer); ++i) {
+ char name[64];
+ snprintf(name, sizeof(name), "GeneralDebugTimer%d", i);
+ _general_debug_timer[i] = ADD_TIMER(_segment_profile, name);
+ }
}
Status OlapScanNode::prepare(RuntimeState* state) {
diff --git a/be/src/exec/olap_scan_node.h b/be/src/exec/olap_scan_node.h
index f9905e2..d57a92d 100644
--- a/be/src/exec/olap_scan_node.h
+++ b/be/src/exec/olap_scan_node.h
@@ -325,6 +325,9 @@ protected:
RuntimeProfile::Counter* _olap_wait_batch_queue_timer = nullptr;
+ // for debugging or profiling, record any info as you want
+ RuntimeProfile::Counter* _general_debug_timer[GENERAL_DEBUG_COUNT] = {};
+
vectorized::VExpr* _dfs_peel_conjunct(vectorized::VExpr* expr, int&
leaf_index);
void _peel_pushed_conjuncts(); // remove pushed expr from conjunct tree
};
diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp
index 2e05c5d..a1efc1d 100644
--- a/be/src/exec/olap_scanner.cpp
+++ b/be/src/exec/olap_scanner.cpp
@@ -541,6 +541,11 @@ void OlapScanner::update_counter() {
COUNTER_UPDATE(_parent->_index_load_timer, stats.index_load_ns);
+ size_t timer_count = sizeof(stats.general_debug_ns) /
sizeof(*stats.general_debug_ns);
+ for (size_t i = 0; i < timer_count; ++i) {
+ COUNTER_UPDATE(_parent->_general_debug_timer[i],
stats.general_debug_ns[i]);
+ }
+
COUNTER_UPDATE(_parent->_total_pages_num_counter, stats.total_pages_num);
COUNTER_UPDATE(_parent->_cached_pages_num_counter, stats.cached_pages_num);
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 64802e5..623528d 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -238,6 +238,8 @@ class Field;
class WrapperField;
using KeyRange = std::pair<WrapperField*, WrapperField*>;
+static const int GENERAL_DEBUG_COUNT = 4;
+
// ReaderStatistics used to collect statistics when scan data from storage
struct OlapReaderStatistics {
int64_t io_ns = 0;
@@ -285,6 +287,17 @@ struct OlapReaderStatistics {
int64_t filtered_segment_number = 0;
// total number of segment
int64_t total_segment_number = 0;
+ // general_debug_ns is designed for the purpose of DEBUG, to record any
infomations of debugging or profiling.
+ // different from specific meaningful timer such as index_load_ns,
general_debug_ns can be used flexibly.
+ // general_debug_ns has associated with OlapScanNode's
_general_debug_timer already.
+ // so general_debug_ns' values will update to _general_debug_timer
automaticly,
+ // the timer result can be checked through QueryProfile web page easily.
+ // when search general_debug_ns, you can find that general_debug_ns has
not been used,
+ // this is because such codes added for debug purpose should not commit,
it's just for debuging.
+ // so, please do not delete general_debug_ns defined here
+ // usage example:
+ // SCOPED_RAW_TIMER(&_stats->general_debug_ns[1]);
+ int64_t general_debug_ns[GENERAL_DEBUG_COUNT] = {};
};
typedef uint32_t ColumnId;
diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h
b/be/src/olap/rowset/segment_v2/binary_plain_page.h
index 3c55d2b..3444cf1 100644
--- a/be/src/olap/rowset/segment_v2/binary_plain_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h
@@ -154,8 +154,6 @@ private:
class BinaryPlainPageDecoder : public PageDecoder {
public:
- BinaryPlainPageDecoder(Slice data) : BinaryPlainPageDecoder(data,
PageDecoderOptions()) {}
-
BinaryPlainPageDecoder(Slice data, const PageDecoderOptions& options)
: _data(data),
_options(options),
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 9b14ff4..4c651f2 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -652,6 +652,7 @@ Status FileColumnIterator::_read_data_page(const
OrdinalPageIndexIterator& iter)
RETURN_IF_ERROR(_reader->read_page(_opts, iter.page(), &handle,
&page_body, &footer));
// parse data page
RETURN_IF_ERROR(ParsedPage::create(std::move(handle), page_body,
footer.data_page_footer(),
+ _opts.stats,
_reader->encoding_info(), iter.page(),
iter.page_index(),
&_page));
@@ -672,7 +673,7 @@ Status FileColumnIterator::_read_data_page(const
OrdinalPageIndexIterator& iter)
&_dict_page_handle,
&dict_data, &dict_footer));
// ignore dict_footer.dict_page_footer().encoding() due to only
// PLAIN_ENCODING is supported for dict page right now
- _dict_decoder.reset(new BinaryPlainPageDecoder(dict_data));
+ _dict_decoder.reset(new BinaryPlainPageDecoder(dict_data,
_opts.stats));
RETURN_IF_ERROR(_dict_decoder->init());
auto* pd_decoder =
(BinaryPlainPageDecoder*)_dict_decoder.get();
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index e5a3bed..27c3d84 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -102,7 +102,7 @@ Status IndexedColumnIterator::_read_data_page(const
PagePointer& pp) {
// parse data page
// note that page_index is not used in IndexedColumnIterator, so we pass 0
return ParsedPage::create(std::move(handle), body,
footer.data_page_footer(),
- _reader->encoding_info(), pp, 0, &_data_page);
+ nullptr, _reader->encoding_info(), pp, 0,
&_data_page);
}
Status IndexedColumnIterator::seek_to_ordinal(ordinal_t idx) {
diff --git a/be/src/olap/rowset/segment_v2/options.h
b/be/src/olap/rowset/segment_v2/options.h
index 0cb2daf..12db113 100644
--- a/be/src/olap/rowset/segment_v2/options.h
+++ b/be/src/olap/rowset/segment_v2/options.h
@@ -32,7 +32,10 @@ struct PageBuilderOptions {
size_t dict_page_size = DEFAULT_PAGE_SIZE;
};
-struct PageDecoderOptions {};
+struct PageDecoderOptions {
+ struct OlapReaderStatistics* stats;
+ PageDecoderOptions(struct OlapReaderStatistics* stats) : stats(stats) {}
+};
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/parsed_page.h
b/be/src/olap/rowset/segment_v2/parsed_page.h
index cf49b89..d7ba5ea 100644
--- a/be/src/olap/rowset/segment_v2/parsed_page.h
+++ b/be/src/olap/rowset/segment_v2/parsed_page.h
@@ -36,6 +36,7 @@ namespace segment_v2 {
// this object
struct ParsedPage {
static Status create(PageHandle handle, const Slice& body, const
DataPageFooterPB& footer,
+ struct OlapReaderStatistics* stats,
const EncodingInfo* encoding, const PagePointer&
page_pointer,
uint32_t page_index, std::unique_ptr<ParsedPage>*
result) {
std::unique_ptr<ParsedPage> page(new ParsedPage);
@@ -51,7 +52,7 @@ struct ParsedPage {
}
Slice data_slice(body.data, body.size - null_size);
- PageDecoderOptions opts;
+ PageDecoderOptions opts(stats);
RETURN_IF_ERROR(encoding->create_page_decoder(data_slice, opts,
&page->data_decoder));
RETURN_IF_ERROR(page->data_decoder->init());
@@ -102,4 +103,4 @@ private:
};
} // namespace segment_v2
-} // namespace doris
\ No newline at end of file
+} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
