This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 68e9a66aa0 [Enchancement](schema scanner) add SchemaScanner profile
(#17230)
68e9a66aa0 is described below
commit 68e9a66aa0abab7724d968a52d832c0e79906848
Author: WenYao <[email protected]>
AuthorDate: Wed Mar 1 08:34:27 2023 +0800
[Enchancement](schema scanner) add SchemaScanner profile (#17230)
Add some profile information to the schema scanner to facilitate
performance optimization.
Example:
SchemaScanner:
- FillBlockTime: 9s131ms
- GetDbTime: 12.816ms
- GetDescribeTime: 1s645ms
- GetTableTime: 25.433ms
---
be/src/exec/schema_scanner.cpp | 7 +++++++
be/src/exec/schema_scanner.h | 7 +++++++
be/src/exec/schema_scanner/schema_backends_scanner.cpp | 1 +
be/src/exec/schema_scanner/schema_charsets_scanner.cpp | 1 +
be/src/exec/schema_scanner/schema_collations_scanner.cpp | 1 +
be/src/exec/schema_scanner/schema_columns_scanner.cpp | 4 ++++
be/src/exec/schema_scanner/schema_files_scanner.cpp | 2 ++
be/src/exec/schema_scanner/schema_partitions_scanner.cpp | 2 ++
be/src/exec/schema_scanner/schema_rowsets_scanner.cpp | 1 +
be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp | 1 +
be/src/exec/schema_scanner/schema_schemata_scanner.cpp | 2 ++
be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp | 2 ++
be/src/exec/schema_scanner/schema_tables_scanner.cpp | 3 +++
be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp | 2 ++
be/src/exec/schema_scanner/schema_variables_scanner.cpp | 1 +
be/src/exec/schema_scanner/schema_views_scanner.cpp | 3 +++
be/src/vec/exec/vschema_scan_node.cpp | 4 ++++
17 files changed, 44 insertions(+)
diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp
index 36c7459d9f..0c84e0c8e4 100644
--- a/be/src/exec/schema_scanner.cpp
+++ b/be/src/exec/schema_scanner.cpp
@@ -90,6 +90,13 @@ Status SchemaScanner::init(SchemaScannerParam* param,
ObjectPool* pool) {
_param = param;
_is_init = true;
+ if (_param->profile) {
+ _get_db_timer = ADD_TIMER(_param->profile, "GetDbTime");
+ _get_table_timer = ADD_TIMER(_param->profile, "GetTableTime");
+ _get_describe_timer = ADD_TIMER(_param->profile, "GetDescribeTime");
+ _fill_block_timer = ADD_TIMER(_param->profile, "FillBlockTime");
+ }
+
return Status::OK();
}
diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h
index dacd84d0d9..216ccb9b70 100644
--- a/be/src/exec/schema_scanner.h
+++ b/be/src/exec/schema_scanner.h
@@ -24,6 +24,7 @@
#include "gen_cpp/Descriptors_types.h"
#include "gen_cpp/Types_types.h"
#include "runtime/mem_pool.h"
+#include "util/runtime_profile.h"
#include "vec/core/block.h"
namespace doris {
@@ -49,6 +50,7 @@ struct SchemaScannerParam {
int64_t thread_id;
const std::vector<TSchemaTableStructure>* table_structure;
const std::string* catalog;
+ std::unique_ptr<RuntimeProfile> profile;
SchemaScannerParam()
: db(nullptr),
@@ -105,6 +107,11 @@ protected:
static DorisServer* _s_doris_server;
TSchemaTableType::type _schema_table_type;
+
+ RuntimeProfile::Counter* _get_db_timer = nullptr;
+ RuntimeProfile::Counter* _get_table_timer = nullptr;
+ RuntimeProfile::Counter* _get_describe_timer = nullptr;
+ RuntimeProfile::Counter* _fill_block_timer = nullptr;
};
} // namespace doris
diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.cpp
b/be/src/exec/schema_scanner/schema_backends_scanner.cpp
index e9c008f8a2..30d00fc2bb 100644
--- a/be/src/exec/schema_scanner/schema_backends_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_backends_scanner.cpp
@@ -84,6 +84,7 @@ Status
SchemaBackendsScanner::get_next_block(vectorized::Block* block, bool* eos
}
Status SchemaBackendsScanner::_fill_block_impl(vectorized::Block* block) {
+ SCOPED_TIMER(_fill_block_timer);
auto row_num = _batch_data.size();
for (size_t col_idx = 0; col_idx < _columns.size(); ++col_idx) {
auto it = _col_name_to_type.find(_columns[col_idx].name);
diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp
b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp
index 56b446fa56..720085d75c 100644
--- a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp
@@ -52,6 +52,7 @@ Status
SchemaCharsetsScanner::get_next_block(vectorized::Block* block, bool* eos
}
Status SchemaCharsetsScanner::_fill_block_impl(vectorized::Block* block) {
+ SCOPED_TIMER(_fill_block_timer);
auto row_num = 0;
while (nullptr != _s_charsets[row_num].charset) {
++row_num;
diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.cpp
b/be/src/exec/schema_scanner/schema_collations_scanner.cpp
index eb35c30f2c..50065b3ca4 100644
--- a/be/src/exec/schema_scanner/schema_collations_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_collations_scanner.cpp
@@ -56,6 +56,7 @@ Status
SchemaCollationsScanner::get_next_block(vectorized::Block* block, bool* e
}
Status SchemaCollationsScanner::_fill_block_impl(vectorized::Block* block) {
+ SCOPED_TIMER(_fill_block_timer);
auto row_num = 0;
while (nullptr != _s_collations[row_num].name) {
++row_num;
diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp
b/be/src/exec/schema_scanner/schema_columns_scanner.cpp
index 8ca5896faf..1f8682d5d6 100644
--- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp
@@ -63,6 +63,7 @@ SchemaColumnsScanner::SchemaColumnsScanner()
SchemaColumnsScanner::~SchemaColumnsScanner() = default;
Status SchemaColumnsScanner::start(RuntimeState* state) {
+ SCOPED_TIMER(_get_db_timer);
if (!_is_init) {
return Status::InternalError("schema columns scanner not inited.");
}
@@ -231,6 +232,7 @@ std::string
SchemaColumnsScanner::_type_to_string(TColumnDesc& desc) {
}
Status SchemaColumnsScanner::_get_new_desc() {
+ SCOPED_TIMER(_get_describe_timer);
TDescribeTableParams desc_params;
desc_params.__set_db(_db_result.dbs[_db_index - 1]);
if (_db_result.__isset.catalogs) {
@@ -259,6 +261,7 @@ Status SchemaColumnsScanner::_get_new_desc() {
}
Status SchemaColumnsScanner::_get_new_table() {
+ SCOPED_TIMER(_get_table_timer);
TGetTablesParams table_params;
table_params.__set_db(_db_result.dbs[_db_index]);
if (_db_result.__isset.catalogs) {
@@ -312,6 +315,7 @@ Status
SchemaColumnsScanner::get_next_block(vectorized::Block* block, bool* eos)
}
Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) {
+ SCOPED_TIMER(_fill_block_timer);
auto columns_num = _desc_result.columns.size();
// TABLE_CATALOG
diff --git a/be/src/exec/schema_scanner/schema_files_scanner.cpp
b/be/src/exec/schema_scanner/schema_files_scanner.cpp
index 762db774f9..1eb8132668 100644
--- a/be/src/exec/schema_scanner/schema_files_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_files_scanner.cpp
@@ -19,6 +19,7 @@
#include "exec/schema_scanner/schema_helper.h"
#include "runtime/primitive_type.h"
+#include "util/runtime_profile.h"
#include "vec/common/string_ref.h"
namespace doris {
@@ -76,6 +77,7 @@ Status SchemaFilesScanner::start(RuntimeState* state) {
if (!_is_init) {
return Status::InternalError("used before initialized.");
}
+ SCOPED_TIMER(_get_db_timer);
TGetDbsParams db_params;
if (nullptr != _param->db) {
db_params.__set_pattern(*(_param->db));
diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
index 41ba26da4b..0f0b9ede9d 100644
--- a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
@@ -20,6 +20,7 @@
#include "exec/schema_scanner/schema_helper.h"
#include "runtime/datetime_value.h"
#include "runtime/primitive_type.h"
+#include "util/runtime_profile.h"
#include "vec/common/string_ref.h"
namespace doris {
@@ -64,6 +65,7 @@ Status SchemaPartitionsScanner::start(RuntimeState* state) {
if (!_is_init) {
return Status::InternalError("used before initialized.");
}
+ SCOPED_TIMER(_get_db_timer);
TGetDbsParams db_params;
if (nullptr != _param->db) {
db_params.__set_pattern(*(_param->db));
diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
index 5ee608acbb..577b7b0093 100644
--- a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
@@ -97,6 +97,7 @@ Status
SchemaRowsetsScanner::get_next_block(vectorized::Block* block, bool* eos)
}
Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) {
+ SCOPED_TIMER(_fill_block_timer);
size_t fill_rowsets_num = std::min(1000ul, rowsets_.size() - _rowsets_idx);
auto fill_idx_begin = _rowsets_idx;
auto fill_idx_end = _rowsets_idx + fill_rowsets_num;
diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp
b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp
index 03072073d2..7b9c3e6652 100644
--- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp
@@ -86,6 +86,7 @@ Status
SchemaSchemaPrivilegesScanner::get_next_block(vectorized::Block* block, b
}
Status SchemaSchemaPrivilegesScanner::_fill_block_impl(vectorized::Block*
block) {
+ SCOPED_TIMER(_fill_block_timer);
auto privileges_num = _priv_result.privileges.size();
// grantee
diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp
b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp
index 647a19a47a..a1d230abbd 100644
--- a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp
@@ -38,6 +38,7 @@ SchemaSchemataScanner::SchemaSchemataScanner()
SchemaSchemataScanner::~SchemaSchemataScanner() = default;
Status SchemaSchemataScanner::start(RuntimeState* state) {
+ SCOPED_TIMER(_get_db_timer);
if (!_is_init) {
return Status::InternalError("used before initial.");
}
@@ -85,6 +86,7 @@ Status
SchemaSchemataScanner::get_next_block(vectorized::Block* block, bool* eos
}
Status SchemaSchemataScanner::_fill_block_impl(vectorized::Block* block) {
+ SCOPED_TIMER(_fill_block_timer);
auto dbs_num = _db_result.dbs.size();
// catalog
diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp
b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp
index d959145168..3f95a7fb84 100644
--- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp
@@ -49,6 +49,7 @@ Status SchemaTablePrivilegesScanner::start(RuntimeState*
state) {
}
Status SchemaTablePrivilegesScanner::_get_new_table() {
+ SCOPED_TIMER(_get_table_timer);
TGetTablesParams table_params;
if (nullptr != _param->wild) {
table_params.__set_pattern(*(_param->wild));
@@ -89,6 +90,7 @@ Status
SchemaTablePrivilegesScanner::get_next_block(vectorized::Block* block, bo
}
Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block*
block) {
+ SCOPED_TIMER(_fill_block_timer);
auto privileges_num = _priv_result.privileges.size();
// grantee
diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp
b/be/src/exec/schema_scanner/schema_tables_scanner.cpp
index 054897b516..2ee16177d0 100644
--- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp
@@ -59,6 +59,7 @@ Status SchemaTablesScanner::start(RuntimeState* state) {
if (!_is_init) {
return Status::InternalError("used before initialized.");
}
+ SCOPED_TIMER(_get_db_timer);
TGetDbsParams db_params;
if (nullptr != _param->db) {
db_params.__set_pattern(*(_param->db));
@@ -87,6 +88,7 @@ Status SchemaTablesScanner::start(RuntimeState* state) {
}
Status SchemaTablesScanner::_get_new_table() {
+ SCOPED_TIMER(_get_table_timer);
TGetTablesParams table_params;
table_params.__set_db(_db_result.dbs[_db_index]);
if (_db_result.__isset.catalogs) {
@@ -117,6 +119,7 @@ Status SchemaTablesScanner::_get_new_table() {
}
Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) {
+ SCOPED_TIMER(_fill_block_timer);
auto table_num = _table_result.tables.size();
// catalog
if (_db_result.__isset.catalogs) {
diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp
b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp
index 4fcdfecfbe..179835528a 100644
--- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp
@@ -45,6 +45,7 @@ Status SchemaUserPrivilegesScanner::start(RuntimeState*
state) {
}
Status SchemaUserPrivilegesScanner::_get_new_table() {
+ SCOPED_TIMER(_get_table_timer);
TGetTablesParams table_params;
if (nullptr != _param->wild) {
table_params.__set_pattern(*(_param->wild));
@@ -85,6 +86,7 @@ Status
SchemaUserPrivilegesScanner::get_next_block(vectorized::Block* block, boo
}
Status SchemaUserPrivilegesScanner::_fill_block_impl(vectorized::Block* block)
{
+ SCOPED_TIMER(_fill_block_timer);
auto privileges_num = _priv_result.privileges.size();
// grantee
diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.cpp
b/be/src/exec/schema_scanner/schema_variables_scanner.cpp
index 38fa4e0af1..dbbef99f6d 100644
--- a/be/src/exec/schema_scanner/schema_variables_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_variables_scanner.cpp
@@ -74,6 +74,7 @@ Status
SchemaVariablesScanner::get_next_block(vectorized::Block* block, bool* eo
}
Status SchemaVariablesScanner::_fill_block_impl(vectorized::Block* block) {
+ SCOPED_TIMER(_fill_block_timer);
// variables names
{
for (auto& it : _var_result.variables) {
diff --git a/be/src/exec/schema_scanner/schema_views_scanner.cpp
b/be/src/exec/schema_scanner/schema_views_scanner.cpp
index fb64337cce..c0273a85b2 100644
--- a/be/src/exec/schema_scanner/schema_views_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_views_scanner.cpp
@@ -46,6 +46,7 @@ Status SchemaViewsScanner::start(RuntimeState* state) {
if (!_is_init) {
return Status::InternalError("used before initialized.");
}
+ SCOPED_TIMER(_get_db_timer);
TGetDbsParams db_params;
if (nullptr != _param->db) {
db_params.__set_pattern(*(_param->db));
@@ -74,6 +75,7 @@ Status SchemaViewsScanner::start(RuntimeState* state) {
}
Status SchemaViewsScanner::_get_new_table() {
+ SCOPED_TIMER(_get_table_timer);
TGetTablesParams table_params;
table_params.__set_db(_db_result.dbs[_db_index++]);
if (nullptr != _param->wild) {
@@ -118,6 +120,7 @@ Status
SchemaViewsScanner::get_next_block(vectorized::Block* block, bool* eos) {
}
Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) {
+ SCOPED_TIMER(_fill_block_timer);
auto tables_num = _table_result.tables.size();
// catalog
diff --git a/be/src/vec/exec/vschema_scan_node.cpp
b/be/src/vec/exec/vschema_scan_node.cpp
index 60fe1d7720..5cede5de72 100644
--- a/be/src/vec/exec/vschema_scan_node.cpp
+++ b/be/src/vec/exec/vschema_scan_node.cpp
@@ -153,6 +153,10 @@ Status VSchemaScanNode::prepare(RuntimeState* state) {
return Status::InternalError("Failed to get schema table descriptor.");
}
+ // init schema scanner profile
+ _scanner_param.profile.reset(new RuntimeProfile("SchemaScanner"));
+ _runtime_profile->add_child(_scanner_param.profile.get(), true, nullptr);
+
// new one scanner
_schema_scanner.reset(SchemaScanner::create(schema_table->schema_table_type()));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
