This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 207327f7417 [pick](branch-3.0) pick #44647 (#45839)
207327f7417 is described below
commit 207327f741705d824752c62314b4482a05dedd57
Author: Xinyi Zou <[email protected]>
AuthorDate: Wed Dec 25 11:50:30 2024 +0800
[pick](branch-3.0) pick #44647 (#45839)
---
be/src/olap/metadata_adder.h | 140 +++++++++++++++++++++++++++++++++++--
be/src/olap/rowset/rowset.cpp | 7 --
be/src/olap/rowset/rowset.h | 5 +-
be/src/runtime/process_profile.cpp | 10 ++-
4 files changed, 146 insertions(+), 16 deletions(-)
diff --git a/be/src/olap/metadata_adder.h b/be/src/olap/metadata_adder.h
index bdc9e7a398d..559c5db873b 100644
--- a/be/src/olap/metadata_adder.h
+++ b/be/src/olap/metadata_adder.h
@@ -20,11 +20,16 @@
#include <bvar/bvar.h>
#include <stdint.h>
+#include "util/runtime_profile.h"
+
namespace doris {
inline bvar::Adder<int64_t>
g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");
+inline bvar::Adder<int64_t>
g_all_rowsets_mem_bytes("doris_all_rowsets_mem_bytes");
+inline bvar::Adder<int64_t> g_all_rowsets_num("doris_all_rowsets_num");
+
inline bvar::Adder<int64_t>
g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");
@@ -37,8 +42,8 @@ inline bvar::Adder<int64_t>
g_tablet_index_num("doris_tablet_index_num");
inline bvar::Adder<int64_t>
g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");
-inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
-inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
+inline bvar::Adder<int64_t>
g_all_segments_mem_bytes("doris_all_segments_mem_bytes");
+inline bvar::Adder<int64_t> g_all_segments_num("doris_all_segments_num");
inline bvar::Adder<int64_t>
g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
@@ -104,6 +109,8 @@ class MetadataAdder {
public:
MetadataAdder();
+ static void dump_metadata_object(RuntimeProfile*
object_heap_dump_snapshot);
+
protected:
MetadataAdder(const MetadataAdder& other);
@@ -159,6 +166,8 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
}
if constexpr (std::is_same_v<T, RowsetMeta>) {
g_rowset_meta_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, Rowset>) {
+ g_all_rowsets_mem_bytes << val;
} else if constexpr (std::is_same_v<T, TabletMeta>) {
g_tablet_meta_mem_bytes << val;
} else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -168,7 +177,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
} else if constexpr (std::is_same_v<T, TabletSchema>) {
g_tablet_schema_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
- g_segment_mem_bytes << val;
+ g_all_segments_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
g_column_reader_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -185,6 +194,9 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
g_ordinal_index_reader_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
g_zone_map_index_reader_mem_bytes << val;
+ } else {
+ LOG(FATAL) << "add_mem_size not match class type: " <<
typeid(T).name() << ", " << val;
+ __builtin_unreachable();
}
}
@@ -195,6 +207,8 @@ void MetadataAdder<T>::add_num(int64_t val) {
}
if constexpr (std::is_same_v<T, RowsetMeta>) {
g_rowset_meta_num << val;
+ } else if constexpr (std::is_same_v<T, Rowset>) {
+ g_all_rowsets_num << val;
} else if constexpr (std::is_same_v<T, TabletMeta>) {
g_tablet_meta_num << val;
} else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -204,7 +218,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
} else if constexpr (std::is_same_v<T, TabletSchema>) {
g_tablet_schema_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
- g_segment_num << val;
+ g_all_segments_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
g_column_reader_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -221,7 +235,123 @@ void MetadataAdder<T>::add_num(int64_t val) {
g_ordinal_index_reader_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
g_zone_map_index_reader_num << val;
+ } else {
+ LOG(FATAL) << "add_num not match class type: " << typeid(T).name() <<
", " << val;
+ __builtin_unreachable();
}
}
-}; // namespace doris
\ No newline at end of file
+template <typename T>
+void MetadataAdder<T>::dump_metadata_object(RuntimeProfile*
object_heap_dump_snapshot) {
+ RuntimeProfile::Counter* rowset_meta_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "RowsetMetaMemBytes",
TUnit::BYTES);
+ RuntimeProfile::Counter* rowset_meta_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "RowsetMetaNum",
TUnit::UNIT);
+ COUNTER_SET(rowset_meta_mem_bytes_counter,
g_rowset_meta_mem_bytes.get_value());
+ COUNTER_SET(rowset_meta_num_counter, g_rowset_meta_num.get_value());
+
+ RuntimeProfile::Counter* all_rowsets_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsMemBytes",
TUnit::BYTES);
+ RuntimeProfile::Counter* all_rowsets_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsNum",
TUnit::UNIT);
+ COUNTER_SET(all_rowsets_mem_bytes_counter,
g_all_rowsets_mem_bytes.get_value());
+ COUNTER_SET(all_rowsets_num_counter, g_all_rowsets_num.get_value());
+
+ RuntimeProfile::Counter* tablet_meta_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaMemBytes",
TUnit::BYTES);
+ RuntimeProfile::Counter* tablet_meta_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaNum",
TUnit::UNIT);
+ COUNTER_SET(tablet_meta_mem_bytes_counter,
g_tablet_meta_mem_bytes.get_value());
+ COUNTER_SET(tablet_meta_num_counter, g_tablet_meta_num.get_value());
+
+ RuntimeProfile::Counter* tablet_column_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "TabletColumnMemBytes",
TUnit::BYTES);
+ RuntimeProfile::Counter* tablet_column_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "TabletColumnNum",
TUnit::UNIT);
+ COUNTER_SET(tablet_column_mem_bytes_counter,
g_tablet_column_mem_bytes.get_value());
+ COUNTER_SET(tablet_column_num_counter, g_tablet_column_num.get_value());
+
+ RuntimeProfile::Counter* tablet_index_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "TabletIndexMemBytes",
TUnit::BYTES);
+ RuntimeProfile::Counter* tablet_index_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "TabletIndexNum",
TUnit::UNIT);
+ COUNTER_SET(tablet_index_mem_bytes_counter,
g_tablet_index_mem_bytes.get_value());
+ COUNTER_SET(tablet_index_num_counter, g_tablet_index_num.get_value());
+
+ RuntimeProfile::Counter* tablet_schema_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "TabletSchemaMemBytes",
TUnit::BYTES);
+ RuntimeProfile::Counter* tablet_schema_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "TabletSchemaNum",
TUnit::UNIT);
+ COUNTER_SET(tablet_schema_mem_bytes_counter,
g_tablet_schema_mem_bytes.get_value());
+ COUNTER_SET(tablet_schema_num_counter, g_tablet_schema_num.get_value());
+
+ RuntimeProfile::Counter* all_segments_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsMemBytes",
TUnit::BYTES);
+ RuntimeProfile::Counter* all_segments_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsNum",
TUnit::UNIT);
+ COUNTER_SET(all_segments_mem_bytes_counter,
g_all_segments_mem_bytes.get_value());
+ COUNTER_SET(all_segments_num_counter, g_all_segments_num.get_value());
+
+ RuntimeProfile::Counter* column_reader_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderMemBytes",
TUnit::BYTES);
+ RuntimeProfile::Counter* column_reader_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderNum",
TUnit::UNIT);
+ COUNTER_SET(column_reader_mem_bytes_counter,
g_column_reader_mem_bytes.get_value());
+ COUNTER_SET(column_reader_num_counter, g_column_reader_num.get_value());
+
+ RuntimeProfile::Counter* bitmap_index_reader_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot,
"BitmapIndexReaderMemBytes", TUnit::BYTES);
+ RuntimeProfile::Counter* bitmap_index_reader_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "BitmapIndexReaderNum",
TUnit::UNIT);
+ COUNTER_SET(bitmap_index_reader_mem_bytes_counter,
g_bitmap_index_reader_mem_bytes.get_value());
+ COUNTER_SET(bitmap_index_reader_num_counter,
g_bitmap_index_reader_num.get_value());
+
+ RuntimeProfile::Counter* bloom_filter_index_reader_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot,
"BloomFilterIndexReaderMemBytes", TUnit::BYTES);
+ RuntimeProfile::Counter* filter_index_reader_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot,
"BloomFilterIndexReaderNum", TUnit::UNIT);
+ COUNTER_SET(bloom_filter_index_reader_mem_bytes_counter,
+ g_bloom_filter_index_reader_mem_bytes.get_value());
+ COUNTER_SET(filter_index_reader_num_counter,
g_bloom_filter_index_reader_num.get_value());
+
+ RuntimeProfile::Counter* index_page_reader_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "IndexPageReaderMemBytes",
TUnit::BYTES);
+ RuntimeProfile::Counter* index_page_reader_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "IndexPageReaderNum",
TUnit::UNIT);
+ COUNTER_SET(index_page_reader_mem_bytes_counter,
g_index_page_reader_mem_bytes.get_value());
+ COUNTER_SET(index_page_reader_num_counter,
g_index_page_reader_num.get_value());
+
+ RuntimeProfile::Counter* indexed_column_reader_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot,
"IndexedColumnReaderMemBytes", TUnit::BYTES);
+ RuntimeProfile::Counter* indexed_column_reader_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "IndexedColumnReaderNum",
TUnit::UNIT);
+ COUNTER_SET(indexed_column_reader_mem_bytes_counter,
+ g_indexed_column_reader_mem_bytes.get_value());
+ COUNTER_SET(indexed_column_reader_num_counter,
g_indexed_column_reader_num.get_value());
+
+ RuntimeProfile::Counter* inverted_index_reader_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot,
"InvertedIndexReaderMemBytes", TUnit::BYTES);
+ RuntimeProfile::Counter* inverted_index_reader_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "InvertedIndexReaderNum",
TUnit::UNIT);
+ COUNTER_SET(inverted_index_reader_mem_bytes_counter,
+ g_inverted_index_reader_mem_bytes.get_value());
+ COUNTER_SET(inverted_index_reader_num_counter,
g_inverted_index_reader_num.get_value());
+
+ RuntimeProfile::Counter* ordinal_index_reader_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot,
"OrdinalIndexReaderMemBytes", TUnit::BYTES);
+ RuntimeProfile::Counter* ordinal_index_reader_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "OrdinalIndexReaderNum",
TUnit::UNIT);
+ COUNTER_SET(ordinal_index_reader_mem_bytes_counter,
+ g_ordinal_index_reader_mem_bytes.get_value());
+ COUNTER_SET(ordinal_index_reader_num_counter,
g_ordinal_index_reader_num.get_value());
+
+ RuntimeProfile::Counter* zone_map_index_reader_mem_bytes_counter =
+ ADD_COUNTER(object_heap_dump_snapshot,
"ZoneMapIndexReaderMemBytes", TUnit::BYTES);
+ RuntimeProfile::Counter* zone_map_index_reader_num_counter =
+ ADD_COUNTER(object_heap_dump_snapshot, "ZoneMapIndexReaderNum",
TUnit::UNIT);
+ COUNTER_SET(zone_map_index_reader_mem_bytes_counter,
+ g_zone_map_index_reader_mem_bytes.get_value());
+ COUNTER_SET(zone_map_index_reader_num_counter,
g_zone_map_index_reader_num.get_value());
+}
+
+}; // namespace doris
diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp
index 256f4d35313..ac3a2a7a1dc 100644
--- a/be/src/olap/rowset/rowset.cpp
+++ b/be/src/olap/rowset/rowset.cpp
@@ -27,8 +27,6 @@
namespace doris {
-static bvar::Adder<size_t> g_total_rowset_num("doris_total_rowset_num");
-
Rowset::Rowset(const TabletSchemaSPtr& schema, RowsetMetaSharedPtr rowset_meta,
std::string tablet_path)
: _rowset_meta(std::move(rowset_meta)),
@@ -56,11 +54,6 @@ Rowset::Rowset(const TabletSchemaSPtr& schema,
RowsetMetaSharedPtr rowset_meta,
}
// build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema
_schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() :
schema;
- g_total_rowset_num << 1;
-}
-
-Rowset::~Rowset() {
- g_total_rowset_num << -1;
}
Status Rowset::load(bool use_cache) {
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index e1a2347f6ae..98d88ba19f2 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -33,6 +33,7 @@
#include "common/logging.h"
#include "common/status.h"
+#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/rowset_meta.h"
#include "olap/tablet_schema.h"
@@ -116,10 +117,8 @@ private:
RowsetState _rowset_state;
};
-class Rowset : public std::enable_shared_from_this<Rowset> {
+class Rowset : public std::enable_shared_from_this<Rowset>, public
MetadataAdder<Rowset> {
public:
- virtual ~Rowset();
-
// Open all segment files in this rowset and load necessary metadata.
// - `use_cache` : whether to use fd cache, only applicable to alpha
rowset now
//
diff --git a/be/src/runtime/process_profile.cpp
b/be/src/runtime/process_profile.cpp
index d91aedbeac2..60b381e9c31 100644
--- a/be/src/runtime/process_profile.cpp
+++ b/be/src/runtime/process_profile.cpp
@@ -19,6 +19,7 @@
#include <memory>
+#include "olap/metadata_adder.h"
#include "runtime/memory/memory_profile.h"
namespace doris {
@@ -37,8 +38,15 @@ void ProcessProfile::refresh_profile() {
std::unique_ptr<RuntimeProfile> process_profile =
std::make_unique<RuntimeProfile>("ProcessProfile");
_memory_profile->make_memory_profile(process_profile.get());
- _process_profile.set(std::move(process_profile));
// TODO make other profile
+
+ // 3. dump object heap
+ RuntimeProfile* object_heap_dump_snapshot =
+ process_profile->create_child("ObjectHeapDump", true, false);
+
MetadataAdder<ProcessProfile>::dump_metadata_object(object_heap_dump_snapshot);
+ // TODO dump other object (block, column, etc.)
+
+ _process_profile.set(std::move(process_profile));
}
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
