This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 981d2d3d75d [fix](segment cache) estimate momory consumed by segment
(#35647)
981d2d3d75d is described below
commit 981d2d3d75df2773a0b627becf30123a7286d46d
Author: Yongqiang YANG <[email protected]>
AuthorDate: Fri May 31 20:35:44 2024 +0800
[fix](segment cache) estimate momory consumed by segment (#35647)
The memory consumed in segment cache is 0 after
https://github.com/apache/doris/pull/35432/files.
The pr also tracks memory usage of column readers.
---
be/src/olap/base_tablet.cpp | 4 ++++
be/src/olap/primary_key_index.cpp | 2 ++
be/src/olap/rowset/rowset.cpp | 7 +++++++
be/src/olap/rowset/rowset.h | 2 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 11 ++++++++++-
be/src/olap/rowset/segment_v2/column_reader.h | 2 +-
.../olap/rowset/segment_v2/indexed_column_reader.cpp | 8 ++++++++
.../olap/rowset/segment_v2/indexed_column_reader.h | 2 ++
be/src/olap/rowset/segment_v2/ordinal_page_index.cpp | 14 ++++++++++++++
be/src/olap/rowset/segment_v2/ordinal_page_index.h | 2 ++
be/src/olap/rowset/segment_v2/segment.cpp | 20 +++++++++++++++-----
be/src/olap/rowset/segment_v2/segment.h | 1 +
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 11 +++++++++++
be/src/olap/rowset/segment_v2/zone_map_index.h | 2 ++
be/src/olap/short_key_index.cpp | 16 ++++++++++++++++
be/src/olap/short_key_index.h | 1 +
be/src/olap/tablet_schema.cpp | 10 ++++++++++
be/src/olap/tablet_schema.h | 4 +++-
18 files changed, 110 insertions(+), 9 deletions(-)
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 9494ce5fd94..63fc989ce45 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -52,6 +52,8 @@ bvar::PerSecond<bvar::Adder<uint64_t>>
g_tablet_pk_not_found_per_second(
"doris_pk", "lookup_not_found_per_second", &g_tablet_pk_not_found, 60);
bvar::LatencyRecorder g_tablet_update_delete_bitmap_latency("doris_pk",
"update_delete_bitmap");
+static bvar::Adder<size_t> g_total_tablet_num("doris_total_tablet_num");
+
// read columns by read plan
// read_index: ori_pos-> block_idx
Status read_columns_by_plan(TabletSchemaSPtr tablet_schema,
@@ -158,10 +160,12 @@ BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta) :
_tablet_meta(std::move
tablet_schema_with_merged_max_schema_version(_tablet_meta->all_rs_metas());
}
DCHECK(_max_version_schema);
+ g_total_tablet_num << 1;
}
BaseTablet::~BaseTablet() {
DorisMetrics::instance()->metric_registry()->deregister_entity(_metric_entity);
+ g_total_tablet_num << -1;
}
TabletSchemaSPtr BaseTablet::tablet_schema_with_merged_max_schema_version(
diff --git a/be/src/olap/primary_key_index.cpp
b/be/src/olap/primary_key_index.cpp
index 6d1b1772a91..6ea4bc30d0a 100644
--- a/be/src/olap/primary_key_index.cpp
+++ b/be/src/olap/primary_key_index.cpp
@@ -32,6 +32,8 @@
namespace doris {
+static bvar::Adder<size_t>
g_primary_key_index_memory_bytes("doris_primary_key_index_memory_bytes");
+
Status PrimaryKeyIndexBuilder::init() {
// TODO(liaoxin) using the column type directly if there's only one column
in unique key columns
const auto* type_info =
get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARCHAR>();
diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp
index 3edba25c018..853b1faf75e 100644
--- a/be/src/olap/rowset/rowset.cpp
+++ b/be/src/olap/rowset/rowset.cpp
@@ -26,6 +26,8 @@
namespace doris {
+static bvar::Adder<size_t> g_total_rowset_num("doris_total_rowset_num");
+
Rowset::Rowset(const TabletSchemaSPtr& schema, const RowsetMetaSharedPtr&
rowset_meta)
: _rowset_meta(rowset_meta), _refs_by_reader(0) {
_is_pending = !_rowset_meta->has_version();
@@ -37,6 +39,11 @@ Rowset::Rowset(const TabletSchemaSPtr& schema, const
RowsetMetaSharedPtr& rowset
}
// build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema
_schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() :
schema;
+ g_total_rowset_num << 1;
+}
+
+Rowset::~Rowset() {
+ g_total_rowset_num << -1;
}
Status Rowset::load(bool use_cache) {
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index 8d1d20145e1..79f454fc495 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -118,7 +118,7 @@ private:
class Rowset : public std::enable_shared_from_this<Rowset> {
public:
- virtual ~Rowset() = default;
+ virtual ~Rowset();
// Open all segment files in this rowset and load necessary metadata.
// - `use_cache` : whether to use fd cache, only applicable to alpha
rowset now
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 4e005c43a7c..a069034cd23 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -77,6 +77,9 @@
namespace doris {
namespace segment_v2 {
+static bvar::Adder<size_t>
g_column_reader_memory_bytes("doris_column_reader_memory_bytes");
+static bvar::Adder<size_t> g_column_reader_num("doris_column_reader_num");
+
Status ColumnReader::create(const ColumnReaderOptions& opts, const
ColumnMetaPB& meta,
uint64_t num_rows, const io::FileReaderSPtr&
file_reader,
std::unique_ptr<ColumnReader>* reader) {
@@ -206,9 +209,15 @@ ColumnReader::ColumnReader(const ColumnReaderOptions&
opts, const ColumnMetaPB&
_meta_is_nullable = meta.is_nullable();
_meta_dict_page = meta.dict_page();
_meta_compression = meta.compression();
+
+ g_column_reader_memory_bytes << sizeof(*this);
+ g_column_reader_num << 1;
}
-ColumnReader::~ColumnReader() = default;
+ColumnReader::~ColumnReader() {
+ g_column_reader_memory_bytes << -sizeof(*this);
+ g_column_reader_num << -1;
+}
Status ColumnReader::init(const ColumnMetaPB* meta) {
_type_info = get_type_info(meta);
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h
b/be/src/olap/rowset/segment_v2/column_reader.h
index ae2b29cf983..94494b4d23c 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -117,7 +117,7 @@ public:
enum DictEncodingType { UNKNOWN_DICT_ENCODING, PARTIAL_DICT_ENCODING,
ALL_DICT_ENCODING };
- ~ColumnReader();
+ virtual ~ColumnReader();
// create a new column iterator. Client should delete returned iterator
Status new_iterator(ColumnIterator** iterator);
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index b5f974fb8a0..59251b5595d 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -56,6 +56,8 @@ static bvar::Adder<uint64_t>
g_index_reader_pk_pages("doris_pk", "index_reader_p
static bvar::PerSecond<bvar::Adder<uint64_t>>
g_index_reader_pk_bytes_per_second(
"doris_pk", "index_reader_pk_pages_per_second",
&g_index_reader_pk_pages, 60);
+static bvar::Adder<uint64_t>
g_index_reader_memory_bytes("doris_index_reader_memory_bytes");
+
using strings::Substitute;
Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory) {
@@ -91,6 +93,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool
kept_in_memory) {
}
}
_num_values = _meta.num_values();
+
+ g_index_reader_memory_bytes << sizeof(*this);
return Status::OK();
}
@@ -134,6 +138,10 @@ Status IndexedColumnReader::read_page(const PagePointer&
pp, PageHandle* handle,
return st;
}
+IndexedColumnReader::~IndexedColumnReader() {
+ g_index_reader_memory_bytes << -sizeof(*this);
+}
+
///////////////////////////////////////////////////////////////////////////////
Status IndexedColumnIterator::_read_data_page(const PagePointer& pp) {
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index ef6e2754609..d156643a21c 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -51,6 +51,8 @@ public:
explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const
IndexedColumnMetaPB& meta)
: _file_reader(std::move(file_reader)), _meta(meta) {}
+ ~IndexedColumnReader();
+
Status load(bool use_page_cache, bool kept_in_memory);
// read a page specified by `pp' from `file' into `handle'
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
index 5761cc42b9d..b93d461d8c7 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
@@ -29,9 +29,13 @@
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/page_handle.h"
#include "olap/rowset/segment_v2/page_io.h"
+#include "ordinal_page_index.h"
#include "util/slice.h"
namespace doris {
+
+static bvar::Adder<size_t>
g_ordinal_index_memory_bytes("doris_ordinal_index_memory_bytes");
+
namespace segment_v2 {
void OrdinalIndexWriter::append_entry(ordinal_t ordinal, const PagePointer&
data_pp) {
@@ -122,6 +126,10 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
_pages[i] = reader.get_value(i);
}
_ordinals[_num_pages] = _num_values;
+
+ g_ordinal_index_memory_bytes << sizeof(*this) + _ordinals.size() *
sizeof(ordinal_t) +
+ _pages.size() *
sizeof(PagePointer) +
+ sizeof(OrdinalIndexReader);
return Status::OK();
}
@@ -146,5 +154,11 @@ OrdinalPageIndexIterator
OrdinalIndexReader::seek_at_or_before(ordinal_t ordinal
return OrdinalPageIndexIterator(this, left);
}
+OrdinalIndexReader::~OrdinalIndexReader() {
+ g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() *
sizeof(ordinal_t) -
+ _pages.size() *
sizeof(PagePointer) -
+ sizeof(OrdinalIndexReader);
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
index 77f0b851816..8f9e0afe1bf 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
@@ -72,6 +72,8 @@ public:
_meta_pb.reset(new OrdinalIndexPB(meta_pb));
}
+ virtual ~OrdinalIndexReader();
+
// load and parse the index page into memory
Status load(bool use_page_cache, bool kept_in_memory);
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index dc934ba5fce..65153f2f162 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -73,7 +73,7 @@
#include "vec/olap/vgeneric_iterators.h"
namespace doris::segment_v2 {
-bvar::Adder<size_t> g_total_segment_num("doris_total_segment_num");
+static bvar::Adder<size_t> g_total_segment_num("doris_total_segment_num");
class InvertedIndexIterator;
Status Segment::open(io::FileSystemSPtr fs, const std::string& path, uint32_t
segment_id,
@@ -112,6 +112,17 @@ Status Segment::_open() {
// DCHECK(footer.has_short_key_index_page());
_sk_index_page = _footer_pb->short_key_index_page();
_num_rows = _footer_pb->num_rows();
+
+ // An estimated memory usage of a segment
+ _meta_mem_usage += _footer_pb->ByteSizeLong();
+ _meta_mem_usage += sizeof(*this);
+ _meta_mem_usage += _tablet_schema->num_columns() *
config::estimated_mem_per_column_reader;
+
+ // 1024 comes from SegmentWriterOptions
+ _meta_mem_usage += (_num_rows + 1023) / 1024 * (36 + 4);
+ // 0.01 comes from PrimaryKeyIndexBuilder::init
+ _meta_mem_usage += BloomFilter::optimal_bit_num(_num_rows, 0.01) / 8;
+
return Status::OK();
}
@@ -298,7 +309,7 @@ Status Segment::_load_pk_bloom_filter() {
auto status = [this]() {
return _load_pk_bf_once.call([this] {
RETURN_IF_ERROR(_pk_index_reader->parse_bf(_file_reader,
*_pk_index_meta));
- _meta_mem_usage += _pk_index_reader->get_bf_memory_size();
+ // _meta_mem_usage += _pk_index_reader->get_bf_memory_size();
return Status::OK();
});
}();
@@ -335,7 +346,7 @@ Status Segment::_load_index_impl() {
if (_tablet_schema->keys_type() == UNIQUE_KEYS && _pk_index_meta !=
nullptr) {
_pk_index_reader.reset(new PrimaryKeyIndexReader());
RETURN_IF_ERROR(_pk_index_reader->parse_index(_file_reader,
*_pk_index_meta));
- _meta_mem_usage += _pk_index_reader->get_memory_size();
+ // _meta_mem_usage += _pk_index_reader->get_memory_size();
return Status::OK();
} else {
// read and parse short key index page
@@ -357,7 +368,7 @@ Status Segment::_load_index_impl() {
DCHECK_EQ(footer.type(), SHORT_KEY_PAGE);
DCHECK(footer.has_short_key_page_footer());
- _meta_mem_usage += body.get_size();
+ // _meta_mem_usage += body.get_size();
_sk_index_decoder.reset(new ShortKeyIndexDecoder);
return _sk_index_decoder->parse(body,
footer.short_key_page_footer());
}
@@ -427,7 +438,6 @@ Status Segment::_create_column_readers(const
SegmentFooterPB& footer) {
RETURN_IF_ERROR(ColumnReader::create(opts,
footer.columns(iter->second), footer.num_rows(),
_file_reader, &reader));
_column_readers.emplace(column.unique_id(), std::move(reader));
- _meta_mem_usage += config::estimated_mem_per_column_reader;
}
// init by column path
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index 8cb45025908..e6c9a169dc8 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -257,6 +257,7 @@ private:
// used to hold short key index page in memory
PageHandle _sk_index_handle;
// short key index decoder
+ // all content is in memory
std::unique_ptr<ShortKeyIndexDecoder> _sk_index_decoder;
// primary key index reader
std::unique_ptr<PrimaryKeyIndexReader> _pk_index_reader;
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
index e232e448ff6..6a1dee39cd7 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
@@ -39,6 +39,8 @@
namespace doris {
struct uint24_t;
+static bvar::Adder<size_t>
g_zone_map_memory_bytes("doris_zone_map_memory_bytes");
+
namespace segment_v2 {
template <PrimitiveType Type>
@@ -173,9 +175,18 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
return Status::Corruption("Failed to parse zone map");
}
}
+
+ g_zone_map_memory_bytes << sizeof(*this) + sizeof(ZoneMapPB) *
_page_zone_maps.size() +
+ sizeof(IndexedColumnMetaPB);
+
return Status::OK();
}
+ZoneMapIndexReader::~ZoneMapIndexReader() {
+ // Maybe wrong due to load failures.
+ g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) *
_page_zone_maps.size() -
+ sizeof(IndexedColumnMetaPB);
+}
#define APPLY_FOR_PRIMITITYPE(M) \
M(TYPE_TINYINT) \
M(TYPE_SMALLINT) \
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.h
b/be/src/olap/rowset/segment_v2/zone_map_index.h
index eeb87eb7da7..923bd2c2046 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.h
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.h
@@ -151,6 +151,8 @@ public:
_page_zone_maps_meta.reset(new IndexedColumnMetaPB(page_zone_maps));
}
+ virtual ~ZoneMapIndexReader();
+
// load all page zone maps into memory
Status load(bool use_page_cache, bool kept_in_memory);
diff --git a/be/src/olap/short_key_index.cpp b/be/src/olap/short_key_index.cpp
index 69622cee454..3ef235e3018 100644
--- a/be/src/olap/short_key_index.cpp
+++ b/be/src/olap/short_key_index.cpp
@@ -22,12 +22,16 @@
#include <ostream>
#include "gutil/strings/substitute.h"
+#include "short_key_index.h"
+#include "util/bvar_helper.h"
#include "util/coding.h"
using strings::Substitute;
namespace doris {
+static bvar::Adder<size_t>
g_short_key_index_memory_bytes("doris_short_key_index_memory_bytes");
+
Status ShortKeyIndexBuilder::add_item(const Slice& key) {
put_varint32(&_offset_buf, _key_buf.size());
_key_buf.append(key.data, key.size);
@@ -85,7 +89,19 @@ Status ShortKeyIndexDecoder::parse(const Slice& body, const
segment_v2::ShortKey
return Status::Corruption("Still has data after parse all key offset");
}
_parsed = true;
+
+ g_short_key_index_memory_bytes << sizeof(_footer) + _key_data.size +
+ _offsets.size() *
sizeof(uint32_t) + sizeof(*this);
+
return Status::OK();
}
+ShortKeyIndexDecoder::~ShortKeyIndexDecoder() {
+ if (_parsed) {
+ g_short_key_index_memory_bytes << -sizeof(_footer) - _key_data.size -
+ _offsets.size() *
sizeof(uint32_t) -
+ sizeof(*this);
+ }
+}
+
} // namespace doris
diff --git a/be/src/olap/short_key_index.h b/be/src/olap/short_key_index.h
index 79303a3edbb..93f4b614908 100644
--- a/be/src/olap/short_key_index.h
+++ b/be/src/olap/short_key_index.h
@@ -135,6 +135,7 @@ private:
class ShortKeyIndexDecoder {
public:
ShortKeyIndexDecoder() : _parsed(false) {}
+ virtual ~ShortKeyIndexDecoder();
// client should assure that body is available when this class is used
Status parse(const Slice& body, const segment_v2::ShortKeyFooterPB&
footer);
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 11d3456653d..ba320521c93 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -53,6 +53,8 @@
namespace doris {
+static bvar::Adder<size_t>
g_total_tablet_schema_num("doris_total_tablet_schema_num");
+
FieldType TabletColumn::get_field_type_by_type(PrimitiveType primitiveType) {
switch (primitiveType) {
case PrimitiveType::INVALID_TYPE:
@@ -820,6 +822,14 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const
{
}
}
+TabletSchema::TabletSchema() {
+ g_total_tablet_schema_num << 1;
+}
+
+TabletSchema::~TabletSchema() {
+ g_total_tablet_schema_num << -1;
+}
+
void TabletSchema::append_column(TabletColumn column, ColumnType col_type) {
if (column.is_key()) {
_num_key_columns++;
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index d08b95ae00f..723748debde 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -268,7 +268,9 @@ public:
// TODO(yingchun): better to make constructor as private to avoid
// manually init members incorrectly, and define a new function like
// void create_from_pb(const TabletSchemaPB& schema, TabletSchema*
tablet_schema).
- TabletSchema() = default;
+ TabletSchema();
+ virtual ~TabletSchema();
+
void init_from_pb(const TabletSchemaPB& schema, bool
ignore_extracted_columns = false);
// Notice: Use deterministic way to serialize protobuf,
// since serialize Map in protobuf may could lead to un-deterministic by
default
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
