This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 807bd672291 [enhancement](memory) add exception check in page builder
to avoid be oom during flush memtable (#35138)
807bd672291 is described below
commit 807bd67229142231e82eced951686e8bc9ff60b0
Author: yiguolei <[email protected]>
AuthorDate: Tue May 21 23:41:11 2024 +0800
[enhancement](memory) add exception check in page builder to avoid be oom
during flush memtable (#35138)
---
be/src/olap/rowset/segment_v2/binary_plain_page.h | 4 +++-
be/src/olap/rowset/segment_v2/binary_prefix_page.cpp | 17 ++++++++++-------
be/src/olap/rowset/segment_v2/bitshuffle_page.h | 4 +++-
be/src/olap/rowset/segment_v2/plain_page.h | 4 +++-
4 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h
b/be/src/olap/rowset/segment_v2/binary_plain_page.h
index 176d5c67f53..cffab96379c 100644
--- a/be/src/olap/rowset/segment_v2/binary_plain_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h
@@ -77,7 +77,9 @@ public:
}
size_t offset = _buffer.size();
_offsets.push_back(offset);
- _buffer.append(src->data, src->size);
+ // This may need a large memory, should return error if could not
allocated
+ // successfully, to avoid BE OOM.
+ RETURN_IF_CATCH_EXCEPTION(_buffer.append(src->data, src->size));
_last_value_size = src->size;
_size_estimate += src->size;
diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
index 183e7bb0853..9d1ecdb9470 100644
--- a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
@@ -71,13 +71,16 @@ Status BinaryPrefixPageBuilder::add(const uint8_t* vals,
size_t* add_count) {
}
}
int non_share_len = entry_len - share_len;
-
- put_varint32(&_buffer, share_len);
- put_varint32(&_buffer, non_share_len);
- _buffer.append(entry + share_len, non_share_len);
-
- _last_entry.clear();
- _last_entry.append(entry, entry_len);
+ // This may need a large memory, should return error if could not
allocated
+ // successfully, to avoid BE OOM.
+ RETURN_IF_CATCH_EXCEPTION({
+ put_varint32(&_buffer, share_len);
+ put_varint32(&_buffer, non_share_len);
+ _buffer.append(entry + share_len, non_share_len);
+
+ _last_entry.clear();
+ _last_entry.append(entry, entry_len);
+ });
++_count;
}
diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_page.h
b/be/src/olap/rowset/segment_v2/bitshuffle_page.h
index 186d8c0a9a0..4e206a7fce0 100644
--- a/be/src/olap/rowset/segment_v2/bitshuffle_page.h
+++ b/be/src/olap/rowset/segment_v2/bitshuffle_page.h
@@ -112,7 +112,9 @@ public:
int to_add = std::min<int>(_remain_element_capacity, *count);
int to_add_size = to_add * SIZE_OF_TYPE;
size_t orig_size = _data.size();
- _data.resize(orig_size + to_add_size);
+ // This may need a large memory, should return error if could not
allocated
+ // successfully, to avoid BE OOM.
+ RETURN_IF_CATCH_EXCEPTION(_data.resize(orig_size + to_add_size));
_count += to_add;
_remain_element_capacity -= to_add;
// return added number through count
diff --git a/be/src/olap/rowset/segment_v2/plain_page.h
b/be/src/olap/rowset/segment_v2/plain_page.h
index 29cec0e8795..cb9236ee315 100644
--- a/be/src/olap/rowset/segment_v2/plain_page.h
+++ b/be/src/olap/rowset/segment_v2/plain_page.h
@@ -48,7 +48,9 @@ public:
return Status::OK();
}
size_t old_size = _buffer.size();
- _buffer.resize(old_size + *count * SIZE_OF_TYPE);
+ // This may need a large memory, should return error if could not
allocated
+ // successfully, to avoid BE OOM.
+ RETURN_IF_CATCH_EXCEPTION(_buffer.resize(old_size + *count *
SIZE_OF_TYPE));
memcpy(&_buffer[old_size], vals, *count * SIZE_OF_TYPE);
_count += *count;
return Status::OK();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
