This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2d10300 [Bug] Fix schema change fail as memory allocation on row
block sorting (#6932)
2d10300 is described below
commit 2d1030054734435f05476144d81ee0dd9e1a0864
Author: GoGoWen <[email protected]>
AuthorDate: Tue Nov 2 16:33:38 2021 +0800
[Bug] Fix schema change fail as memory allocation on row block sorting
(#6932)
schema change fail as memory allocation fail on row block sorting.
however, it should do internal sorting first before schema change fail
as memory allocation fail on row block sorting in case there are enough
memory after internal sorting.
---
be/src/olap/schema_change.cpp | 22 ++++++++++++++++++++++
be/src/olap/schema_change.h | 1 +
2 files changed, 23 insertions(+)
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 8a1eb34..8b3fc49 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -56,6 +56,9 @@ class RowBlockSorter {
public:
explicit RowBlockSorter(RowBlockAllocator* allocator);
virtual ~RowBlockSorter();
+ size_t num_rows() {
+ return _swap_row_block->capacity();
+ }
bool sort(RowBlock** row_block);
@@ -800,6 +803,15 @@ void RowBlockAllocator::release(RowBlock* row_block) {
delete row_block;
}
+bool RowBlockAllocator::is_memory_enough_for_sorting(size_t num_rows, size_t
allocated_rows){
+ if (num_rows <= allocated_rows) {
+ return true;
+ }
+ size_t row_block_size = _row_len * (num_rows - allocated_rows);
+ return _mem_tracker->consumption() + row_block_size < _memory_limitation;
+}
+
+
RowBlockMerger::RowBlockMerger(TabletSharedPtr tablet) : _tablet(tablet) {}
RowBlockMerger::~RowBlockMerger() {}
@@ -1184,6 +1196,16 @@ OLAPStatus
SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader,
true)) {
LOG(WARNING) << "failed to allocate RowBlock.";
return OLAP_ERR_INPUT_PARAMETER_ERROR;
+ } else {
+ // do memory check for sorting, in case schema change task fail at
row block sorting because of
+ // not doing internal sorting first
+ if
(!_row_block_allocator->is_memory_enough_for_sorting(ref_row_block->row_block_info().row_num,
+
row_block_sorter.num_rows())) {
+ if (new_row_block != nullptr) {
+ _row_block_allocator->release(new_row_block);
+ new_row_block = nullptr;
+ }
+ }
}
if (new_row_block == nullptr) {
diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h
index 73f784a..e0d42e8 100644
--- a/be/src/olap/schema_change.h
+++ b/be/src/olap/schema_change.h
@@ -81,6 +81,7 @@ public:
OLAPStatus allocate(RowBlock** row_block, size_t num_rows, bool
null_supported);
void release(RowBlock* row_block);
+ bool is_memory_enough_for_sorting(size_t num_rows, size_t allocated_rows);
private:
const TabletSchema& _tablet_schema;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
