This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 30934ff0c9e [branch-2.0-fix](partial update) Don't use delete bitmap
to mark delete for rows with delete sign when sequence column doesn't exist
(#28800)
30934ff0c9e is described below
commit 30934ff0c9eb2965a5891463801a288e87b5b7af
Author: bobhan1 <[email protected]>
AuthorDate: Thu Dec 21 20:15:24 2023 +0800
[branch-2.0-fix](partial update) Don't use delete bitmap to mark delete for
rows with delete sign when sequence column doesn't exist (#28800)
---
be/src/olap/rowset/segment_v2/segment_writer.cpp | 30 -----
be/src/olap/tablet.cpp | 8 --
be/src/olap/tablet_meta.h | 1 -
.../test_partial_update_delete_sign.out | 2 -
.../test_delete_sign_delete_bitmap.out | 54 --------
.../test_delete_sign_delete_bitmap.groovy | 140 ++++++++++-----------
6 files changed, 70 insertions(+), 165 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 02ea943818d..29ec7e02bc5 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -444,13 +444,6 @@ Status
SegmentWriter::append_block_with_partial_content(const vectorized::Block*
// mark key with delete sign as deleted.
bool have_delete_sign =
(delete_sign_column_data != nullptr &&
delete_sign_column_data[block_pos] != 0);
- if (have_delete_sign && !_tablet_schema->has_sequence_col() &&
!have_input_seq_column) {
- // we can directly use delete bitmap to mark the rows with delete
sign as deleted
- // if sequence column doesn't exist to eliminate reading delete
sign columns in later reads
- _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id,
_segment_id,
-
DeleteBitmap::TEMP_VERSION_FOR_DELETE_SIGN},
- segment_pos);
- }
RowLocation loc;
// save rowset shared ptr so this rowset wouldn't delete
@@ -699,29 +692,6 @@ Status SegmentWriter::append_block(const
vectorized::Block* block, size_t row_po
_serialize_block_to_row_column(*const_cast<vectorized::Block*>(block));
}
- if (_opts.write_type == DataWriteType::TYPE_DIRECT &&
_opts.enable_unique_key_merge_on_write &&
- !_tablet_schema->has_sequence_col() &&
_tablet_schema->delete_sign_idx() != -1) {
- const vectorized::ColumnWithTypeAndName& delete_sign_column =
- block->get_by_position(_tablet_schema->delete_sign_idx());
- auto& delete_sign_col =
- reinterpret_cast<const
vectorized::ColumnInt8&>(*(delete_sign_column.column));
- if (delete_sign_col.size() >= row_pos + num_rows) {
- const vectorized::Int8* delete_sign_column_data =
delete_sign_col.get_data().data();
- uint32_t segment_start_pos =
-
_column_writers[_tablet_schema->delete_sign_idx()]->get_next_rowid();
- for (size_t block_pos = row_pos, seg_pos = segment_start_pos;
- seg_pos < segment_start_pos + num_rows; block_pos++,
seg_pos++) {
- // we can directly use delete bitmap to mark the rows with
delete sign as deleted
- // if sequence column doesn't exist to eliminate reading
delete sign columns in later reads
- if (delete_sign_column_data[block_pos]) {
-
_mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id,
-
DeleteBitmap::TEMP_VERSION_FOR_DELETE_SIGN},
- seg_pos);
- }
- }
- }
- }
-
_olap_data_convertor->set_source_content(block, row_pos, num_rows);
// find all row pos for short key indexes
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 4c1ebd721de..611e32828bb 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -3075,14 +3075,6 @@ Status
Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
continue;
}
if (is_partial_update && rowset_writer != nullptr) {
- if (delete_bitmap->contains(
- {rowset_id, seg->id(),
DeleteBitmap::TEMP_VERSION_FOR_DELETE_SIGN},
- row_id)) {
- LOG(INFO)
- << "DEBUG: skip a delete sign column while
calc_segment_delete_bitmap "
- << "processing confict for partial update";
- continue;
- }
// In publish version, record rows to be deleted for
concurrent update
// For example, if version 5 and 6 update a row, but version 6
only see
// version 4 when write, and when publish version, version 5's
value will
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 11dc3532514..0ef058760ea 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -343,7 +343,6 @@ public:
// tablet's delete bitmap we can use arbitary version number in BitmapKey.
Here we define some version numbers
// for specific usage during this periods to avoid conflicts
constexpr static inline uint64_t TEMP_VERSION_COMMON = 0;
- constexpr static inline uint64_t TEMP_VERSION_FOR_DELETE_SIGN = 1;
/**
*
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
index f14434b2f9f..8d3e69bbe26 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
@@ -27,7 +27,6 @@
-- !2 --
-- !3 --
-1 2 \N
-- !1 --
1 1 1 1
@@ -65,7 +64,6 @@
-- !2 --
-- !3 --
-1 2 \N
-- !1 --
1 1 1 1
diff --git
a/regression-test/data/unique_with_mow_p0/test_delete_sign_delete_bitmap.out
b/regression-test/data/unique_with_mow_p0/test_delete_sign_delete_bitmap.out
deleted file mode 100644
index 687aeab54a1..00000000000
--- a/regression-test/data/unique_with_mow_p0/test_delete_sign_delete_bitmap.out
+++ /dev/null
@@ -1,54 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !sql --
-1 1 1 1 1
-2 2 2 2 2
-3 3 3 3 3
-4 4 4 4 4
-5 5 5 5 5
-
--- !after_delete --
-2 2 2 2 2
-4 4 4 4 4
-
--- !1 --
-1 1 1 1 1 0
-1 1 1 1 1 1
-2 2 2 2 2 0
-3 3 3 3 3 0
-3 3 3 3 3 1
-4 4 4 4 4 0
-5 5 5 5 5 0
-5 5 5 5 5 1
-
--- !2 --
-2 2 2 2 2 0
-4 4 4 4 4 0
-
--- !sql --
-1 1 1 1 1
-2 2 2 2 2
-3 3 3 3 3
-4 4 4 4 4
-5 5 5 5 5
-
--- !after_delete --
-2 2 2 2 2
-4 4 4 4 4
-
--- !1 --
-1 1 1 1 1 0
-1 1 1 1 1 1
-2 2 2 2 2 0
-3 3 3 3 3 0
-3 3 3 3 3 1
-4 4 4 4 4 0
-5 5 5 5 5 0
-5 5 5 5 5 1
-
--- !2 --
-1 1 1 1 1 1
-2 2 2 2 2 0
-3 3 3 3 3 1
-4 4 4 4 4 0
-5 5 5 5 5 1
-
diff --git
a/regression-test/suites/unique_with_mow_p0/test_delete_sign_delete_bitmap.groovy
b/regression-test/suites/unique_with_mow_p0/test_delete_sign_delete_bitmap.groovy
index f9b89c1eea7..be6324d2ec7 100644
---
a/regression-test/suites/unique_with_mow_p0/test_delete_sign_delete_bitmap.groovy
+++
b/regression-test/suites/unique_with_mow_p0/test_delete_sign_delete_bitmap.groovy
@@ -17,80 +17,80 @@
suite('test_delete_sign_delete_bitmap') {
- def tableName1 = "test_delete_sign_delete_bitmap1"
- sql "DROP TABLE IF EXISTS ${tableName1};"
- sql """ CREATE TABLE IF NOT EXISTS ${tableName1} (
- `k1` int NOT NULL,
- `c1` int,
- `c2` int,
- `c3` int,
- `c4` int
- )UNIQUE KEY(k1)
- DISTRIBUTED BY HASH(k1) BUCKETS 1
- PROPERTIES (
- "enable_unique_key_merge_on_write" = "true",
- "disable_auto_compaction" = "true",
- "replication_num" = "1"
- );"""
+ // def tableName1 = "test_delete_sign_delete_bitmap1"
+ // sql "DROP TABLE IF EXISTS ${tableName1};"
+ // sql """ CREATE TABLE IF NOT EXISTS ${tableName1} (
+ // `k1` int NOT NULL,
+ // `c1` int,
+ // `c2` int,
+ // `c3` int,
+ // `c4` int
+ // )UNIQUE KEY(k1)
+ // DISTRIBUTED BY HASH(k1) BUCKETS 1
+ // PROPERTIES (
+ // "enable_unique_key_merge_on_write" = "true",
+ // "disable_auto_compaction" = "true",
+ // "replication_num" = "1"
+ // );"""
- sql "insert into ${tableName1}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
- qt_sql "select * from ${tableName1} order by k1,c1,c2,c3,c4;"
- // sql "insert into ${tableName1}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__)
select k1,c1,c2,c3,c4,1 from ${tableName1} where k1 in (1,3,5);"
- sql """insert into ${tableName1}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__)
values(1,1,1,1,1,1),(3,3,3,3,3,1),(5,5,5,5,5,1);"""
- sql "sync"
- qt_after_delete "select * from ${tableName1} order by k1,c1,c2,c3,c4;"
- sql "set skip_delete_sign=true;"
- sql "set skip_storage_engine_merge=true;"
- sql "set skip_delete_bitmap=true;"
- sql "sync"
- // skip_delete_bitmap=true, skip_delete_sign=true
- qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ // sql "insert into ${tableName1}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
+ // qt_sql "select * from ${tableName1} order by k1,c1,c2,c3,c4;"
+ // // sql "insert into ${tableName1}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__)
select k1,c1,c2,c3,c4,1 from ${tableName1} where k1 in (1,3,5);"
+ // sql """insert into ${tableName1}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__)
values(1,1,1,1,1,1),(3,3,3,3,3,1),(5,5,5,5,5,1);"""
+ // sql "sync"
+ // qt_after_delete "select * from ${tableName1} order by k1,c1,c2,c3,c4;"
+ // sql "set skip_delete_sign=true;"
+ // sql "set skip_storage_engine_merge=true;"
+ // sql "set skip_delete_bitmap=true;"
+ // sql "sync"
+ // // skip_delete_bitmap=true, skip_delete_sign=true
+ // qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1}
order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
- sql "set skip_delete_sign=true;"
- sql "set skip_delete_bitmap=false;"
- sql "sync"
- // skip_delete_bitmap=false, skip_delete_sign=true
- qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
- sql "drop table if exists ${tableName1};"
+ // sql "set skip_delete_sign=true;"
+ // sql "set skip_delete_bitmap=false;"
+ // sql "sync"
+ // // skip_delete_bitmap=false, skip_delete_sign=true
+ // qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1}
order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ // sql "drop table if exists ${tableName1};"
- sql "set skip_delete_sign=false;"
- sql "set skip_storage_engine_merge=false;"
- sql "set skip_delete_bitmap=false;"
- sql "sync"
- def tableName2 = "test_delete_sign_delete_bitmap2"
- sql "DROP TABLE IF EXISTS ${tableName2};"
- sql """ CREATE TABLE IF NOT EXISTS ${tableName2} (
- `k1` int NOT NULL,
- `c1` int,
- `c2` int,
- `c3` int,
- `c4` int
- )UNIQUE KEY(k1)
- DISTRIBUTED BY HASH(k1) BUCKETS 1
- PROPERTIES (
- "enable_unique_key_merge_on_write" = "true",
- "disable_auto_compaction" = "true",
- "replication_num" = "1",
- "function_column.sequence_col" = 'c4'
- );"""
+ // sql "set skip_delete_sign=false;"
+ // sql "set skip_storage_engine_merge=false;"
+ // sql "set skip_delete_bitmap=false;"
+ // sql "sync"
+ // def tableName2 = "test_delete_sign_delete_bitmap2"
+ // sql "DROP TABLE IF EXISTS ${tableName2};"
+ // sql """ CREATE TABLE IF NOT EXISTS ${tableName2} (
+ // `k1` int NOT NULL,
+ // `c1` int,
+ // `c2` int,
+ // `c3` int,
+ // `c4` int
+ // )UNIQUE KEY(k1)
+ // DISTRIBUTED BY HASH(k1) BUCKETS 1
+ // PROPERTIES (
+ // "enable_unique_key_merge_on_write" = "true",
+ // "disable_auto_compaction" = "true",
+ // "replication_num" = "1",
+ // "function_column.sequence_col" = 'c4'
+ // );"""
- sql "insert into ${tableName2}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
- qt_sql "select * from ${tableName2} order by k1,c1,c2,c3,c4;"
- sql """insert into ${tableName2}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__)
values(1,1,1,1,1,1),(3,3,3,3,3,1),(5,5,5,5,5,1);"""
- sql "sync"
- qt_after_delete "select * from ${tableName2} order by k1,c1,c2,c3,c4;"
- sql "set skip_delete_sign=true;"
- sql "set skip_storage_engine_merge=true;"
- sql "set skip_delete_bitmap=true;"
- sql "sync"
- // skip_delete_bitmap=true, skip_delete_sign=true
- qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ // sql "insert into ${tableName2}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
+ // qt_sql "select * from ${tableName2} order by k1,c1,c2,c3,c4;"
+ // sql """insert into ${tableName2}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__)
values(1,1,1,1,1,1),(3,3,3,3,3,1),(5,5,5,5,5,1);"""
+ // sql "sync"
+ // qt_after_delete "select * from ${tableName2} order by k1,c1,c2,c3,c4;"
+ // sql "set skip_delete_sign=true;"
+ // sql "set skip_storage_engine_merge=true;"
+ // sql "set skip_delete_bitmap=true;"
+ // sql "sync"
+ // // skip_delete_bitmap=true, skip_delete_sign=true
+ // qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2}
order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
- sql "set skip_delete_sign=true;"
- sql "set skip_delete_bitmap=false;"
- sql "sync"
- // skip_delete_bitmap=false, skip_delete_sign=true
- qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order
by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
- sql "drop table if exists ${tableName2};"
+ // sql "set skip_delete_sign=true;"
+ // sql "set skip_delete_bitmap=false;"
+ // sql "sync"
+ // // skip_delete_bitmap=false, skip_delete_sign=true
+ // qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2}
order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
+ // sql "drop table if exists ${tableName2};"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
