This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2783267599 [feature-wip](unique-key-merge-on-write) update bitmap
after compaction, DSIP-018 (#11289)
2783267599 is described below
commit 2783267599cfc253ef770a8c71d4edee0d644eed
Author: Xin Liao <[email protected]>
AuthorDate: Sat Jul 30 19:50:47 2022 +0800
[feature-wip](unique-key-merge-on-write) update bitmap after compaction,
DSIP-018 (#11289)
---
be/src/olap/compaction.cpp | 8 ++++++++
be/src/olap/merger.cpp | 1 +
be/src/olap/tablet_meta.cpp | 40 ++++++++++++++++++++++++++++++++++++++++
be/src/olap/tablet_meta.h | 4 ++++
4 files changed, 53 insertions(+)
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index a456370f85..9a0e50d591 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -252,6 +252,14 @@ Status Compaction::modify_rowsets() {
std::vector<RowsetSharedPtr> output_rowsets;
output_rowsets.push_back(_output_rowset);
std::lock_guard<std::shared_mutex> wrlock(_tablet->get_header_lock());
+
+ // update dst rowset delete bitmap
+ if (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
+ _tablet->enable_unique_key_merge_on_write()) {
+ _tablet->tablet_meta()->update_delete_bitmap(_input_rowsets,
_output_rs_writer->version(),
+ _rowid_conversion);
+ }
+
RETURN_NOT_OK(_tablet->modify_rowsets(output_rowsets, _input_rowsets,
true));
_tablet->save_meta();
return Status::OK();
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index 26ac92e1ff..e6f769c6c5 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -103,6 +103,7 @@ Status Merger::vmerge_rowsets(TabletSharedPtr tablet,
ReaderType reader_type,
reader_params.rs_readers = src_rowset_readers;
reader_params.version = dst_rowset_writer->version();
reader_params.tablet_schema = cur_tablet_schema;
+ reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap();
if (stats_output && stats_output->rowid_conversion) {
reader_params.record_rowids = true;
}
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 2b6ccc40b5..06f04a4578 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -670,6 +670,8 @@ void TabletMeta::delete_stale_rs_meta_by_version(const
Version& version) {
while (it != _stale_rs_metas.end()) {
if ((*it)->version() == version) {
it = _stale_rs_metas.erase(it);
+ // remove rowset delete bitmap
+ delete_bitmap().remove({(*it)->rowset_id(), 0, 0},
{(*it)->rowset_id(), UINT32_MAX, 0});
} else {
it++;
}
@@ -755,6 +757,44 @@ Status TabletMeta::set_partition_id(int64_t partition_id) {
return Status::OK();
}
+// We take a delete bitmap's snapshot of origin rowset at the beginning of
+// compaction, some keys of origin rowsets might be deleted during compaction,
+// but exist in dest rowset. so we need to update the bitmap of dest rowset
+// after compaction.
+// ANNT: should take a tablet lock before calling the function
+void TabletMeta::update_delete_bitmap(const std::vector<RowsetSharedPtr>&
input_rowsets,
+ const Version& version,
+ const RowIdConversion& rowid_conversion)
{
+ RowLocation src;
+ RowLocation dst;
+ DeleteBitmap output_rowset_delete_bitmap(_tablet_id);
+ for (auto& rowset : input_rowsets) {
+ src.rowset_id = rowset->rowset_id();
+ for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) {
+ src.segment_id = seg_id;
+ DeleteBitmap upper_map(_tablet_id);
+ delete_bitmap().subset({rowset->rowset_id(), seg_id,
version.second},
+ {rowset->rowset_id(), seg_id, INT64_MAX},
&upper_map);
+ // traverse all versions and convert rowid
+ for (auto iter = upper_map.delete_bitmap.begin(); iter !=
upper_map.delete_bitmap.end();
+ ++iter) {
+ auto cur_version = std::get<2>(iter->first);
+ for (auto index = iter->second.begin(); index !=
iter->second.end(); ++index) {
+ src.row_id = *index;
+ if (rowid_conversion.get(src, &dst) != 0) {
+ LOG(WARNING) << "Can't find rowid, may be deleted by
the delete_handler.";
+ continue;
+ }
+ output_rowset_delete_bitmap.add({dst.rowset_id,
dst.segment_id, cur_version},
+ dst.row_id);
+ }
+ }
+ }
+ }
+ // update output rowset delete bitmap
+ delete_bitmap().merge(output_rowset_delete_bitmap);
+}
+
bool operator==(const TabletMeta& a, const TabletMeta& b) {
if (a._table_id != b._table_id) return false;
if (a._partition_id != b._partition_id) return false;
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index ec3ad1f60f..9e589e9fdd 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -29,6 +29,7 @@
#include "olap/delete_handler.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
+#include "olap/rowid_conversion.h"
#include "olap/rowset/rowset.h"
#include "olap/rowset/rowset_meta.h"
#include "olap/tablet_schema.h"
@@ -205,6 +206,9 @@ public:
bool enable_unique_key_merge_on_write() const { return
_enable_unique_key_merge_on_write; }
+ void update_delete_bitmap(const std::vector<RowsetSharedPtr>&
input_rowsets,
+ const Version& version, const RowIdConversion&
rowid_conversion);
+
private:
Status _save_meta(DataDir* data_dir);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
