This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2a9e748073 [enhancement](merge-on-write) do compaction with merge on
read (#16799)
2a9e748073 is described below
commit 2a9e7480733d8e908c8045eb73f9a99a677a0ea9
Author: Xin Liao <[email protected]>
AuthorDate: Thu Feb 16 19:20:15 2023 +0800
[enhancement](merge-on-write) do compaction with merge on read (#16799)
To avoid data irrecoverable due to delete bitmap calculation error，do
compaction with merge on read. Through this way ，even if the delete bitmap
calculation is wrong, the data can be recovered by full compaction.
---
be/src/olap/compaction.cpp | 35 ++++++++++++++++++++++++++---------
be/src/olap/merger.cpp | 7 -------
be/src/olap/tablet.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++
be/src/olap/tablet.h | 5 +++++
be/src/olap/tablet_meta.cpp | 38 --------------------------------------
be/src/olap/tablet_meta.h | 3 ---
6 files changed, 74 insertions(+), 57 deletions(-)
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index b7fedd2203..49526b6382 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -413,19 +413,36 @@ Status Compaction::construct_input_rowset_readers() {
Status Compaction::modify_rowsets() {
std::vector<RowsetSharedPtr> output_rowsets;
output_rowsets.push_back(_output_rowset);
- {
- std::lock_guard<std::mutex> wrlock_(_tablet->get_rowset_update_lock());
- std::lock_guard<std::shared_mutex> wrlock(_tablet->get_header_lock());
- // update dst rowset delete bitmap
- if (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
- _tablet->enable_unique_key_merge_on_write()) {
- _tablet->tablet_meta()->update_delete_bitmap(
- _input_rowsets, _output_rs_writer->version(),
_rowid_conversion);
+ if (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
+ _tablet->enable_unique_key_merge_on_write()) {
+ Version version = _tablet->max_version();
+ DeleteBitmap output_rowset_delete_bitmap(_tablet->tablet_id());
+ // Convert the delete bitmap of the input rowsets to output rowset.
+ // New loads are not blocked, so some keys of input rowsets might
+ // be deleted during the time. We need to deal with delete bitmap
+ // of incremental data later.
+ _tablet->calc_compaction_output_rowset_delete_bitmap(_input_rowsets,
_rowid_conversion, 0,
+ version.second +
1,
+
&output_rowset_delete_bitmap);
+ {
+ std::lock_guard<std::mutex>
wrlock_(_tablet->get_rowset_update_lock());
+ std::lock_guard<std::shared_mutex>
wrlock(_tablet->get_header_lock());
+
+ // Convert the delete bitmap of the input rowsets to output rowset
for
+ // incremental data.
+
_tablet->calc_compaction_output_rowset_delete_bitmap(_input_rowsets,
_rowid_conversion,
+
version.second, UINT64_MAX,
+
&output_rowset_delete_bitmap);
+
+ _tablet->merge_delete_bitmap(output_rowset_delete_bitmap);
+ RETURN_NOT_OK(_tablet->modify_rowsets(output_rowsets,
_input_rowsets, true));
}
-
+ } else {
+ std::lock_guard<std::shared_mutex> wrlock(_tablet->get_header_lock());
RETURN_NOT_OK(_tablet->modify_rowsets(output_rowsets, _input_rowsets,
true));
}
+
{
std::shared_lock rlock(_tablet->get_header_lock());
_tablet->save_meta();
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index b350882e4b..7e2a620d95 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -57,9 +57,6 @@ Status Merger::vmerge_rowsets(TabletSharedPtr tablet,
ReaderType reader_type,
merge_tablet_schema->merge_dropped_columns(tablet->tablet_schema(del_pred_rs->version()));
}
reader_params.tablet_schema = merge_tablet_schema;
- if (tablet->enable_unique_key_merge_on_write()) {
- reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap();
- }
if (stats_output && stats_output->rowid_conversion) {
reader_params.record_rowids = true;
@@ -193,10 +190,6 @@ Status Merger::vertical_compact_one_group(
}
reader_params.tablet_schema = merge_tablet_schema;
- if (tablet->enable_unique_key_merge_on_write()) {
- reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap();
- }
-
reader_params.return_columns = column_group;
reader_params.origin_return_columns = &reader_params.return_columns;
RETURN_NOT_OK(reader.init(reader_params));
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index f7b36a2d40..f0ea777713 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2542,6 +2542,49 @@ Status Tablet::update_delete_bitmap(const
RowsetSharedPtr& rowset, DeleteBitmapP
return Status::OK();
}
+void Tablet::calc_compaction_output_rowset_delete_bitmap(
+ const std::vector<RowsetSharedPtr>& input_rowsets, const
RowIdConversion& rowid_conversion,
+ uint64_t start_version, uint64_t end_version, DeleteBitmap*
output_rowset_delete_bitmap) {
+ RowLocation src;
+ RowLocation dst;
+ for (auto& rowset : input_rowsets) {
+ src.rowset_id = rowset->rowset_id();
+ for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) {
+ src.segment_id = seg_id;
+ DeleteBitmap subset_map(tablet_id());
+ _tablet_meta->delete_bitmap().subset({rowset->rowset_id(), seg_id,
start_version},
+ {rowset->rowset_id(), seg_id,
end_version},
+ &subset_map);
+ // traverse all versions and convert rowid
+ for (auto iter = subset_map.delete_bitmap.begin();
+ iter != subset_map.delete_bitmap.end(); ++iter) {
+ auto cur_version = std::get<2>(iter->first);
+ for (auto index = iter->second.begin(); index !=
iter->second.end(); ++index) {
+ src.row_id = *index;
+ if (rowid_conversion.get(src, &dst) != 0) {
+ VLOG_CRITICAL << "Can't find rowid, may be deleted by
the delete_handler, "
+ << " src loaction: |" << src.rowset_id
<< "|"
+ << src.segment_id << "|" << src.row_id
+ << " version: " << cur_version;
+ continue;
+ }
+ VLOG_DEBUG << "calc_compaction_output_rowset_delete_bitmap
dst location: |"
+ << dst.rowset_id << "|" << dst.segment_id <<
"|" << dst.row_id
+ << " src location: |" << src.rowset_id << "|"
<< src.segment_id
+ << "|" << src.row_id << " start version: " <<
start_version
+ << "end version" << end_version;
+ output_rowset_delete_bitmap->add({dst.rowset_id,
dst.segment_id, cur_version},
+ dst.row_id);
+ }
+ }
+ }
+ }
+}
+
+void Tablet::merge_delete_bitmap(const DeleteBitmap& delete_bitmap) {
+ _tablet_meta->delete_bitmap().merge(delete_bitmap);
+}
+
RowsetIdUnorderedSet Tablet::all_rs_id(int64_t max_version) const {
RowsetIdUnorderedSet rowset_ids;
for (const auto& rs_it : _rs_version_map) {
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index f40450b947..ed8ada05c7 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -363,6 +363,11 @@ public:
Status update_delete_bitmap_without_lock(const RowsetSharedPtr& rowset);
Status update_delete_bitmap(const RowsetSharedPtr& rowset, DeleteBitmapPtr
delete_bitmap,
const RowsetIdUnorderedSet& pre_rowset_ids);
+ void calc_compaction_output_rowset_delete_bitmap(
+ const std::vector<RowsetSharedPtr>& input_rowsets,
+ const RowIdConversion& rowid_conversion, uint64_t start_version,
uint64_t end_version,
+ DeleteBitmap* output_rowset_delete_bitmap);
+ void merge_delete_bitmap(const DeleteBitmap& delete_bitmap);
RowsetIdUnorderedSet all_rs_id(int64_t max_version) const;
bool check_all_rowset_segment();
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 1fc57239d1..c8bcd4818a 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -834,44 +834,6 @@ Status TabletMeta::set_partition_id(int64_t partition_id) {
return Status::OK();
}
-// We take a delete bitmap's snapshot of origin rowset at the beginning of
-// compaction, some keys of origin rowsets might be deleted during compaction,
-// but exist in dest rowset. so we need to update the bitmap of dest rowset
-// after compaction.
-// ANNT: should take a tablet lock before calling the function
-void TabletMeta::update_delete_bitmap(const std::vector<RowsetSharedPtr>&
input_rowsets,
- const Version& version,
- const RowIdConversion& rowid_conversion)
{
- RowLocation src;
- RowLocation dst;
- DeleteBitmap output_rowset_delete_bitmap(_tablet_id);
- for (auto& rowset : input_rowsets) {
- src.rowset_id = rowset->rowset_id();
- for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) {
- src.segment_id = seg_id;
- DeleteBitmap upper_map(_tablet_id);
- delete_bitmap().subset({rowset->rowset_id(), seg_id,
version.second},
- {rowset->rowset_id(), seg_id, INT64_MAX},
&upper_map);
- // traverse all versions and convert rowid
- for (auto iter = upper_map.delete_bitmap.begin(); iter !=
upper_map.delete_bitmap.end();
- ++iter) {
- auto cur_version = std::get<2>(iter->first);
- for (auto index = iter->second.begin(); index !=
iter->second.end(); ++index) {
- src.row_id = *index;
- if (rowid_conversion.get(src, &dst) != 0) {
- VLOG_CRITICAL << "Can't find rowid, may be deleted by
the delete_handler.";
- continue;
- }
- output_rowset_delete_bitmap.add({dst.rowset_id,
dst.segment_id, cur_version},
- dst.row_id);
- }
- }
- }
- }
- // update output rowset delete bitmap
- delete_bitmap().merge(output_rowset_delete_bitmap);
-}
-
bool operator==(const TabletMeta& a, const TabletMeta& b) {
if (a._table_id != b._table_id) return false;
if (a._partition_id != b._partition_id) return false;
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 96adc4baad..0fa3cffe26 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -205,9 +205,6 @@ public:
bool enable_unique_key_merge_on_write() const { return
_enable_unique_key_merge_on_write; }
- void update_delete_bitmap(const std::vector<RowsetSharedPtr>&
input_rowsets,
- const Version& version, const RowIdConversion&
rowid_conversion);
-
private:
Status _save_meta(DataDir* data_dir);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
