This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit c99577568d9fabd740a4550e35762505e8f68ed2 Author: Xin Liao <[email protected]> AuthorDate: Thu Feb 16 19:20:15 2023 +0800 [enhancement](merge-on-write) do compaction with merge on read (#16799) To avoid data irrecoverable due to delete bitmap calculation error，do compaction with merge on read. Through this way ，even if the delete bitmap calculation is wrong, the data can be recovered by full compaction. --- be/src/olap/compaction.cpp | 35 ++++++++++++++++++++++++++--------- be/src/olap/merger.cpp | 7 ------- be/src/olap/tablet.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++ be/src/olap/tablet.h | 5 +++++ be/src/olap/tablet_meta.cpp | 38 -------------------------------------- be/src/olap/tablet_meta.h | 3 --- 6 files changed, 74 insertions(+), 57 deletions(-) diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index c7029e9722..e56738b3e1 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -434,19 +434,36 @@ Status Compaction::construct_input_rowset_readers() { Status Compaction::modify_rowsets() { std::vector<RowsetSharedPtr> output_rowsets; output_rowsets.push_back(_output_rowset); - { - std::lock_guard<std::mutex> wrlock_(_tablet->get_rowset_update_lock()); - std::lock_guard<std::shared_mutex> wrlock(_tablet->get_header_lock()); - // update dst rowset delete bitmap - if (_tablet->keys_type() == KeysType::UNIQUE_KEYS && - _tablet->enable_unique_key_merge_on_write()) { - _tablet->tablet_meta()->update_delete_bitmap( - _input_rowsets, _output_rs_writer->version(), _rowid_conversion); + if (_tablet->keys_type() == KeysType::UNIQUE_KEYS && + _tablet->enable_unique_key_merge_on_write()) { + Version version = _tablet->max_version(); + DeleteBitmap output_rowset_delete_bitmap(_tablet->tablet_id()); + // Convert the delete bitmap of the input rowsets to output rowset. + // New loads are not blocked, so some keys of input rowsets might + // be deleted during the time. We need to deal with delete bitmap + // of incremental data later. + _tablet->calc_compaction_output_rowset_delete_bitmap(_input_rowsets, _rowid_conversion, 0, + version.second + 1, + &output_rowset_delete_bitmap); + { + std::lock_guard<std::mutex> wrlock_(_tablet->get_rowset_update_lock()); + std::lock_guard<std::shared_mutex> wrlock(_tablet->get_header_lock()); + + // Convert the delete bitmap of the input rowsets to output rowset for + // incremental data. + _tablet->calc_compaction_output_rowset_delete_bitmap(_input_rowsets, _rowid_conversion, + version.second, UINT64_MAX, + &output_rowset_delete_bitmap); + + _tablet->merge_delete_bitmap(output_rowset_delete_bitmap); + RETURN_NOT_OK(_tablet->modify_rowsets(output_rowsets, _input_rowsets, true)); } - + } else { + std::lock_guard<std::shared_mutex> wrlock(_tablet->get_header_lock()); RETURN_NOT_OK(_tablet->modify_rowsets(output_rowsets, _input_rowsets, true)); } + { std::shared_lock rlock(_tablet->get_header_lock()); _tablet->save_meta(); diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index 58d9df0158..ded6060b3d 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -134,9 +134,6 @@ Status Merger::vmerge_rowsets(TabletSharedPtr tablet, ReaderType reader_type, merge_tablet_schema->merge_dropped_columns(tablet->tablet_schema(del_pred_rs->version())); } reader_params.tablet_schema = merge_tablet_schema; - if (tablet->enable_unique_key_merge_on_write()) { - reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap(); - } if (stats_output && stats_output->rowid_conversion) { reader_params.record_rowids = true; @@ -267,10 +264,6 @@ Status Merger::vertical_compact_one_group( } reader_params.tablet_schema = merge_tablet_schema; - if (tablet->enable_unique_key_merge_on_write()) { - reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap(); - } - reader_params.return_columns = column_group; reader_params.origin_return_columns = &reader_params.return_columns; RETURN_NOT_OK(reader.init(reader_params)); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 0834ce87d6..35f82c8317 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2212,6 +2212,49 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, DeleteBitmapP return Status::OK(); } +void Tablet::calc_compaction_output_rowset_delete_bitmap( + const std::vector<RowsetSharedPtr>& input_rowsets, const RowIdConversion& rowid_conversion, + uint64_t start_version, uint64_t end_version, DeleteBitmap* output_rowset_delete_bitmap) { + RowLocation src; + RowLocation dst; + for (auto& rowset : input_rowsets) { + src.rowset_id = rowset->rowset_id(); + for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) { + src.segment_id = seg_id; + DeleteBitmap subset_map(tablet_id()); + _tablet_meta->delete_bitmap().subset({rowset->rowset_id(), seg_id, start_version}, + {rowset->rowset_id(), seg_id, end_version}, + &subset_map); + // traverse all versions and convert rowid + for (auto iter = subset_map.delete_bitmap.begin(); + iter != subset_map.delete_bitmap.end(); ++iter) { + auto cur_version = std::get<2>(iter->first); + for (auto index = iter->second.begin(); index != iter->second.end(); ++index) { + src.row_id = *index; + if (rowid_conversion.get(src, &dst) != 0) { + VLOG_CRITICAL << "Can't find rowid, may be deleted by the delete_handler, " + << " src loaction: |" << src.rowset_id << "|" + << src.segment_id << "|" << src.row_id + << " version: " << cur_version; + continue; + } + VLOG_DEBUG << "calc_compaction_output_rowset_delete_bitmap dst location: |" + << dst.rowset_id << "|" << dst.segment_id << "|" << dst.row_id + << " src location: |" << src.rowset_id << "|" << src.segment_id + << "|" << src.row_id << " start version: " << start_version + << "end version" << end_version; + output_rowset_delete_bitmap->add({dst.rowset_id, dst.segment_id, cur_version}, + dst.row_id); + } + } + } + } +} + +void Tablet::merge_delete_bitmap(const DeleteBitmap& delete_bitmap) { + _tablet_meta->delete_bitmap().merge(delete_bitmap); +} + RowsetIdUnorderedSet Tablet::all_rs_id(int64_t max_version) const { RowsetIdUnorderedSet rowset_ids; for (const auto& rs_it : _rs_version_map) { diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 4b444f13ec..a9293bfd54 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -353,6 +353,11 @@ public: Status update_delete_bitmap_without_lock(const RowsetSharedPtr& rowset); Status update_delete_bitmap(const RowsetSharedPtr& rowset, DeleteBitmapPtr delete_bitmap, const RowsetIdUnorderedSet& pre_rowset_ids); + void calc_compaction_output_rowset_delete_bitmap( + const std::vector<RowsetSharedPtr>& input_rowsets, + const RowIdConversion& rowid_conversion, uint64_t start_version, uint64_t end_version, + DeleteBitmap* output_rowset_delete_bitmap); + void merge_delete_bitmap(const DeleteBitmap& delete_bitmap); RowsetIdUnorderedSet all_rs_id(int64_t max_version) const; void remove_self_owned_remote_rowsets(); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 010b2217c6..5999c268e2 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -795,44 +795,6 @@ Status TabletMeta::set_partition_id(int64_t partition_id) { return Status::OK(); } -// We take a delete bitmap's snapshot of origin rowset at the beginning of -// compaction, some keys of origin rowsets might be deleted during compaction, -// but exist in dest rowset. so we need to update the bitmap of dest rowset -// after compaction. -// ANNT: should take a tablet lock before calling the function -void TabletMeta::update_delete_bitmap(const std::vector<RowsetSharedPtr>& input_rowsets, - const Version& version, - const RowIdConversion& rowid_conversion) { - RowLocation src; - RowLocation dst; - DeleteBitmap output_rowset_delete_bitmap(_tablet_id); - for (auto& rowset : input_rowsets) { - src.rowset_id = rowset->rowset_id(); - for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) { - src.segment_id = seg_id; - DeleteBitmap upper_map(_tablet_id); - delete_bitmap().subset({rowset->rowset_id(), seg_id, version.second}, - {rowset->rowset_id(), seg_id, INT64_MAX}, &upper_map); - // traverse all versions and convert rowid - for (auto iter = upper_map.delete_bitmap.begin(); iter != upper_map.delete_bitmap.end(); - ++iter) { - auto cur_version = std::get<2>(iter->first); - for (auto index = iter->second.begin(); index != iter->second.end(); ++index) { - src.row_id = *index; - if (rowid_conversion.get(src, &dst) != 0) { - VLOG_CRITICAL << "Can't find rowid, may be deleted by the delete_handler."; - continue; - } - output_rowset_delete_bitmap.add({dst.rowset_id, dst.segment_id, cur_version}, - dst.row_id); - } - } - } - } - // update output rowset delete bitmap - delete_bitmap().merge(output_rowset_delete_bitmap); -} - bool operator==(const TabletMeta& a, const TabletMeta& b) { if (a._table_id != b._table_id) return false; if (a._partition_id != b._partition_id) return false; diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 3655b71602..92284c83aa 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -207,9 +207,6 @@ public: bool enable_unique_key_merge_on_write() const { return _enable_unique_key_merge_on_write; } - void update_delete_bitmap(const std::vector<RowsetSharedPtr>& input_rowsets, - const Version& version, const RowIdConversion& rowid_conversion); - private: Status _save_meta(DataDir* data_dir); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
