This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 87b69cf9ab6 [enhancement](rowset-meta) Remove rowset meta from olap
meta directly when rowsets deleted (#41716)
87b69cf9ab6 is described below
commit 87b69cf9ab662b33731ea397c658001610ced0cf
Author: Siyang Tang <[email protected]>
AuthorDate: Mon Oct 14 21:44:56 2024 +0800
[enhancement](rowset-meta) Remove rowset meta from olap meta directly when
rowsets deleted (#41716)
## Proposed changes
As title.
---
be/src/olap/cold_data_compaction.cpp | 3 ++-
be/src/olap/schema_change.cpp | 2 +-
be/src/olap/tablet.cpp | 21 ++++++++++++---------
be/src/olap/tablet.h | 2 +-
be/src/olap/tablet_meta.cpp | 1 +
5 files changed, 17 insertions(+), 12 deletions(-)
diff --git a/be/src/olap/cold_data_compaction.cpp
b/be/src/olap/cold_data_compaction.cpp
index 3c618199034..54e21d7d7bc 100644
--- a/be/src/olap/cold_data_compaction.cpp
+++ b/be/src/olap/cold_data_compaction.cpp
@@ -28,6 +28,7 @@
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/config.h"
+#include "common/status.h"
#include "io/fs/remote_file_system.h"
#include "olap/compaction.h"
#include "olap/olap_common.h"
@@ -97,7 +98,7 @@ Status ColdDataCompaction::modify_rowsets() {
std::lock_guard wlock(_tablet->get_header_lock());
SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
// Merged cooldowned rowsets MUST NOT be managed by version graph,
they will be reclaimed by `remove_unused_remote_files`.
- tablet()->delete_rowsets(_input_rowsets, false);
+ RETURN_IF_ERROR(tablet()->delete_rowsets(_input_rowsets, false));
tablet()->add_rowsets({_output_rowset});
// TODO(plat1ko): process primary key
_tablet->tablet_meta()->set_cooldown_meta_id(cooldown_meta_id);
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index d40a44fd710..3b77ac19e2e 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -899,7 +899,7 @@ Status SchemaChangeJob::_do_process_alter_tablet(const
TAlterTabletReqV2& reques
}
}
std::vector<RowsetSharedPtr> empty_vec;
- _new_tablet->delete_rowsets(rowsets_to_delete, false);
+ RETURN_IF_ERROR(_new_tablet->delete_rowsets(rowsets_to_delete,
false));
// inherit cumulative_layer_point from base_tablet
// check if new_tablet.ce_point > base_tablet.ce_point?
_new_tablet->set_cumulative_layer_point(-1);
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 1e8518b47dc..9dfb7940dcc 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -424,7 +424,7 @@ Status Tablet::revise_tablet_meta(const
std::vector<RowsetSharedPtr>& to_add,
// error handling
if (!calc_bm_status.ok()) {
if (is_incremental_clone) {
- delete_rowsets(to_add, false);
+ RETURN_IF_ERROR(delete_rowsets(to_add, false));
LOG(WARNING) << "incremental clone on tablet: " << tablet_id() <<
" failed due to "
<< calc_bm_status.msg() << ", revert " <<
to_add.size()
<< " rowsets added before.";
@@ -437,7 +437,7 @@ Status Tablet::revise_tablet_meta(const
std::vector<RowsetSharedPtr>& to_add,
// full clone, calculate delete bitmap succeeded, update rowset
if (!is_incremental_clone) {
- delete_rowsets(to_delete, false);
+ RETURN_IF_ERROR(delete_rowsets(to_delete, false));
add_rowsets(to_add);
// reconstruct from tablet meta
_timestamped_version_tracker.construct_versioned_tracker(_tablet_meta->all_rs_metas());
@@ -622,30 +622,33 @@ void Tablet::add_rowsets(const
std::vector<RowsetSharedPtr>& to_add) {
_tablet_meta->modify_rs_metas(rs_metas, {});
}
-void Tablet::delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete,
bool move_to_stale) {
+Status Tablet::delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete,
bool move_to_stale) {
if (to_delete.empty()) {
- return;
+ return Status::OK();
}
std::vector<RowsetMetaSharedPtr> rs_metas;
rs_metas.reserve(to_delete.size());
- for (auto& rs : to_delete) {
+ for (const auto& rs : to_delete) {
rs_metas.push_back(rs->rowset_meta());
_rs_version_map.erase(rs->version());
}
_tablet_meta->modify_rs_metas({}, rs_metas, !move_to_stale);
if (move_to_stale) {
- for (auto& rs : to_delete) {
+ for (const auto& rs : to_delete) {
_stale_rs_version_map[rs->version()] = rs;
}
_timestamped_version_tracker.add_stale_path_version(rs_metas);
} else {
- for (auto& rs : to_delete) {
+ for (const auto& rs : to_delete) {
_timestamped_version_tracker.delete_version(rs->version());
if (rs->is_local()) {
_engine.add_unused_rowset(rs);
+
RETURN_IF_ERROR(RowsetMetaManager::remove(_data_dir->get_meta(), tablet_uid(),
+
rs->rowset_meta()->rowset_id()));
}
}
}
+ return Status::OK();
}
RowsetSharedPtr Tablet::_rowset_with_largest_size() {
@@ -2050,7 +2053,7 @@ Status Tablet::_cooldown_data(RowsetSharedPtr rowset) {
std::unique_lock meta_wlock(_meta_lock);
SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
if (tablet_state() == TABLET_RUNNING) {
- delete_rowsets({std::move(old_rowset)}, false);
+ RETURN_IF_ERROR(delete_rowsets({std::move(old_rowset)}, false));
add_rowsets({std::move(new_rowset)});
// TODO(plat1ko): process primary key
_tablet_meta->set_cooldown_meta_id(cooldown_meta_id);
@@ -2269,7 +2272,7 @@ Status Tablet::_follow_cooldowned_data() {
to_add.push_back(std::move(rs));
}
// Note: We CANNOT call `modify_rowsets` here because `modify_rowsets`
cannot process version graph correctly.
- delete_rowsets(to_delete, false);
+ RETURN_IF_ERROR(delete_rowsets(to_delete, false));
add_rowsets(to_add);
// TODO(plat1ko): process primary key
_tablet_meta->set_cooldown_meta_id(cooldown_meta_pb.cooldown_meta_id());
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index d7d10978d5a..2b4daa5a4c3 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -362,7 +362,7 @@ public:
// MUST hold EXCLUSIVE `_meta_lock`
void add_rowsets(const std::vector<RowsetSharedPtr>& to_add);
// MUST hold EXCLUSIVE `_meta_lock`
- void delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete, bool
move_to_stale);
+ Status delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete, bool
move_to_stale);
// MUST hold SHARED `_meta_lock`
const auto& rowset_map() const { return _rs_version_map; }
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 12f4b5d6ae5..97e74211504 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -42,6 +42,7 @@
#include "olap/olap_common.h"
#include "olap/olap_define.h"
#include "olap/rowset/rowset.h"
+#include "olap/rowset/rowset_meta_manager.h"
#include "olap/tablet_meta_manager.h"
#include "olap/utils.h"
#include "util/debug_points.h"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
