This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 4031808f00b [enhancement](rowset-meta) Remove rowset meta from olap
meta directly when rowsets deleted (#41716) (#43183)
4031808f00b is described below
commit 4031808f00b20c27e9d6154cac144419272f1e17
Author: Siyang Tang <[email protected]>
AuthorDate: Tue Nov 5 00:40:12 2024 +0800
[enhancement](rowset-meta) Remove rowset meta from olap meta directly when
rowsets deleted (#41716) (#43183)
pick: #41716
---
be/src/olap/cold_data_compaction.cpp | 3 ++-
be/src/olap/schema_change.cpp | 2 +-
be/src/olap/tablet.cpp | 21 ++++++++++++---------
be/src/olap/tablet.h | 2 +-
be/src/olap/tablet_meta.cpp | 2 ++
5 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/be/src/olap/cold_data_compaction.cpp
b/be/src/olap/cold_data_compaction.cpp
index b4bd95ea80f..d46e3c5b5e6 100644
--- a/be/src/olap/cold_data_compaction.cpp
+++ b/be/src/olap/cold_data_compaction.cpp
@@ -28,6 +28,7 @@
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/config.h"
+#include "common/status.h"
#include "olap/compaction.h"
#include "olap/olap_common.h"
#include "olap/rowset/rowset.h"
@@ -85,7 +86,7 @@ Status ColdDataCompaction::modify_rowsets(const
Merger::Statistics* stats) {
std::lock_guard wlock(_tablet->get_header_lock());
SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
// Merged cooldowned rowsets MUST NOT be managed by version graph,
they will be reclaimed by `remove_unused_remote_files`.
- _tablet->delete_rowsets(_input_rowsets, false);
+ RETURN_IF_ERROR(_tablet->delete_rowsets(_input_rowsets, false));
_tablet->add_rowsets({_output_rowset});
// TODO(plat1ko): process primary key
_tablet->tablet_meta()->set_cooldown_meta_id(cooldown_meta_id);
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 38e7bba52c6..b0701edf09d 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -881,7 +881,7 @@ Status
SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
}
}
std::vector<RowsetSharedPtr> empty_vec;
- new_tablet->delete_rowsets(rowsets_to_delete, false);
+ RETURN_IF_ERROR(new_tablet->delete_rowsets(rowsets_to_delete,
false));
// inherit cumulative_layer_point from base_tablet
// check if new_tablet.ce_point > base_tablet.ce_point?
new_tablet->set_cumulative_layer_point(-1);
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 9f07f17ab8a..ad8550709a0 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -439,7 +439,7 @@ Status Tablet::revise_tablet_meta(const
std::vector<RowsetSharedPtr>& to_add,
// error handling
if (!calc_bm_status.ok()) {
if (is_incremental_clone) {
- delete_rowsets(to_add, false);
+ RETURN_IF_ERROR(delete_rowsets(to_add, false));
LOG(WARNING) << "incremental clone on tablet: " << tablet_id() <<
" failed due to "
<< calc_bm_status.msg() << ", revert " <<
to_add.size()
<< " rowsets added before.";
@@ -452,7 +452,7 @@ Status Tablet::revise_tablet_meta(const
std::vector<RowsetSharedPtr>& to_add,
// full clone, calculate delete bitmap succeeded, update rowset
if (!is_incremental_clone) {
- delete_rowsets(to_delete, false);
+ RETURN_IF_ERROR(delete_rowsets(to_delete, false));
add_rowsets(to_add);
// reconstruct from tablet meta
_timestamped_version_tracker.construct_versioned_tracker(_tablet_meta->all_rs_metas());
@@ -638,28 +638,31 @@ void Tablet::add_rowsets(const
std::vector<RowsetSharedPtr>& to_add) {
_tablet_meta->modify_rs_metas(rs_metas, {});
}
-void Tablet::delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete,
bool move_to_stale) {
+Status Tablet::delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete,
bool move_to_stale) {
if (to_delete.empty()) {
- return;
+ return Status::OK();
}
std::vector<RowsetMetaSharedPtr> rs_metas;
rs_metas.reserve(to_delete.size());
- for (auto& rs : to_delete) {
+ for (const auto& rs : to_delete) {
rs_metas.push_back(rs->rowset_meta());
_rs_version_map.erase(rs->version());
}
_tablet_meta->modify_rs_metas({}, rs_metas, !move_to_stale);
if (move_to_stale) {
- for (auto& rs : to_delete) {
+ for (const auto& rs : to_delete) {
_stale_rs_version_map[rs->version()] = rs;
}
_timestamped_version_tracker.add_stale_path_version(rs_metas);
} else {
- for (auto& rs : to_delete) {
+ for (const auto& rs : to_delete) {
_timestamped_version_tracker.delete_version(rs->version());
_engine.add_unused_rowset(rs);
+ RETURN_IF_ERROR(RowsetMetaManager::remove(_data_dir->get_meta(),
tablet_uid(),
+
rs->rowset_meta()->rowset_id()));
}
}
+ return Status::OK();
}
// snapshot manager may call this api to check if version exists, so that
@@ -2292,7 +2295,7 @@ Status Tablet::_cooldown_data(RowsetSharedPtr rowset) {
std::unique_lock meta_wlock(_meta_lock);
SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
if (tablet_state() == TABLET_RUNNING) {
- delete_rowsets({std::move(old_rowset)}, false);
+ RETURN_IF_ERROR(delete_rowsets({std::move(old_rowset)}, false));
add_rowsets({std::move(new_rowset)});
// TODO(plat1ko): process primary key
_tablet_meta->set_cooldown_meta_id(cooldown_meta_id);
@@ -2506,7 +2509,7 @@ Status Tablet::_follow_cooldowned_data() {
to_add.push_back(std::move(rs));
}
// Note: We CANNOT call `modify_rowsets` here because `modify_rowsets`
cannot process version graph correctly.
- delete_rowsets(to_delete, false);
+ RETURN_IF_ERROR(delete_rowsets(to_delete, false));
add_rowsets(to_add);
// TODO(plat1ko): process primary key
_tablet_meta->set_cooldown_meta_id(cooldown_meta_pb.cooldown_meta_id());
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index fdfad28a0de..fc2d7a21cab 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -376,7 +376,7 @@ public:
// MUST hold EXCLUSIVE `_meta_lock`
void add_rowsets(const std::vector<RowsetSharedPtr>& to_add);
// MUST hold EXCLUSIVE `_meta_lock`
- void delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete, bool
move_to_stale);
+ Status delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete, bool
move_to_stale);
// MUST hold SHARED `_meta_lock`
const auto& rowset_map() const { return _rs_version_map; }
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 83a92faf903..dd17041b6b1 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -37,6 +37,8 @@
#include "olap/file_header.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
+#include "olap/rowset/rowset.h"
+#include "olap/rowset/rowset_meta_manager.h"
#include "olap/tablet_meta_manager.h"
#include "olap/utils.h"
#include "util/debug_points.h"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
