This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 15a94ba691d [fix](partition) Fix be tablet partition id eq 0 By report
tablet #32179 (#37588)
15a94ba691d is described below
commit 15a94ba691d282c7dab28bd4004875da9806e225
Author: deardeng <[email protected]>
AuthorDate: Sat Jul 13 22:13:06 2024 +0800
[fix](partition) Fix be tablet partition id eq 0 By report tablet #32179
(#37588)
cherry pick from #32179
---
be/src/agent/task_worker_pool.cpp | 17 +++++++++++++++++
be/src/olap/tablet_manager.cpp | 20 ++++++++++++++++++++
be/src/olap/tablet_manager.h | 3 +++
be/src/olap/tablet_meta.cpp | 4 ++--
.../main/java/org/apache/doris/common/Config.java | 2 ++
.../apache/doris/catalog/TabletInvertedIndex.java | 9 +++++++++
6 files changed, 53 insertions(+), 2 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index 5d67a5bac08..ea56c2a8a22 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -417,6 +417,23 @@ void
TaskWorkerPool::_update_tablet_meta_worker_thread_callback() {
continue;
}
bool need_to_save = false;
+ if (tablet_meta_info.__isset.partition_id) {
+ // for fix partition_id = 0
+ LOG(WARNING) << "change be tablet id: " <<
tablet->tablet_meta()->tablet_id()
+ << "partition id from : " <<
tablet->tablet_meta()->partition_id()
+ << " to : " << tablet_meta_info.partition_id;
+ auto succ =
StorageEngine::instance()->tablet_manager()->update_tablet_partition_id(
+ tablet_meta_info.partition_id,
tablet->tablet_meta()->tablet_id());
+ if (!succ) {
+ std::string err_msg = fmt::format(
+ "change be tablet id : {} partition_id : {}
failed",
+ tablet->tablet_meta()->tablet_id(),
tablet_meta_info.partition_id);
+ LOG(WARNING) << err_msg;
+ status = Status::InvalidArgument(err_msg);
+ continue;
+ }
+ need_to_save = true;
+ }
if (tablet_meta_info.__isset.storage_policy_id) {
tablet->tablet_meta()->set_storage_policy_id(tablet_meta_info.storage_policy_id);
need_to_save = true;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 61c7809cf54..573ec920160 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1653,4 +1653,24 @@ std::set<int64_t>
TabletManager::check_all_tablet_segment(bool repair) {
return bad_tablets;
}
+bool TabletManager::update_tablet_partition_id(::doris::TPartitionId
partition_id,
+ ::doris::TTabletId tablet_id) {
+ std::shared_lock rdlock(_get_tablets_shard_lock(tablet_id));
+ TabletSharedPtr tablet = _get_tablet_unlocked(tablet_id);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "get tablet err partition_id: " << partition_id
+ << " tablet_id:" << tablet_id;
+ return false;
+ }
+ _remove_tablet_from_partition(tablet);
+ auto st = tablet->tablet_meta()->set_partition_id(partition_id);
+ if (!st.ok()) {
+ LOG(WARNING) << "set partition id err partition_id: " << partition_id
+ << " tablet_id:" << tablet_id;
+ return false;
+ }
+ _add_tablet_to_partition(tablet);
+ return true;
+}
+
} // end namespace doris
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 07c9d563b87..d33cd6e4bc4 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -166,6 +166,9 @@ public:
std::set<int64_t> check_all_tablet_segment(bool repair);
+ bool update_tablet_partition_id(::doris::TPartitionId partition_id,
+ ::doris::TTabletId tablet_id);
+
private:
// Add a tablet pointer to StorageEngine
// If force, drop the existing tablet add this new one
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index adbfd4d6f38..0d9eb5316f1 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -879,8 +879,8 @@ std::string TabletMeta::full_name() const {
Status TabletMeta::set_partition_id(int64_t partition_id) {
if ((_partition_id > 0 && _partition_id != partition_id) || partition_id <
1) {
- LOG(FATAL) << "cur partition id=" << _partition_id << " new partition
id=" << partition_id
- << " not equal";
+ LOG(WARNING) << "cur partition id=" << _partition_id << " new
partition id=" << partition_id
+ << " not equal";
}
_partition_id = partition_id;
return Status::OK();
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index f1b1908fc2d..e7c5fdcdbf1 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2408,4 +2408,6 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static boolean enable_create_bitmap_index_as_inverted_index = true;
+ @ConfField(mutable = true)
+ public static boolean fix_tablet_partition_id_eq_0 = false;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
index 1c337740de9..667a7d27882 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
@@ -165,6 +165,15 @@ public class TabletInvertedIndex {
tabletMetaInfo.setIsInMemory(!backendTabletInfo.isIsInMemory());
}
}
+ if (Config.fix_tablet_partition_id_eq_0
+ && tabletMeta.getPartitionId() > 0
+ && backendTabletInfo.getPartitionId() ==
0) {
+ LOG.warn("be report tablet partition id not eq
fe, in be {} but in fe {}",
+ backendTabletInfo, tabletMeta);
+ // Need to update partition id in BE
+ tabletMetaInfo = new TTabletMetaInfo();
+
tabletMetaInfo.setPartitionId(tabletMeta.getPartitionId());
+ }
// 1. (intersection)
if (needSync(replica, backendTabletInfo)) {
// need sync
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
