This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new ab467f53dbf [fix](partition) Fix be tablet partition id eq 0 By report
tablet (#32179) (#32667)
ab467f53dbf is described below
commit ab467f53dbf0cac53ab3637991f6934ac3c476ec
Author: deardeng <[email protected]>
AuthorDate: Fri Mar 22 15:38:58 2024 +0800
[fix](partition) Fix be tablet partition id eq 0 By report tablet (#32179)
(#32667)
---
be/src/agent/task_worker_pool.cpp | 17 +++++++++++++++++
be/src/olap/tablet_manager.cpp | 20 ++++++++++++++++++++
be/src/olap/tablet_manager.h | 3 +++
be/src/olap/tablet_meta.cpp | 4 ++--
.../main/java/org/apache/doris/common/Config.java | 4 ++++
.../apache/doris/catalog/TabletInvertedIndex.java | 9 +++++++++
6 files changed, 55 insertions(+), 2 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index 34918a5b0a2..4921f4585f5 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -677,6 +677,23 @@ void update_tablet_meta_callback(StorageEngine& engine,
const TAgentTaskRequest&
continue;
}
bool need_to_save = false;
+ if (tablet_meta_info.__isset.partition_id) {
+ // for fix partition_id = 0
+ LOG(WARNING) << "change be tablet id: " <<
tablet->tablet_meta()->tablet_id()
+ << "partition id from : " <<
tablet->tablet_meta()->partition_id()
+ << " to : " << tablet_meta_info.partition_id;
+ auto succ = engine.tablet_manager()->update_tablet_partition_id(
+ tablet_meta_info.partition_id,
tablet->tablet_meta()->tablet_id());
+ if (!succ) {
+ std::string err_msg = fmt::format(
+ "change be tablet id : {} partition_id : {} failed",
+ tablet->tablet_meta()->tablet_id(),
tablet_meta_info.partition_id);
+ LOG(WARNING) << err_msg;
+ status = Status::InvalidArgument(err_msg);
+ continue;
+ }
+ need_to_save = true;
+ }
if (tablet_meta_info.__isset.storage_policy_id) {
tablet->tablet_meta()->set_storage_policy_id(tablet_meta_info.storage_policy_id);
need_to_save = true;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 5293a13bf84..1f5411b9de1 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1536,4 +1536,24 @@ std::set<int64_t>
TabletManager::check_all_tablet_segment(bool repair) {
return bad_tablets;
}
+bool TabletManager::update_tablet_partition_id(::doris::TPartitionId
partition_id,
+ ::doris::TTabletId tablet_id) {
+ std::shared_lock rdlock(_get_tablets_shard_lock(tablet_id));
+ TabletSharedPtr tablet = _get_tablet_unlocked(tablet_id);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "get tablet err partition_id: " << partition_id
+ << " tablet_id:" << tablet_id;
+ return false;
+ }
+ _remove_tablet_from_partition(tablet);
+ auto st = tablet->tablet_meta()->set_partition_id(partition_id);
+ if (!st.ok()) {
+ LOG(WARNING) << "set partition id err partition_id: " << partition_id
+ << " tablet_id:" << tablet_id;
+ return false;
+ }
+ _add_tablet_to_partition(tablet);
+ return true;
+}
+
} // end namespace doris
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 178ba50d9eb..9f8164b853f 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -163,6 +163,9 @@ public:
std::set<int64_t> check_all_tablet_segment(bool repair);
+ bool update_tablet_partition_id(::doris::TPartitionId partition_id,
+ ::doris::TTabletId tablet_id);
+
private:
// Add a tablet pointer to StorageEngine
// If force, drop the existing tablet add this new one
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index cc9b9dc0b9c..d8235191b93 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -875,8 +875,8 @@ RowsetMetaSharedPtr
TabletMeta::acquire_stale_rs_meta_by_version(const Version&
Status TabletMeta::set_partition_id(int64_t partition_id) {
if ((_partition_id > 0 && _partition_id != partition_id) || partition_id <
1) {
- LOG(FATAL) << "cur partition id=" << _partition_id << " new partition
id=" << partition_id
- << " not equal";
+ LOG(WARNING) << "cur partition id=" << _partition_id << " new
partition id=" << partition_id
+ << " not equal";
}
_partition_id = partition_id;
return Status::OK();
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 7f4f579cddd..5fd9a969bc5 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2505,6 +2505,9 @@ public class Config extends ConfigBase {
options = {"default", "ranger-doris"})
public static String access_controller_type = "default";
+ @ConfField(mutable = true)
+ public static boolean fix_tablet_partition_id_eq_0 = false;
+
@ConfField(mutable = true, masterOnly = true, description = {
"倒排索引默认存储格式",
"Default storage format of inverted index, the default value is
V1."
@@ -2517,6 +2520,7 @@ public class Config extends ConfigBase {
})
public static boolean enable_proxy_protocol = false;
+
//==========================================================================
// begin of cloud config
//==========================================================================
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
index b5b8cc6166c..c4650cb5e05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
@@ -167,6 +167,15 @@ public class TabletInvertedIndex {
tabletMetaInfo.setIsInMemory(!backendTabletInfo.isIsInMemory());
}
}
+ if (Config.fix_tablet_partition_id_eq_0
+ && tabletMeta.getPartitionId() > 0
+ && backendTabletInfo.getPartitionId() ==
0) {
+ LOG.warn("be report tablet partition id not eq
fe, in be {} but in fe {}",
+ backendTabletInfo, tabletMeta);
+ // Need to update partition id in BE
+ tabletMetaInfo = new TTabletMetaInfo();
+
tabletMetaInfo.setPartitionId(tabletMeta.getPartitionId());
+ }
// 1. (intersection)
if (needSync(replica, backendTabletInfo)) {
// need sync
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
