This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9d2fc78bd5 [fix](cooldown) Fix potential data loss when clone task's
dst tablet is cooldown replica (#17644)
9d2fc78bd5 is described below
commit 9d2fc78bd5d268a37a5cc61472bdb2c6cc559c27
Author: plat1ko <[email protected]>
AuthorDate: Fri Sep 1 15:27:52 2023 +0800
[fix](cooldown) Fix potential data loss when clone task's dst tablet is
cooldown replica (#17644)
Co-authored-by: Yongqiang YANG
<[email protected]>
Co-authored-by: Kang <[email protected]>
---
be/src/olap/tablet.cpp | 3 ++-
be/src/olap/task/engine_clone_task.cpp | 17 ++++++++++++++++-
.../java/org/apache/doris/clone/TabletSchedCtx.java | 3 +++
3 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 45d94f857f..afae3fa7e2 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2144,7 +2144,8 @@ Status Tablet::_read_cooldown_meta(const
std::shared_ptr<io::RemoteFileSystem>&
RETURN_IF_ERROR(tablet_meta_reader->read_at(0, {buf.get(), file_size},
&bytes_read));
tablet_meta_reader->close();
if (!tablet_meta_pb->ParseFromArray(buf.get(), file_size)) {
- return Status::InternalError("malformed tablet meta");
+ return Status::InternalError("malformed tablet meta, path={}/{}",
fs->root_path().native(),
+ remote_meta_path);
}
return Status::OK();
}
diff --git a/be/src/olap/task/engine_clone_task.cpp
b/be/src/olap/task/engine_clone_task.cpp
index 9b03375756..75e17e433e 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -735,7 +735,22 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet,
RETURN_IF_ERROR(tablet->create_rowset(rs_meta, &rs));
to_add.push_back(std::move(rs));
}
-
tablet->tablet_meta()->set_cooldown_meta_id(cloned_tablet_meta->cooldown_meta_id());
+ {
+ std::shared_lock cooldown_conf_rlock(tablet->get_cooldown_conf_lock());
+ if (tablet->cooldown_conf_unlocked().first == tablet->replica_id()) {
+ // If this replica is cooldown replica, MUST generate a new
`cooldown_meta_id` to avoid use `cooldown_meta_id`
+ // generated in old cooldown term which may lead to such situation:
+ // Replica A is cooldown replica, cooldown_meta_id=2,
+ // Replica B: cooldown_replica=A, cooldown_meta_id=1
+ // Replica A: full clone Replica A, cooldown_meta_id=1, but remote
cooldown_meta is still with cooldown_meta_id=2
+ // After tablet report. FE finds all replicas' cooldowned data is
consistent
+ // Replica A: confirm_unused_remote_files, delete some cooldowned
data of cooldown_meta_id=2
+ // Replica B: follow_cooldown_data, cooldown_meta_id=2, data lost
+ tablet->tablet_meta()->set_cooldown_meta_id(UniqueId::gen_uid());
+ } else {
+
tablet->tablet_meta()->set_cooldown_meta_id(cloned_tablet_meta->cooldown_meta_id());
+ }
+ }
if (tablet->enable_unique_key_merge_on_write()) {
tablet->tablet_meta()->delete_bitmap() =
cloned_tablet_meta->delete_bitmap();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
index 5b1ad25e68..f3c03b3e1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
@@ -925,6 +925,8 @@ public class TabletSchedCtx implements
Comparable<TabletSchedCtx> {
committedVersion, /* use committed version as last failed
version */
-1 /* last success version */);
+ LOG.info("create clone task to make new replica, tabletId={},
replicaId={}", tabletId,
+ cloneReplica.getId());
// addReplica() method will add this replica to tablet inverted
index too.
tablet.addReplica(cloneReplica);
replicaId = cloneReplica.getId();
@@ -950,6 +952,7 @@ public class TabletSchedCtx implements
Comparable<TabletSchedCtx> {
replicaId, schemaHash, Lists.newArrayList(tSrcBe),
storageMedium,
visibleVersion, (int) (taskTimeoutMs / 1000));
cloneTask.setPathHash(srcPathHash, destPathHash);
+ LOG.info("create clone task to repair replica, tabletId={},
replicaId={}", tabletId, replicaId);
this.state = State.RUNNING;
return cloneTask;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
