This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new d723aa3e789 branch-2.1: [feat](backup) GetSnapshot returns snapshot
expiration time (#43830)
d723aa3e789 is described below
commit d723aa3e7899638ac43776b6ed02cca6dd51c37c
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Nov 13 17:47:47 2024 +0800
branch-2.1: [feat](backup) GetSnapshot returns snapshot expiration time
(#43830)
Cherry-picked from #43731
Co-authored-by: walter <[email protected]>
---
.../main/java/org/apache/doris/backup/BackupJob.java | 8 +++++++-
.../src/main/java/org/apache/doris/backup/Snapshot.java | 17 ++++++++++++++---
.../org/apache/doris/service/FrontendServiceImpl.java | 9 +++++++--
gensrc/thrift/FrontendService.thrift | 1 +
gensrc/thrift/Status.thrift | 2 ++
.../org/apache/doris/regression/suite/Syncer.groovy | 2 ++
6 files changed, 33 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 9e932d6f8fc..71f6d4a9dd8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -1032,12 +1032,18 @@ public class BackupJob extends AbstractJob {
return null;
}
+ // Avoid loading expired meta.
+ long expiredAt = createTime + timeoutMs;
+ if (System.currentTimeMillis() >= expiredAt) {
+ return new Snapshot(label, new byte[0], new byte[0], expiredAt);
+ }
+
try {
File metaInfoFile = new File(localMetaInfoFilePath);
File jobInfoFile = new File(localJobInfoFilePath);
byte[] metaInfoBytes = Files.readAllBytes(metaInfoFile.toPath());
byte[] jobInfoBytes = Files.readAllBytes(jobInfoFile.toPath());
- return new Snapshot(label, metaInfoBytes, jobInfoBytes);
+ return new Snapshot(label, metaInfoBytes, jobInfoBytes, expiredAt);
} catch (IOException e) {
LOG.warn("failed to load meta info and job info file, meta info
file {}, job info file {}: ",
localMetaInfoFilePath, localJobInfoFilePath, e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java
index e31309b19a5..c4c93548177 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java
@@ -31,16 +31,19 @@ public class Snapshot {
@SerializedName(value = "jobInfo")
private byte[] jobInfo = null;
+ @SerializedName(value = "expired_at")
+ private long expiredAt = 0;
+
public Snapshot() {
}
- public Snapshot(String label, byte[] meta, byte[] jobInfo) {
+ public Snapshot(String label, byte[] meta, byte[] jobInfo, long expiredAt)
{
this.label = label;
this.meta = meta;
this.jobInfo = jobInfo;
+ this.expiredAt = expiredAt;
}
-
public byte[] getMeta() {
return meta;
}
@@ -49,17 +52,25 @@ public class Snapshot {
return jobInfo;
}
+ public long getExpiredAt() {
+ return expiredAt;
+ }
+
+ public boolean isExpired() {
+ return System.currentTimeMillis() > expiredAt;
+ }
+
public String toJson() {
return GsonUtils.GSON.toJson(this);
}
@Override
public String toString() {
- // return toJson();
return "Snapshot{"
+ "label='" + label + '\''
+ ", meta=" + meta
+ ", jobInfo=" + jobInfo
+ + ", expiredAt=" + expiredAt
+ '}';
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 5f5e1e92d1a..4b15e839178 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -3027,12 +3027,16 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
if (snapshot == null) {
result.getStatus().setStatusCode(TStatusCode.SNAPSHOT_NOT_EXIST);
result.getStatus().addToErrorMsgs(String.format("snapshot %s not
exist", label));
+ } else if (snapshot.isExpired()) {
+ result.getStatus().setStatusCode(TStatusCode.SNAPSHOT_EXPIRED);
+ result.getStatus().addToErrorMsgs(String.format("snapshot %s is
expired", label));
} else {
byte[] meta = snapshot.getMeta();
byte[] jobInfo = snapshot.getJobInfo();
+ long expiredAt = snapshot.getExpiredAt();
- LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info
size: {}",
- label, meta.length, jobInfo.length);
+ LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info
size: {}, expired at: {}",
+ label, meta.length, jobInfo.length, expiredAt);
if (request.isEnableCompress()) {
meta = GZIPUtils.compress(meta);
jobInfo = GZIPUtils.compress(jobInfo);
@@ -3044,6 +3048,7 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
}
result.setMeta(meta);
result.setJobInfo(jobInfo);
+ result.setExpiredAt(expiredAt);
}
return result;
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index dbb19e53380..cf89fb987a9 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1196,6 +1196,7 @@ struct TGetSnapshotResult {
3: optional binary job_info
4: optional Types.TNetworkAddress master_address
5: optional bool compressed;
+ 6: optional i64 expiredAt; // in millis
}
struct TTableRef {
diff --git a/gensrc/thrift/Status.thrift b/gensrc/thrift/Status.thrift
index 53f50aff333..31393650fa8 100644
--- a/gensrc/thrift/Status.thrift
+++ b/gensrc/thrift/Status.thrift
@@ -105,6 +105,8 @@ enum TStatusCode {
OBTAIN_LOCK_FAILED = 74,
+ SNAPSHOT_EXPIRED = 75,
+
// used for cloud
DELETE_BITMAP_LOCK_ERROR = 100,
// Not be larger than 200, see status.h
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
index 354cebc835f..3a952b6edad 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
@@ -463,6 +463,8 @@ class Syncer {
logger.error("TGetSnapshotResult meta is unset.")
} else if (!result.isSetJobInfo()) {
logger.error("TGetSnapshotResult job info is
unset.")
+ } else if (!result.isSetExpiredAt()) {
+ logger.error("TGetSnapshotResult expiredAt is
unset.")
} else {
isCheckedOK = true
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
