This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 182bf4d3234 [chore](fe) Returns dropped tables in GetMeta request
(#38541)
182bf4d3234 is described below
commit 182bf4d3234ea40edca651bac44b5422362c6c93
Author: walter <[email protected]>
AuthorDate: Wed Jul 31 10:57:00 2024 +0800
[chore](fe) Returns dropped tables in GetMeta request (#38541)
Cherry-pick #38019
---
be/src/runtime/snapshot_loader.cpp | 1 -
.../org/apache/doris/binlog/BinlogManager.java | 14 ++++++
.../java/org/apache/doris/binlog/DBBinlog.java | 50 +++++++++++++++++-----
.../org/apache/doris/binlog/DropTableRecord.java | 4 ++
.../main/java/org/apache/doris/catalog/Env.java | 1 +
gensrc/thrift/FrontendService.thrift | 1 +
6 files changed, 60 insertions(+), 11 deletions(-)
diff --git a/be/src/runtime/snapshot_loader.cpp
b/be/src/runtime/snapshot_loader.cpp
index a5061c4decf..1764e3d4322 100644
--- a/be/src/runtime/snapshot_loader.cpp
+++ b/be/src/runtime/snapshot_loader.cpp
@@ -464,7 +464,6 @@ Status SnapshotLoader::remote_http_download(
}
// Step 3: Validate remote tablet snapshot paths && remote files map
- // TODO(Drogon): Add md5sum check
// key is remote snapshot paths, value is filelist
// get all these use http download action
//
http://172.16.0.14:6781/api/_tablet/_download?token=e804dd27-86da-4072-af58-70724075d2a4&file=/home/ubuntu/doris_master/output/be/storage/snapshot/20230410102306.9.180//2774718/217609978/2774718.hdr
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
index 77f2bf74e66..fc3115e2b92 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
@@ -369,6 +369,20 @@ public class BinlogManager {
}
}
+ // get the dropped tables of the db.
+ public List<Long> getDroppedTables(long dbId) {
+ lock.readLock().lock();
+ try {
+ DBBinlog dbBinlog = dbBinlogMap.get(dbId);
+ if (dbBinlog == null) {
+ return Lists.newArrayList();
+ }
+ return dbBinlog.getDroppedTables();
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
public List<BinlogTombstone> gc() {
LOG.info("begin gc binlog");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
index b43805b06d5..502491004e5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
@@ -62,6 +62,8 @@ public class DBBinlog {
// The commit seq of the dropped partitions
private List<Pair<Long, Long>> droppedPartitions;
+ // The commit seq of the dropped tables
+ private List<Pair<Long, Long>> droppedTables;
private List<TBinlog> tableDummyBinlogs;
@@ -79,6 +81,7 @@ public class DBBinlog {
tableBinlogMap = Maps.newHashMap();
timestamps = Lists.newArrayList();
droppedPartitions = Lists.newArrayList();
+ droppedTables = Lists.newArrayList();
TBinlog dummy;
if (binlog.getType() == TBinlogType.DUMMY) {
@@ -121,6 +124,11 @@ public class DBBinlog {
if (info != null && info.getPartitionId() > 0) {
droppedPartitions.add(Pair.of(info.getPartitionId(),
binlog.getCommitSeq()));
}
+ } else if (binlog.getType() == TBinlogType.DROP_TABLE) {
+ DropTableRecord record = DropTableRecord.fromJson(binlog.data);
+ if (record != null && record.getTableId() > 0) {
+ droppedTables.add(Pair.of(record.getTableId(),
binlog.getCommitSeq()));
+ }
}
if (tableIds == null) {
@@ -174,6 +182,19 @@ public class DBBinlog {
if (!binlog.isSetType()) {
return;
}
+
+ if (binlog.getType() == TBinlogType.DROP_PARTITION && raw
instanceof DropPartitionInfo) {
+ long partitionId = ((DropPartitionInfo) raw).getPartitionId();
+ if (partitionId > 0) {
+ droppedPartitions.add(Pair.of(partitionId,
binlog.getCommitSeq()));
+ }
+ } else if (binlog.getType() == TBinlogType.DROP_TABLE && raw
instanceof DropTableRecord) {
+ long tableId = ((DropTableRecord) raw).getTableId();
+ if (tableId > 0) {
+ droppedTables.add(Pair.of(tableId, binlog.getCommitSeq()));
+ }
+ }
+
switch (binlog.getType()) {
case CREATE_TABLE:
return;
@@ -183,13 +204,6 @@ public class DBBinlog {
break;
}
- if (binlog.getType() == TBinlogType.DROP_PARTITION && raw
instanceof DropPartitionInfo) {
- long partitionId = ((DropPartitionInfo) raw).getPartitionId();
- if (partitionId > 0) {
- droppedPartitions.add(Pair.of(partitionId,
binlog.getCommitSeq()));
- }
- }
-
for (long tableId : tableIds) {
TableBinlog tableBinlog = getTableBinlog(binlog, tableId,
dbBinlogEnable);
if (tableBinlog != null) {
@@ -237,6 +251,18 @@ public class DBBinlog {
}
}
+ // Get the dropped tables of the db.
+ public List<Long> getDroppedTables() {
+ lock.readLock().lock();
+ try {
+ return droppedTables.stream()
+ .map(v -> v.first)
+ .collect(Collectors.toList());
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
public Pair<TStatus, Long> getBinlogLag(long tableId, long prevCommitSeq) {
TStatus status = new TStatus(TStatusCode.OK);
lock.readLock().lock();
@@ -354,7 +380,7 @@ public class DBBinlog {
}
}
- gcDroppedPartitions(largestExpiredCommitSeq);
+ gcDroppedPartitionAndTables(largestExpiredCommitSeq);
if (lastCommitSeq != -1) {
dummy.setCommitSeq(lastCommitSeq);
}
@@ -392,7 +418,7 @@ public class DBBinlog {
timeIter.remove();
}
- gcDroppedPartitions(lastExpiredBinlog.getCommitSeq());
+ gcDroppedPartitionAndTables(lastExpiredBinlog.getCommitSeq());
}
return lastExpiredBinlog;
@@ -502,11 +528,15 @@ public class DBBinlog {
}
}
- private void gcDroppedPartitions(long commitSeq) {
+ private void gcDroppedPartitionAndTables(long commitSeq) {
Iterator<Pair<Long, Long>> iter = droppedPartitions.iterator();
while (iter.hasNext() && iter.next().second < commitSeq) {
iter.remove();
}
+ iter = droppedTables.iterator();
+ while (iter.hasNext() && iter.next().second < commitSeq) {
+ iter.remove();
+ }
}
// not thread safety, do this without lock
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java
b/fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java
index 155898cbc9c..4417edeb973 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java
@@ -58,6 +58,10 @@ public class DropTableRecord {
return GsonUtils.GSON.toJson(this);
}
+ public static DropTableRecord fromJson(String json) {
+ return GsonUtils.GSON.fromJson(json, DropTableRecord.class);
+ }
+
@Override
public String toString() {
return toJson();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index f908442361f..1bb4de57f8d 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -6122,6 +6122,7 @@ public class Env {
if (Config.enable_feature_binlog) {
BinlogManager binlogManager =
Env.getCurrentEnv().getBinlogManager();
dbMeta.setDroppedPartitions(binlogManager.getDroppedPartitions(db.getId()));
+
dbMeta.setDroppedTables(binlogManager.getDroppedTables(db.getId()));
}
result.setDbMeta(dbMeta);
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index d0b45d72647..3ad3250bffe 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1414,6 +1414,7 @@ struct TGetMetaDBMeta {
2: optional string name
3: optional list<TGetMetaTableMeta> tables
4: optional list<i64> dropped_partitions
+ 5: optional list<i64> dropped_tables
}
struct TGetMetaResult {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
