This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 749e1df28a8b9f6206cf2d0f4a866d9e76aab7ad Author: lsy3993 <[email protected]> AuthorDate: Wed Jul 3 14:41:47 2024 +0800 [opt](binlog) support rename binlog (#37031) Rename operator doesn't have binlog now. This PR will create binlog when execute rename. The rename operator means : 1. rename table : ALTER TABLE table1 RENAME table2; 2. rename rollup : ALTER TABLE example_table RENAME ROLLUP rollup1 rollup2; 3. rename partition : ALTER TABLE example_table RENAME PARTITION p1 p2; After SQL analyzing, we can get the old and new table name (rollup name or partition name), then record the info to binlog, so we can use the info from binlog. --- .../org/apache/doris/binlog/BinlogManager.java | 11 ++++++ .../main/java/org/apache/doris/catalog/Env.java | 8 +++-- .../java/org/apache/doris/persist/EditLog.java | 12 +++++-- .../java/org/apache/doris/persist/TableInfo.java | 39 +++++++++++++++++++--- gensrc/thrift/FrontendService.thrift | 1 + 5 files changed, 61 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java index 454f678e2e1..8426f2bdec3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java @@ -32,6 +32,7 @@ import org.apache.doris.persist.DropPartitionInfo; import org.apache.doris.persist.ModifyTablePropertyOperationLog; import org.apache.doris.persist.ReplacePartitionOperationLog; import org.apache.doris.persist.TableAddOrDropColumnsInfo; +import org.apache.doris.persist.TableInfo; import org.apache.doris.persist.TruncateTableInfo; import org.apache.doris.thrift.TBinlog; import org.apache.doris.thrift.TBinlogType; @@ -319,6 +320,16 @@ public class BinlogManager { addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false); } + public void addTableRename(TableInfo info, long commitSeq) { + long dbId = info.getDbId(); + List<Long> tableIds = Lists.newArrayList(); + tableIds.add(info.getTableId()); + long timestamp = -1; + TBinlogType type = TBinlogType.RENAME_TABLE; + String data = info.toJson(); + addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false); + } + // get binlog by dbId, return first binlog.version > version public Pair<TStatus, TBinlog> getBinlog(long dbId, long tableId, long prevCommitSeq) { TStatus status = new TStatus(TStatusCode.OK); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index cda6f3b4602..700b51b8a9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -4637,7 +4637,8 @@ public class Env { db.unregisterTable(oldTableName); db.registerTable(table); - TableInfo tableInfo = TableInfo.createForTableRename(db.getId(), table.getId(), newTableName); + TableInfo tableInfo = TableInfo.createForTableRename(db.getId(), table.getId(), oldTableName, + newTableName); editLog.logTableRename(tableInfo); LOG.info("rename table[{}] to {}", oldTableName, newTableName); } finally { @@ -4824,7 +4825,8 @@ public class Env { indexNameToIdMap.put(newRollupName, indexId); // log - TableInfo tableInfo = TableInfo.createForRollupRename(db.getId(), table.getId(), indexId, newRollupName); + TableInfo tableInfo = TableInfo.createForRollupRename(db.getId(), table.getId(), indexId, + rollupName, newRollupName); editLog.logRollupRename(tableInfo); LOG.info("rename rollup[{}] to {}", rollupName, newRollupName); } finally { @@ -4883,7 +4885,7 @@ public class Env { // log TableInfo tableInfo = TableInfo.createForPartitionRename(db.getId(), table.getId(), partition.getId(), - newPartitionName); + partitionName, newPartitionName); editLog.logPartitionRename(tableInfo); LOG.info("rename partition[{}] to {}", partitionName, newPartitionName); } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 170ba1b9bb6..eb26bbc04f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -1535,7 +1535,9 @@ public class EditLog { } public void logTableRename(TableInfo tableInfo) { - logEdit(OperationType.OP_RENAME_TABLE, tableInfo); + long logId = logEdit(OperationType.OP_RENAME_TABLE, tableInfo); + LOG.info("log table rename, logId : {}, infos: {}", logId, tableInfo); + Env.getCurrentEnv().getBinlogManager().addTableRename(tableInfo, logId); } public void logModifyViewDef(AlterViewInfo alterViewInfo) { @@ -1543,11 +1545,15 @@ public class EditLog { } public void logRollupRename(TableInfo tableInfo) { - logEdit(OperationType.OP_RENAME_ROLLUP, tableInfo); + long logId = logEdit(OperationType.OP_RENAME_ROLLUP, tableInfo); + LOG.info("log rollup rename, logId : {}, infos: {}", logId, tableInfo); + Env.getCurrentEnv().getBinlogManager().addTableRename(tableInfo, logId); } public void logPartitionRename(TableInfo tableInfo) { - logEdit(OperationType.OP_RENAME_PARTITION, tableInfo); + long logId = logEdit(OperationType.OP_RENAME_PARTITION, tableInfo); + LOG.info("log partition rename, logId : {}, infos: {}", logId, tableInfo); + Env.getCurrentEnv().getBinlogManager().addTableRename(tableInfo, logId); } public void logColumnRename(TableRenameColumnInfo info) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TableInfo.java index 8a210508fc2..a24655bcd74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/TableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TableInfo.java @@ -42,10 +42,16 @@ public class TableInfo implements Writable { @SerializedName("nT") private String newTableName; + @SerializedName("oT") + private String oldTableName; @SerializedName("nR") private String newRollupName; + @SerializedName("oR") + private String oldRollupName; @SerializedName("nP") private String newPartitionName; + @SerializedName("oP") + private String oldPartitionName; public TableInfo() { // for persist @@ -63,17 +69,38 @@ public class TableInfo implements Writable { this.newPartitionName = newPartitionName; } + private TableInfo(long dbId, long tableId, long indexId, long partitionId, + String newTableName, String oldTableName, String newRollupName, String oldRollupName, + String newPartitionName, String oldPartitionName) { + this.dbId = dbId; + this.tableId = tableId; + this.indexId = indexId; + this.partitionId = partitionId; + + this.newTableName = newTableName; + this.oldTableName = oldTableName; + this.newRollupName = newRollupName; + this.oldRollupName = oldRollupName; + this.newPartitionName = newPartitionName; + this.oldPartitionName = oldPartitionName; + } + public static TableInfo createForTableRename(long dbId, long tableId, String newTableName) { return new TableInfo(dbId, tableId, -1L, -1L, newTableName, "", ""); } - public static TableInfo createForRollupRename(long dbId, long tableId, long indexId, String newRollupName) { - return new TableInfo(dbId, tableId, indexId, -1L, "", newRollupName, ""); + public static TableInfo createForTableRename(long dbId, long tableId, String oldTableName, String newTableName) { + return new TableInfo(dbId, tableId, -1L, -1L, newTableName, oldTableName, "", "", "", ""); + } + + public static TableInfo createForRollupRename(long dbId, long tableId, long indexId, String oldRollupName, + String newRollupName) { + return new TableInfo(dbId, tableId, indexId, -1L, "", "", newRollupName, oldRollupName, "", ""); } public static TableInfo createForPartitionRename(long dbId, long tableId, long partitionId, - String newPartitionName) { - return new TableInfo(dbId, tableId, -1L, partitionId, "", "", newPartitionName); + String oldPartitionName, String newPartitionName) { + return new TableInfo(dbId, tableId, -1L, partitionId, "", "", "", "", newPartitionName, oldPartitionName); } public static TableInfo createForModifyDistribution(long dbId, long tableId) { @@ -134,4 +161,8 @@ public class TableInfo implements Writable { newRollupName = Text.readString(in); newPartitionName = Text.readString(in); } + + public String toJson() { + return GsonUtils.GSON.toJson(this); + } } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index d25f5c0ac2b..6b11402f299 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1143,6 +1143,7 @@ enum TBinlogType { MODIFY_PARTITIONS = 11, REPLACE_PARTITIONS = 12, TRUNCATE_TABLE = 13, + RENAME_TABLE = 14, } struct TBinlog { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
