This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0d80456869 [enhancement](backup) teach fe to acquire a consistent
backup between be and fe (#21014)
0d80456869 is described below
commit 0d80456869b3833a1234da292deaa8e92afea6c7
Author: Yongqiang YANG <[email protected]>
AuthorDate: Tue Jun 20 15:37:41 2023 +0800
[enhancement](backup) teach fe to acquire a consistent backup between be
and fe (#21014)
---
.../java/org/apache/doris/backup/BackupJob.java | 249 ++++++++++-----------
1 file changed, 122 insertions(+), 127 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 60d486c310..61eda11a79 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -32,7 +32,6 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.View;
import org.apache.doris.common.io.Text;
@@ -358,6 +357,9 @@ public class BackupJob extends AbstractJob {
unfinishedTaskIds.clear();
taskProgress.clear();
taskErrMsg.clear();
+ // copy all related schema at this moment
+ List<Table> copiedTables = Lists.newArrayList();
+ List<Resource> copiedResources = Lists.newArrayList();
AgentBatchTask batchTask = new AgentBatchTask();
for (TableRef tableRef : tableRefs) {
String tblName = tableRef.getName().getTbl();
@@ -366,37 +368,34 @@ public class BackupJob extends AbstractJob {
status = new Status(ErrCode.NOT_FOUND, "table " + tblName + "
does not exist");
return;
}
- switch (tbl.getType()) {
- case OLAP:
- checkOlapTable((OlapTable) tbl, tableRef);
- if (getContent() == BackupContent.ALL) {
- prepareSnapshotTaskForOlapTable((OlapTable) tbl,
tableRef, batchTask);
- }
- break;
- case VIEW:
- break;
- case ODBC:
- OdbcTable odbcTable = (OdbcTable) tbl;
- if (odbcTable.getOdbcCatalogResourceName() != null) {
- String odbcResourceName =
odbcTable.getOdbcCatalogResourceName();
- Resource resource =
Env.getCurrentEnv().getResourceMgr()
- .getResource(odbcResourceName);
- if (resource == null) {
- status = new Status(ErrCode.NOT_FOUND, "resource "
+ odbcResourceName
- + " related to " + tblName + "does not
exist.");
- return;
+ tbl.readLock();
+ try {
+ switch (tbl.getType()) {
+ case OLAP:
+ OlapTable olapTable = (OlapTable) tbl;
+ checkOlapTable(olapTable, tableRef);
+ if (getContent() == BackupContent.ALL) {
+
prepareSnapshotTaskForOlapTableWithoutLock((OlapTable) tbl, tableRef,
batchTask);
}
- }
- break;
- default:
- status = new Status(ErrCode.COMMON_ERROR,
- "backup job does not support this type of table "
+ tblName);
- return;
+ prepareBackupMetaForOlapTableWithoutLock(tableRef,
olapTable, copiedTables);
+ break;
+ case VIEW:
+ prepareBackupMetaForViewWithoutLock((View) tbl,
copiedTables);
+ break;
+ case ODBC:
+ prepareBackupMetaForOdbcTableWithoutLock((OdbcTable)
tbl, copiedTables, copiedResources);
+ break;
+ default:
+ status = new Status(ErrCode.COMMON_ERROR,
+ "backup job does not support this type of
table " + tblName);
+ return;
+ }
+ } finally {
+ tbl.readUnlock();
}
}
- // copy all related schema at this moment
- prepareBackupMeta(db);
+ backupMeta = new BackupMeta(copiedTables, copiedResources);
// send tasks
for (AgentTask task : batchTask.getAllTasks()) {
@@ -430,121 +429,117 @@ public class BackupJob extends AbstractJob {
}
}
- private void prepareSnapshotTaskForOlapTable(OlapTable olapTable,
+ private void prepareSnapshotTaskForOlapTableWithoutLock(OlapTable
olapTable,
TableRef backupTableRef, AgentBatchTask batchTask) {
- olapTable.readLock();
- try {
- // check backup table again
- if (backupTableRef.getPartitionNames() != null) {
- for (String partName :
backupTableRef.getPartitionNames().getPartitionNames()) {
- Partition partition = olapTable.getPartition(partName);
- if (partition == null) {
- status = new Status(ErrCode.NOT_FOUND, "partition " +
partName
- + " does not exist in table" +
backupTableRef.getName().getTbl());
- return;
- }
+ // check backup table again
+ if (backupTableRef.getPartitionNames() != null) {
+ for (String partName :
backupTableRef.getPartitionNames().getPartitionNames()) {
+ Partition partition = olapTable.getPartition(partName);
+ if (partition == null) {
+ status = new Status(ErrCode.NOT_FOUND, "partition " +
partName
+ + " does not exist in table" +
backupTableRef.getName().getTbl());
+ return;
}
}
+ }
- // create snapshot tasks
- List<Partition> partitions = Lists.newArrayList();
- if (backupTableRef.getPartitionNames() == null) {
- partitions.addAll(olapTable.getPartitions());
- } else {
- for (String partName :
backupTableRef.getPartitionNames().getPartitionNames()) {
- Partition partition = olapTable.getPartition(partName);
- partitions.add(partition);
- }
+ // create snapshot tasks
+ List<Partition> partitions = Lists.newArrayList();
+ if (backupTableRef.getPartitionNames() == null) {
+ partitions.addAll(olapTable.getPartitions());
+ } else {
+ for (String partName :
backupTableRef.getPartitionNames().getPartitionNames()) {
+ Partition partition = olapTable.getPartition(partName);
+ partitions.add(partition);
}
+ }
- // snapshot partitions
- for (Partition partition : partitions) {
- long visibleVersion = partition.getVisibleVersion();
- List<MaterializedIndex> indexes =
partition.getMaterializedIndices(IndexExtState.VISIBLE);
- for (MaterializedIndex index : indexes) {
- int schemaHash =
olapTable.getSchemaHashByIndexId(index.getId());
- List<Tablet> tablets = index.getTablets();
- for (Tablet tablet : tablets) {
- Replica replica = chooseReplica(tablet,
visibleVersion);
- if (replica == null) {
- status = new Status(ErrCode.COMMON_ERROR,
- "failed to choose replica to make snapshot
for tablet " + tablet.getId()
- + ". visible version: " +
visibleVersion);
- return;
- }
- SnapshotTask task = new SnapshotTask(null,
replica.getBackendId(), tablet.getId(),
- jobId, dbId, olapTable.getId(),
partition.getId(),
- index.getId(), tablet.getId(),
- visibleVersion,
- schemaHash, timeoutMs, false /* not restore
task */);
- batchTask.addTask(task);
- unfinishedTaskIds.put(tablet.getId(),
replica.getBackendId());
+ // snapshot partitions
+ for (Partition partition : partitions) {
+ long visibleVersion = partition.getVisibleVersion();
+ List<MaterializedIndex> indexes =
partition.getMaterializedIndices(IndexExtState.VISIBLE);
+ for (MaterializedIndex index : indexes) {
+ int schemaHash =
olapTable.getSchemaHashByIndexId(index.getId());
+ List<Tablet> tablets = index.getTablets();
+ for (Tablet tablet : tablets) {
+ Replica replica = chooseReplica(tablet, visibleVersion);
+ if (replica == null) {
+ status = new Status(ErrCode.COMMON_ERROR,
+ "failed to choose replica to make snapshot for
tablet " + tablet.getId()
+ + ". visible version: " +
visibleVersion);
+ return;
}
+ SnapshotTask task = new SnapshotTask(null,
replica.getBackendId(), tablet.getId(),
+ jobId, dbId, olapTable.getId(), partition.getId(),
+ index.getId(), tablet.getId(),
+ visibleVersion,
+ schemaHash, timeoutMs, false /* not restore task
*/);
+ batchTask.addTask(task);
+ unfinishedTaskIds.put(tablet.getId(),
replica.getBackendId());
}
+ }
- LOG.info("snapshot for partition {}, version: {}",
- partition.getId(), visibleVersion);
+ LOG.info("snapshot for partition {}, version: {}",
+ partition.getId(), visibleVersion);
+ }
+ }
+
+ private void checkResourceForOdbcTable(OdbcTable odbcTable) {
+ if (odbcTable.getOdbcCatalogResourceName() != null) {
+ String odbcResourceName = odbcTable.getOdbcCatalogResourceName();
+ Resource resource = Env.getCurrentEnv().getResourceMgr()
+ .getResource(odbcResourceName);
+ if (resource == null) {
+ status = new Status(ErrCode.NOT_FOUND, "resource " +
odbcResourceName
+ + " related to " + odbcTable.getName() + "does not
exist.");
+ return;
}
- } finally {
- olapTable.readUnlock();
}
}
- private void prepareBackupMeta(Database db) {
- // copy all related schema at this moment
- List<Table> copiedTables = Lists.newArrayList();
- List<Resource> copiedResources = Lists.newArrayList();
- for (TableRef tableRef : tableRefs) {
- String tblName = tableRef.getName().getTbl();
- Table table = db.getTableNullable(tblName);
- table.readLock();
- try {
- if (table.getType() == TableType.OLAP) {
- OlapTable olapTable = (OlapTable) table;
- // only copy visible indexes
- List<String> reservedPartitions =
tableRef.getPartitionNames() == null ? null
- : tableRef.getPartitionNames().getPartitionNames();
- OlapTable copiedTbl =
olapTable.selectiveCopy(reservedPartitions, IndexExtState.VISIBLE, true);
- if (copiedTbl == null) {
- status = new Status(ErrCode.COMMON_ERROR, "failed to
copy table: " + tblName);
- return;
- }
+ private void prepareBackupMetaForOlapTableWithoutLock(TableRef tableRef,
OlapTable olapTable,
+ List<Table>
copiedTables) {
+ // only copy visible indexes
+ List<String> reservedPartitions = tableRef.getPartitionNames() == null
? null
+ : tableRef.getPartitionNames().getPartitionNames();
+ OlapTable copiedTbl = olapTable.selectiveCopy(reservedPartitions,
IndexExtState.VISIBLE, true);
+ if (copiedTbl == null) {
+ status = new Status(ErrCode.COMMON_ERROR, "failed to copy table: "
+ olapTable.getName());
+ return;
+ }
- removeUnsupportProperties(copiedTbl);
- copiedTables.add(copiedTbl);
- } else if (table.getType() == TableType.VIEW) {
- View view = (View) table;
- View copiedView = view.clone();
- if (copiedView == null) {
- status = new Status(ErrCode.COMMON_ERROR, "failed to
copy view: " + tblName);
- return;
- }
- copiedTables.add(copiedView);
- } else if (table.getType() == TableType.ODBC) {
- OdbcTable odbcTable = (OdbcTable) table;
- OdbcTable copiedOdbcTable = odbcTable.clone();
- if (copiedOdbcTable == null) {
- status = new Status(ErrCode.COMMON_ERROR, "failed to
copy odbc table: " + tblName);
- return;
- }
- copiedTables.add(copiedOdbcTable);
- if (copiedOdbcTable.getOdbcCatalogResourceName() != null) {
- Resource resource =
Env.getCurrentEnv().getResourceMgr()
-
.getResource(copiedOdbcTable.getOdbcCatalogResourceName());
- Resource copiedResource = resource.clone();
- if (copiedResource == null) {
- status = new Status(ErrCode.COMMON_ERROR, "failed
to copy odbc resource: "
- + resource.getName());
- return;
- }
- copiedResources.add(copiedResource);
- }
- }
- } finally {
- table.readUnlock();
+ removeUnsupportProperties(copiedTbl);
+ copiedTables.add(copiedTbl);
+ }
+
+ private void prepareBackupMetaForViewWithoutLock(View view, List<Table>
copiedTables) {
+ View copiedView = view.clone();
+ if (copiedView == null) {
+ status = new Status(ErrCode.COMMON_ERROR, "failed to copy view: "
+ view.getName());
+ return;
+ }
+ copiedTables.add(copiedView);
+ }
+
+ private void prepareBackupMetaForOdbcTableWithoutLock(OdbcTable odbcTable,
List<Table> copiedTables,
+ List<Resource> copiedResources) {
+ OdbcTable copiedOdbcTable = odbcTable.clone();
+ if (copiedOdbcTable == null) {
+ status = new Status(ErrCode.COMMON_ERROR, "failed to copy odbc
table: " + odbcTable.getName());
+ return;
+ }
+ copiedTables.add(copiedOdbcTable);
+ if (copiedOdbcTable.getOdbcCatalogResourceName() != null) {
+ Resource resource = Env.getCurrentEnv().getResourceMgr()
+ .getResource(copiedOdbcTable.getOdbcCatalogResourceName());
+ Resource copiedResource = resource.clone();
+ if (copiedResource == null) {
+ status = new Status(ErrCode.COMMON_ERROR, "failed to copy odbc
resource: "
+ + resource.getName());
+ return;
}
+ copiedResources.add(copiedResource);
}
- backupMeta = new BackupMeta(copiedTables, copiedResources);
}
private void removeUnsupportProperties(OlapTable tbl) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
