This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit b2db5a4ded100d13225ff7b1457b25ce05376ab2 Author: xueweizhang <[email protected]> AuthorDate: Fri Mar 22 10:34:54 2024 +0800 [fix](restore) fix restore odbc resource bug (#31989) if some odbc tables use the same resource, and restore not all odbc tables, it will not retain the resource. and check some conf for backup/restore Signed-off-by: nextdreamblue <[email protected]> --- .../src/main/java/org/apache/doris/backup/BackupJob.java | 5 ++++- .../src/main/java/org/apache/doris/backup/BackupJobInfo.java | 7 ++++--- .../src/main/java/org/apache/doris/backup/RestoreJob.java | 10 ++++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index dbb96a96f03..597dacc3c07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -643,7 +643,10 @@ public class BackupJob extends AbstractJob { for (Long beId : beToSnapshots.keySet()) { List<SnapshotInfo> infos = beToSnapshots.get(beId); int totalNum = infos.size(); - int batchNum = Math.min(totalNum, Config.backup_upload_task_num_per_be); + int batchNum = totalNum; + if (Config.backup_upload_task_num_per_be > 0) { + batchNum = Math.min(totalNum, Config.backup_upload_task_num_per_be); + } // each task contains several upload sub tasks int taskNumPerBatch = Math.max(totalNum / batchNum, 1); LOG.info("backend {} has {} batch, total {} tasks, {}", beId, batchNum, totalNum, this); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java index f154ff72d70..c24d6c03295 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java @@ -277,17 +277,18 @@ public class BackupJobInfo implements Writable { public void retainOdbcTables(Set<String> odbcTableNames) { Iterator<BackupOdbcTableInfo> odbcIter = newBackupObjects.odbcTables.listIterator(); - Set<String> removedOdbcResourceNames = Sets.newHashSet(); + Set<String> retainOdbcResourceNames = Sets.newHashSet(); while (odbcIter.hasNext()) { BackupOdbcTableInfo backupOdbcTableInfo = odbcIter.next(); if (!odbcTableNames.contains(backupOdbcTableInfo.dorisTableName)) { - removedOdbcResourceNames.add(backupOdbcTableInfo.resourceName); odbcIter.remove(); + } else { + retainOdbcResourceNames.add(backupOdbcTableInfo.resourceName); } } Iterator<BackupOdbcResourceInfo> resourceIter = newBackupObjects.odbcResources.listIterator(); while (resourceIter.hasNext()) { - if (removedOdbcResourceNames.contains(resourceIter.next().name)) { + if (!retainOdbcResourceNames.contains(resourceIter.next().name)) { resourceIter.remove(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 2e6bbb162ac..d5f2b726ca0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1381,7 +1381,10 @@ public class RestoreJob extends AbstractJob { for (Long beId : beToSnapshots.keySet()) { List<SnapshotInfo> beSnapshotInfos = beToSnapshots.get(beId); int totalNum = beSnapshotInfos.size(); - int batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be); + int batchNum = totalNum; + if (Config.restore_download_task_num_per_be > 0) { + batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be); + } // each task contains several upload sub tasks int taskNumPerBatch = Math.max(totalNum / batchNum, 1); if (LOG.isDebugEnabled()) { @@ -1537,7 +1540,10 @@ public class RestoreJob extends AbstractJob { for (Long beId : beToSnapshots.keySet()) { List<SnapshotInfo> beSnapshotInfos = beToSnapshots.get(beId); int totalNum = beSnapshotInfos.size(); - int batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be); + int batchNum = totalNum; + if (Config.restore_download_task_num_per_be > 0) { + batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be); + } // each task contains several upload sub tasks int taskNumPerBatch = Math.max(totalNum / batchNum, 1); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
