This is an automated email from the ASF dual-hosted git repository. dahn pushed a commit to branch backup-framework-merge-problems in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit aeb07108fa1943ee39334c52b605dadf17b77c9d Author: Daan Hoogland <[email protected]> AuthorDate: Mon Feb 24 20:52:04 2025 +0100 resolve merge problems in the backup framework --- .../apache/cloudstack/backup/BackupProvider.java | 7 ++ .../cloudstack/backup/DummyBackupProvider.java | 4 + .../cloudstack/backup/NASBackupProvider.java | 6 ++ .../cloudstack/backup/NetworkerBackupProvider.java | 89 +++++++++++++++++++++- .../cloudstack/backup/VeeamBackupProvider.java | 27 ++++++- 5 files changed, 131 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index e3a6c3a62bd..dd67a247827 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -116,4 +116,11 @@ public interface BackupProvider { * @param metric */ Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric); + + /** + * This method should reconcile and create backup entries for any backups created out-of-band + * @param vm + * @param metric + */ + void syncBackups(VirtualMachine vm, Backup.Metric metric); } diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index 6935d177c72..7de682b3eb3 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -146,4 +146,8 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { public boolean deleteBackup(Backup backup, boolean forced) { return true; } + + @Override + public void syncBackups(VirtualMachine vm, Backup.Metric metric) { + } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index f148c53e614..3198f1a4925 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -36,6 +36,7 @@ import com.cloud.utils.component.AdapterBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; + import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.BackupRepositoryDao; @@ -414,6 +415,11 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co return false; } + @Override + public void syncBackups(VirtualMachine vm, Backup.Metric metric) { + // TODO: check and sum/return backups metrics on per VM basis + } + @Override public List<BackupOffering> listBackupOfferings(Long zoneId) { final List<BackupRepository> repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 822688a86a3..0aaebf56701 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -21,6 +21,7 @@ import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.script.Script; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; @@ -29,11 +30,16 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; + +import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl; import org.apache.cloudstack.backup.networker.NetworkerClient; @@ -44,7 +50,9 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.apache.xml.utils.URI; import org.apache.cloudstack.backup.networker.api.NetworkerBackup; + import javax.inject.Inject; + import java.net.URISyntaxException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; @@ -60,7 +68,6 @@ import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import com.cloud.utils.script.Script; public class NetworkerBackupProvider extends AdapterBase implements BackupProvider, Configurable { @@ -560,6 +567,86 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid return metrics; } + @Override + public void syncBackups(VirtualMachine vm, Backup.Metric metric) { + final Long zoneId = vm.getDataCenterId(); + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final List<Backup> backupsInDb = backupDao.listByVmId(null, vm.getId()); + final ArrayList<String> backupsInNetworker = getClient(zoneId).getBackupsForVm(vm); + final List<Long> removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); + for (final String networkerBackupId : backupsInNetworker ) { + Long vmBackupSize=0L; + boolean backupExists = false; + for (final Backup backupInDb : backupsInDb) { + LOG.debug(String.format("Checking if Backup %s with external ID %s for VM %s is valid", backupsInDb, backupInDb.getName(), vm)); + if ( networkerBackupId.equals(backupInDb.getExternalId()) ) { + LOG.debug(String.format("Found Backup %s in both Database and Networker", backupInDb)); + backupExists = true; + removeList.remove(backupInDb.getId()); + if (metric != null) { + LOG.debug(String.format("Update backup [%s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].", + backupInDb, backupInDb.getSize(), backupInDb.getProtectedSize(), + metric.getBackupSize(), metric.getDataSize())); + ((BackupVO) backupInDb).setSize(metric.getBackupSize()); + ((BackupVO) backupInDb).setProtectedSize(metric.getDataSize()); + backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb)); + } + break; + } + } + if (backupExists) { + continue; + } + // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts + // with the proper parameters. So we will register any backups taken on the Networker side from + // outside Cloudstack. If ever Networker will support KVM out of the box this functionality also will + // ensure that SLA like backups will be found and registered. + NetworkerBackup strayNetworkerBackup = getClient(vm.getDataCenterId()).getNetworkerBackupInfo(networkerBackupId); + // Since running backups are already present in Networker Server but not completed + // make sure the backup is not in progress at this time. + if ( strayNetworkerBackup.getCompletionTime() != null) { + BackupVO strayBackup = new BackupVO(); + strayBackup.setVmId(vm.getId()); + strayBackup.setExternalId(strayNetworkerBackup.getId()); + strayBackup.setType(strayNetworkerBackup.getType()); + SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + try { + strayBackup.setDate(formatterDateTime.parse(strayNetworkerBackup.getSaveTime())); + } catch (ParseException e) { + String msg = String.format("Unable to parse date [%s].", strayNetworkerBackup.getSaveTime()); + LOG.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + strayBackup.setStatus(Backup.Status.BackedUp); + for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { + vmBackupSize += (thisVMVol.getSize() / 1024L /1024L); + } + strayBackup.setSize(vmBackupSize); + strayBackup.setProtectedSize(strayNetworkerBackup.getSize().getValue() / 1024L ); + strayBackup.setBackupOfferingId(vm.getBackupOfferingId()); + strayBackup.setAccountId(vm.getAccountId()); + strayBackup.setDomainId(vm.getDomainId()); + strayBackup.setZoneId(vm.getDataCenterId()); + LOG.debug(String.format("Creating a new entry in backups: [id: %s, uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " + + "domain_id: %s, zone_id: %s].", strayBackup.getId(), strayBackup.getUuid(), strayBackup.getVmId(), strayBackup.getExternalId(), + strayBackup.getType(), strayBackup.getDate(), strayBackup.getBackupOfferingId(), strayBackup.getAccountId(), + strayBackup.getDomainId(), strayBackup.getZoneId())); + backupDao.persist(strayBackup); + LOG.warn("Added backup found in provider [" + strayBackup + "]"); + } else { + LOG.debug ("Backup is in progress, skipping addition for this run"); + } + } + for (final Long backupIdToRemove : removeList) { + LOG.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); + backupDao.remove(backupIdToRemove); + } + } + }); + } + @Override public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index a7ce0c09cc6..579c707ec48 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -29,6 +29,8 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.veeam.VeeamClient; @@ -40,15 +42,21 @@ import org.apache.commons.lang3.BooleanUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; +import com.cloud.event.EventVO; import com.cloud.hypervisor.Hypervisor; import com.cloud.dc.VmwareDatacenter; -import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; import com.cloud.dc.dao.VmwareDatacenterDao; +import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.User; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; @@ -344,6 +352,23 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName()); } + private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(List<Backup> backupsInDb, Backup.RestorePoint restorePoint, Backup.Metric metric) { + for (final Backup backup : backupsInDb) { + if (restorePoint.getId().equals(backup.getExternalId())) { + if (metric != null) { + logger.debug("Update backup with [id: {}, uuid: {}, name: {}, external id: {}] from [size: {}, protected size: {}] to [size: {}, protected size: {}].", + backup.getId(), backup.getUuid(), backup.getName(), backup.getExternalId(), backup.getSize(), backup.getProtectedSize(), metric.getBackupSize(), metric.getDataSize()); + + ((BackupVO) backup).setSize(metric.getBackupSize()); + ((BackupVO) backup).setProtectedSize(metric.getDataSize()); + backupDao.update(backup.getId(), ((BackupVO) backup)); + } + return backup; + } + } + return null; + } + @Override public void syncBackups(VirtualMachine vm, Backup.Metric metric) { List<Backup.RestorePoint> restorePoints = listRestorePoints(vm);
