This is an automated email from the ASF dual-hosted git repository.
sureshanaparti pushed a commit to branch 4.22
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.22 by this push:
new 0d380a015a5 Linstor add support for incremental nasbackup (#14194)
0d380a015a5 is described below
commit 0d380a015a588da5e0e09700b1711bec4e672a21
Author: Rene Peinthor <[email protected]>
AuthorDate: Fri Oct 23 10:41:25 2026 +0200
Linstor add support for incremental nasbackup (#14194)
---
.../cloudstack/backup/TakeBackupCommand.java | 9 +
.../cloudstack/backup/NASBackupChainKeys.java | 18 ++
.../cloudstack/backup/NASBackupProvider.java | 147 ++++++++-
.../cloudstack/backup/NASBackupProviderTest.java | 226 ++++++++++++++
.../LibvirtRestoreBackupCommandWrapper.java | 14 +-
.../wrapper/LibvirtTakeBackupCommandWrapper.java | 213 ++++++++++++-
.../hypervisor/kvm/storage/KVMStoragePool.java | 33 ++
.../LibvirtRestoreBackupCommandWrapperTest.java | 67 ++++
.../LibvirtTakeBackupCommandWrapperTest.java | 280 +++++++++++++++++
.../kvm/storage/LinstorStorageAdaptor.java | 225 +++++++++++++-
.../hypervisor/kvm/storage/LinstorStoragePool.java | 21 ++
.../kvm/storage/LinstorStorageAdaptorTest.java | 344 +++++++++++++++++++++
scripts/vm/hypervisor/kvm/nasbackup.sh | 285 ++++++++++++++++-
13 files changed, 1854 insertions(+), 28 deletions(-)
diff --git
a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java
b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java
index 7c2d3fc76fc..3928b63dc77 100644
--- a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java
@@ -39,6 +39,7 @@ public class TakeBackupCommand extends Command {
// Incremental backup fields (NAS provider; null/empty for legacy
full-only callers).
private String mode; // "full" or "incremental"; null => legacy
behaviour (script default)
+ private String method; // "qcow2" (bitmaps, push mode) or "content"
(pull mode); null => "qcow2"
private String bitmapNew; // Checkpoint/bitmap name to create with
this backup (timestamp-based)
private String bitmapParent; // Incremental: parent bitmap to read
changes since
@@ -134,6 +135,14 @@ public class TakeBackupCommand extends Command {
this.mode = mode;
}
+ public String getMethod() {
+ return method;
+ }
+
+ public void setMethod(String method) {
+ this.method = method;
+ }
+
public String getBitmapNew() {
return bitmapNew;
}
diff --git
a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java
b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java
index 511f0ccb711..28779311449 100644
---
a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java
+++
b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java
@@ -48,6 +48,24 @@ public final class NASBackupChainKeys {
// "legacy-full" mode token (which sets make_checkpoint=0).
public static final String TYPE_LEGACY_FULL = "legacy-full";
+ /**
+ * In-memory backup-method sentinels used by {@code ChainDecision.method}:
how a backup is
+ * taken, independent of whether it is a full or an incremental ({@code
ChainDecision.mode}).
+ * Both methods write the same chain of qcow2 files, so restore and delete
do not care.
+ *
+ * <p>{@link #METHOD_QCOW2}: libvirt push-mode backup; incrementals come
from QEMU persistent
+ * dirty bitmaps (libvirt checkpoints), which only qcow2 volumes can
carry.</p>
+ */
+ public static final String METHOD_QCOW2 = "qcow2";
+
+ /**
+ * Content-based libvirt pull-mode backup, used for raw block-device
storage such as
+ * LINSTOR/DRBD which cannot carry persistent dirty bitmaps. A full is
written as a sparse
+ * qcow2 in a single pass; an incremental is a qcow2 overlay on the disk's
NBD export,
+ * safe-rebased onto the parent so it holds only the clusters that differ.
+ */
+ public static final String METHOD_CONTENT = "content";
+
/**
* VM-scoped detail (stored in {@code vm_instance_details}) holding the
QEMU dirty-bitmap
* name that currently exists on the running VM and is therefore the only
valid parent
diff --git
a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java
b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java
index 60c9983f016..ff59e0f587a 100644
---
a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java
+++
b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java
@@ -224,11 +224,12 @@ public class NASBackupProvider extends AdapterBase
implements BackupProvider, Co
/**
* Returned by {@link #decideChain(VirtualMachine)} to describe the next
backup's place in
- * the chain: full vs incremental, the bitmap name to create, and (for
incrementals) the
- * parent bitmap and parent file path.
+ * the chain: full vs incremental, how it is taken, the bitmap name to
create, and (for
+ * incrementals) the parent bitmap and parent file path.
*/
static final class ChainDecision {
final String mode; // "full" or "incremental"
+ final String method; // NASBackupChainKeys.METHOD_QCOW2 or
METHOD_CONTENT
final String bitmapNew;
final String bitmapParent; // null for full
// Per-volume parent backup file paths, one per current VM volume in
deviceId order.
@@ -237,15 +238,26 @@ public class NASBackupProvider extends AdapterBase
implements BackupProvider, Co
final List<String> parentPaths;
final String chainId; // chain identifier this backup belongs
to
final int chainPosition; // 0 for full, N for the Nth incremental
in the chain
+ // Content-based chains have no bitmap to look the parent up by, so
the parent backup's
+ // uuid is resolved when the decision is made. null for bitmap-based
decisions.
+ final String parentBackupUuid;
private ChainDecision(String mode, String bitmapNew, String
bitmapParent, List<String> parentPaths,
String chainId, int chainPosition) {
+ this(mode, NASBackupChainKeys.METHOD_QCOW2, bitmapNew,
bitmapParent, parentPaths, chainId,
+ chainPosition, null);
+ }
+
+ private ChainDecision(String mode, String method, String bitmapNew,
String bitmapParent,
+ List<String> parentPaths, String chainId, int
chainPosition, String parentBackupUuid) {
this.mode = mode;
+ this.method = method;
this.bitmapNew = bitmapNew;
this.bitmapParent = bitmapParent;
this.parentPaths = parentPaths;
this.chainId = chainId;
this.chainPosition = chainPosition;
+ this.parentBackupUuid = parentBackupUuid;
}
static ChainDecision fullStart(String bitmapName) {
@@ -268,10 +280,30 @@ public class NASBackupProvider extends AdapterBase
implements BackupProvider, Co
parentPaths, chainId, chainPosition);
}
+ /**
+ * Full backup on the content-based (pull mode) path: no bitmap, no
checkpoint, but it
+ * does anchor a chain so later content-incrementals can hang off it.
+ */
+ static ChainDecision contentFull() {
+ return new ChainDecision(NASBackupChainKeys.TYPE_FULL,
NASBackupChainKeys.METHOD_CONTENT, null, null,
+ null, UUID.randomUUID().toString(), 0, null);
+ }
+
+ static ChainDecision contentIncremental(List<String> parentPaths,
String chainId,
+ int chainPosition, String
parentBackupUuid) {
+ return new ChainDecision(NASBackupChainKeys.TYPE_INCREMENTAL,
NASBackupChainKeys.METHOD_CONTENT, null,
+ null, parentPaths, chainId, chainPosition,
parentBackupUuid);
+ }
+
boolean isIncremental() {
return NASBackupChainKeys.TYPE_INCREMENTAL.equals(mode);
}
+ /** True for the pull-mode content-diff path (no dirty bitmaps
involved). */
+ boolean isContentBased() {
+ return NASBackupChainKeys.METHOD_CONTENT.equals(method);
+ }
+
boolean isLegacyFull() {
return NASBackupChainKeys.TYPE_LEGACY_FULL.equals(mode);
}
@@ -293,15 +325,24 @@ public class NASBackupProvider extends AdapterBase
implements BackupProvider, Co
// Master switch — when the operator disables incrementals at the zone
level the backup
// behaves exactly like the pre-incremental full-only path: no bitmap
is generated and no
// chain/checkpoint metadata is created, sent to the agent, or
persisted (legacy-full).
+ // This holds for every storage, so enabling incrementals is also the
opt-in to the
+ // content-based path below and the scratch space it takes from the
primary storage.
Boolean incrementalEnabled =
NASBackupIncrementalEnabled.valueIn(vm.getDataCenterId());
if (incrementalEnabled == null || !incrementalEnabled) {
return ChainDecision.legacyFull();
}
+ // Raw block-device storage (LINSTOR/DRBD) cannot carry QEMU
persistent dirty bitmaps, so
+ // libvirt checkpoints — and with them the push-mode incremental path
— are unavailable.
+ // Those VMs take the content-based pull-mode path instead.
+ if (allVolumesOnContentDiffCapableStorage(vm)) {
+ return decideContentChain(vm);
+ }
+
// Incremental backups rely on QEMU dirty bitmaps / libvirt
checkpoints, which only exist
- // on file-based qcow2 storage. Storage such as Ceph-RBD and Linstor
cannot carry per-disk
- // checkpoints, so a VM with any volume on such a pool must stay on
the full-only (legacy)
- // path — otherwise an incremental attempt would fail or regress those
storages.
+ // on file-based qcow2 storage. Storage such as Ceph-RBD cannot carry
per-disk checkpoints
+ // and has no content-diff path either, so such a VM stays on the
full-only (legacy) path —
+ // otherwise an incremental attempt would fail or regress those
storages.
if (!allVolumesOnCheckpointCapableStorage(vm)) {
return ChainDecision.legacyFull();
}
@@ -365,6 +406,74 @@ public class NASBackupProvider extends AdapterBase
implements BackupProvider, Co
parentChainId, parentChainPosition + 1);
}
+ /**
+ * Chain decision for content-based (pull mode) storage such as
LINSTOR/DRBD.
+ *
+ * <p>Unlike the bitmap path there is no host-side state to anchor on. The
delta is derived by
+ * comparing the disk's point-in-time NBD export against the parent
backup, so the chain cannot
+ * be invalidated by a VM restart, live migration or restore — the
comparison is stateless. The
+ * parent is therefore simply the most recent BackedUp backup of the same
chain, and the agent
+ * independently verifies the parent files still exist on the NAS,
degrading to a full if they
+ * do not (INCREMENTAL_FALLBACK).</p>
+ */
+ protected ChainDecision decideContentChain(VirtualMachine vm) {
+ // Stopped VMs are backed up straight from the disk with qemu-img
convert; that path has no
+ // point-in-time export to diff against a parent, so it is always a
full.
+ if (VirtualMachine.State.Stopped.equals(vm.getState())) {
+ return ChainDecision.contentFull();
+ }
+
+ Integer fullEvery = NASBackupFullEvery.valueIn(vm.getDataCenterId());
+ if (fullEvery == null || fullEvery <= 1) {
+ return ChainDecision.contentFull();
+ }
+
+ Backup parent = findLatestBackedUpBackup(vm.getId());
+ if (parent == null) {
+ return ChainDecision.contentFull();
+ }
+
+ String parentChainId = readDetail(parent, NASBackupChainKeys.CHAIN_ID);
+ int parentChainPosition = chainPosition(parent);
+ if (parentChainId == null || parentChainPosition == Integer.MAX_VALUE)
{
+ return ChainDecision.contentFull();
+ }
+
+ // Force a fresh full when the chain has reached the configured length.
+ if (parentChainPosition + 1 >= fullEvery) {
+ return ChainDecision.contentFull();
+ }
+
+ List<String> parentPaths = composeParentBackupPaths(parent,
vm.getId());
+ if (parentPaths == null) {
+ LOG.debug("VM {} parent backup {} volume layout no longer matches
current VM — forcing full",
+ vm.getInstanceName(), parent.getUuid());
+ return ChainDecision.contentFull();
+ }
+ return ChainDecision.contentIncremental(parentPaths, parentChainId,
parentChainPosition + 1,
+ parent.getUuid());
+ }
+
+ /**
+ * True when EVERY volume of the VM sits on storage whose backups must be
derived by content
+ * comparison instead of dirty bitmaps — currently LINSTOR, whose volumes
are raw DRBD block
+ * devices and so cannot persist a bitmap (persistence is a qcow2-only
feature). A VM with a
+ * mix of such volumes and others is not eligible, since one backup run
uses a single mode.
+ */
+ protected boolean allVolumesOnContentDiffCapableStorage(VirtualMachine vm)
{
+ List<VolumeVO> volumes = volumeDao.findByInstance(vm.getId());
+ if (CollectionUtils.isEmpty(volumes)) {
+ return false;
+ }
+ for (VolumeVO volume : volumes) {
+ StoragePoolVO pool =
primaryDataStoreDao.findById(volume.getPoolId());
+ if (pool == null ||
!Storage.StoragePoolType.Linstor.equals(pool.getPoolType())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* Incremental backups require QEMU dirty bitmaps / libvirt checkpoints,
which are only
* possible on file-based qcow2 storage. Returns {@code true} only when
EVERY volume of the
@@ -502,7 +611,10 @@ public class NASBackupProvider extends AdapterBase
implements BackupProvider, Co
// source of truth. Not duplicated into backup_details.
if (decision.isIncremental()) {
// Resolve the parent backup's UUID so restore can walk the chain
by id, not by path.
- String parentUuid = lookupParentBackupUuid(backup.getVmId(),
decision.bitmapParent);
+ // Content-based decisions carry the parent uuid directly (no
bitmap to look it up by).
+ String parentUuid = decision.parentBackupUuid != null
+ ? decision.parentBackupUuid
+ : lookupParentBackupUuid(backup.getVmId(),
decision.bitmapParent);
if (parentUuid != null) {
backupDetailsDao.persist(new BackupDetailVO(backup.getId(),
NASBackupChainKeys.PARENT_BACKUP_ID, parentUuid, true));
}
@@ -593,11 +705,14 @@ public class NASBackupProvider extends AdapterBase
implements BackupProvider, Co
command.setQuiesce(quiesceVM);
command.setQuiesceTimeout(NASBackupQuiesceAgentTimeout.valueIn(vm.getDataCenterId()));
command.setMode(decision.mode);
+ command.setMethod(decision.method);
command.setBitmapNew(decision.bitmapNew);
command.setBitmapParent(decision.bitmapParent);
command.setParentPaths(decision.parentPaths);
- if (VirtualMachine.State.Stopped.equals(vm.getState())) {
+ // A stopped VM is copied straight from its volumes. A running VM
backed up with the content method
+ // takes the scratch space of its pull-mode backup from the volumes'
primary storage.
+ if (VirtualMachine.State.Stopped.equals(vm.getState()) ||
decision.isContentBased()) {
List<VolumeVO> vmVolumes = volumeDao.findByInstance(vm.getId());
vmVolumes.sort(Comparator.comparing(Volume::getDeviceId));
Pair<List<PrimaryDataStoreTO>, List<String>> volumePoolsAndPaths =
getVolumePoolsAndPaths(vmVolumes);
@@ -628,7 +743,9 @@ public class NASBackupProvider extends AdapterBase
implements BackupProvider, Co
// backup as a full and start a new chain.
ChainDecision effective = decision;
if (answer.getIncrementalFallback()) {
- effective = ChainDecision.fullStart(decision.bitmapNew);
+ effective = decision.isContentBased()
+ ? ChainDecision.contentFull()
+ : ChainDecision.fullStart(decision.bitmapNew);
backupVO.setType("FULL");
}
List<Volume> volumes = new
ArrayList<>(volumeDao.findByInstance(vm.getId()));
@@ -642,11 +759,15 @@ public class NASBackupProvider extends AdapterBase
implements BackupProvider, Co
// created — the only valid parent for the next
incremental (see decideChain).
// If the agent reports no bitmap (bitmapCreated=null),
clear any stale detail
// so the next backup starts a fresh full.
- String confirmedBitmap = answer.getBitmapCreated();
- if (confirmedBitmap != null) {
- upsertVmActiveCheckpoint(vm.getId(), confirmedBitmap);
- } else {
- clearVmActiveCheckpoint(vm.getId());
+ // The content-based path creates no bitmaps, so there is
no active
+ // checkpoint to track — its chain is anchored purely on
backup history.
+ if (!decision.isContentBased()) {
+ String confirmedBitmap = answer.getBitmapCreated();
+ if (confirmedBitmap != null) {
+ upsertVmActiveCheckpoint(vm.getId(),
confirmedBitmap);
+ } else {
+ clearVmActiveCheckpoint(vm.getId());
+ }
}
}
return new Pair<>(true, backupVO);
diff --git
a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java
b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java
index fbd681e726f..857fd167d82 100644
---
a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java
+++
b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java
@@ -19,6 +19,7 @@ package org.apache.cloudstack.backup;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@@ -49,6 +50,7 @@ import com.cloud.resource.ResourceManager;
import com.cloud.storage.DiskOfferingVO;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage;
+import java.util.Date;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.DiskOfferingDao;
@@ -275,6 +277,37 @@ public class NASBackupProviderTest {
Mockito.verify(agentManager).send(anyLong(),
Mockito.any(TakeBackupCommand.class));
}
+ @Test
+ public void takeBackupSendsVolumePoolsForContentMethodOnRunningVm() throws
AgentUnavailableException, OperationTimedoutException {
+ VMInstanceVO vm = mockRunningVmForBackup(1L, 2L, 3L, 4L, 5L, 6L);
+
Mockito.doReturn(NASBackupProvider.ChainDecision.contentFull()).when(nasBackupProvider).decideChain(vm);
+ VolumeVO volume = volumeDao.findByInstance(1L).get(0);
+ Mockito.when(volume.getPoolId()).thenReturn(11L);
+ Mockito.when(volume.getPath()).thenReturn("vol-uuid");
+ Mockito.when(volumeDao.findByInstance(1L)).thenReturn(new
ArrayList<>(List.of(volume)));
+ StoragePoolVO pool = mock(StoragePoolVO.class);
+ Mockito.when(pool.getId()).thenReturn(11L);
+
Mockito.when(pool.getPoolType()).thenReturn(Storage.StoragePoolType.Linstor);
+ Mockito.when(storagePoolDao.findById(11L)).thenReturn(pool);
+ DataStore dataStore = mock(DataStore.class);
+ PrimaryDataStoreTO dataStoreTO = mock(PrimaryDataStoreTO.class);
+ Mockito.when(dataStore.getTO()).thenReturn(dataStoreTO);
+ Mockito.when(dataStoreMgr.getDataStore(11L,
DataStoreRole.Primary)).thenReturn(dataStore);
+
+ BackupAnswer answer = mock(BackupAnswer.class);
+ Mockito.when(answer.getResult()).thenReturn(false);
+ ArgumentCaptor<TakeBackupCommand> commandCaptor =
ArgumentCaptor.forClass(TakeBackupCommand.class);
+ Mockito.when(agentManager.send(anyLong(),
commandCaptor.capture())).thenReturn(answer);
+
Mockito.when(backupDao.persist(Mockito.any(BackupVO.class))).thenAnswer(invocation
-> invocation.getArgument(0));
+
+ nasBackupProvider.takeBackup(vm, false);
+
+ TakeBackupCommand command = commandCaptor.getValue();
+ Assert.assertEquals("content", command.getMethod());
+ Assert.assertEquals(List.of(dataStoreTO), command.getVolumePools());
+ Assert.assertEquals(List.of("/dev/drbd/by-res/cs-vol-uuid/0"),
command.getVolumePaths());
+ }
+
@Test
public void takeBackupFailureNeedingCleanupReturnsErrorBackup() throws
AgentUnavailableException, OperationTimedoutException {
VMInstanceVO vm = mockRunningVmForBackup(1L, 2L, 3L, 4L, 5L, 6L);
@@ -570,6 +603,7 @@ public class NASBackupProviderTest {
NASBackupProvider.ChainDecision decision =
nasBackupProvider.decideChain(vm);
Assert.assertNotNull(decision);
Assert.assertEquals(NASBackupChainKeys.TYPE_FULL, decision.mode);
+ Assert.assertEquals(NASBackupChainKeys.METHOD_QCOW2, decision.method);
Assert.assertNull(decision.bitmapParent);
Assert.assertEquals(0, decision.chainPosition);
}
@@ -1165,4 +1199,196 @@ public class NASBackupProviderTest {
Mockito.verify(backupDao, Mockito.never()).remove(51L);
Mockito.verify(backupDao).remove(50L);
}
+
+ // -- content-based (pull mode) chain decisions for raw block storage
-----------------
+
+ /**
+ * With the incremental master switch off, a LINSTOR-backed VM keeps the
legacy push-mode full
+ * like every other VM: enabling incrementals is the opt-in to the
content-based path and its
+ * host-local scratch space. The storage is not even looked at.
+ */
+ @Test
+ public void
decideChainReturnsLegacyFullForLinstorWhenIncrementalDisabled() {
+ Long vmId = 70L;
+ VMInstanceVO vm = mock(VMInstanceVO.class);
+ Mockito.lenient().when(vm.getId()).thenReturn(vmId);
+ Mockito.lenient().when(vm.getDataCenterId()).thenReturn(1L);
+
+ ReflectionTestUtils.setField(nasBackupProvider,
"NASBackupIncrementalEnabled",
+ new
org.apache.cloudstack.framework.config.ConfigKey<>("Advanced", Boolean.class,
+ "nas.backup.incremental.enabled", "false",
+ "test override — disabled", true,
+
org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone));
+
+ NASBackupProvider.ChainDecision decision =
nasBackupProvider.decideChain(vm);
+ Assert.assertTrue(decision.isLegacyFull());
+ Assert.assertEquals(NASBackupChainKeys.METHOD_QCOW2, decision.method);
+ Assert.assertFalse(decision.isContentBased());
+ Assert.assertNull("legacy full anchors no chain", decision.chainId);
+ Mockito.verify(volumeDao, Mockito.never()).findByInstance(vmId);
+ }
+
+ /**
+ * Stopped VMs are imaged straight from the disk by qemu-img, with no
point-in-time export to
+ * diff against a parent, so a LINSTOR VM that is stopped must take a
content full.
+ */
+ @Test
+ public void decideChainReturnsContentFullForStoppedLinstorVm() {
+ Long vmId = 71L;
+ VMInstanceVO vm = mock(VMInstanceVO.class);
+ Mockito.when(vm.getId()).thenReturn(vmId);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+ Mockito.when(vm.getState()).thenReturn(VMInstanceVO.State.Stopped);
+ stubAllVolumesOnLinstor(vmId, 1);
+ enableIncrementals();
+
+ NASBackupProvider.ChainDecision decision =
nasBackupProvider.decideChain(vm);
+ Assert.assertEquals(NASBackupChainKeys.TYPE_FULL, decision.mode);
+ Assert.assertEquals(NASBackupChainKeys.METHOD_CONTENT,
decision.method);
+ }
+
+ /**
+ * Running LINSTOR VM with a healthy parent inside the cadence => content
incremental, carrying
+ * the parent's chain id, the next position, the parent uuid (resolved
directly since there is
+ * no bitmap to look it up by) and the per-volume parent paths the script
rebases onto.
+ */
+ @Test
+ public void
decideChainReturnsContentIncrementalForLinstorWithValidParent() {
+ Long vmId = 72L;
+ VMInstanceVO vm = mock(VMInstanceVO.class);
+ Mockito.when(vm.getId()).thenReturn(vmId);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+ Mockito.when(vm.getState()).thenReturn(VMInstanceVO.State.Running);
+ stubAllVolumesOnLinstor(vmId, 1);
+ enableIncrementals();
+ ReflectionTestUtils.setField(nasBackupProvider, "NASBackupFullEvery",
+ new
org.apache.cloudstack.framework.config.ConfigKey<>("Advanced", Integer.class,
+ "nas.backup.full.every", "3", "test override", true,
+
org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone));
+
+ Backup parent = mock(Backup.class);
+ Mockito.when(parent.getId()).thenReturn(900L);
+ Mockito.when(parent.getStatus()).thenReturn(Backup.Status.BackedUp);
+ Mockito.lenient().when(parent.getDate()).thenReturn(new Date());
+ Mockito.when(parent.getUuid()).thenReturn("parent-uuid");
+
Mockito.when(parent.getExternalId()).thenReturn("i-2-3-VM/2026.01.01.00.00.00");
+ Backup.VolumeInfo pv = mock(Backup.VolumeInfo.class);
+ Mockito.when(pv.getPath()).thenReturn("volpath1");
+
Mockito.<List<Backup.VolumeInfo>>when(parent.getBackedUpVolumes()).thenReturn(List.of(pv));
+ Mockito.when(backupDao.listByVmId(null,
vmId)).thenReturn(List.of(parent));
+
+ BackupDetailVO chainId = mock(BackupDetailVO.class);
+ Mockito.when(chainId.getValue()).thenReturn("chain-1");
+ BackupDetailVO chainPos = mock(BackupDetailVO.class);
+ Mockito.when(chainPos.getValue()).thenReturn("0");
+ Mockito.when(backupDetailsDao.findDetail(900L,
NASBackupChainKeys.CHAIN_ID)).thenReturn(chainId);
+ Mockito.when(backupDetailsDao.findDetail(900L,
NASBackupChainKeys.CHAIN_POSITION)).thenReturn(chainPos);
+
+ NASBackupProvider.ChainDecision decision =
nasBackupProvider.decideChain(vm);
+ Assert.assertEquals(NASBackupChainKeys.TYPE_INCREMENTAL,
decision.mode);
+ Assert.assertEquals(NASBackupChainKeys.METHOD_CONTENT,
decision.method);
+ Assert.assertTrue(decision.isIncremental());
+ Assert.assertTrue(decision.isContentBased());
+ Assert.assertNull("a content incremental carries no bitmap",
decision.bitmapNew);
+ Assert.assertNull(decision.bitmapParent);
+ Assert.assertEquals("chain-1", decision.chainId);
+ Assert.assertEquals(1, decision.chainPosition);
+ Assert.assertEquals("parent-uuid", decision.parentBackupUuid);
+
Assert.assertEquals(List.of("i-2-3-VM/2026.01.01.00.00.00/root.volpath1.qcow2"),
+ decision.parentPaths);
+ }
+
+ /**
+ * Once the chain reaches nas.backup.full.every, the next LINSTOR backup
must anchor a new
+ * chain with a content full rather than extending the existing one.
+ */
+ @Test
+ public void decideChainReturnsContentFullForLinstorAtChainEnd() {
+ Long vmId = 73L;
+ VMInstanceVO vm = mock(VMInstanceVO.class);
+ Mockito.when(vm.getId()).thenReturn(vmId);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+ Mockito.when(vm.getState()).thenReturn(VMInstanceVO.State.Running);
+ stubAllVolumesOnLinstor(vmId, 1);
+ enableIncrementals();
+ ReflectionTestUtils.setField(nasBackupProvider, "NASBackupFullEvery",
+ new
org.apache.cloudstack.framework.config.ConfigKey<>("Advanced", Integer.class,
+ "nas.backup.full.every", "2", "test override", true,
+
org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone));
+
+ Backup parent = mock(Backup.class);
+ Mockito.when(parent.getId()).thenReturn(901L);
+ Mockito.when(parent.getStatus()).thenReturn(Backup.Status.BackedUp);
+ Mockito.lenient().when(parent.getDate()).thenReturn(new Date());
+ Mockito.when(backupDao.listByVmId(null,
vmId)).thenReturn(List.of(parent));
+ BackupDetailVO chainId = mock(BackupDetailVO.class);
+ Mockito.when(chainId.getValue()).thenReturn("chain-9");
+ BackupDetailVO chainPos = mock(BackupDetailVO.class);
+ Mockito.when(chainPos.getValue()).thenReturn("1"); // 1 + 1 >= 2 =>
new full
+ Mockito.when(backupDetailsDao.findDetail(901L,
NASBackupChainKeys.CHAIN_ID)).thenReturn(chainId);
+ Mockito.when(backupDetailsDao.findDetail(901L,
NASBackupChainKeys.CHAIN_POSITION)).thenReturn(chainPos);
+
+ NASBackupProvider.ChainDecision decision =
nasBackupProvider.decideChain(vm);
+ Assert.assertEquals(NASBackupChainKeys.TYPE_FULL, decision.mode);
+ Assert.assertEquals(NASBackupChainKeys.METHOD_CONTENT,
decision.method);
+ }
+
+ /**
+ * A VM straddling LINSTOR and file-based storage is not content-diff
capable: one backup run
+ * uses a single mode, so it must not be routed down the content path.
+ */
+ @Test
+ public void allVolumesOnContentDiffCapableStorageFalseForMixedStorage() {
+ Long vmId = 74L;
+ VMInstanceVO vm = mock(VMInstanceVO.class);
+ Mockito.when(vm.getId()).thenReturn(vmId);
+
+ VolumeVO linVol = mock(VolumeVO.class);
+ Mockito.when(linVol.getPoolId()).thenReturn(1L);
+ VolumeVO nfsVol = mock(VolumeVO.class);
+ Mockito.when(nfsVol.getPoolId()).thenReturn(2L);
+
Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(linVol,
nfsVol));
+
+ StoragePoolVO lin = mock(StoragePoolVO.class);
+
Mockito.when(lin.getPoolType()).thenReturn(Storage.StoragePoolType.Linstor);
+ StoragePoolVO nfs = mock(StoragePoolVO.class);
+
Mockito.when(nfs.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem);
+ Mockito.when(storagePoolDao.findById(1L)).thenReturn(lin);
+ Mockito.when(storagePoolDao.findById(2L)).thenReturn(nfs);
+
+
Assert.assertFalse(nasBackupProvider.allVolumesOnContentDiffCapableStorage(vm));
+ }
+
+ /** A VM with no volumes is not content-diff capable (safe default). */
+ @Test
+ public void allVolumesOnContentDiffCapableStorageFalseForNoVolumes() {
+ Long vmId = 75L;
+ VMInstanceVO vm = mock(VMInstanceVO.class);
+ Mockito.when(vm.getId()).thenReturn(vmId);
+ Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of());
+
Assert.assertFalse(nasBackupProvider.allVolumesOnContentDiffCapableStorage(vm));
+ }
+
+ private void enableIncrementals() {
+ ReflectionTestUtils.setField(nasBackupProvider,
"NASBackupIncrementalEnabled",
+ new
org.apache.cloudstack.framework.config.ConfigKey<>("Advanced", Boolean.class,
+ "nas.backup.incremental.enabled", "true",
+ "test override — enabled", true,
+
org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone));
+ }
+
+ /** Point every volume of {@code vmId} at a LINSTOR pool. */
+ private void stubAllVolumesOnLinstor(Long vmId, int count) {
+ List<VolumeVO> vols = new java.util.ArrayList<>();
+ StoragePoolVO lin = mock(StoragePoolVO.class);
+
Mockito.lenient().when(lin.getPoolType()).thenReturn(Storage.StoragePoolType.Linstor);
+ for (int i = 0; i < count; i++) {
+ VolumeVO v = mock(VolumeVO.class);
+ long poolId = 100L + i;
+ Mockito.when(v.getPoolId()).thenReturn(poolId);
+ Mockito.when(storagePoolDao.findById(poolId)).thenReturn(lin);
+ vols.add(v);
+ }
+ Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(vols);
+ }
}
diff --git
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java
index d067a29d31a..dcd4cf8bcf3 100644
---
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java
+++
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java
@@ -63,7 +63,7 @@ public class LibvirtRestoreBackupCommandWrapper extends
CommandWrapper<RestoreBa
private static final String QEMU_IMG_HAS_BACKING_COMMAND =
"qemu-img info --output=json %s 2>/dev/null | grep -q
'\"backing-filename\"'";
- private String getVolumeUuidFromPath(String volumePath, PrimaryDataStoreTO
volumePool) {
+ static String getVolumeUuidFromPath(String volumePath, PrimaryDataStoreTO
volumePool) {
if (Storage.StoragePoolType.Linstor.equals(volumePool.getPoolType())) {
Path path = Paths.get(volumePath);
String rscName = path.getParent().getFileName().toString();
@@ -330,15 +330,22 @@ public class LibvirtRestoreBackupCommandWrapper extends
CommandWrapper<RestoreBa
KVMStoragePool volumeStoragePool =
storagePoolMgr.getStoragePool(volumePool.getPoolType(), volumePool.getUuid());
QemuImg qemu;
try {
- qemu = new QemuImg(timeout, true, false);
String volumeUuid = getVolumeUuidFromPath(volumePath, volumePool);
KVMPhysicalDisk disk = null;
+ boolean targetExists = true;
+ // Let qemu-img skip the backup's zero regions (--target-is-zero)
only on a volume that
+ // provably reads as zeroes. An in-place restore overwrites a
volume that still holds its
+ // old data, and a new LINSTOR volume reads as zeroes only on thin
or ZFS backed pools.
+ boolean targetReadsAsZero = false;
if (createTargetVolume) {
if
(Storage.StoragePoolType.Linstor.equals(volumePool.getPoolType())) {
if (size == null) {
throw new CloudRuntimeException("Restore volume size
is required for Linstor pool when creating target volume");
}
disk = volumeStoragePool.createPhysicalDisk(volumeUuid,
QemuImg.PhysicalDiskFormat.RAW, Storage.ProvisioningType.THIN, size, null);
+ targetReadsAsZero =
volumeStoragePool.newVolumeReadsAsZero(volumeUuid);
+ } else {
+ targetExists = false;
}
} else {
if
(Storage.StoragePoolType.Linstor.equals(volumePool.getPoolType())) {
@@ -346,8 +353,9 @@ public class LibvirtRestoreBackupCommandWrapper extends
CommandWrapper<RestoreBa
} else {
disk = volumeStoragePool.getPhysicalDisk(volumePath);
}
- qemu.setSkipTargetVolumeCreation(true);
}
+ qemu = new QemuImg(timeout, targetReadsAsZero, false);
+ qemu.setSkipTargetVolumeCreation(targetExists);
if (disk != null) {
logger.debug("Restoring volume: {}", disk.toString());
}
diff --git
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java
index b7e60958765..92a4d3af8c7 100644
---
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java
+++
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java
@@ -21,6 +21,7 @@ package com.cloud.hypervisor.kvm.resource.wrapper;
import com.cloud.agent.api.Answer;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
@@ -28,17 +29,25 @@ import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import com.cloud.storage.Storage;
import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import org.apache.cloudstack.backup.BackupAnswer;
import org.apache.cloudstack.backup.TakeBackupCommand;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.libvirt.Connect;
+import org.libvirt.Domain;
+import org.libvirt.DomainInfo;
+import org.libvirt.LibvirtException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
+import java.util.stream.Collectors;
@ResourceWrapper(handles = TakeBackupCommand.class)
public class LibvirtTakeBackupCommandWrapper extends
CommandWrapper<TakeBackupCommand, Answer, LibvirtComputingResource> {
@@ -53,13 +62,18 @@ public class LibvirtTakeBackupCommandWrapper extends
CommandWrapper<TakeBackupCo
// chain metadata. Matches nasbackup.sh's "legacy-full" mode
(make_checkpoint=0).
private static final String MODE_LEGACY_FULL = "legacy-full";
+ // Backup methods, orthogonal to the mode. qcow2 is the push-mode path
with QEMU dirty bitmaps,
+ // also assumed when the management server sends none. content is libvirt
pull mode, used for
+ // raw block-device storage such as LINSTOR/DRBD which cannot carry
persistent dirty bitmaps:
+ // it needs no bitmap names, an incremental only its per-disk parent paths.
+ private static final String METHOD_QCOW2 = "qcow2";
+ private static final String METHOD_CONTENT = "content";
+ // Fixed part of the qcow2 metadata on a scratch device, with room to
spare.
+ private static final long SCRATCH_EXTRA_BYTES = 4L * 1024 * 1024;
+
@Override
public Answer execute(TakeBackupCommand command, LibvirtComputingResource
libvirtComputingResource) {
final String vmName = command.getVmName();
- final String backupPath = command.getBackupPath();
- final String backupRepoType = command.getBackupRepoType();
- final String backupRepoAddress = command.getBackupRepoAddress();
- final String mountOptions = command.getMountOptions();
List<PrimaryDataStoreTO> volumePools = command.getVolumePools();
final List<String> volumePaths = command.getVolumePaths();
KVMStoragePoolManager storagePoolMgr =
libvirtComputingResource.getStoragePoolMgr();
@@ -87,12 +101,67 @@ public class LibvirtTakeBackupCommandWrapper extends
CommandWrapper<TakeBackupCo
}
}
- Pair<Integer, String> result =
runBackupScript(libvirtComputingResource, command, vmName, backupRepoType,
backupRepoAddress,
- mountOptions, backupPath, diskPaths, command.getMode(),
- command.getBitmapNew(), command.getBitmapParent(),
command.getParentPaths(), timeout);
+ // A stopped VM is copied straight from its volumes, but a LINSTOR
volume only has a device on
+ // the hosts that hold a resource of it, and once the VM is stopped
its last host may hold none.
+ List<Pair<PrimaryDataStoreTO, String>> connectedVolumes;
+ try {
+ connectedVolumes = connectMissingLinstorVolumes(storagePoolMgr,
volumePools, volumePaths);
+ } catch (CloudRuntimeException e) {
+ logger.warn("Failed to make the volumes of VM {} available for its
backup", vmName, e);
+ return new BackupAnswer(command, false,
+ String.format("Failed to make the volumes of VM %s
available: %s", vmName, e.getMessage()));
+ }
+
+ KVMStoragePool scratchPool = null;
+ Pair<String, List<String>> scratch = null;
+ try {
+ // A pull-mode backup of a running VM needs a copy-before-write
scratch device per disk, provided by
+ // the VM's primary storage. A stopped VM is copied straight from
its disks and needs none.
+ if (METHOD_CONTENT.equals(command.getMethod())) {
+ try {
+ List<Long> diskSizes =
getRunningVmDiskSizes(libvirtComputingResource, vmName);
+ if (diskSizes != null) {
+ if (CollectionUtils.isEmpty(volumePools)) {
+ return new BackupAnswer(command, false, "No
volumes to back up for VM " + vmName);
+ }
+ PrimaryDataStoreTO scratchPoolTO = volumePools.get(0);
+ scratchPool =
storagePoolMgr.getStoragePool(scratchPoolTO.getPoolType(),
scratchPoolTO.getUuid());
+ scratch = scratchPool.createBackupScratch(vmName,
+
diskSizes.stream().map(LibvirtTakeBackupCommandWrapper::scratchSize).collect(Collectors.toList()));
+ }
+ } catch (LibvirtException | CloudRuntimeException e) {
+ logger.warn("Failed to create the backup scratch space for
VM {}", vmName, e);
+ return new BackupAnswer(command, false,
+ String.format("Failed to create the backup scratch
space for VM %s: %s", vmName, e.getMessage()));
+ }
+ }
+
+ return runBackup(libvirtComputingResource, command, diskPaths,
scratch != null ? scratch.second() : null, timeout);
+ } finally {
+ if (scratch != null) {
+ try {
+ scratchPool.deleteBackupScratch(scratch.first());
+ } catch (CloudRuntimeException e) {
+ logger.warn("Failed to delete the backup scratch space {}
of VM {}, delete it manually: {}",
+ scratch.first(), vmName, e.getMessage());
+ }
+ }
+ disconnectVolumes(storagePoolMgr, connectedVolumes);
+ }
+ }
+
+ private Answer runBackup(LibvirtComputingResource
libvirtComputingResource, TakeBackupCommand command,
+ List<String> diskPaths, List<String> scratchDevices, int timeout) {
+ final String vmName = command.getVmName();
+ Pair<Integer, String> result =
runBackupScript(libvirtComputingResource, command, vmName,
command.getBackupRepoType(),
+ command.getBackupRepoAddress(), command.getMountOptions(),
command.getBackupPath(), diskPaths, command.getMode(),
+ command.getBitmapNew(), command.getBitmapParent(),
command.getParentPaths(), scratchDevices, timeout);
if (result.first() != 0) {
logger.debug("Failed to take VM backup: " + result.second());
+ if (Script.ERR_TIMEOUT.equals(result.second()) &&
METHOD_CONTENT.equals(command.getMethod())) {
+ abortPullBackupJob(vmName);
+ }
BackupAnswer answer = new BackupAnswer(command, false,
StringUtils.trimToEmpty(result.second()));
if (EXIT_CLEANUP_FAILED.equals(result.first())) {
logger.debug("Backup cleanup failed");
@@ -118,6 +187,104 @@ public class LibvirtTakeBackupCommandWrapper extends
CommandWrapper<TakeBackupCo
return answer;
}
+ /**
+ * Makes each LINSTOR volume whose device does not exist on this host
available here; connecting
+ * returns once the device exists. A volume of a running VM, or one this
host holds a replica of,
+ * already has its device and is left alone. Returns the pool and uuid of
every volume it connected,
+ * for {@link #disconnectVolumes}.
+ */
+ private List<Pair<PrimaryDataStoreTO, String>>
connectMissingLinstorVolumes(KVMStoragePoolManager storagePoolMgr,
+ List<PrimaryDataStoreTO> volumePools, List<String> volumePaths) {
+ List<Pair<PrimaryDataStoreTO, String>> connected = new ArrayList<>();
+ if (volumePaths == null) {
+ return connected;
+ }
+ try {
+ for (int idx = 0; idx < volumePaths.size(); idx++) {
+ PrimaryDataStoreTO volumePool = volumePools.get(idx);
+ String volumePath = volumePaths.get(idx);
+ if (volumePool.getPoolType() !=
Storage.StoragePoolType.Linstor || deviceExists(volumePath)) {
+ continue;
+ }
+ String volumeUuid =
LibvirtRestoreBackupCommandWrapper.getVolumeUuidFromPath(volumePath,
volumePool);
+ logger.debug("Making LINSTOR volume {} available for the
backup", volumeUuid);
+ storagePoolMgr.connectPhysicalDisk(volumePool.getPoolType(),
volumePool.getUuid(), volumeUuid, null);
+ connected.add(new Pair<>(volumePool, volumeUuid));
+ }
+ } catch (CloudRuntimeException e) {
+ disconnectVolumes(storagePoolMgr, connected);
+ throw e;
+ }
+ return connected;
+ }
+
+ private void disconnectVolumes(KVMStoragePoolManager storagePoolMgr,
List<Pair<PrimaryDataStoreTO, String>> volumes) {
+ for (Pair<PrimaryDataStoreTO, String> volume : volumes) {
+ try {
+
storagePoolMgr.disconnectPhysicalDisk(volume.first().getPoolType(),
volume.first().getUuid(), volume.second());
+ } catch (CloudRuntimeException e) {
+ logger.warn("Failed to disconnect volume {} after the backup:
{}", volume.second(), e.getMessage());
+ }
+ }
+ }
+
+ boolean deviceExists(String path) {
+ return Files.exists(Paths.get(path));
+ }
+
+ /**
+ * A pull-mode backup job never ends on its own: nasbackup.sh ends it once
the data is read,
+ * and its EXIT trap does so on failure. A timed-out script is killed with
SIGKILL, which
+ * no trap can catch, so end the job here. Otherwise it keeps the NBD
server and the scratch
+ * devices busy and blocks every later backup or migration of the VM.
+ */
+ private void abortPullBackupJob(String vmName) {
+ logger.warn("Backup of VM {} timed out, aborting its pull-mode backup
job", vmName);
+ int exitValue =
Script.executeCommandForExitValue(Script.getExecutableAbsolutePath("virsh"),
+ "-c", "qemu:///system", "domjobabort", vmName);
+ if (exitValue != 0) {
+ logger.warn("Failed to abort the backup job of VM {} (exit code
{})", vmName, exitValue);
+ }
+ }
+
+ /**
+ * The capacity of each disk of the VM in domain order, which is the order
nasbackup.sh lists them in, or
+ * {@code null} if the VM is not running here. nasbackup.sh takes a
stopped VM's backup without pull mode.
+ */
+ private List<Long> getRunningVmDiskSizes(LibvirtComputingResource
libvirtComputingResource, String vmName)
+ throws LibvirtException {
+ Connect conn =
libvirtComputingResource.getLibvirtUtilitiesHelper().getConnectionByVmName(vmName);
+ Domain dm;
+ try {
+ dm = libvirtComputingResource.getDomain(conn, vmName);
+ } catch (LibvirtException e) {
+ logger.debug("VM {} is not defined on this host: {}", vmName,
e.getMessage());
+ return null;
+ }
+ try {
+ if (dm.getInfo().state !=
DomainInfo.DomainState.VIR_DOMAIN_RUNNING) {
+ return null;
+ }
+ List<Long> sizes = new ArrayList<>();
+ for (DiskDef disk : libvirtComputingResource.getDisks(conn,
vmName)) {
+ if (disk.getDeviceType() == DiskDef.DeviceType.DISK) {
+ sizes.add(dm.blockInfo(disk.getDiskLabel()).getCapacity());
+ }
+ }
+ return sizes;
+ } finally {
+ dm.free();
+ }
+ }
+
+ /**
+ * Size of a scratch device for a disk of {@code diskSize} bytes. libvirt
formats it as a qcow2 that may
+ * end up holding every block of the disk, plus metadata: about 0.015% of
the disk with 64 KiB clusters.
+ */
+ static long scratchSize(long diskSize) {
+ return diskSize + diskSize / 4096 + SCRATCH_EXTRA_BYTES;
+ }
+
/** Remove nasbackup.sh's stdout signalling marker lines so they don't
pollute size parsing. */
private String stripMarkerLines(String stdout) {
if (StringUtils.isBlank(stdout)) {
@@ -142,7 +309,7 @@ public class LibvirtTakeBackupCommandWrapper extends
CommandWrapper<TakeBackupCo
private Pair<Integer, String> runBackupScript(LibvirtComputingResource
libvirtComputingResource,
TakeBackupCommand command, String vmName, String backupRepoType,
String backupRepoAddress,
String mountOptions, String backupPath, List<String> diskPaths,
String mode,
- String bitmapNew, String bitmapParent, List<String> parentPaths,
int timeout) {
+ String bitmapNew, String bitmapParent, List<String> parentPaths,
List<String> scratchDevices, int timeout) {
boolean quiesce = command.getQuiesce() != null && command.getQuiesce();
List<String> argv = new ArrayList<>(Arrays.asList(
libvirtComputingResource.getNasBackupPath(),
@@ -164,6 +331,10 @@ public class LibvirtTakeBackupCommandWrapper extends
CommandWrapper<TakeBackupCo
argv.add("-M");
argv.add(mode);
}
+ if (StringUtils.isNotBlank(command.getMethod())) {
+ argv.add("--method");
+ argv.add(command.getMethod());
+ }
if (StringUtils.isNotBlank(bitmapNew)) {
argv.add("--bitmap-new");
argv.add(bitmapNew);
@@ -176,6 +347,10 @@ public class LibvirtTakeBackupCommandWrapper extends
CommandWrapper<TakeBackupCo
argv.add("--parent-paths");
argv.add(String.join(",", parentPaths));
}
+ if (CollectionUtils.isNotEmpty(scratchDevices)) {
+ argv.add("--scratch-devs");
+ argv.add(String.join(",", scratchDevices));
+ }
List<String[]> commands = new ArrayList<>();
commands.add(argv.toArray(new String[0]));
@@ -188,6 +363,13 @@ public class LibvirtTakeBackupCommandWrapper extends
CommandWrapper<TakeBackupCo
*/
private String validateBackupArgs(TakeBackupCommand command) {
String mode = command.getMode();
+ String method = StringUtils.defaultIfBlank(command.getMethod(),
METHOD_QCOW2);
+ if (METHOD_CONTENT.equals(method)) {
+ return validateContentBackupArgs(command);
+ }
+ if (!METHOD_QCOW2.equals(method)) {
+ return "Unknown backup method: " + method;
+ }
if (StringUtils.isBlank(mode)) {
return null; // legacy full-only, no extra args expected
}
@@ -215,6 +397,21 @@ public class LibvirtTakeBackupCommandWrapper extends
CommandWrapper<TakeBackupCo
return "Unknown backup mode: " + mode;
}
+ private String validateContentBackupArgs(TakeBackupCommand command) {
+ String mode = command.getMode();
+ if (MODE_FULL.equals(mode)) {
+ return null; // pull-mode full — no bitmap, no parent
+ }
+ if (MODE_INCREMENTAL.equals(mode)) {
+ // No bitmap is involved: the delta comes from comparing the disk
against the parent.
+ if (CollectionUtils.isEmpty(command.getParentPaths())) {
+ return "content incremental mode requires parentPaths";
+ }
+ return null;
+ }
+ return "Unsupported mode for the content backup method: " + mode;
+ }
+
/**
* Sum the per-disk size lines emitted by nasbackup.sh. Single-volume mode
emits one
* line containing just the byte count; multi-volume mode emits one line
per disk
diff --git
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java
index 3e35ed9476b..46e57d701a0 100644
---
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java
+++
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java
@@ -30,6 +30,8 @@ import com.cloud.agent.api.to.HostTO;
import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.StoragePoolType;
+import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
public interface KVMStoragePool {
@@ -125,4 +127,35 @@ public interface KVMStoragePool {
default void customizeLibvirtDiskDef(LibvirtVMDef.DiskDef disk) {
}
+
+ /**
+ * Whether the volume {@code volumeUuid}, freshly created in this pool,
already reads as zeroes,
+ * so a copy onto it may skip zero regions (qemu-img --target-is-zero).
Says nothing about a
+ * volume that has been written to: that still holds its old data.
+ */
+ default boolean newVolumeReadsAsZero(String volumeUuid) {
+ return false;
+ }
+
+ /**
+ * Creates the scratch space for a pull-mode backup of the running VM
{@code vmName} on this host: one
+ * block device per entry of {@code sizes}, at least that many bytes
large. libvirt formats each device
+ * as the copy-before-write target of one disk, keeping the guest data the
backup has not read yet.
+ *
+ * @return the name to pass to {@link #deleteBackupScratch(String)} and
the device paths, in the order
+ * of {@code sizes}
+ */
+ default Pair<String, List<String>> createBackupScratch(String vmName,
List<Long> sizes) {
+ throw new CloudRuntimeException(String.format("Storage pool %s of type
%s cannot provide backup scratch space",
+ getUuid(), getType()));
+ }
+
+ /**
+ * Removes scratch space created by {@link #createBackupScratch(String,
List)}. Only valid once the
+ * backup job has ended and libvirt has released the devices.
+ */
+ default void deleteBackupScratch(String name) {
+ throw new CloudRuntimeException(String.format("Storage pool %s of type
%s cannot provide backup scratch space",
+ getUuid(), getType()));
+ }
}
diff --git
a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java
b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java
index 138b57aef0a..56497c3c14f 100644
---
a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java
+++
b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java
@@ -22,6 +22,7 @@ import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.mockConstruction;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.when;
@@ -35,10 +36,13 @@ import java.util.List;
import org.apache.cloudstack.backup.BackupAnswer;
import org.apache.cloudstack.backup.RestoreBackupCommand;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.MockedConstruction;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
@@ -871,4 +875,67 @@ public class LibvirtRestoreBackupCommandWrapperTest {
}
}
}
+
+ /**
+ * Restores a backup onto a LINSTOR volume through
replaceBlockDeviceWithBackup and returns
+ * the QemuImg instance used for the copy together with its constructor
arguments.
+ */
+ private Pair<QemuImg, List<?>> restoreOntoLinstorVolume(boolean
createTargetVolume, boolean newVolumeReadsAsZero)
+ throws Exception {
+ Method method =
LibvirtRestoreBackupCommandWrapper.class.getDeclaredMethod("replaceBlockDeviceWithBackup",
+ KVMStoragePoolManager.class, PrimaryDataStoreTO.class,
String.class, String.class, int.class,
+ boolean.class, Long.class);
+ method.setAccessible(true);
+
+ PrimaryDataStoreTO volumePool = Mockito.mock(PrimaryDataStoreTO.class);
+
when(volumePool.getPoolType()).thenReturn(Storage.StoragePoolType.Linstor);
+ when(volumePool.getUuid()).thenReturn("pool-uuid");
+ KVMStoragePoolManager storagePoolMgr =
Mockito.mock(KVMStoragePoolManager.class);
+ KVMStoragePool linstorPool = Mockito.mock(KVMStoragePool.class);
+
lenient().when(linstorPool.newVolumeReadsAsZero("volume-123")).thenReturn(newVolumeReadsAsZero);
+ when(storagePoolMgr.getStoragePool(Storage.StoragePoolType.Linstor,
"pool-uuid")).thenReturn(linstorPool);
+
+ final List<?>[] ctorArgs = new List<?>[1];
+ try (MockedConstruction<QemuImg> qemuMock =
mockConstruction(QemuImg.class,
+ (mock, context) -> ctorArgs[0] = context.arguments())) {
+ boolean result = (boolean) method.invoke(wrapper, storagePoolMgr,
volumePool,
+ "/dev/drbd/by-res/cs-volume-123/0",
"/mnt/backup/root.volume-123.qcow2", 60,
+ createTargetVolume, 1024L);
+
+ Assert.assertTrue(result);
+ Assert.assertEquals(1, qemuMock.constructed().size());
+ QemuImg qemu = qemuMock.constructed().get(0);
+ Mockito.verify(qemu).convert(any(QemuImgFile.class),
any(QemuImgFile.class));
+ if (!createTargetVolume) {
+ Mockito.verify(linstorPool,
Mockito.never()).newVolumeReadsAsZero(anyString());
+ }
+ return new Pair<>(qemu, ctorArgs[0]);
+ }
+ }
+
+ @Test
+ public void testInPlaceLinstorRestoreDoesNotSkipZeroes() throws Exception {
+ // Even on a thin pool: the volume still holds its old data, so the
backup's zero regions
+ // must be written.
+ Pair<QemuImg, List<?>> restore = restoreOntoLinstorVolume(false, true);
+
+ Assert.assertEquals(Boolean.FALSE, restore.second().get(1));
+ Mockito.verify(restore.first()).setSkipTargetVolumeCreation(true);
+ }
+
+ @Test
+ public void testLinstorRestoreToNewThickVolumeDoesNotSkipZeroes() throws
Exception {
+ Pair<QemuImg, List<?>> restore = restoreOntoLinstorVolume(true, false);
+
+ Assert.assertEquals(Boolean.FALSE, restore.second().get(1));
+ Mockito.verify(restore.first()).setSkipTargetVolumeCreation(true);
+ }
+
+ @Test
+ public void testLinstorRestoreToNewThinVolumeSkipsZeroes() throws
Exception {
+ Pair<QemuImg, List<?>> restore = restoreOntoLinstorVolume(true, true);
+
+ Assert.assertEquals(Boolean.TRUE, restore.second().get(1));
+ Mockito.verify(restore.first()).setSkipTargetVolumeCreation(true);
+ }
}
diff --git
a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapperTest.java
b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapperTest.java
index 7ef64fd53b5..2feaba13528 100644
---
a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapperTest.java
+++
b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapperTest.java
@@ -18,25 +18,41 @@ package com.cloud.hypervisor.kvm.resource.wrapper;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.when;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.apache.cloudstack.backup.BackupAnswer;
import org.apache.cloudstack.backup.TakeBackupCommand;
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.libvirt.Connect;
+import org.libvirt.Domain;
+import org.libvirt.DomainBlockInfo;
+import org.libvirt.DomainInfo;
+import org.libvirt.LibvirtException;
import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
+import com.cloud.storage.Storage;
import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
@RunWith(MockitoJUnitRunner.class)
@@ -99,4 +115,268 @@ public class LibvirtTakeBackupCommandWrapperTest {
Assert.assertFalse(runAndCaptureArgv().contains("--quiesce-timeout"));
}
+
+ private void runFailing(Pair<Integer, String> result, boolean
expectJobAbort) {
+ try (MockedStatic<Script> scriptMock = mockStatic(Script.class)) {
+ scriptMock.when(() -> Script.executePipedCommands(anyList(),
anyLong())).thenReturn(result);
+ scriptMock.when(() ->
Script.getExecutableAbsolutePath("virsh")).thenReturn("/usr/bin/virsh");
+
+ BackupAnswer answer = (BackupAnswer) wrapper.execute(command,
libvirtComputingResource);
+ Assert.assertFalse(answer.getResult());
+
+ if (expectJobAbort) {
+ scriptMock.verify(() ->
Script.executeCommandForExitValue("/usr/bin/virsh", "-c", "qemu:///system",
+ "domjobabort", "i-2-3-VM"));
+ } else {
+ scriptMock.verify(() ->
Script.executeCommandForExitValue(anyString(), anyString(), anyString(),
+ anyString(), anyString()), never());
+ }
+ }
+ }
+
+ private static final String LINSTOR_DEVICE = "/dev/drbd/by-res/cs-vol-1/0";
+
+ /** A stopped VM with one volume on a LINSTOR pool; its device exists here
as given. */
+ private KVMStoragePoolManager mockStoppedVmOnLinstor(boolean deviceExists)
{
+ PrimaryDataStoreTO pool = Mockito.mock(PrimaryDataStoreTO.class);
+ when(pool.getPoolType()).thenReturn(Storage.StoragePoolType.Linstor);
+ Mockito.lenient().when(pool.getUuid()).thenReturn("pool-uuid");
+ command.setVolumePools(Collections.singletonList(pool));
+ command.setVolumePaths(Collections.singletonList(LINSTOR_DEVICE));
+ KVMStoragePoolManager storagePoolMgr =
Mockito.mock(KVMStoragePoolManager.class);
+
when(libvirtComputingResource.getStoragePoolMgr()).thenReturn(storagePoolMgr);
+ wrapper = Mockito.spy(wrapper);
+
Mockito.doReturn(deviceExists).when(wrapper).deviceExists(LINSTOR_DEVICE);
+ return storagePoolMgr;
+ }
+
+ @Test
+ public void executeConnectsLinstorVolumeMissingOnThisHost() {
+ KVMStoragePoolManager storagePoolMgr = mockStoppedVmOnLinstor(false);
+
+ List<String> argv = runAndCaptureArgv();
+
+ Assert.assertEquals(LINSTOR_DEVICE, argv.get(argv.indexOf("-d") + 1));
+ InOrder inOrder = Mockito.inOrder(storagePoolMgr);
+
inOrder.verify(storagePoolMgr).connectPhysicalDisk(Storage.StoragePoolType.Linstor,
"pool-uuid", "vol-1", null);
+
inOrder.verify(storagePoolMgr).disconnectPhysicalDisk(Storage.StoragePoolType.Linstor,
"pool-uuid", "vol-1");
+ }
+
+ @Test
+ public void executeLeavesLinstorVolumeWithDeviceOnThisHostAlone() {
+ KVMStoragePoolManager storagePoolMgr = mockStoppedVmOnLinstor(true);
+
+ runAndCaptureArgv();
+
+ Mockito.verify(storagePoolMgr,
never()).connectPhysicalDisk(Mockito.any(Storage.StoragePoolType.class),
+ anyString(), anyString(), Mockito.any());
+ Mockito.verify(storagePoolMgr,
never()).disconnectPhysicalDisk(Mockito.any(Storage.StoragePoolType.class),
+ anyString(), anyString());
+ }
+
+ @Test
+ public void executeDisconnectsConnectedVolumeWhenBackupFails() {
+ KVMStoragePoolManager storagePoolMgr = mockStoppedVmOnLinstor(false);
+
+ runFailing(new Pair<>(1, "qemu-img convert failed"), false);
+
+
Mockito.verify(storagePoolMgr).disconnectPhysicalDisk(Storage.StoragePoolType.Linstor,
"pool-uuid", "vol-1");
+ }
+
+ @Test
+ public void executeFailsWhenLinstorVolumeCannotBeConnected() {
+ KVMStoragePoolManager storagePoolMgr = mockStoppedVmOnLinstor(false);
+
when(storagePoolMgr.connectPhysicalDisk(Storage.StoragePoolType.Linstor,
"pool-uuid", "vol-1", null))
+ .thenThrow(new CloudRuntimeException("resource not found"));
+
+ try (MockedStatic<Script> scriptMock = mockStatic(Script.class)) {
+ BackupAnswer answer = (BackupAnswer) wrapper.execute(command,
libvirtComputingResource);
+
+ Assert.assertFalse(answer.getResult());
+ Assert.assertTrue(answer.getDetails().contains("resource not
found"));
+ scriptMock.verify(() -> Script.executePipedCommands(anyList(),
anyLong()), never());
+ }
+ Mockito.verify(storagePoolMgr,
never()).disconnectPhysicalDisk(Mockito.any(Storage.StoragePoolType.class),
+ anyString(), anyString());
+ }
+
+ /** The VM runs here with disks of the given sizes on a pool that hands
out scratch devices. */
+ private KVMStoragePool mockRunningVmOnScratchPool(long... diskSizes)
throws LibvirtException {
+ LibvirtUtilitiesHelper libvirtUtilitiesHelper =
Mockito.mock(LibvirtUtilitiesHelper.class);
+ Connect conn = Mockito.mock(Connect.class);
+ Domain dm = Mockito.mock(Domain.class);
+ DomainInfo info = new DomainInfo();
+ info.state = DomainInfo.DomainState.VIR_DOMAIN_RUNNING;
+
when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
+
when(libvirtUtilitiesHelper.getConnectionByVmName("i-2-3-VM")).thenReturn(conn);
+ when(libvirtComputingResource.getDomain(conn,
"i-2-3-VM")).thenReturn(dm);
+ when(dm.getInfo()).thenReturn(info);
+
+ List<DiskDef> disks = new ArrayList<>();
+ List<PrimaryDataStoreTO> pools = new ArrayList<>();
+ List<String> volumePaths = new ArrayList<>();
+ List<String> scratchDevices = new ArrayList<>();
+ for (int i = 0; i < diskSizes.length; i++) {
+ String label = "vd" + (char) ('a' + i);
+ DiskDef disk = Mockito.mock(DiskDef.class);
+ when(disk.getDeviceType()).thenReturn(DiskDef.DeviceType.DISK);
+ when(disk.getDiskLabel()).thenReturn(label);
+ DomainBlockInfo blockInfo = Mockito.mock(DomainBlockInfo.class);
+ when(blockInfo.getCapacity()).thenReturn(diskSizes[i]);
+ when(dm.blockInfo(label)).thenReturn(blockInfo);
+ disks.add(disk);
+
+ PrimaryDataStoreTO pool = Mockito.mock(PrimaryDataStoreTO.class);
+
when(pool.getPoolType()).thenReturn(Storage.StoragePoolType.Linstor);
+ pools.add(pool);
+ volumePaths.add("/dev/drbd/by-res/cs-vol-" + i + "/0");
+ scratchDevices.add("/dev/scratch/" + label);
+ }
+ DiskDef cdrom = Mockito.mock(DiskDef.class);
+ when(cdrom.getDeviceType()).thenReturn(DiskDef.DeviceType.CDROM);
+ disks.add(cdrom);
+ when(libvirtComputingResource.getDisks(conn,
"i-2-3-VM")).thenReturn(disks);
+ when(pools.get(0).getUuid()).thenReturn("pool-uuid");
+ command.setVolumePools(pools);
+ command.setVolumePaths(volumePaths);
+ wrapper = Mockito.spy(wrapper);
+ Mockito.doReturn(true).when(wrapper).deviceExists(anyString());
+
+ KVMStoragePoolManager storagePoolMgr =
Mockito.mock(KVMStoragePoolManager.class);
+ KVMStoragePool scratchPool = Mockito.mock(KVMStoragePool.class);
+
when(libvirtComputingResource.getStoragePoolMgr()).thenReturn(storagePoolMgr);
+ when(storagePoolMgr.getStoragePool(Storage.StoragePoolType.Linstor,
"pool-uuid")).thenReturn(scratchPool);
+ when(scratchPool.createBackupScratch(Mockito.eq("i-2-3-VM"),
anyList()))
+ .thenReturn(new Pair<>("cs-bkscratch-1", scratchDevices));
+ return scratchPool;
+ }
+
+ @Test
+ public void executeAbortsPullBackupJobOnTimeout() throws LibvirtException {
+ command.setMode("full");
+ command.setMethod("content");
+ KVMStoragePool scratchPool = mockRunningVmOnScratchPool(1L << 30);
+
+ runFailing(new Pair<>(-1, Script.ERR_TIMEOUT), true);
+
+ Mockito.verify(scratchPool).deleteBackupScratch("cs-bkscratch-1");
+ }
+
+ @Test
+ public void executeLeavesPullBackupJobToScriptOnFailure() throws
LibvirtException {
+ command.setMode("full");
+ command.setMethod("content");
+ KVMStoragePool scratchPool = mockRunningVmOnScratchPool(1L << 30);
+
+ runFailing(new Pair<>(1, "qemu-img convert failed"), false);
+
+ Mockito.verify(scratchPool).deleteBackupScratch("cs-bkscratch-1");
+ }
+
+ @Test
+ public void executeFailsWhenScratchCannotBeCreated() throws
LibvirtException {
+ command.setMode("full");
+ command.setMethod("content");
+ KVMStoragePool scratchPool = mockRunningVmOnScratchPool(1L << 30);
+ when(scratchPool.createBackupScratch(Mockito.eq("i-2-3-VM"),
anyList()))
+ .thenThrow(new CloudRuntimeException("no storage pool"));
+
+ try (MockedStatic<Script> scriptMock = mockStatic(Script.class)) {
+ BackupAnswer answer = (BackupAnswer) wrapper.execute(command,
libvirtComputingResource);
+
+ Assert.assertFalse(answer.getResult());
+ Assert.assertTrue(answer.getDetails().contains("no storage pool"));
+ scriptMock.verify(() -> Script.executePipedCommands(anyList(),
anyLong()), never());
+ }
+ Mockito.verify(scratchPool, never()).deleteBackupScratch(anyString());
+ }
+
+ @Test
+ public void executeKeepsBackupWhenScratchCannotBeDeleted() throws
LibvirtException {
+ command.setMode("full");
+ command.setMethod("content");
+ KVMStoragePool scratchPool = mockRunningVmOnScratchPool(1L << 30);
+ Mockito.doThrow(new CloudRuntimeException("in
use")).when(scratchPool).deleteBackupScratch("cs-bkscratch-1");
+
+ Assert.assertTrue(runAndCaptureArgv().contains("--scratch-devs"));
+ }
+
+ @Test
+ public void executeTakesStoppedVmBackupWithoutScratch() throws
LibvirtException {
+ command.setMode("full");
+ command.setMethod("content");
+ LibvirtUtilitiesHelper libvirtUtilitiesHelper =
Mockito.mock(LibvirtUtilitiesHelper.class);
+ Connect conn = Mockito.mock(Connect.class);
+
when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
+
when(libvirtUtilitiesHelper.getConnectionByVmName("i-2-3-VM")).thenReturn(conn);
+ when(libvirtComputingResource.getDomain(conn,
"i-2-3-VM")).thenThrow(Mockito.mock(LibvirtException.class));
+
+ Assert.assertFalse(runAndCaptureArgv().contains("--scratch-devs"));
+ }
+
+ @Test
+ public void scratchSizeHoldsAFullyAllocatedQcow2() {
+ // qemu-img measure -O qcow2 --size <n>, "fully allocated size"
+
Assert.assertTrue(LibvirtTakeBackupCommandWrapper.scratchSize(540909568L) >=
541327360L);
+
Assert.assertTrue(LibvirtTakeBackupCommandWrapper.scratchSize(5371715584L) >=
5372837888L);
+
Assert.assertTrue(LibvirtTakeBackupCommandWrapper.scratchSize(1099511627776L)
>= 1099679662080L);
+ }
+
+ @Test
+ public void executeDoesNotAbortPushBackupJobOnTimeout() {
+ command.setMode("legacy-full");
+
+ runFailing(new Pair<>(-1, Script.ERR_TIMEOUT), false);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void executePassesContentMethodToScript() throws LibvirtException {
+ command.setMode("incremental");
+ command.setMethod("content");
+
command.setParentPaths(Arrays.asList("i-2-3-VM/2026.10.03.00.00.00/root.vol-0.qcow2",
+ "i-2-3-VM/2026.10.03.00.00.00/datadisk.vol-1.qcow2"));
+ KVMStoragePool scratchPool = mockRunningVmOnScratchPool(1L << 30, 5L
<< 30);
+
+ List<String> argv = runAndCaptureArgv();
+
+ Assert.assertEquals("incremental", argv.get(argv.indexOf("-M") + 1));
+ Assert.assertEquals("content", argv.get(argv.indexOf("--method") + 1));
+ Assert.assertFalse(argv.contains("--bitmap-new"));
+ // One scratch device per disk, in disk order, each large enough for
the whole disk.
+ Assert.assertEquals("/dev/scratch/vda,/dev/scratch/vdb",
argv.get(argv.indexOf("--scratch-devs") + 1));
+ ArgumentCaptor<List<Long>> sizes = ArgumentCaptor.forClass(List.class);
+
Mockito.verify(scratchPool).createBackupScratch(Mockito.eq("i-2-3-VM"),
sizes.capture());
+
Assert.assertEquals(Arrays.asList(LibvirtTakeBackupCommandWrapper.scratchSize(1L
<< 30),
+ LibvirtTakeBackupCommandWrapper.scratchSize(5L << 30)),
sizes.getValue());
+ Mockito.verify(scratchPool).deleteBackupScratch("cs-bkscratch-1");
+ }
+
+ @Test
+ public void executeOmitsScratchDevicesForPushMode() {
+ command.setMode("legacy-full");
+
+ Assert.assertFalse(runAndCaptureArgv().contains("--scratch-devs"));
+ }
+
+ @Test
+ public void executeRejectsContentIncrementalWithoutParentPaths() {
+ command.setMode("incremental");
+ command.setMethod("content");
+
+ BackupAnswer answer = (BackupAnswer) wrapper.execute(command,
libvirtComputingResource);
+
+ Assert.assertFalse(answer.getResult());
+ Assert.assertEquals("content incremental mode requires parentPaths",
answer.getDetails());
+ }
+
+ @Test
+ public void executeRejectsLegacyFullWithContentMethod() {
+ command.setMode("legacy-full");
+ command.setMethod("content");
+
+ BackupAnswer answer = (BackupAnswer) wrapper.execute(command,
libvirtComputingResource);
+
+ Assert.assertFalse(answer.getResult());
+ }
}
diff --git
a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
index 848bdc60d66..94878684a99 100644
---
a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
+++
b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
@@ -19,15 +19,20 @@ package com.cloud.hypervisor.kvm.storage;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import com.cloud.storage.Storage;
+import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import
org.apache.cloudstack.storage.datastore.util.LinstorConfigurationManager;
@@ -46,18 +51,24 @@ import com.linbit.linstor.api.ApiException;
import com.linbit.linstor.api.DevelopersApi;
import com.linbit.linstor.api.model.ApiCallRc;
import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.AutoPlaceRequest;
+import com.linbit.linstor.api.model.AutoSelectFilter;
+import com.linbit.linstor.api.model.LayerType;
import com.linbit.linstor.api.model.Node;
import com.linbit.linstor.api.model.Properties;
import com.linbit.linstor.api.model.ProviderKind;
import com.linbit.linstor.api.model.Resource;
import com.linbit.linstor.api.model.ResourceConnectionModify;
import com.linbit.linstor.api.model.ResourceDefinition;
+import com.linbit.linstor.api.model.ResourceDefinitionCreate;
import com.linbit.linstor.api.model.ResourceDefinitionModify;
import com.linbit.linstor.api.model.ResourceGroupSpawn;
import com.linbit.linstor.api.model.ResourceMakeAvailable;
import com.linbit.linstor.api.model.ResourceWithVolumes;
import com.linbit.linstor.api.model.StoragePool;
+import com.linbit.linstor.api.model.Volume;
import com.linbit.linstor.api.model.VolumeDefinition;
+import com.linbit.linstor.api.model.VolumeDefinitionCreate;
import java.io.File;
import java.io.FileInputStream;
@@ -69,9 +80,14 @@ import java.nio.file.Paths;
public class LinstorStorageAdaptor implements StorageAdaptor {
protected Logger logger = LogManager.getLogger(getClass());
private static final Map<String, KVMStoragePool>
MapStorageUuidToStoragePool = new HashMap<>();
+ static final String BACKUP_SCRATCH_PREFIX = "cs-bkscratch-";
+ static final String BACKUP_SCRATCH_VM_PROP = "Aux/cs-backup-vm";
+ private static final long DEVICE_WAIT_MS = 10000;
+ // Controllers this agent has already removed its leftover backup scratch
from, see deleteLeftoverBackupScratch().
+ private static final Set<String> BackupScratchSweptControllers =
ConcurrentHashMap.newKeySet();
private final String localNodeName;
- private DevelopersApi getLinstorAPI(KVMStoragePool pool) {
+ DevelopersApi getLinstorAPI(KVMStoragePool pool) {
// Use a fresh client per pool so a self-signed/insecure pool can't
weaken the TLS settings
// of another pool sharing this agent.
ApiClient client = new ApiClient();
@@ -186,6 +202,11 @@ public class LinstorStorageAdaptor implements
StorageAdaptor {
:
Boolean.parseBoolean(LinstorConfigurationManager.InsecureSsl.defaultValue());
LinstorStoragePool storagePool = new LinstorStoragePool(name, host,
port, userInfo, type, this, insecureSsl);
+ // Pools are registered again whenever the agent reconnects, but only
the first registration of a
+ // controller precedes every backup that could be using its scratch
space.
+ if (BackupScratchSweptControllers.add(host)) {
+ deleteLeftoverBackupScratch(storagePool);
+ }
MapStorageUuidToStoragePool.put(name, storagePool);
return storagePool;
@@ -208,6 +229,206 @@ public class LinstorStorageAdaptor implements
StorageAdaptor {
ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName,
localNodeName, rma);
handleLinstorApiAnswers(answers,
String.format("Linstor: Unable to make resource %s available on
node: %s", rscName, localNodeName));
+ waitForLocalDevice(api, rscName);
+ }
+
+ /**
+ * Waits until the device of resource {@code rscName} exists on this node.
Making a resource available
+ * returns before udev has created its /dev/drbd/by-res link, so opening
the device right away can fail
+ * with "No such file or directory".
+ */
+ private void waitForLocalDevice(DevelopersApi api, String rscName) throws
ApiException {
+ List<ResourceWithVolumes> resources =
api.viewResources(Collections.singletonList(localNodeName),
+ Collections.singletonList(rscName), Collections.emptyList(),
null, null, null);
+ if (resources.isEmpty() || resources.get(0).getVolumes().isEmpty()) {
+ throw new CloudRuntimeException(String.format("Linstor: resource
%s has no volumes on %s",
+ rscName, localNodeName));
+ }
+ waitForDevice(LinstorUtil.getDevicePathFromResource(resources.get(0)));
+ }
+
+ void waitForDevice(String device) {
+ long deadline = System.currentTimeMillis() + DEVICE_WAIT_MS;
+ while (!Files.exists(Paths.get(device))) {
+ if (System.currentTimeMillis() > deadline) {
+ throw new CloudRuntimeException(String.format("Linstor: device
%s did not appear within %d ms",
+ device, DEVICE_WAIT_MS));
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new CloudRuntimeException("Linstor: interrupted while
waiting for device " + device, e);
+ }
+ }
+ }
+
+ /**
+ * Creates the scratch space of a pull-mode backup as one resource with a
volume per size. The resource
+ * definition is created in the pool's resource group, so the group's
properties and storage pool selection
+ * apply; spawning it from the group would limit the volumes to the
group's volume groups.
+ *
+ * Copy-before-write runs inline with the guest's writes, so the scratch
is placed on a storage pool of this
+ * node if there is one, as a plain STORAGE resource: it only lives for
one backup and needs no replication.
+ * Otherwise, e.g. on a compute node without storage, it is a single DRBD
replica on another node that this
+ * node attaches diskless.
+ *
+ * The agent deletes the scratch once the backup has ended. If it never
gets there, e.g. because the agent
+ * crashed or the host went down, it deletes the leftover the next time it
starts, see
+ * {@link #deleteLeftoverBackupScratch(KVMStoragePool)}; the {@value
#BACKUP_SCRATCH_VM_PROP} property
+ * names the VM.
+ */
+ public Pair<String, List<String>> createBackupScratch(KVMStoragePool pool,
String vmName, List<Long> sizes) {
+ final LinstorStoragePool lpool = (LinstorStoragePool) pool;
+ final DevelopersApi api = getLinstorAPI(pool);
+ // LINSTOR caps resource names at 48 characters.
+ final String rscName = BACKUP_SCRATCH_PREFIX +
UUID.randomUUID().toString().replace("-", "");
+ logger.info("Linstor: creating backup scratch {} for VM {} with volume
sizes {}", rscName, vmName, sizes);
+ try {
+ ResourceDefinition rd = new ResourceDefinition();
+ rd.setName(rscName);
+ rd.setResourceGroupName(lpool.getResourceGroup());
+ Properties props = new Properties();
+ props.put(BACKUP_SCRATCH_VM_PROP, vmName);
+ rd.setProps(props);
+ ResourceDefinitionCreate rdCreate = new ResourceDefinitionCreate();
+ rdCreate.setResourceDefinition(rd);
+ checkLinstorAnswersThrow(api.resourceDefinitionCreate(rdCreate));
+ } catch (ApiException apiEx) {
+ throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+ }
+
+ try {
+ for (int vlmNr = 0; vlmNr < sizes.size(); vlmNr++) {
+ VolumeDefinition vd = new VolumeDefinition();
+ vd.setVolumeNumber(vlmNr);
+ vd.setSizeKib((sizes.get(vlmNr) + 1023) / 1024);
+ VolumeDefinitionCreate vdCreate = new VolumeDefinitionCreate();
+ vdCreate.setVolumeDefinition(vd);
+ checkLinstorAnswersThrow(api.volumeDefinitionCreate(rscName,
vdCreate));
+ }
+ placeBackupScratch(api, rscName);
+ return new Pair<>(rscName, getBackupScratchDevices(api, rscName));
+ } catch (ApiException | CloudRuntimeException ex) {
+ try {
+ deleteBackupScratch(pool, rscName);
+ } catch (CloudRuntimeException deleteEx) {
+ logger.warn("Linstor: unable to delete backup scratch {},
delete it manually: {}",
+ rscName, deleteEx.getMessage());
+ }
+ if (ex instanceof ApiException) {
+ throw new CloudRuntimeException(((ApiException)
ex).getBestMessage(), ex);
+ }
+ throw (CloudRuntimeException) ex;
+ }
+ }
+
+ private void placeBackupScratch(DevelopersApi api, String rscName) throws
ApiException {
+ String localError;
+ try {
+ // A failed placement usually comes back as an HTTP error rather
than as error answers.
+ ApiCallRcList answers = api.resourceAutoplace(rscName,
backupScratchPlacement(
+ Collections.singletonList(localNodeName),
LayerType.STORAGE));
+ if (!answers.hasError()) {
+ logLinstorAnswers(answers);
+ return;
+ }
+ localError = LinstorUtil.getBestErrorMessage(answers);
+ } catch (ApiException apiEx) {
+ localError = apiEx.getBestMessage();
+ }
+ logger.info("Linstor: cannot place backup scratch {} on {} ({}),
attaching a replica on another node diskless",
+ rscName, localNodeName, localError);
+ checkLinstorAnswersThrow(api.resourceAutoplace(rscName,
backupScratchPlacement(
+ Collections.emptyList(), LayerType.DRBD, LayerType.STORAGE)));
+ makeResourceAvailable(api, rscName, false);
+ }
+
+ private static AutoPlaceRequest backupScratchPlacement(List<String>
nodeNames, LayerType... layers) {
+ List<String> layerStack =
Arrays.stream(layers).map(LayerType::toString).collect(Collectors.toList());
+ AutoSelectFilter filter = new AutoSelectFilter();
+ filter.setPlaceCount(1);
+ filter.setNodeNameList(nodeNames);
+ filter.setLayerStack(layerStack);
+ filter.setDisklessOnRemaining(false);
+ AutoPlaceRequest request = new AutoPlaceRequest();
+ request.setSelectFilter(filter);
+ request.setLayerList(Arrays.asList(layers));
+ request.setDisklessOnRemaining(false);
+ return request;
+ }
+
+ /**
+ * The device paths of the scratch volumes on this node, in volume number
order, once udev has created them.
+ */
+ private List<String> getBackupScratchDevices(DevelopersApi api, String
rscName) throws ApiException {
+ List<ResourceWithVolumes> resources =
api.viewResources(Collections.singletonList(localNodeName),
+ Collections.singletonList(rscName), Collections.emptyList(),
null, null, null);
+ if (resources.isEmpty()) {
+ throw new CloudRuntimeException(String.format("Linstor: backup
scratch %s has no resource on %s",
+ rscName, localNodeName));
+ }
+ ResourceWithVolumes rsc = resources.get(0);
+ boolean drbd = rsc.getLayerObject().getDrbd() != null;
+ List<String> devices = rsc.getVolumes().stream()
+ .sorted(Comparator.comparing(Volume::getVolumeNumber))
+ .map(vlm -> drbd ? String.format("/dev/drbd/by-res/%s/%d",
rscName, vlm.getVolumeNumber())
+ : vlm.getDevicePath())
+ .collect(Collectors.toList());
+ for (String device : devices) {
+ waitForDevice(device);
+ }
+ return devices;
+ }
+
+ /**
+ * Deletes the backup scratch resources an earlier run of this agent left
on this node, e.g. because it
+ * crashed or was restarted during a backup. Called when the first pool of
a controller is registered,
+ * before any backup can use it, so none of them can belong to a running
backup. A scratch always has a
+ * resource on the node whose agent created it, diskless if its data lives
on another node. Failures are
+ * logged only: a leftover must not keep the pool from being registered.
+ */
+ void deleteLeftoverBackupScratch(KVMStoragePool pool) {
+ final DevelopersApi api = getLinstorAPI(pool);
+ try {
+ Map<String, ResourceDefinition> scratchRds =
LinstorUtil.getRDListStartingWith(api, BACKUP_SCRATCH_PREFIX)
+
.stream().collect(Collectors.toMap(ResourceDefinition::getName, rd -> rd));
+ if (scratchRds.isEmpty()) {
+ return;
+ }
+ List<String> leftovers =
api.viewResources(Collections.singletonList(localNodeName),
+ new ArrayList<>(scratchRds.keySet()),
Collections.emptyList(), null, null, null)
+
.stream().map(Resource::getName).distinct().collect(Collectors.toList());
+ for (String rscName : leftovers) {
+ Properties props = scratchRds.get(rscName).getProps();
+ String vmName = props != null ?
props.get(BACKUP_SCRATCH_VM_PROP) : null;
+ logger.warn("Linstor: deleting backup scratch {} of VM {} left
behind on {}", rscName, vmName,
+ localNodeName);
+ try {
+ deleteBackupScratch(pool, rscName);
+ } catch (CloudRuntimeException e) {
+ // Typically after a crash during a backup: the VM's
pull-mode backup job outlived the agent
+ // and still holds the scratch devices.
+ logger.warn("Linstor: unable to delete leftover backup
scratch {}: {}. If VM {} still has a backup "
+ + "job, end it with 'virsh domjobabort {}', then
delete the resource definition manually.",
+ rscName, e.getMessage(), vmName, vmName);
+ }
+ }
+ } catch (ApiException apiEx) {
+ logger.warn("Linstor: unable to look for leftover backup scratch
on {}: {}", localNodeName,
+ apiEx.getBestMessage());
+ } catch (CloudRuntimeException e) {
+ logger.warn("Linstor: unable to look for leftover backup scratch
on {}: {}", localNodeName, e.getMessage());
+ }
+ }
+
+ public void deleteBackupScratch(KVMStoragePool pool, String rscName) {
+ logger.info("Linstor: deleting backup scratch {}", rscName);
+ try {
+
checkLinstorAnswersThrow(getLinstorAPI(pool).resourceDefinitionDelete(rscName));
+ } catch (ApiException apiEx) {
+ throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+ }
}
/**
@@ -340,7 +561,7 @@ public class LinstorStorageAdaptor implements
StorageAdaptor {
ResourceMakeAvailable rma = new ResourceMakeAvailable();
ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName,
localNodeName, rma);
checkLinstorAnswersThrow(answers);
-
+ waitForLocalDevice(api, rscName);
} catch (ApiException apiEx) {
logger.error(apiEx);
throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
diff --git
a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java
b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java
index 8a02560fd46..2e9a0bbe98b 100644
---
a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java
+++
b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java
@@ -24,6 +24,7 @@ import com.cloud.agent.properties.AgentProperties;
import com.cloud.agent.properties.AgentPropertiesFileHandler;
import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool;
import com.cloud.storage.Storage;
+import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
@@ -33,6 +34,7 @@ import com.google.gson.JsonIOException;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonSyntaxException;
+import org.apache.cloudstack.storage.datastore.util.LinstorUtil;
import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -374,6 +376,25 @@ public class LinstorStoragePool implements KVMStoragePool {
return false;
}
+ /**
+ * Thin LVM and ZFS backed resources start out unallocated and read as
zeroes, thick LVM hands out
+ * whatever was on the disk before.
+ */
+ @Override
+ public boolean newVolumeReadsAsZero(String volumeUuid) {
+ return LinstorUtil.resourceSupportZeroBlocks(this,
LinstorUtil.RSC_PREFIX + volumeUuid);
+ }
+
+ @Override
+ public Pair<String, List<String>> createBackupScratch(String vmName,
List<Long> sizes) {
+ return ((LinstorStorageAdaptor)
_storageAdaptor).createBackupScratch(this, vmName, sizes);
+ }
+
+ @Override
+ public void deleteBackupScratch(String name) {
+ ((LinstorStorageAdaptor) _storageAdaptor).deleteBackupScratch(this,
name);
+ }
+
@Override
public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration
activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath,
long duration) {
LOGGER.trace(String.format("Linstor.vmActivityCheck: %s, %s",
pool.getPoolIp(), host.getPrivateNetwork().getIp()));
diff --git
a/plugins/storage/volume/linstor/src/test/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptorTest.java
b/plugins/storage/volume/linstor/src/test/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptorTest.java
new file mode 100644
index 00000000000..054d6b8eac1
--- /dev/null
+++
b/plugins/storage/volume/linstor/src/test/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptorTest.java
@@ -0,0 +1,344 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.hypervisor.kvm.storage;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import com.cloud.storage.Storage;
+import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.linbit.linstor.api.ApiConsts;
+import com.linbit.linstor.api.ApiException;
+import com.linbit.linstor.api.DevelopersApi;
+import com.linbit.linstor.api.model.ApiCallRc;
+import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.AutoPlaceRequest;
+import com.linbit.linstor.api.model.DrbdResource;
+import com.linbit.linstor.api.model.LayerType;
+import com.linbit.linstor.api.model.Properties;
+import com.linbit.linstor.api.model.ResourceDefinition;
+import com.linbit.linstor.api.model.ResourceDefinitionCreate;
+import com.linbit.linstor.api.model.ResourceLayer;
+import com.linbit.linstor.api.model.ResourceMakeAvailable;
+import com.linbit.linstor.api.model.ResourceWithVolumes;
+import com.linbit.linstor.api.model.StorageResource;
+import com.linbit.linstor.api.model.Volume;
+import com.linbit.linstor.api.model.VolumeDefinitionCreate;
+
+@RunWith(MockitoJUnitRunner.class)
+public class LinstorStorageAdaptorTest {
+
+ private static final String DEVICE = "/dev/drbd/by-res/cs-vol-1/0";
+
+ private LinstorStorageAdaptor adaptor;
+ private LinstorStoragePool pool;
+ private DevelopersApi api;
+ private String localNode;
+
+ @Before
+ public void setUp() {
+ adaptor = spy(new LinstorStorageAdaptor());
+ pool = new LinstorStoragePool("pool-uuid", "http://controller:3370";,
3370, "cloudstack",
+ Storage.StoragePoolType.Linstor, adaptor, false);
+ api = mock(DevelopersApi.class);
+ doReturn(api).when(adaptor).getLinstorAPI(any());
+ doNothing().when(adaptor).waitForDevice(anyString());
+ localNode = LinstorStoragePool.getHostname();
+ }
+
+ private static ApiCallRcList answers(long retCode, String message) {
+ ApiCallRc rc = new ApiCallRc();
+ rc.setRetCode(retCode);
+ rc.setMessage(message);
+ ApiCallRcList list = new ApiCallRcList();
+ list.add(rc);
+ return list;
+ }
+
+ private static ApiCallRcList success() {
+ return answers(ApiConsts.MASK_SUCCESS, "ok");
+ }
+
+ private static ApiCallRcList error(String message) {
+ return answers(ApiConsts.MASK_ERROR, message);
+ }
+
+ private void mockDefinitionsCreated() throws ApiException {
+ when(api.resourceDefinitionCreate(any())).thenReturn(success());
+ when(api.volumeDefinitionCreate(anyString(),
any())).thenReturn(success());
+ }
+
+ private static ResourceWithVolumes drbdResource() {
+ ResourceLayer layer = new ResourceLayer();
+ layer.setDrbd(new DrbdResource());
+ Volume vlm = new Volume();
+ vlm.setVolumeNumber(0);
+ vlm.setDevicePath("/dev/drbd1000");
+ vlm.setAllocatedSizeKib(0L);
+ ResourceWithVolumes rsc = new ResourceWithVolumes();
+ rsc.setName("cs-vol-1");
+ rsc.setLayerObject(layer);
+ rsc.setVolumes(Collections.singletonList(vlm));
+ return rsc;
+ }
+
+ /** How the controller reports a failed request: an HTTP error carrying
the answers in its body. */
+ private static ApiException httpError(String message) {
+ return new ApiException(500, Collections.emptyMap(),
+ String.format("[{\"ret_code\": %d, \"message\": \"%s\"}]",
ApiConsts.MASK_ERROR, message));
+ }
+
+ private static Volume volume(int number, String devicePath) {
+ Volume vlm = new Volume();
+ vlm.setVolumeNumber(number);
+ vlm.setDevicePath(devicePath);
+ return vlm;
+ }
+
+ private void mockLocalResource(boolean drbd, Volume... volumes) throws
ApiException {
+ ResourceLayer layer = new ResourceLayer();
+ if (drbd) {
+ layer.setDrbd(new DrbdResource());
+ } else {
+ layer.setStorage(new StorageResource());
+ }
+ ResourceWithVolumes rsc = new ResourceWithVolumes();
+ rsc.setLayerObject(layer);
+ rsc.setVolumes(Arrays.asList(volumes));
+ when(api.viewResources(eq(Collections.singletonList(localNode)),
any(), any(), any(), any(), any()))
+ .thenReturn(Collections.singletonList(rsc));
+ }
+
+ @Test
+ public void createBackupScratchPlacesStorageOnlyResourceLocally() throws
ApiException {
+ mockDefinitionsCreated();
+ when(api.resourceAutoplace(anyString(), any())).thenReturn(success());
+ mockLocalResource(false, volume(1, "/dev/storage/scratch_00001"),
volume(0, "/dev/storage/scratch_00000"));
+
+ Pair<String, List<String>> scratch = adaptor.createBackupScratch(pool,
"i-2-3-VM", Arrays.asList(1025L, 4096L));
+
+
Assert.assertTrue(scratch.first().startsWith(LinstorStorageAdaptor.BACKUP_SCRATCH_PREFIX));
+ Assert.assertTrue(scratch.first().length() <= 48);
+ Assert.assertEquals(Arrays.asList("/dev/storage/scratch_00000",
"/dev/storage/scratch_00001"), scratch.second());
+
+ ArgumentCaptor<ResourceDefinitionCreate> rdCreate =
ArgumentCaptor.forClass(ResourceDefinitionCreate.class);
+ verify(api).resourceDefinitionCreate(rdCreate.capture());
+ ResourceDefinition rd = rdCreate.getValue().getResourceDefinition();
+ Assert.assertEquals(scratch.first(), rd.getName());
+ Assert.assertEquals("cloudstack", rd.getResourceGroupName());
+ Assert.assertEquals("i-2-3-VM",
rd.getProps().get(LinstorStorageAdaptor.BACKUP_SCRATCH_VM_PROP));
+
+ // One volume per disk, sizes rounded up to KiB.
+ ArgumentCaptor<VolumeDefinitionCreate> vdCreate =
ArgumentCaptor.forClass(VolumeDefinitionCreate.class);
+ verify(api, times(2)).volumeDefinitionCreate(eq(scratch.first()),
vdCreate.capture());
+ Assert.assertEquals(Integer.valueOf(0),
vdCreate.getAllValues().get(0).getVolumeDefinition().getVolumeNumber());
+ Assert.assertEquals(Long.valueOf(2L),
vdCreate.getAllValues().get(0).getVolumeDefinition().getSizeKib());
+ Assert.assertEquals(Integer.valueOf(1),
vdCreate.getAllValues().get(1).getVolumeDefinition().getVolumeNumber());
+ Assert.assertEquals(Long.valueOf(4L),
vdCreate.getAllValues().get(1).getVolumeDefinition().getSizeKib());
+
+ ArgumentCaptor<AutoPlaceRequest> placement =
ArgumentCaptor.forClass(AutoPlaceRequest.class);
+ verify(api).resourceAutoplace(eq(scratch.first()),
placement.capture());
+ Assert.assertEquals(Collections.singletonList(localNode),
placement.getValue().getSelectFilter().getNodeNameList());
+ Assert.assertEquals(Collections.singletonList(LayerType.STORAGE),
placement.getValue().getLayerList());
+ Assert.assertEquals(Integer.valueOf(1),
placement.getValue().getSelectFilter().getPlaceCount());
+ verify(api, never()).resourceMakeAvailableOnNode(anyString(),
anyString(), any());
+ }
+
+ @Test
+ public void
createBackupScratchAttachesRemoteReplicaDisklessWithoutLocalStoragePool()
throws ApiException {
+ mockDefinitionsCreated();
+ when(api.resourceAutoplace(anyString(), any()))
+ .thenThrow(httpError("Not enough available nodes"))
+ .thenReturn(success());
+ when(api.resourceMakeAvailableOnNode(anyString(), eq(localNode),
any())).thenReturn(success());
+ mockLocalResource(true, volume(0, "/dev/drbd1000"), volume(1,
"/dev/drbd1001"));
+
+ Pair<String, List<String>> scratch = adaptor.createBackupScratch(pool,
"i-2-3-VM", Arrays.asList(1024L, 1024L));
+
+ String rscName = scratch.first();
+ Assert.assertEquals(Arrays.asList("/dev/drbd/by-res/" + rscName +
"/0", "/dev/drbd/by-res/" + rscName + "/1"),
+ scratch.second());
+
+ ArgumentCaptor<AutoPlaceRequest> placement =
ArgumentCaptor.forClass(AutoPlaceRequest.class);
+ verify(api, times(2)).resourceAutoplace(eq(rscName),
placement.capture());
+ AutoPlaceRequest remote = placement.getAllValues().get(1);
+ Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.STORAGE),
remote.getLayerList());
+
Assert.assertTrue(remote.getSelectFilter().getNodeNameList().isEmpty());
+ Assert.assertEquals(Integer.valueOf(1),
remote.getSelectFilter().getPlaceCount());
+
+ ArgumentCaptor<ResourceMakeAvailable> available =
ArgumentCaptor.forClass(ResourceMakeAvailable.class);
+ verify(api).resourceMakeAvailableOnNode(eq(rscName), eq(localNode),
available.capture());
+ Assert.assertFalse(available.getValue().isDiskful());
+ verify(api, never()).resourceDefinitionDelete(anyString());
+ }
+
+ @Test
+ public void
createBackupScratchDeletesResourceDefinitionWhenItCannotBePlaced() throws
ApiException {
+ mockDefinitionsCreated();
+ when(api.resourceAutoplace(anyString(), any()))
+ .thenReturn(error("Not enough available nodes"))
+ .thenThrow(httpError("No storage pool with enough space"));
+ when(api.resourceDefinitionDelete(anyString())).thenReturn(success());
+
+ try {
+ adaptor.createBackupScratch(pool, "i-2-3-VM",
Collections.singletonList(1024L));
+ Assert.fail("expected CloudRuntimeException");
+ } catch (CloudRuntimeException e) {
+ Assert.assertTrue(e.getMessage().contains("No storage pool with
enough space"));
+ }
+
+ ArgumentCaptor<ResourceDefinitionCreate> rdCreate =
ArgumentCaptor.forClass(ResourceDefinitionCreate.class);
+ verify(api).resourceDefinitionCreate(rdCreate.capture());
+
verify(api).resourceDefinitionDelete(rdCreate.getValue().getResourceDefinition().getName());
+ }
+
+ @Test(expected = CloudRuntimeException.class)
+ public void deleteBackupScratchFailsWhenStillInUse() throws ApiException {
+
when(api.resourceDefinitionDelete("cs-bkscratch-1")).thenReturn(error("Resource
is still in use"));
+
+ adaptor.deleteBackupScratch(pool, "cs-bkscratch-1");
+ }
+
+ @Test
+ public void connectPhysicalDiskWaitsForTheLocalDevice() throws
ApiException {
+ when(api.resourceMakeAvailableOnNode(eq("cs-vol-1"), eq(localNode),
any())).thenReturn(success());
+ when(api.viewResources(Collections.singletonList(localNode),
Collections.singletonList("cs-vol-1"),
+ Collections.emptyList(), null, null,
null)).thenReturn(Collections.singletonList(drbdResource()));
+
+ Assert.assertTrue(adaptor.connectPhysicalDisk("vol-1", pool, null,
false));
+
+ InOrder inOrder = Mockito.inOrder(api, adaptor);
+ inOrder.verify(api).resourceMakeAvailableOnNode(eq("cs-vol-1"),
eq(localNode), any());
+ inOrder.verify(adaptor).waitForDevice(DEVICE);
+ }
+
+ @Test
+ public void createPhysicalDiskWaitsForTheLocalDevice() throws ApiException
{
+ when(api.resourceDefinitionList(null, false, null, null,
null)).thenReturn(Collections.emptyList());
+ when(api.resourceGroupSpawn(eq("cloudstack"),
any())).thenReturn(success());
+ when(api.viewResources(any(),
eq(Collections.singletonList("cs-vol-1")), any(), any(), any(), any()))
+ .thenReturn(Collections.singletonList(drbdResource()));
+ when(api.resourceMakeAvailableOnNode(eq("cs-vol-1"), eq(localNode),
any())).thenReturn(success());
+
+ KVMPhysicalDisk disk = adaptor.createPhysicalDisk("vol-1", pool,
QemuImg.PhysicalDiskFormat.RAW,
+ Storage.ProvisioningType.THIN, 1024L * 1024, null);
+
+ Assert.assertEquals(DEVICE, disk.getPath());
+ InOrder inOrder = Mockito.inOrder(api, adaptor);
+ inOrder.verify(api).resourceMakeAvailableOnNode(eq("cs-vol-1"),
eq(localNode), any());
+ inOrder.verify(adaptor).waitForDevice(DEVICE);
+ }
+
+ @Test(expected = CloudRuntimeException.class)
+ public void connectPhysicalDiskFailsWithoutLocalVolume() throws
ApiException {
+ when(api.resourceMakeAvailableOnNode(eq("cs-vol-1"), eq(localNode),
any())).thenReturn(success());
+ when(api.viewResources(Collections.singletonList(localNode),
Collections.singletonList("cs-vol-1"),
+ Collections.emptyList(), null, null,
null)).thenReturn(Collections.emptyList());
+
+ adaptor.connectPhysicalDisk("vol-1", pool, null, false);
+ }
+
+ @Test
+ public void waitForDeviceReturnsForAnExistingDevice() {
+ new LinstorStorageAdaptor().waitForDevice("/dev/null");
+ }
+
+ private static ResourceDefinition scratchRd(String name, String vmName) {
+ ResourceDefinition rd = new ResourceDefinition();
+ rd.setName(name);
+ Properties props = new Properties();
+ props.put(LinstorStorageAdaptor.BACKUP_SCRATCH_VM_PROP, vmName);
+ rd.setProps(props);
+ return rd;
+ }
+
+ private static ResourceWithVolumes resourceOf(String name) {
+ ResourceWithVolumes rsc = new ResourceWithVolumes();
+ rsc.setName(name);
+ return rsc;
+ }
+
+ // The controllers this agent has swept are remembered per process, so
every test uses its own one.
+ private KVMStoragePool registerPool(String controller) {
+ return adaptor.createStoragePool("pool-" + controller, controller,
3370, "cloudstack", null,
+ Storage.StoragePoolType.Linstor, null, true);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void createStoragePoolDeletesLeftoverScratchOfThisNode() throws
ApiException {
+ ResourceDefinition volume = new ResourceDefinition();
+ volume.setName("cs-vol-1");
+ when(api.resourceDefinitionList(null, false, null, null,
null)).thenReturn(Arrays.asList(
+ scratchRd("cs-bkscratch-here", "i-2-3-VM"),
scratchRd("cs-bkscratch-elsewhere", "i-2-4-VM"), volume));
+ ArgumentCaptor<List<String>> rscNames =
ArgumentCaptor.forClass(List.class);
+ when(api.viewResources(eq(Collections.singletonList(localNode)),
rscNames.capture(), any(), any(), any(), any()))
+
.thenReturn(Collections.singletonList(resourceOf("cs-bkscratch-here")));
+
when(api.resourceDefinitionDelete("cs-bkscratch-here")).thenReturn(success());
+
+ Assert.assertNotNull(registerPool("http://sweep-1:3370";));
+
+ Assert.assertEquals(2, rscNames.getValue().size());
+ Assert.assertFalse(rscNames.getValue().contains("cs-vol-1"));
+ verify(api).resourceDefinitionDelete("cs-bkscratch-here");
+ verify(api,
never()).resourceDefinitionDelete("cs-bkscratch-elsewhere");
+ }
+
+ @Test
+ public void createStoragePoolSweepsEachControllerOnlyOnce() throws
ApiException {
+ when(api.resourceDefinitionList(null, false, null, null,
null)).thenReturn(Collections.emptyList());
+
+ registerPool("http://sweep-2:3370";);
+ registerPool("http://sweep-2:3370";);
+
+ verify(api, times(1)).resourceDefinitionList(null, false, null, null,
null);
+ }
+
+ @Test
+ public void createStoragePoolRegistersPoolWhenControllerIsUnreachable()
throws ApiException {
+ when(api.resourceDefinitionList(null, false, null, null,
null)).thenThrow(new ApiException("connection refused"));
+
+ KVMStoragePool registered = registerPool("http://sweep-3:3370";);
+
+ Assert.assertSame(registered,
adaptor.getStoragePool("pool-http://sweep-3:3370";));
+ verify(api, never()).resourceDefinitionDelete(anyString());
+ }
+}
diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh
b/scripts/vm/hypervisor/kvm/nasbackup.sh
index 04cc7cd86f2..8999ede1c01 100755
--- a/scripts/vm/hypervisor/kvm/nasbackup.sh
+++ b/scripts/vm/hypervisor/kvm/nasbackup.sh
@@ -35,6 +35,8 @@ QUIESCE=""
QUIESCE_TIMEOUT="" # Seconds to wait for the guest agent freeze/thaw; empty
=> libvirt default
# Incremental backup parameters (all optional; legacy callers omit them)
MODE="" # "full" or "incremental"; empty => legacy full-only
behavior (no checkpoint created)
+METHOD="qcow2" # How the backup is taken: "qcow2" (push mode,
incrementals from dirty bitmaps) or
+ # "content" (pull mode, incrementals by content
comparison; raw block storage)
BITMAP_NEW="" # Bitmap/checkpoint name to create with this backup
(e.g. "backup-1711586400")
BITMAP_PARENT="" # For incremental: parent bitmap name to read changes
since
PARENT_PATHS="" # For incremental: comma-separated list of parent backup
file paths,
@@ -42,10 +44,15 @@ PARENT_PATHS="" # For incremental: comma-separated
list of parent backup f
# is rebased onto its corresponding parent file.
Required because
# data-disk backup files don't share the root volume's
UUID, so
# each disk must be rebased onto its own parent.
+SCRATCH_DEVS="" # Pull mode: comma-separated list of copy-before-write
scratch block devices, one
+ # per VM volume in the same order as DISK_PATHS, created
by the agent on the
+ # VM's primary storage and at least as large as the disk
plus qcow2 metadata.
+PULL_WORK_PARENT="/var/tmp" # Parent dir for the pull-mode work dir (NBD
socket + backup XML)
logFile="/var/log/cloudstack/agent/agent.log"
EXIT_CLEANUP_FAILED=20
BACKUP_JOB_ACTIVE=0
+PULL_WORK_DIR="" # Work dir of a running pull-mode backup, removed by
on_pull_backup_exit()
log() {
[[ "$verb" -eq 1 ]] && builtin echo "$@"
@@ -183,6 +190,236 @@ thaw_guest() {
return 1
}
+# ---------------------------------------------------------------------------
+# Content-based backup path (libvirt pull mode).
+#
+# Used for raw block-device disks such as LINSTOR/DRBD. Those cannot carry QEMU
+# persistent dirty bitmaps -- persistence is a qcow2-only feature -- so libvirt
+# checkpoints, and with them the push-mode incremental path, are unavailable.
+#
+# Instead libvirt is asked for a pull-mode backup: it starts an NBD server
exposing a
+# point-in-time view of each disk, held consistent by copy-before-write into a
scratch
+# block device per disk, which the agent creates on the VM's primary storage
and libvirt
+# formats as qcow2. The backup is then derived from that export:
+#
+# full qemu-img convert -> sparse standalone qcow2. Zero detection
happens
+# while streaming, so the fully allocated push-mode output and
its
+# separate re-convert pass are both avoided.
+# incremental a qcow2 overlay backed by the NBD export, then a SAFE rebase
onto the
+# parent backup. That merges exactly the clusters which differ
between
+# export and parent into the overlay, yielding a delta qcow2
backed by the
+# parent -- the same chain shape the push-mode incremental
produces, so
+# restore/delete are unchanged.
+#
+# Unlike a storage-snapshot based diff there is no trailing DRBD metadata to
clip: the
+# DRBD device, and therefore the NBD export, is already net-sized.
+#
+# Assumes mount_operation() already ran, so $dest and $mount_point are set.
+# ---------------------------------------------------------------------------
+
+# EXIT trap of the pull-mode path, armed once the work dir exists and cleared
once the
+# backup is complete. Any exit in between -- an explicit failure, an
unexpected error under
+# "set -e", or SIGINT/SIGTERM/SIGHUP -- has to end the pull job: unlike a push
job it never
+# finishes on its own, so it would keep the NBD server and the scratch devices
busy and
+# block every later backup or migration of the VM. cleanup() aborts the job
(via
+# BACKUP_JOB_ACTIVE) before removing the partial backup. SIGKILL cannot be
trapped, so the
+# agent wrapper aborts the job itself when the script times out.
+on_pull_backup_exit() {
+ local rc=$?
+ trap - EXIT INT TERM HUP
+ [[ $rc -ne 0 ]] || rc=1
+ # The NBD socket belongs to the job: only remove it once the job is gone.
+ if abort_backup_job; then
+ rm -rf "$PULL_WORK_DIR"
+ fi
+ cleanup
+ exit $rc
+}
+
+backup_running_vm_pull() {
+ local effective_mode="$1"
+
+ # Work dir for the NBD socket and the backup XML. 711 lets the qemu process
traverse in
+ # to the socket.
+ local work_dir
+ work_dir=$(mktemp -d "$PULL_WORK_PARENT/csbackup-pull.XXXXXX") || {
+ echo "Failed to create work directory under $PULL_WORK_PARENT"; cleanup;
exit 1;
+ }
+ chmod 711 "$work_dir"
+ PULL_WORK_DIR="$work_dir"
+ trap on_pull_backup_exit EXIT
+ trap 'exit 129' HUP
+ trap 'exit 130' INT
+ trap 'exit 143' TERM
+ local nbd_sock="$work_dir/nbd.sock"
+ local backupxml="$work_dir/backup.xml"
+
+ # Snapshot the disk list once so the backup XML and the read-back loop stay
in sync.
+ local -a disks
+ mapfile -t disks < <(
+ virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk
'$2=="disk"{print $3, $4}'
+ )
+ if [[ ${#disks[@]} -eq 0 ]]; then
+ echo "No disks found for vm $VM"
+ exit 1
+ fi
+
+ # Copy-before-write saves each guest block overwritten during the backup
into the disk's
+ # scratch device, at most once per block. The agent sizes each device for
the whole disk,
+ # so it cannot fill up and pause the VM.
+ local -a scratch_devs=()
+ IFS=',' read -ra scratch_devs <<< "$SCRATCH_DEVS"
+ if [[ ${#scratch_devs[@]} -ne ${#disks[@]} ]]; then
+ echo "Pull-mode backup of vm $VM needs one scratch device per disk:" \
+ "got ${#scratch_devs[@]} for ${#disks[@]} disks"
+ exit 1
+ fi
+ local scratch_dev
+ for scratch_dev in "${scratch_devs[@]}"; do
+ if [[ ! -b "$scratch_dev" ]]; then
+ echo "Scratch device $scratch_dev for the backup of vm $VM is not a
block device"
+ exit 1
+ fi
+ done
+
+ # Resolve each disk's parent backup up front. PARENT_PATHS is comma
separated, one entry
+ # per volume in the same order as the disk list. Anything missing degrades
this run to a
+ # full backup rather than failing it, mirroring the push-mode fallback.
+ local -a parent_paths_arr=()
+ if [[ "$effective_mode" == "incremental" ]]; then
+ if [[ -z "$PARENT_PATHS" ]]; then
+ log -e "content incremental: no parent paths supplied -- falling back to
full"
+ echo "INCREMENTAL_FALLBACK=true"
+ effective_mode="full"
+ else
+ IFS=',' read -ra parent_paths_arr <<< "$PARENT_PATHS"
+ if [[ ${#parent_paths_arr[@]} -lt ${#disks[@]} ]]; then
+ log -e "content incremental: parent path list shorter than disk list
-- falling back to full"
+ echo "INCREMENTAL_FALLBACK=true"
+ effective_mode="full"
+ else
+ local pp
+ for pp in "${parent_paths_arr[@]}"; do
+ if [[ ! -f "$mount_point/$pp" ]]; then
+ log -e "content incremental: parent $mount_point/$pp missing on
NAS -- falling back to full"
+ echo "INCREMENTAL_FALLBACK=true"
+ effective_mode="full"
+ break
+ fi
+ done
+ fi
+ fi
+ fi
+
+ # Pull-mode backup XML: one NBD export and one scratch device per disk.
+ {
+ echo "<domainbackup mode='pull'>"
+ echo "<server transport='unix' socket='$nbd_sock'/>"
+ echo "<disks>"
+ local entry idx=0
+ for entry in "${disks[@]}"; do
+ echo "<disk name='${entry%% *}' backup='yes' type='block'
backupmode='full'><scratch dev='${scratch_devs[$idx]}'/></disk>"
+ idx=$((idx + 1))
+ done
+ echo "</disks></domainbackup>"
+ } > "$backupxml"
+
+ local thaw=0
+ local freeze_ok=0
+ if [[ ${QUIESCE} == "true" ]]; then
+ # Always thaw after a freeze attempt, see backup_running_vm().
+ thaw=1
+ if freeze_err=$(qemu_agent_command '{"execute":"guest-fsfreeze-freeze"}'
2>&1 > /dev/null); then
+ freeze_ok=1
+ else
+ log -e "Failed to freeze the filesystem for vm $VM, continuing without
quiescing: $freeze_err"
+ fi
+ fi
+
+ # backup-begin establishes the point-in-time and starts the NBD server.
+ local backup_begin=0
+ local backup_out
+ if backup_out=$(virsh -c qemu:///system backup-begin --domain "$VM"
--backupxml "$backupxml" 2>&1); then
+ backup_begin=1
+ BACKUP_JOB_ACTIVE=1
+ fi
+
+ if [[ $thaw -eq 1 ]] && ! thaw_guest "$freeze_ok"; then
+ exit 1
+ fi
+
+ if [[ $backup_begin -ne 1 ]]; then
+ echo "Failed to start pull-mode backup for $VM: $backup_out"
+ exit 1
+ fi
+
+ # Backup domain information
+ virsh -c qemu:///system dumpxml $VM > $dest/domain-config.xml 2>/dev/null
+ virsh -c qemu:///system dominfo $VM > $dest/dominfo.xml 2>/dev/null
+ virsh -c qemu:///system domiflist $VM > $dest/domiflist.xml 2>/dev/null
+ virsh -c qemu:///system domblklist $VM > $dest/domblklist.xml 2>/dev/null
+
+ local name="root"
+ local disk_idx=0
+ local entry disk fullpath volUuid output export_uri parent_abs parent_rel
+ for entry in "${disks[@]}"; do
+ disk="${entry%% *}"
+ fullpath="${entry#* }"
+ if [[ "$fullpath" == /dev/drbd/by-res/* ]]; then
+ volUuid=$(get_linstor_uuid_from_path "$fullpath")
+ elif [[ "$fullpath" == /dev/drbd[0-9]* ]]; then
+ if ! volUuid=$(get_linstor_uuid_from_device "$fullpath"); then
+ echo "Failed to resolve LINSTOR volume UUID for $fullpath"
+ exit 1
+ fi
+ else
+ volUuid="${fullpath##*/}"
+ fi
+ output="$dest/$name.$volUuid.qcow2"
+ # The export name defaults to the disk target (e.g. vda) and presents raw
bytes.
+ export_uri="nbd+unix:///$disk?socket=$nbd_sock"
+
+ if [[ "$effective_mode" == "incremental" ]]; then
+ parent_abs="$mount_point/${parent_paths_arr[$disk_idx]}"
+ parent_rel=$(realpath --relative-to="$dest" "$parent_abs")
+ # Overlay over the export, safe-rebase onto the parent to merge the
differing
+ # clusters in, then a metadata-only rebase to store the parent as a
relative path
+ # so the chain survives being mounted at a different mount point.
+ if ! qemu-img create -f qcow2 -b "$export_uri" -F raw "$output" >>
"$logFile" 2> >(cat >&2) \
+ || ! qemu-img rebase -b "$parent_abs" -F qcow2 "$output" >>
"$logFile" 2> >(cat >&2) \
+ || ! qemu-img rebase -u -b "$parent_rel" -F qcow2 "$output" >>
"$logFile" 2> >(cat >&2); then
+ echo "qemu-img incremental delta failed for $disk -> $output"
+ exit 1
+ fi
+ else
+ # Zero detection while streaming gives a sparse qcow2 in a single pass.
+ if ! qemu-img convert -f raw -O qcow2 "$export_uri" "$output" >>
"$logFile" 2> >(cat >&2); then
+ echo "qemu-img convert failed for pull export $disk -> $output"
+ exit 1
+ fi
+ fi
+ name="datadisk"
+ disk_idx=$((disk_idx + 1))
+ done
+
+ # End the pull job. Aborting the active job is how a pull backup finishes
(there is no
+ # backup-end command): libvirt tears down the NBD server and releases the
scratch devices,
+ # which the agent then deletes. The data already pulled is a complete
backup, so from here
+ # on a failure must no longer discard it.
+ trap - EXIT INT TERM HUP
+ if ! abort_backup_job; then
+ log -e "warning: failed to end (domjobabort) backup job for $VM"
+ fi
+ rm -rf "$work_dir"
+ sync
+
+ # Statistics -- the size on the last stdout line is parsed by the Java
wrapper.
+ du -sb $dest | cut -f1
+
+ umount $mount_point
+ rmdir $mount_point
+}
+
backup_running_vm() {
mount_operation
mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit
1; }
@@ -193,6 +430,30 @@ backup_running_vm() {
# invoking the script; the case below is a defensive fallback for direct
invocations.
local effective_mode="${MODE:-legacy-full}"
local make_checkpoint=0
+ case "$METHOD" in
+ qcow2)
+ ;;
+ content)
+ # Raw block-device storage (LINSTOR/DRBD) cannot carry persistent dirty
bitmaps,
+ # so checkpoints are unavailable. Use the content-based pull-mode path
instead.
+ case "$effective_mode" in
+ full|incremental)
+ backup_running_vm_pull "$effective_mode"
+ return
+ ;;
+ *)
+ echo "Mode $effective_mode is not supported by the content method"
+ cleanup
+ exit 1
+ ;;
+ esac
+ ;;
+ *)
+ echo "Unknown method: $METHOD"
+ cleanup
+ exit 1
+ ;;
+ esac
case "$effective_mode" in
incremental|full)
make_checkpoint=1
@@ -513,9 +774,10 @@ backup_stopped_vm() {
volUuid="${disk##*/}"
fi
output="$dest/$name.$volUuid.qcow2"
- if ! qemu-img convert -O qcow2 "$disk" "$output" > "$logFile" 2> >(cat
>&2); then
+ if ! qemu-img convert -O qcow2 "$disk" "$output" >> "$logFile" 2> >(cat
>&2); then
echo "qemu-img convert failed for $disk $output"
cleanup
+ exit 1
fi
# Pre-seed a persistent bitmap on the source disk so the NEXT backup (taken
@@ -616,7 +878,7 @@ function usage {
echo ""
echo "Usage: $0 -o <operation> -v|--vm <domain name> -t <storage type> -s
<storage address> -m <mount options> -p <backup path> -d <disks path>
-q|--quiesce <true|false>"
echo " [--quiesce-timeout <seconds>]"
- echo " [-M|--mode <full|incremental>] [--bitmap-new <name>]
[--bitmap-parent <name>] [--parent-paths <p1,p2,...>]"
+ echo " [-M|--mode <full|incremental>] [--method <qcow2|content>]
[--bitmap-new <name>] [--bitmap-parent <name>] [--parent-paths <p1,p2,...>]"
echo ""
echo "Incremental backup options (running VMs only; requires QEMU >= 4.2 and
libvirt >= 7.2):"
echo " -M|--mode full Take a full backup AND create a checkpoint
(--bitmap-new required) for future incrementals."
@@ -624,6 +886,15 @@ function usage {
echo " Requires --bitmap-parent, --bitmap-new, and
--parent-paths (comma-separated list, one"
echo " parent qcow2 path per disk:
root.<uuid>.qcow2, datadisk.<uuid>.qcow2, … same order"
echo " as -d|--disks)."
+ echo " --method qcow2 Default. Push-mode backup;
incrementals come from QEMU dirty bitmaps."
+ echo " --method content Libvirt pull-mode backup, for raw
block-device storage (LINSTOR/DRBD) which"
+ echo " cannot carry the persistent dirty
bitmaps checkpoints need. A full is written"
+ echo " as a sparse qcow2; an incremental is a
qcow2 overlay on the NBD export,"
+ echo " safe-rebased onto the parent, keeping
only the clusters that differ."
+ echo " Takes no bitmap names; an incremental
requires --parent-paths."
+ echo " --scratch-devs <d1,d2,...> Copy-before-write scratch block
devices for --method content on a running"
+ echo " VM, one per disk in the same order as
-d|--disks, each large enough for"
+ echo " the whole disk plus qcow2 metadata.
libvirt formats them."
echo " Without -M, behaves as legacy full-only backup with no checkpoint
creation."
echo ""
exit 1
@@ -681,6 +952,11 @@ while [[ $# -gt 0 ]]; do
shift
shift
;;
+ --method)
+ METHOD="$2"
+ shift
+ shift
+ ;;
--bitmap-new)
BITMAP_NEW="$2"
shift
@@ -696,6 +972,11 @@ while [[ $# -gt 0 ]]; do
shift
shift
;;
+ --scratch-devs)
+ SCRATCH_DEVS="$2"
+ shift
+ shift
+ ;;
-h|--help)
usage
shift
