JoaoJandre commented on code in PR #14697:
URL: https://github.com/apache/cloudstack/pull/14697#discussion_r4563869076


##########
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java:
##########
@@ -3609,4 +3665,231 @@ public KVMStoragePool 
getTemplateSourcePoolUsingMigrationOptions(KVMStoragePool
 
         return 
storagePoolMgr.getStoragePool(migrationOptions.getSrcPoolType(), 
migrationOptions.getSrcPoolUuid());
     }
+
+    /**
+     * Write a marker file before creating a snapshot overlay.
+     * Marker format: 
poolUuid|volumePath|snapshotName|vmName|diskLabel|overlayPath|timestamp
+     */
+    protected void writeSnapshotMarker(KVMStoragePool pool, String volumePath, 
String snapshotName) {
+        writeSnapshotMarker(pool, volumePath, snapshotName, null, null, null);
+    }
+
+    protected void writeSnapshotMarker(KVMStoragePool pool, String volumePath, 
String snapshotName,
+                                       String vmName, String diskLabel, String 
overlayPath) {
+        try {
+            File markerDir = new File(SNAPSHOT_MARKER_DIR);
+            if (!markerDir.exists()) {
+                markerDir.mkdirs();
+            }
+            String markerName = SNAPSHOT_MARKER_PREFIX + pool.getUuid() + "_" 
+ volumePath.replace("/", "_") + "_" + snapshotName;
+            File markerFile = new File(markerDir, markerName);
+            String content = String.join("|",
+                    pool.getUuid(),
+                    volumePath,
+                    snapshotName,
+                    StringUtils.defaultString(vmName),
+                    StringUtils.defaultString(diskLabel),
+                    StringUtils.defaultString(overlayPath),
+                    String.valueOf(System.currentTimeMillis()));
+            try (FileOutputStream fos = new FileOutputStream(markerFile)) {
+                fos.write(content.getBytes());
+            }
+            logger.debug("Wrote snapshot marker: {}", 
markerFile.getAbsolutePath());
+        } catch (IOException e) {
+            logger.warn("Failed to write snapshot marker for volume {} 
snapshot {}: {}", volumePath, snapshotName, e.getMessage());
+        }
+    }
+
+    protected void updateSnapshotMarker(KVMStoragePool pool, String 
volumePath, String snapshotName,
+                                        String vmName, String diskLabel, 
String overlayPath) {
+        writeSnapshotMarker(pool, volumePath, snapshotName, vmName, diskLabel, 
overlayPath);
+    }
+
+    /**
+     * Remove the snapshot marker after successful merge.
+     */
+    protected void removeSnapshotMarker(KVMStoragePool pool, String 
volumePath, String snapshotName) {
+        try {
+            String markerName = SNAPSHOT_MARKER_PREFIX + pool.getUuid() + "_" 
+ volumePath.replace("/", "_") + "_" + snapshotName;
+            File markerFile = new File(SNAPSHOT_MARKER_DIR, markerName);
+            if (markerFile.exists()) {
+                markerFile.delete();
+                logger.debug("Removed snapshot marker: {}", 
markerFile.getAbsolutePath());
+            }
+        } catch (Exception e) {
+            logger.warn("Failed to remove snapshot marker for volume {} 
snapshot {}: {}", volumePath, snapshotName, e.getMessage());
+        }
+    }
+
+    /**
+     * Reconcile incomplete snapshots by checking for marker files on agent 
startup.
+     * If a marker exists, the agent died during a snapshot and the merge may 
not have completed.
+     *
+     * Strategy per marker:
+     * - Overlay gone: snapshot completed or was cleaned up, remove marker
+     * - VM still running on overlay: blockcommit --active --pivot back to base
+     * - VM already on base: overlay is orphaned, remove marker
+     * - Domain gone, overlay exists: offline qemu-img commit
+     * - Base modified since marker: cannot safely commit, alert and keep 
marker
+     */
+    protected void reconcileSnapshotMarkers() {
+        File markerDir = new File(SNAPSHOT_MARKER_DIR);
+        if (!markerDir.exists() || !markerDir.isDirectory()) {
+            return;
+        }
+
+        File[] markers = markerDir.listFiles((dir, name) -> 
name.startsWith(SNAPSHOT_MARKER_PREFIX));
+        if (markers == null || markers.length == 0) {
+            return;
+        }
+
+        logger.info("Found {} snapshot marker(s) to reconcile", 
markers.length);
+        for (File marker : markers) {
+            reconcileSingleMarker(marker);
+        }
+    }
+
+    protected void reconcileSingleMarker(File marker) {
+        try {
+            String content = new 
String(Files.readAllBytes(marker.toPath())).trim();
+            String[] parts = content.split("\\|", -1);
+            if (parts.length < 3) {
+                logger.error("Invalid snapshot marker format in {}: {}", 
marker.getName(), content);
+                return;
+            }
+
+            String poolUuid = parts[0];
+            String volumePath = parts[1];
+            String snapshotName = parts[2];
+            String vmName = parts.length > 3 ? parts[3] : "";
+            String diskLabel = parts.length > 4 ? parts[4] : "";
+            String overlayPath = parts.length > 5 ? parts[5] : "";
+
+            if (StringUtils.isEmpty(overlayPath)) {
+                overlayPath = resource.getSnapshotTemporaryPath(volumePath, 
snapshotName);
+            }
+
+            logger.warn("Reconciling incomplete snapshot: pool={}, volume={}, 
snapshot={}, vm={}, disk={}, overlay={}",
+                    poolUuid, volumePath, snapshotName, vmName, diskLabel, 
overlayPath);
+
+            File overlayFile = new File(overlayPath);
+            if (!overlayFile.exists()) {
+                logger.info("Overlay file {} no longer exists. Snapshot may 
have completed or been cleaned up. Removing marker.", overlayPath);
+                marker.delete();
+                return;
+            }
+
+            if (StringUtils.isNotEmpty(vmName)) {
+                if (reconcileWithRunningDomain(vmName, diskLabel, volumePath, 
overlayPath, marker)) {
+                    return;
+                }
+            }
+
+            reconcileOffline(volumePath, overlayPath, marker);
+
+        } catch (IOException e) {
+            logger.error("Failed to read snapshot marker {}: {}", 
marker.getName(), e.getMessage());
+        }
+    }
+
+    private boolean reconcileWithRunningDomain(String vmName, String 
diskLabel, String volumePath,
+                                               String overlayPath, File 
marker) {
+        try {
+            Connect conn = LibvirtConnection.getConnection();
+            Domain vm = conn.domainLookupByName(vmName);
+            if (vm == null) {
+                return false;
+            }
+
+            String xmlDesc = vm.getXMLDesc(0);
+            LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser();
+            parser.parseDomainXML(xmlDesc);
+
+            for (DiskDef disk : parser.getDisks()) {
+                String currentSource = disk.getDiskPath();
+                if (currentSource != null && 
currentSource.equals(overlayPath)) {
+                    logger.warn("VM [{}] disk [{}] is still on overlay [{}]. 
Attempting blockcommit to merge back to base [{}].",
+                            vmName, disk.getDiskLabel(), overlayPath, 
volumePath);
+
+                    String actualDiskLabel = StringUtils.isNotEmpty(diskLabel) 
? diskLabel : disk.getDiskLabel();
+                    try {
+                        resource.mergeDeltaIntoBaseFile(vm, actualDiskLabel, 
volumePath, null, true, "reconcile", null, conn);

Review Comment:
   Informing 'reconcile' here will fail on 
`mergeSnapshotIntoBaseFileWithoutEvents` during the overlay delete section. 
Since the snapshot name is used to determine its path.



##########
agent/src/main/java/com/cloud/agent/Agent.java:
##########
@@ -663,6 +663,35 @@ protected void reconnect(final Link link, String 
preferredMSHost, boolean forTra
         logger.info("Connected to the host: {}", shell.getConnectedHost());
     }
 
+    protected void waitForCommandsAndReconnect(final Link link) {
+        int inProgress = commandsInProgress.get();
+        if (inProgress > 0) {
+            logger.warn("Agent disconnected with {} command(s) still in 
progress. Waiting for them to complete before reconnecting.", inProgress);
+
+            long maxWaitMs = 300000;
+            long waitedMs = 0;
+            while (commandsInProgress.get() > 0 && waitedMs < maxWaitMs) {
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    break;
+                }
+                waitedMs += 1000;
+            }
+
+            if (commandsInProgress.get() > 0) {
+                logger.warn("Timed out waiting for {} command(s) to complete 
after {} ms. Proceeding with reconnect — "
+                        + "in-progress commands (e.g. snapshot merges) may not 
have completed.", commandsInProgress.get(), waitedMs);
+            } else {
+                logger.info("All in-progress commands completed. Proceeding 
with reconnect.");
+            }
+        }
+
+        logger.info("Reconnecting instead of exiting to avoid losing in-flight 
work (e.g. snapshot overlays).");
+        reconnect(link);

Review Comment:
   When the management server sees that the agent has disconnected, it will 
immediately stop waiting for command responses from the agent. This does not 
seem like an effective change without any changes to the management server as 
well.



##########
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java:
##########
@@ -3609,4 +3665,231 @@ public KVMStoragePool 
getTemplateSourcePoolUsingMigrationOptions(KVMStoragePool
 
         return 
storagePoolMgr.getStoragePool(migrationOptions.getSrcPoolType(), 
migrationOptions.getSrcPoolUuid());
     }
+
+    /**
+     * Write a marker file before creating a snapshot overlay.
+     * Marker format: 
poolUuid|volumePath|snapshotName|vmName|diskLabel|overlayPath|timestamp
+     */
+    protected void writeSnapshotMarker(KVMStoragePool pool, String volumePath, 
String snapshotName) {
+        writeSnapshotMarker(pool, volumePath, snapshotName, null, null, null);
+    }
+
+    protected void writeSnapshotMarker(KVMStoragePool pool, String volumePath, 
String snapshotName,
+                                       String vmName, String diskLabel, String 
overlayPath) {
+        try {
+            File markerDir = new File(SNAPSHOT_MARKER_DIR);
+            if (!markerDir.exists()) {
+                markerDir.mkdirs();
+            }
+            String markerName = SNAPSHOT_MARKER_PREFIX + pool.getUuid() + "_" 
+ volumePath.replace("/", "_") + "_" + snapshotName;
+            File markerFile = new File(markerDir, markerName);
+            String content = String.join("|",
+                    pool.getUuid(),
+                    volumePath,
+                    snapshotName,
+                    StringUtils.defaultString(vmName),
+                    StringUtils.defaultString(diskLabel),
+                    StringUtils.defaultString(overlayPath),
+                    String.valueOf(System.currentTimeMillis()));
+            try (FileOutputStream fos = new FileOutputStream(markerFile)) {
+                fos.write(content.getBytes());
+            }
+            logger.debug("Wrote snapshot marker: {}", 
markerFile.getAbsolutePath());
+        } catch (IOException e) {
+            logger.warn("Failed to write snapshot marker for volume {} 
snapshot {}: {}", volumePath, snapshotName, e.getMessage());
+        }
+    }
+
+    protected void updateSnapshotMarker(KVMStoragePool pool, String 
volumePath, String snapshotName,
+                                        String vmName, String diskLabel, 
String overlayPath) {
+        writeSnapshotMarker(pool, volumePath, snapshotName, vmName, diskLabel, 
overlayPath);
+    }
+
+    /**
+     * Remove the snapshot marker after successful merge.
+     */
+    protected void removeSnapshotMarker(KVMStoragePool pool, String 
volumePath, String snapshotName) {
+        try {
+            String markerName = SNAPSHOT_MARKER_PREFIX + pool.getUuid() + "_" 
+ volumePath.replace("/", "_") + "_" + snapshotName;
+            File markerFile = new File(SNAPSHOT_MARKER_DIR, markerName);
+            if (markerFile.exists()) {
+                markerFile.delete();
+                logger.debug("Removed snapshot marker: {}", 
markerFile.getAbsolutePath());
+            }
+        } catch (Exception e) {
+            logger.warn("Failed to remove snapshot marker for volume {} 
snapshot {}: {}", volumePath, snapshotName, e.getMessage());
+        }
+    }
+
+    /**
+     * Reconcile incomplete snapshots by checking for marker files on agent 
startup.
+     * If a marker exists, the agent died during a snapshot and the merge may 
not have completed.
+     *
+     * Strategy per marker:
+     * - Overlay gone: snapshot completed or was cleaned up, remove marker
+     * - VM still running on overlay: blockcommit --active --pivot back to base
+     * - VM already on base: overlay is orphaned, remove marker
+     * - Domain gone, overlay exists: offline qemu-img commit
+     * - Base modified since marker: cannot safely commit, alert and keep 
marker
+     */
+    protected void reconcileSnapshotMarkers() {
+        File markerDir = new File(SNAPSHOT_MARKER_DIR);
+        if (!markerDir.exists() || !markerDir.isDirectory()) {
+            return;
+        }
+
+        File[] markers = markerDir.listFiles((dir, name) -> 
name.startsWith(SNAPSHOT_MARKER_PREFIX));
+        if (markers == null || markers.length == 0) {
+            return;
+        }
+
+        logger.info("Found {} snapshot marker(s) to reconcile", 
markers.length);
+        for (File marker : markers) {
+            reconcileSingleMarker(marker);
+        }
+    }
+
+    protected void reconcileSingleMarker(File marker) {
+        try {
+            String content = new 
String(Files.readAllBytes(marker.toPath())).trim();
+            String[] parts = content.split("\\|", -1);
+            if (parts.length < 3) {
+                logger.error("Invalid snapshot marker format in {}: {}", 
marker.getName(), content);
+                return;
+            }
+
+            String poolUuid = parts[0];
+            String volumePath = parts[1];
+            String snapshotName = parts[2];
+            String vmName = parts.length > 3 ? parts[3] : "";
+            String diskLabel = parts.length > 4 ? parts[4] : "";
+            String overlayPath = parts.length > 5 ? parts[5] : "";
+
+            if (StringUtils.isEmpty(overlayPath)) {
+                overlayPath = resource.getSnapshotTemporaryPath(volumePath, 
snapshotName);
+            }
+
+            logger.warn("Reconciling incomplete snapshot: pool={}, volume={}, 
snapshot={}, vm={}, disk={}, overlay={}",
+                    poolUuid, volumePath, snapshotName, vmName, diskLabel, 
overlayPath);
+
+            File overlayFile = new File(overlayPath);
+            if (!overlayFile.exists()) {
+                logger.info("Overlay file {} no longer exists. Snapshot may 
have completed or been cleaned up. Removing marker.", overlayPath);
+                marker.delete();
+                return;
+            }
+
+            if (StringUtils.isNotEmpty(vmName)) {
+                if (reconcileWithRunningDomain(vmName, diskLabel, volumePath, 
overlayPath, marker)) {
+                    return;
+                }
+            }
+
+            reconcileOffline(volumePath, overlayPath, marker);
+
+        } catch (IOException e) {
+            logger.error("Failed to read snapshot marker {}: {}", 
marker.getName(), e.getMessage());
+        }
+    }
+
+    private boolean reconcileWithRunningDomain(String vmName, String 
diskLabel, String volumePath,
+                                               String overlayPath, File 
marker) {
+        try {
+            Connect conn = LibvirtConnection.getConnection();
+            Domain vm = conn.domainLookupByName(vmName);
+            if (vm == null) {
+                return false;
+            }
+
+            String xmlDesc = vm.getXMLDesc(0);
+            LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser();
+            parser.parseDomainXML(xmlDesc);
+
+            for (DiskDef disk : parser.getDisks()) {
+                String currentSource = disk.getDiskPath();
+                if (currentSource != null && 
currentSource.equals(overlayPath)) {
+                    logger.warn("VM [{}] disk [{}] is still on overlay [{}]. 
Attempting blockcommit to merge back to base [{}].",
+                            vmName, disk.getDiskLabel(), overlayPath, 
volumePath);
+
+                    String actualDiskLabel = StringUtils.isNotEmpty(diskLabel) 
? diskLabel : disk.getDiskLabel();
+                    try {
+                        resource.mergeDeltaIntoBaseFile(vm, actualDiskLabel, 
volumePath, null, true, "reconcile", null, conn);
+                        logger.info("Successfully reconciled overlay [{}] back 
to base [{}] for VM [{}].", overlayPath, volumePath, vmName);
+                        marker.delete();
+                        return true;
+                    } catch (Exception e) {
+                        logger.error("Failed to blockcommit overlay [{}] for 
VM [{}]: {}. Manual intervention required.",
+                                overlayPath, vmName, e.getMessage(), e);
+                        return true;
+                    }
+                }
+
+                if (currentSource != null && currentSource.equals(volumePath)) 
{
+                    logger.info("VM [{}] disk [{}] is already on base [{}]. 
Overlay [{}] is orphaned. Removing marker.",
+                            vmName, disk.getDiskLabel(), volumePath, 
overlayPath);
+                    marker.delete();

Review Comment:
   Shouldn't we check if the overlay file exists and delete it if needed?



##########
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java:
##########
@@ -2528,10 +2565,29 @@ private SnapshotObjectTO 
takeFullVolumeSnapshotOfRunningVm(CreateObjectCommand c
 
             String convertResult = 
convertBaseFileToSnapshotFileInStorageDir(ObjectUtils.defaultIfNull(secondaryPool,
 primaryPool), disk, snapshotPath, directoryPath, volume, cmd.getWait());
 
-            resource.mergeDeltaIntoBaseFile(vm, diskLabel, diskPath, null, 
true, snapshotName, volume, conn);
+            ReentrantLock vmLock = getVmBlockJobLock(vmName);
+            boolean lockAcquired = false;
+            try {
+                lockAcquired = vmLock.tryLock(incrementalSnapshotTimeout, 
TimeUnit.MILLISECONDS);
+                if (!lockAcquired) {
+                    throw new CloudRuntimeException(String.format("Could not 
acquire block job lock for VM [%s] within timeout [%d ms]. Another block job 
(snapshot/backup) may be in progress.", vmName, incrementalSnapshotTimeout));
+                }
+
+                resource.mergeDeltaIntoBaseFile(vm, diskLabel, diskPath, null, 
true, snapshotName, volume, conn);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new CloudRuntimeException("Interrupted while waiting for 
block job lock for VM [" + vmName + "]", e);
+            } finally {
+                if (lockAcquired) {
+                    vmLock.unlock();
+                }
+            }
+
+            removeSnapshotMarker(primaryPool, diskPath, snapshotName);
 
             validateConvertResult(convertResult, snapshotPath);
         } catch (LibvirtException e) {
+            removeSnapshotMarker(primaryPool, diskPath, snapshotName);

Review Comment:
   shouldn't we leave the marker in case of exceptions?



##########
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java:
##########
@@ -3609,4 +3665,231 @@ public KVMStoragePool 
getTemplateSourcePoolUsingMigrationOptions(KVMStoragePool
 
         return 
storagePoolMgr.getStoragePool(migrationOptions.getSrcPoolType(), 
migrationOptions.getSrcPoolUuid());
     }
+
+    /**
+     * Write a marker file before creating a snapshot overlay.
+     * Marker format: 
poolUuid|volumePath|snapshotName|vmName|diskLabel|overlayPath|timestamp
+     */
+    protected void writeSnapshotMarker(KVMStoragePool pool, String volumePath, 
String snapshotName) {
+        writeSnapshotMarker(pool, volumePath, snapshotName, null, null, null);
+    }
+
+    protected void writeSnapshotMarker(KVMStoragePool pool, String volumePath, 
String snapshotName,
+                                       String vmName, String diskLabel, String 
overlayPath) {
+        try {
+            File markerDir = new File(SNAPSHOT_MARKER_DIR);
+            if (!markerDir.exists()) {
+                markerDir.mkdirs();
+            }
+            String markerName = SNAPSHOT_MARKER_PREFIX + pool.getUuid() + "_" 
+ volumePath.replace("/", "_") + "_" + snapshotName;
+            File markerFile = new File(markerDir, markerName);
+            String content = String.join("|",
+                    pool.getUuid(),
+                    volumePath,
+                    snapshotName,
+                    StringUtils.defaultString(vmName),
+                    StringUtils.defaultString(diskLabel),
+                    StringUtils.defaultString(overlayPath),
+                    String.valueOf(System.currentTimeMillis()));
+            try (FileOutputStream fos = new FileOutputStream(markerFile)) {
+                fos.write(content.getBytes());
+            }
+            logger.debug("Wrote snapshot marker: {}", 
markerFile.getAbsolutePath());
+        } catch (IOException e) {
+            logger.warn("Failed to write snapshot marker for volume {} 
snapshot {}: {}", volumePath, snapshotName, e.getMessage());
+        }
+    }
+
+    protected void updateSnapshotMarker(KVMStoragePool pool, String 
volumePath, String snapshotName,
+                                        String vmName, String diskLabel, 
String overlayPath) {
+        writeSnapshotMarker(pool, volumePath, snapshotName, vmName, diskLabel, 
overlayPath);
+    }
+
+    /**
+     * Remove the snapshot marker after successful merge.
+     */
+    protected void removeSnapshotMarker(KVMStoragePool pool, String 
volumePath, String snapshotName) {
+        try {
+            String markerName = SNAPSHOT_MARKER_PREFIX + pool.getUuid() + "_" 
+ volumePath.replace("/", "_") + "_" + snapshotName;
+            File markerFile = new File(SNAPSHOT_MARKER_DIR, markerName);
+            if (markerFile.exists()) {
+                markerFile.delete();
+                logger.debug("Removed snapshot marker: {}", 
markerFile.getAbsolutePath());
+            }
+        } catch (Exception e) {
+            logger.warn("Failed to remove snapshot marker for volume {} 
snapshot {}: {}", volumePath, snapshotName, e.getMessage());
+        }
+    }
+
+    /**
+     * Reconcile incomplete snapshots by checking for marker files on agent 
startup.
+     * If a marker exists, the agent died during a snapshot and the merge may 
not have completed.
+     *
+     * Strategy per marker:
+     * - Overlay gone: snapshot completed or was cleaned up, remove marker
+     * - VM still running on overlay: blockcommit --active --pivot back to base
+     * - VM already on base: overlay is orphaned, remove marker
+     * - Domain gone, overlay exists: offline qemu-img commit
+     * - Base modified since marker: cannot safely commit, alert and keep 
marker
+     */
+    protected void reconcileSnapshotMarkers() {
+        File markerDir = new File(SNAPSHOT_MARKER_DIR);
+        if (!markerDir.exists() || !markerDir.isDirectory()) {
+            return;
+        }
+
+        File[] markers = markerDir.listFiles((dir, name) -> 
name.startsWith(SNAPSHOT_MARKER_PREFIX));
+        if (markers == null || markers.length == 0) {
+            return;
+        }
+
+        logger.info("Found {} snapshot marker(s) to reconcile", 
markers.length);
+        for (File marker : markers) {
+            reconcileSingleMarker(marker);
+        }
+    }
+
+    protected void reconcileSingleMarker(File marker) {
+        try {
+            String content = new 
String(Files.readAllBytes(marker.toPath())).trim();
+            String[] parts = content.split("\\|", -1);
+            if (parts.length < 3) {
+                logger.error("Invalid snapshot marker format in {}: {}", 
marker.getName(), content);
+                return;
+            }
+
+            String poolUuid = parts[0];
+            String volumePath = parts[1];
+            String snapshotName = parts[2];
+            String vmName = parts.length > 3 ? parts[3] : "";
+            String diskLabel = parts.length > 4 ? parts[4] : "";
+            String overlayPath = parts.length > 5 ? parts[5] : "";
+
+            if (StringUtils.isEmpty(overlayPath)) {
+                overlayPath = resource.getSnapshotTemporaryPath(volumePath, 
snapshotName);
+            }
+
+            logger.warn("Reconciling incomplete snapshot: pool={}, volume={}, 
snapshot={}, vm={}, disk={}, overlay={}",
+                    poolUuid, volumePath, snapshotName, vmName, diskLabel, 
overlayPath);
+
+            File overlayFile = new File(overlayPath);
+            if (!overlayFile.exists()) {
+                logger.info("Overlay file {} no longer exists. Snapshot may 
have completed or been cleaned up. Removing marker.", overlayPath);
+                marker.delete();
+                return;
+            }
+
+            if (StringUtils.isNotEmpty(vmName)) {
+                if (reconcileWithRunningDomain(vmName, diskLabel, volumePath, 
overlayPath, marker)) {
+                    return;
+                }
+            }
+
+            reconcileOffline(volumePath, overlayPath, marker);
+
+        } catch (IOException e) {
+            logger.error("Failed to read snapshot marker {}: {}", 
marker.getName(), e.getMessage());
+        }
+    }
+
+    private boolean reconcileWithRunningDomain(String vmName, String 
diskLabel, String volumePath,
+                                               String overlayPath, File 
marker) {
+        try {
+            Connect conn = LibvirtConnection.getConnection();
+            Domain vm = conn.domainLookupByName(vmName);
+            if (vm == null) {
+                return false;
+            }
+
+            String xmlDesc = vm.getXMLDesc(0);
+            LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser();
+            parser.parseDomainXML(xmlDesc);
+
+            for (DiskDef disk : parser.getDisks()) {
+                String currentSource = disk.getDiskPath();
+                if (currentSource != null && 
currentSource.equals(overlayPath)) {
+                    logger.warn("VM [{}] disk [{}] is still on overlay [{}]. 
Attempting blockcommit to merge back to base [{}].",
+                            vmName, disk.getDiskLabel(), overlayPath, 
volumePath);
+
+                    String actualDiskLabel = StringUtils.isNotEmpty(diskLabel) 
? diskLabel : disk.getDiskLabel();
+                    try {
+                        resource.mergeDeltaIntoBaseFile(vm, actualDiskLabel, 
volumePath, null, true, "reconcile", null, conn);
+                        logger.info("Successfully reconciled overlay [{}] back 
to base [{}] for VM [{}].", overlayPath, volumePath, vmName);
+                        marker.delete();
+                        return true;
+                    } catch (Exception e) {
+                        logger.error("Failed to blockcommit overlay [{}] for 
VM [{}]: {}. Manual intervention required.",
+                                overlayPath, vmName, e.getMessage(), e);
+                        return true;
+                    }
+                }
+
+                if (currentSource != null && currentSource.equals(volumePath)) 
{
+                    logger.info("VM [{}] disk [{}] is already on base [{}]. 
Overlay [{}] is orphaned. Removing marker.",
+                            vmName, disk.getDiskLabel(), volumePath, 
overlayPath);
+                    marker.delete();
+                    return true;
+                }
+            }
+
+            return false;
+        } catch (LibvirtException e) {
+            logger.debug("VM [{}] not found or not running: {}. Will attempt 
offline reconciliation.", vmName, e.getMessage());
+            return false;
+        }
+    }
+
+    private void reconcileOffline(String volumePath, String overlayPath, File 
marker) {
+        File baseFile = new File(volumePath);
+        File overlayFile = new File(overlayPath);
+
+        if (!overlayFile.exists()) {
+            logger.info("Overlay [{}] no longer exists. Removing marker.", 
overlayPath);
+            marker.delete();
+            return;
+        }
+
+        if (!baseFile.exists()) {
+            logger.error("Base file [{}] does not exist but overlay [{}] does. 
Cannot reconcile. Manual intervention required.", volumePath, overlayPath);
+            return;
+        }
+
+        long markerTimestamp = 0;
+        try {
+            String content = new 
String(Files.readAllBytes(marker.toPath())).trim();
+            String[] parts = content.split("\\|", -1);
+            if (parts.length > 6) {
+                markerTimestamp = Long.parseLong(parts[6]);
+            }
+        } catch (Exception e) {
+            logger.debug("Could not parse marker timestamp", e);
+        }
+
+        if (markerTimestamp > 0 && baseFile.lastModified() > markerTimestamp) {
+            logger.error("Base file [{}] has been modified since snapshot 
marker was written (base modified={}, marker={}). "
+                    + "The overlay [{}] cannot be safely committed. Manual 
intervention required — keeping marker.",
+                    volumePath, baseFile.lastModified(), markerTimestamp, 
overlayPath);
+            return;
+        }

Review Comment:
   To me it seems like this will always be the case. We write the marker before 
starting the snapshot process, no?



##########
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java:
##########
@@ -3609,4 +3665,231 @@ public KVMStoragePool 
getTemplateSourcePoolUsingMigrationOptions(KVMStoragePool
 
         return 
storagePoolMgr.getStoragePool(migrationOptions.getSrcPoolType(), 
migrationOptions.getSrcPoolUuid());
     }
+
+    /**
+     * Write a marker file before creating a snapshot overlay.
+     * Marker format: 
poolUuid|volumePath|snapshotName|vmName|diskLabel|overlayPath|timestamp
+     */
+    protected void writeSnapshotMarker(KVMStoragePool pool, String volumePath, 
String snapshotName) {
+        writeSnapshotMarker(pool, volumePath, snapshotName, null, null, null);
+    }
+
+    protected void writeSnapshotMarker(KVMStoragePool pool, String volumePath, 
String snapshotName,
+                                       String vmName, String diskLabel, String 
overlayPath) {
+        try {
+            File markerDir = new File(SNAPSHOT_MARKER_DIR);
+            if (!markerDir.exists()) {
+                markerDir.mkdirs();
+            }
+            String markerName = SNAPSHOT_MARKER_PREFIX + pool.getUuid() + "_" 
+ volumePath.replace("/", "_") + "_" + snapshotName;
+            File markerFile = new File(markerDir, markerName);
+            String content = String.join("|",
+                    pool.getUuid(),
+                    volumePath,
+                    snapshotName,
+                    StringUtils.defaultString(vmName),
+                    StringUtils.defaultString(diskLabel),
+                    StringUtils.defaultString(overlayPath),
+                    String.valueOf(System.currentTimeMillis()));
+            try (FileOutputStream fos = new FileOutputStream(markerFile)) {
+                fos.write(content.getBytes());
+            }
+            logger.debug("Wrote snapshot marker: {}", 
markerFile.getAbsolutePath());
+        } catch (IOException e) {
+            logger.warn("Failed to write snapshot marker for volume {} 
snapshot {}: {}", volumePath, snapshotName, e.getMessage());
+        }
+    }
+
+    protected void updateSnapshotMarker(KVMStoragePool pool, String 
volumePath, String snapshotName,
+                                        String vmName, String diskLabel, 
String overlayPath) {
+        writeSnapshotMarker(pool, volumePath, snapshotName, vmName, diskLabel, 
overlayPath);
+    }
+
+    /**
+     * Remove the snapshot marker after successful merge.
+     */
+    protected void removeSnapshotMarker(KVMStoragePool pool, String 
volumePath, String snapshotName) {
+        try {
+            String markerName = SNAPSHOT_MARKER_PREFIX + pool.getUuid() + "_" 
+ volumePath.replace("/", "_") + "_" + snapshotName;
+            File markerFile = new File(SNAPSHOT_MARKER_DIR, markerName);
+            if (markerFile.exists()) {
+                markerFile.delete();
+                logger.debug("Removed snapshot marker: {}", 
markerFile.getAbsolutePath());
+            }
+        } catch (Exception e) {
+            logger.warn("Failed to remove snapshot marker for volume {} 
snapshot {}: {}", volumePath, snapshotName, e.getMessage());
+        }
+    }
+
+    /**
+     * Reconcile incomplete snapshots by checking for marker files on agent 
startup.
+     * If a marker exists, the agent died during a snapshot and the merge may 
not have completed.
+     *
+     * Strategy per marker:
+     * - Overlay gone: snapshot completed or was cleaned up, remove marker
+     * - VM still running on overlay: blockcommit --active --pivot back to base
+     * - VM already on base: overlay is orphaned, remove marker
+     * - Domain gone, overlay exists: offline qemu-img commit
+     * - Base modified since marker: cannot safely commit, alert and keep 
marker
+     */
+    protected void reconcileSnapshotMarkers() {

Review Comment:
   Is this called anywhere?



##########
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java:
##########
@@ -3609,4 +3665,231 @@ public KVMStoragePool 
getTemplateSourcePoolUsingMigrationOptions(KVMStoragePool
 
         return 
storagePoolMgr.getStoragePool(migrationOptions.getSrcPoolType(), 
migrationOptions.getSrcPoolUuid());
     }
+
+    /**
+     * Write a marker file before creating a snapshot overlay.
+     * Marker format: 
poolUuid|volumePath|snapshotName|vmName|diskLabel|overlayPath|timestamp
+     */
+    protected void writeSnapshotMarker(KVMStoragePool pool, String volumePath, 
String snapshotName) {
+        writeSnapshotMarker(pool, volumePath, snapshotName, null, null, null);
+    }
+
+    protected void writeSnapshotMarker(KVMStoragePool pool, String volumePath, 
String snapshotName,
+                                       String vmName, String diskLabel, String 
overlayPath) {
+        try {
+            File markerDir = new File(SNAPSHOT_MARKER_DIR);
+            if (!markerDir.exists()) {
+                markerDir.mkdirs();
+            }
+            String markerName = SNAPSHOT_MARKER_PREFIX + pool.getUuid() + "_" 
+ volumePath.replace("/", "_") + "_" + snapshotName;
+            File markerFile = new File(markerDir, markerName);
+            String content = String.join("|",
+                    pool.getUuid(),
+                    volumePath,
+                    snapshotName,
+                    StringUtils.defaultString(vmName),
+                    StringUtils.defaultString(diskLabel),
+                    StringUtils.defaultString(overlayPath),
+                    String.valueOf(System.currentTimeMillis()));
+            try (FileOutputStream fos = new FileOutputStream(markerFile)) {
+                fos.write(content.getBytes());
+            }
+            logger.debug("Wrote snapshot marker: {}", 
markerFile.getAbsolutePath());
+        } catch (IOException e) {
+            logger.warn("Failed to write snapshot marker for volume {} 
snapshot {}: {}", volumePath, snapshotName, e.getMessage());
+        }
+    }
+
+    protected void updateSnapshotMarker(KVMStoragePool pool, String 
volumePath, String snapshotName,
+                                        String vmName, String diskLabel, 
String overlayPath) {
+        writeSnapshotMarker(pool, volumePath, snapshotName, vmName, diskLabel, 
overlayPath);
+    }
+
+    /**
+     * Remove the snapshot marker after successful merge.
+     */
+    protected void removeSnapshotMarker(KVMStoragePool pool, String 
volumePath, String snapshotName) {
+        try {
+            String markerName = SNAPSHOT_MARKER_PREFIX + pool.getUuid() + "_" 
+ volumePath.replace("/", "_") + "_" + snapshotName;
+            File markerFile = new File(SNAPSHOT_MARKER_DIR, markerName);
+            if (markerFile.exists()) {
+                markerFile.delete();
+                logger.debug("Removed snapshot marker: {}", 
markerFile.getAbsolutePath());
+            }
+        } catch (Exception e) {
+            logger.warn("Failed to remove snapshot marker for volume {} 
snapshot {}: {}", volumePath, snapshotName, e.getMessage());
+        }
+    }
+
+    /**
+     * Reconcile incomplete snapshots by checking for marker files on agent 
startup.
+     * If a marker exists, the agent died during a snapshot and the merge may 
not have completed.
+     *
+     * Strategy per marker:
+     * - Overlay gone: snapshot completed or was cleaned up, remove marker
+     * - VM still running on overlay: blockcommit --active --pivot back to base
+     * - VM already on base: overlay is orphaned, remove marker
+     * - Domain gone, overlay exists: offline qemu-img commit
+     * - Base modified since marker: cannot safely commit, alert and keep 
marker
+     */
+    protected void reconcileSnapshotMarkers() {
+        File markerDir = new File(SNAPSHOT_MARKER_DIR);
+        if (!markerDir.exists() || !markerDir.isDirectory()) {
+            return;
+        }
+
+        File[] markers = markerDir.listFiles((dir, name) -> 
name.startsWith(SNAPSHOT_MARKER_PREFIX));
+        if (markers == null || markers.length == 0) {
+            return;
+        }
+
+        logger.info("Found {} snapshot marker(s) to reconcile", 
markers.length);
+        for (File marker : markers) {
+            reconcileSingleMarker(marker);
+        }
+    }
+
+    protected void reconcileSingleMarker(File marker) {

Review Comment:
   this seems to be called only by the reconcileSnapshotMarkers method, which 
does not seem to be called anywhere



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to