Adam Litke has uploaded a new change for review. Change subject: Live Merge: Resolve unknown merge status in vdsm after host crash ......................................................................
Live Merge: Resolve unknown merge status in vdsm after host crash When a host crashes during a live merge operation resulting in the VM crashing, we need to take some special steps in vdsm to ensure that we can recover using shared storage only. This involves two changes: 1. For active layer merges, mark the leaf volume as ILLEGAL before pivoting. This allows engine to determine that this leaf has been merged and can be discarded. Since active layer pivot now calls the IRS broker, it should be run in the LiveMergeCleanupThread just like a regular cleanup. 2. Allow a live merge to be resubmitted on a new host using blank volume uuids. This will tell vdsm to skip the libvirt part of the merge but to rerun synchronization operations. Change-Id: I1ca5e4ad67216d30c3149a53c2ea65cc97601bfc Signed-off-by: Adam Litke <[email protected]> Bug-Url: https://bugzilla.redhat.com/show_bug.cgi?id=1127294 --- M vdsm/storage/hsm.py M vdsm/virt/vm.py 2 files changed, 67 insertions(+), 24 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/67/31267/1 diff --git a/vdsm/storage/hsm.py b/vdsm/storage/hsm.py index 105e28b..7f0caef 100644 --- a/vdsm/storage/hsm.py +++ b/vdsm/storage/hsm.py @@ -1831,9 +1831,14 @@ return dstParent = sdDom.produceVolume(imgUUID, subChain[0]).getParent() - children = sdDom.produceVolume(imgUUID, subChain[-1]).getChildren() + subChainTailVol = sdDom.produceVolume(imgUUID, subChain[-1]) + children = subChainTailVol.getChildren() for childID in children: sdDom.produceVolume(imgUUID, childID).setParentMeta(dstParent) + if len(children) == 0: + self.log.debug("Leaf volume is being removed from the chain. " + "Marking it ILLEGAL to prevent data corruption") + subChainTailVol.setLegality(volume.ILLEGAL_VOL) @public def mergeSnapshots(self, sdUUID, spUUID, vmUUID, imgUUID, ancestor, diff --git a/vdsm/virt/vm.py b/vdsm/virt/vm.py index 9e93425..25d75eb 100644 --- a/vdsm/virt/vm.py +++ b/vdsm/virt/vm.py @@ -5277,8 +5277,8 @@ return False def queryBlockJobs(self): - def startCleanup(job, drive): - t = LiveMergeCleanupThread(self, job['jobID'], drive) + def startCleanup(job, drive, mode): + t = LiveMergeCleanupThread(self, job['jobID'], drive, mode) t.start() self._liveMergeCleanupThreads[job['jobID']] = t @@ -5290,7 +5290,7 @@ for storedJob in self.conf['_blockJobs'].values(): jobID = storedJob['jobID'] cleanThread = self._liveMergeCleanupThreads.get(jobID) - if cleanThread and cleanThread.isSuccessful(): + if cleanThread and cleanThread.isFinalized(): # Handle successfully cleaned jobs early because the job # just needs to be untracked and the stored disk info might # be stale anyway (ie. after active layer commit). @@ -5312,32 +5312,32 @@ jobsRet[jobID] = entry continue + mode = None if liveInfo: entry['bandwidth'] = liveInfo['bandwidth'] entry['cur'] = str(liveInfo['cur']) entry['end'] = str(liveInfo['end']) if self._activeLayerCommitReady(liveInfo): - try: - self.handleBlockJobEvent(jobID, drive, 'pivot') - except Exception: - # Just log it. We will retry next time - self.log.error("Pivot failed for job %s", jobID) + mode = LiveMergeCleanupThread.MODE_PIVOT else: # Libvirt has stopped reporting this job so we know it will # never report it again. + mode = LiveMergeCleanupThread.MODE_CLEANUP storedJob['gone'] = True + if mode: if not cleanThread: # There is no cleanup thread so the job must have just # ended. Spawn an async cleanup. - startCleanup(storedJob, drive) + startCleanup(storedJob, drive, mode) elif cleanThread.isAlive(): # Let previously started cleanup thread continue self.log.debug("Still waiting for block job %s to be " - "cleaned up", jobID) - elif not cleanThread.isSuccessful(): + "synchronized", jobID) + elif not cleanThread.isFinalized(): # At this point we know the thread is not alive and the - # cleanup failed. Retry it with a new thread. - startCleanup(storedJob, drive) + # cleanup failed or we have another step to complete. + # Continue or retry it with a new thread. + startCleanup(storedJob, drive, mode) jobsRet[jobID] = entry return jobsRet @@ -5360,6 +5360,20 @@ self.log.error("merge: libvirt does not support volume chain " "monitoring. Unable to perform live merge.") return errCode['mergeErr'] + + # If engine passes empty values for baseVolUUID and topVolUUID then we + # will just try to recover from a previous merge. Just create a fake + # block job and allow the cleanup logic to run again. + if not baseVolUUID and not topVolUUID: + with self._jobsLock: + try: + self.trackBlockJob(jobUUID, drive, baseVolUUID, topVolUUID, + 'commit') + except BlockJobExistsError: + self.log.error("A block job is active on this disk") + return errCode['mergeErr'] + self._vmStats.sampleVmJobs() + return {'status': doneCode} base = top = None for v in drive.volumeChain: @@ -5514,10 +5528,20 @@ device['volumeChain'] = drive.volumeChain = newChain def handleBlockJobEvent(self, jobID, drive, mode): - if mode == 'finished': + if mode == LiveMergeCleanupThread.MODE_CLEANUP: self.log.info("Live merge job completed (job %s)", jobID) self._syncVolumeChain(drive) - elif mode == 'pivot': + elif mode == LiveMergeCleanupThread.MODE_PIVOT: + # We call imageSyncVolumeChain which will mark the current leaf + # ILLEGAL. We do this before requesting a pivot so that we can + # properly recover the VM in case we crash. At this point the + # active layer contains the same data as its parent so the ILLEGAL + # flag indicates that the VM should be restarted using the parent. + curVols = set([x['volumeID'] for x in drive.volumeChain]) + newVols = curVols - set((drive.volumeID,)) + self.cif.irs.imageSyncVolumeChain(drive.domainID, drive.imageID, + drive['volumeID'], newVols) + self.log.info("Requesting pivot to complete active layer commit " "(job %s)", jobID) flags = libvirt.VIR_DOMAIN_BLOCK_JOB_ABORT_PIVOT @@ -5562,32 +5586,46 @@ class LiveMergeCleanupThread(threading.Thread): - def __init__(self, vm, jobId, drive): + MODE_CLEANUP = 'cleanup' + MODE_PIVOT = 'pivot' + + def __init__(self, vm, jobId, drive, mode): threading.Thread.__init__(self) self.setDaemon(True) self.vm = vm self.jobId = jobId self.drive = drive + self.mode = mode self.success = False @utils.traceback() def run(self): - self.vm.log.info("Starting live merge cleanup for job %s", - self.jobId) + self.vm.log.info("Starting live merge %s for job %s", + self.mode, self.jobId) try: - self.vm.handleBlockJobEvent(self.jobId, self.drive, 'finished') + self.vm.handleBlockJobEvent(self.jobId, self.drive, self.mode) except Exception: - self.vm.log.warning("Cleanup failed for live merge job %s", - self.jobId) + self.vm.log.warning("%s failed for live merge job %s", + self.mode, self.jobId) raise else: self.success = True - self.vm.log.info("Cleanup completed for live merge job %s", - self.jobId) + self.vm.log.info("%s completed for live merge job %s", + self.mode, self.jobId) def isSuccessful(self): + """ + Returns True if this phase completed successfully. + """ return self.success + def isFinalized(self): + """ + Returns True if the last cleanup phase succeeded and the job can be + removed. + """ + return bool(self.success and self.mode == self.MODE_CLEANUP) + def _devicesWithAlias(domXML): return vmxml.filter_devices_with_alias(vmxml.all_devices(domXML)) -- To view, visit http://gerrit.ovirt.org/31267 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I1ca5e4ad67216d30c3149a53c2ea65cc97601bfc Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Adam Litke <[email protected]> _______________________________________________ vdsm-patches mailing list [email protected] https://lists.fedorahosted.org/mailman/listinfo/vdsm-patches
