CLOUDSTACK-1846, CLOUDSTACK-1845 - KVM Storage, sometimes KVMHA will remount deleted NFS pools, causing failures when defining new storage pools. Sometimes a storage pool has never been used on a host, and getStoragePool fails when copying templates or in storage migration. deleteStoragePool(pool) often fails silently, leaving no pool defined in libvirt, but a mountpoint left behind. This patch handles some of these exceptions and brings forward any issues via logging.
Signed-off-by: Marcus Sorensen <mar...@betterservers.com> 1364603486 -0600 Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/5a16e70d Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/5a16e70d Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/5a16e70d Branch: refs/heads/kvm-vnc-listen Commit: 5a16e70de9d7bfd93f71bf476de8e23f84e52152 Parents: cf72aa3 Author: Marcus Sorensen <mar...@betterservers.com> Authored: Fri Mar 29 18:31:26 2013 -0600 Committer: Marcus Sorensen <mar...@betterservers.com> Committed: Fri Mar 29 18:31:26 2013 -0600 ---------------------------------------------------------------------- .../kvm/resource/LibvirtComputingResource.java | 34 ++++++++++- .../kvm/storage/LibvirtStorageAdaptor.java | 46 ++++++++++---- 2 files changed, 63 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5a16e70d/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java ---------------------------------------------------------------------- diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index f786f88..6852e2c 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1218,10 +1218,23 @@ ServerResource { StorageFilerTO pool = cmd.getPool(); String secondaryStorageUrl = cmd.getSecondaryStorageURL(); KVMStoragePool secondaryStoragePool = null; + KVMStoragePool primaryPool = null; try { - KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool( - pool.getType(), - pool.getUuid()); + try { + primaryPool = _storagePoolMgr.getStoragePool( + pool.getType(), + pool.getUuid()); + } catch (CloudRuntimeException e) { + if (e.getMessage().contains("not found")) { + primaryPool = _storagePoolMgr.createStoragePool(cmd.getPool().getUuid(), + cmd.getPool().getHost(), cmd.getPool().getPort(), + cmd.getPool().getPath(), cmd.getPool().getUserInfo(), + cmd.getPool().getType()); + } else { + return new CopyVolumeAnswer(cmd, false, e.getMessage(), null, null); + } + } + String volumeName = UUID.randomUUID().toString(); if (copyToSecondary) { @@ -2155,6 +2168,7 @@ ServerResource { String secondaryStorageURL = cmd.getSecondaryStorageUrl(); KVMStoragePool secondaryStorage = null; + KVMStoragePool primary = null; try { Connect conn = LibvirtConnection.getConnection(); String templateFolder = cmd.getAccountId() + File.separator @@ -2164,9 +2178,21 @@ ServerResource { secondaryStorage = _storagePoolMgr.getStoragePoolByURI( secondaryStorageURL); - KVMStoragePool primary = _storagePoolMgr.getStoragePool( + try { + primary = _storagePoolMgr.getStoragePool( cmd.getPool().getType(), cmd.getPrimaryStoragePoolNameLabel()); + } catch (CloudRuntimeException e) { + if (e.getMessage().contains("not found")) { + primary = _storagePoolMgr.createStoragePool(cmd.getPool().getUuid(), + cmd.getPool().getHost(), cmd.getPool().getPort(), + cmd.getPool().getPath(), cmd.getPool().getUserInfo(), + cmd.getPool().getType()); + } else { + return new CreatePrivateTemplateAnswer(cmd, false, e.getMessage()); + } + } + KVMPhysicalDisk disk = primary.getPhysicalDisk(cmd.getVolumePath()); String tmpltPath = secondaryStorage.getLocalPath() + File.separator + templateInstallFolder; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5a16e70d/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java ---------------------------------------------------------------------- diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index ca5da5c..5e83ef6 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -124,6 +124,23 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return sp; } catch (LibvirtException e) { s_logger.error(e.toString()); + // if error is that pool is mounted, try to handle it + if (e.toString().contains("already mounted")) { + s_logger.error("Attempting to unmount old mount libvirt is unaware of at "+targetPath); + String result = Script.runSimpleBashScript("umount " + targetPath ); + if (result == null) { + s_logger.error("Succeeded in unmounting " + targetPath); + try { + sp = conn.storagePoolCreateXML(spd.toString(), 0); + s_logger.error("Succeeded in redefining storage"); + return sp; + } catch (LibvirtException l) { + s_logger.error("Target was already mounted, unmounted it but failed to redefine storage:" + l); + } + } else { + s_logger.error("Failed in unmounting and redefining storage"); + } + } if (sp != null) { try { if (sp.isPersistent() == 1) { @@ -134,8 +151,8 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } sp.free(); } catch (LibvirtException l) { - s_logger.debug("Failed to define nfs storage pool with: " - + l.toString()); + s_logger.debug("Failed to undefine nfs storage pool with: " + + l.toString()); } } return null; @@ -541,6 +558,19 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } return true; } catch (LibvirtException e) { + // handle ebusy error when pool is quickly destroyed + if (e.toString().contains("exit status 16")) { + String targetPath = _mountPoint + File.separator + uuid; + s_logger.error("deleteStoragePool removed pool from libvirt, but libvirt had trouble" + + "unmounting the pool. Trying umount location " + targetPath + + "again in a few seconds"); + String result = Script.runSimpleBashScript("sleep 5 && umount " + targetPath ); + if (result == null) { + s_logger.error("Succeeded in unmounting " + targetPath); + return true; + } + s_logger.error("failed in umount retry"); + } throw new CloudRuntimeException(e.toString()); } } @@ -766,17 +796,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { @Override public boolean deleteStoragePool(KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; - StoragePool virtPool = libvirtPool.getPool(); - try { - virtPool.destroy(); - virtPool.undefine(); - virtPool.free(); - } catch (LibvirtException e) { - return false; - } - - return true; + return deleteStoragePool(pool.getUuid()); } public boolean deleteVbdByPath(String diskPath) {
