Updated Branches: refs/heads/4.0 c412a7a2d -> 3541aa86e
CLOUDSTACK-1846, CLOUDSTACK-1845 - KVM Storage, sometimes KVMHA will remount deleted NFS pools, causing failures when defining new storage pools. Sometimes a storage pool has never been used on a host, and getStoragePool fails when copying templates or in storage migration. deleteStoragePool(pool) often fails silently, leaving no pool defined in libvirt, but a mountpoint left behind. This patch handles some of these exceptions and brings forward any issues via logging. Signed-off-by: Chip Childers <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/3541aa86 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/3541aa86 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/3541aa86 Branch: refs/heads/4.0 Commit: 3541aa86e34605e5ac4a99df84c47e6f6e7c71ee Parents: c412a7a Author: Marcus Sorensen <[email protected]> Authored: Tue Apr 2 20:46:53 2013 +0100 Committer: Chip Childers <[email protected]> Committed: Tue Apr 2 20:46:53 2013 +0100 ---------------------------------------------------------------------- .../kvm/storage/LibvirtStorageAdaptor.java | 46 ++++++++++---- 1 files changed, 33 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3541aa86/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java ---------------------------------------------------------------------- diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 131c37e..06ffebd 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -124,6 +124,23 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return sp; } catch (LibvirtException e) { s_logger.error(e.toString()); + // if error is that pool is mounted, try to handle it + if (e.toString().contains("already mounted")) { + s_logger.error("Attempting to unmount old mount libvirt is unaware of at "+targetPath); + String result = Script.runSimpleBashScript("umount " + targetPath ); + if (result == null) { + s_logger.error("Succeeded in unmounting " + targetPath); + try { + sp = conn.storagePoolCreateXML(spd.toString(), 0); + s_logger.error("Succeeded in redefining storage"); + return sp; + } catch (LibvirtException l) { + s_logger.error("Target was already mounted, unmounted it but failed to redefine storage:" + l); + } + } else { + s_logger.error("Failed in unmounting and redefining storage"); + } + } if (sp != null) { try { if (sp.isPersistent() == 1) { @@ -134,8 +151,8 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } sp.free(); } catch (LibvirtException l) { - s_logger.debug("Failed to define nfs storage pool with: " - + l.toString()); + s_logger.debug("Failed to undefine nfs storage pool with: " + + l.toString()); } } return null; @@ -593,6 +610,19 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } return true; } catch (LibvirtException e) { + // handle ebusy error when pool is quickly destroyed + if (e.toString().contains("exit status 16")) { + String targetPath = _mountPoint + File.separator + uuid; + s_logger.error("deleteStoragePool removed pool from libvirt, but libvirt had trouble" + + "unmounting the pool. Trying umount location " + targetPath + + "again in a few seconds"); + String result = Script.runSimpleBashScript("sleep 5 && umount " + targetPath ); + if (result == null) { + s_logger.error("Succeeded in unmounting " + targetPath); + return true; + } + s_logger.error("failed in umount retry"); + } throw new CloudRuntimeException(e.toString()); } } @@ -844,17 +874,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { @Override public boolean deleteStoragePool(KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; - StoragePool virtPool = libvirtPool.getPool(); - try { - virtPool.destroy(); - virtPool.undefine(); - virtPool.free(); - } catch (LibvirtException e) { - return false; - } - - return true; + return deleteStoragePool(pool.getUuid()); } public boolean deleteVbdByPath(String diskPath) {
