Successful completion of bhyvectl --suspend alone is not sufficient to confirm suspension. Remove stale snapshot files before suspending and require the image, .kern, and .meta files to exist afterward.
On failure, preserve the error, destroy the remaining VM instance, remove partial snapshot files, and mark the domain shut off with a failed reason. Keep successfully suspended domains active by retaining their domain ID. Allow destroying suspended domains without requiring a running process, and remove snapshot files after destruction or successful resume. Signed-off-by: Roman Bogorodskiy <[email protected]> --- src/bhyve/bhyve_process.c | 186 ++++++++++++++++++++++++++++++++++---- 1 file changed, 168 insertions(+), 18 deletions(-) diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c index 0cac9e841a..001ee59130 100644 --- a/src/bhyve/bhyve_process.c +++ b/src/bhyve/bhyve_process.c @@ -535,6 +535,106 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver, return ret; } + +static int +bhyveProcessRemoveSuspendFile(const char *path, + bool report) +{ + if (unlink(path) < 0 && errno != ENOENT) { + if (report) { + virReportSystemError(errno, + _("Unable to remove bhyve snapshot file '%1$s'"), + path); + } else { + VIR_WARN("Unable to remove bhyve snapshot file '%s': %s", + path, g_strerror(errno)); + } + return -1; + } + + return 0; +} + + +static int +bhyveProcessRemoveSuspendFiles(bhyveConn *driver, + virDomainDef *def, + bool report) +{ + g_autofree char *snapshot_dir = NULL; + g_autofree char *snapshot_path = NULL; + g_autofree char *kern_path = NULL; + g_autofree char *meta_path = NULL; + int ret = 0; + + snapshot_dir = g_build_filename(driver->config->snapshotDir, def->name, NULL); + snapshot_path = g_build_filename(snapshot_dir, "snapshot", NULL); + kern_path = g_strdup_printf("%s.kern", snapshot_path); + meta_path = g_strdup_printf("%s.meta", snapshot_path); + + if (bhyveProcessRemoveSuspendFile(snapshot_path, report) < 0) + ret = -1; + if (bhyveProcessRemoveSuspendFile(kern_path, report) < 0) + ret = -1; + if (bhyveProcessRemoveSuspendFile(meta_path, report) < 0) + ret = -1; + + if (rmdir(snapshot_dir) < 0 && errno != ENOENT) { + if (report) { + virReportSystemError(errno, + _("Unable to remove bhyve snapshot directory '%1$s'"), + snapshot_dir); + } else { + VIR_WARN("Unable to remove bhyve snapshot directory '%s': %s", + snapshot_dir, g_strerror(errno)); + } + ret = -1; + } + + return ret; +} + + +static int +bhyveProcessCheckSuspendFile(const char *path) +{ + struct stat sb; + + if (stat(path, &sb) < 0) { + virReportSystemError(errno, + _("Unable to stat bhyve snapshot file '%1$s'"), + path); + return -1; + } + + return 0; +} + + +static int +bhyveProcessCheckSuspend(bhyveConn *driver, + virDomainDef *def) +{ + g_autofree char *snapshot_path = NULL; + g_autofree char *kern_path = NULL; + g_autofree char *meta_path = NULL; + + /* bhyvectl stores guest memory in the requested file and guest device + * and vCPU state in files with the .kern and .meta suffixes */ + snapshot_path = g_build_filename(driver->config->snapshotDir, def->name, + "snapshot", NULL); + kern_path = g_strdup_printf("%s.kern", snapshot_path); + meta_path = g_strdup_printf("%s.meta", snapshot_path); + + if (bhyveProcessCheckSuspendFile(snapshot_path) < 0 || + bhyveProcessCheckSuspendFile(kern_path) < 0 || + bhyveProcessCheckSuspendFile(meta_path) < 0) + return -1; + + return 0; +} + + int bhyveProcessPrepareDomain(bhyveConn *driver, virDomainObj *vm, @@ -746,6 +846,7 @@ bhyveProcessRemoveIPCSocket(struct _bhyveConn *driver, * @reason: shutoff reason * @forceCleanup: boolean controlling cleanup * @restoreDef: whether to restore the persistent definition after cleanup + * @suspend: whether to suspend instead of shutoff * * Stops the domain and cleans up its resources. * It could be used whether as a direct call or as a cleanup routine. @@ -768,8 +869,11 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, int ret = 0; size_t i = 0; g_autoptr(virCommand) cmd = NULL; + g_autoptr(virCommand) cleanup_cmd = NULL; bhyveDomainObjPrivate *priv = vm->privateData; bool vm_started = false; + bool suspended = false; + bool suspendSucceeded = false; virErrorPtr save_err = NULL; virErrorPreserveLast(&save_err); @@ -777,13 +881,16 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, if (vm->pid != 0) vm_started = true; + if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_PAUSED && vm->pid == 0) + suspended = true; + if (!forceCleanup) { if (!virDomainObjIsActive(vm)) { VIR_DEBUG("VM '%s' not active", vm->def->name); goto cleanup; } - if (!vm_started) { + if (!vm_started && !suspended) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Invalid PID %1$d for VM"), (int)vm->pid); @@ -796,6 +903,13 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, g_autofree char *snapshot_dir = g_build_filename(driver->config->snapshotDir, vm->def->name, NULL); + if (bhyveProcessRemoveSuspendFiles(driver, vm->def, true) < 0) { + ret = -1; + if (!save_err) + virErrorPreserveLast(&save_err); + goto cleanup; + } + if (g_mkdir_with_parents(snapshot_dir, 0700) < 0) { virReportSystemError(errno, _("Failed to mkdir %1$s"), snapshot_dir); @@ -806,24 +920,50 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, } } - /* Destroy monitor before running the actual destroy command to prevent + /* Destroy monitor before running the actual stop command to prevent * it from detecting VM shutdown and entering this cleanup routine again */ if ((priv != NULL) && (priv->mon != NULL)) g_clear_pointer(&priv->mon, bhyveMonitorClose); virBhyveDomainObjStopWorker(vm); - if (suspend) - cmd = virBhyveProcessBuildSuspendCmd(driver, vm->def); - else - cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); - if (virCommandRun(cmd, NULL) < 0) { - /* Only failure of the actual destroy command warrants unsuccessful return code, - * other failures are not considered critical */ - ret = -1; - VIR_WARN("Failed to run the domain destroy command"); + if (!suspended) { + if (suspend) { + cmd = virBhyveProcessBuildSuspendCmd(driver, vm->def); + } else { + cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); + } + + if (virCommandRun(cmd, NULL) < 0) { + /* Only failure of the actual stop command warrants unsuccessful + * return code; other cleanup failures are not considered critical. */ + ret = -1; + VIR_WARN("Failed to run the domain %s command", + suspend ? "suspend" : "destroy"); + if (suspend && !save_err) + virErrorPreserveLast(&save_err); + } + + if (suspend && ret == 0) { + if (bhyveProcessCheckSuspend(driver, vm->def) < 0) { + ret = -1; + if (!save_err) + virErrorPreserveLast(&save_err); + } else { + suspendSucceeded = true; + } + } + + if (suspend && !suspendSucceeded) { + cleanup_cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); + if (virCommandRun(cleanup_cmd, NULL) < 0) + VIR_WARN("Failed to destroy domain after unsuccessful suspend"); + } } + if (suspended) + goto state_cleanup; + if (vm_started) bhyveProcessStopHook(driver, vm, VIR_HOOK_BHYVE_OP_STOPPED); @@ -852,13 +992,19 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, virFileRemove(channel->source->data.nix.path, 0, 0); } - virCloseCallbacksDomainRemove(vm, NULL, bhyveProcessAutoDestroy); + state_cleanup: + if (!suspendSucceeded) + virCloseCallbacksDomainRemove(vm, NULL, bhyveProcessAutoDestroy); + + if (suspended || (suspend && !suspendSucceeded)) + ignore_value(bhyveProcessRemoveSuspendFiles(driver, vm->def, false)); virDomainObjSetState(vm, - suspend ? VIR_DOMAIN_PAUSED : VIR_DOMAIN_SHUTOFF, - reason); + suspendSucceeded ? VIR_DOMAIN_PAUSED : VIR_DOMAIN_SHUTOFF, + suspend && !suspendSucceeded ? VIR_DOMAIN_SHUTOFF_FAILED : reason); vm->pid = 0; - vm->def->id = -1; + if (!suspendSucceeded) + vm->def->id = -1; if (vm_started) bhyveProcessStopHook(driver, vm, VIR_HOOK_BHYVE_OP_RELEASE); @@ -866,7 +1012,7 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, bhyveProcessRemoveDomainStatus(driver->config->stateDir, vm->def->name); bhyveProcessRemoveIPCSocket(driver, vm->def->name); - if (restoreDef && !suspend) + if (restoreDef && !suspendSucceeded) virDomainObjRemoveTransientDef(vm); cleanup: @@ -943,8 +1089,12 @@ int virBhyveProcessResume(struct _bhyveConn *driver, virDomainObj *vm) { - return virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_UNPAUSED, - true); + if (virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_UNPAUSED, + true) < 0) + return -1; + + ignore_value(bhyveProcessRemoveSuspendFiles(driver, vm->def, false)); + return 0; } int -- 2.56.0
