Hi everyone,
This is the first commit in the second part of the patch series that
cannot exist without part 1. All previous commits are independent and
do not depend on part 1.
On Tue, Sep 2, 2025 at 1:10 AM Mykola Kvach <xakep.ama...@gmail.com> wrote:
>
> From: Mirela Simonovic <mirela.simono...@aggios.com>
>
> Trigger Xen suspend when the hardware domain initiates suspend via
> SHUTDOWN_suspend. Redirect system suspend to CPU#0 to ensure the
> suspend logic runs on the boot CPU, as required.
>
> Introduce full suspend/resume infrastructure gated by CONFIG_SYSTEM_SUSPEND,
> including logic to:
> - disable and enable non-boot physical CPUs
> - freeze and thaw domains
> - suspend and resume the GIC, timer, iommu and console
> - maintain system state before and after suspend
>
> On boot, init_ttbr is normally initialized during secondary CPU hotplug.
> On uniprocessor systems, this would leave init_ttbr uninitialized,
> causing resume to fail. To address this, the boot CPU now sets init_ttbr
> during suspend.
>
> Remove the restriction in the vPSCI interface preventing suspend from the
> hardware domain.
>
> Select HAS_SYSTEM_SUSPEND for ARM_64.
>
> Signed-off-by: Mirela Simonovic <mirela.simono...@aggios.com>
> Signed-off-by: Saeed Nowshadi <saeed.nowsh...@xilinx.com>
> Signed-off-by: Mykyta Poturai <mykyta_potu...@epam.com>
> Signed-off-by: Mykola Kvach <mykola_kv...@epam.com>
> ---
> Changes in v6:
> - Minor changes in comments.
> - The implementation now uses own tasklet instead of
> continue_hypercall_on_cpu,
> as the latter rewrites user registers and would tie system suspend status
> to guest suspend status.
> - The order of calls when suspending devices has been updated.
>
> Changes in v5:
> - select HAS_SYSTEM_SUSPEND in ARM_64 instead of ARM
> - check llc_coloring_enabled instead of LLC_COLORING during the selection
> of HAS_SYSTEM_SUSPEND config
> - call host_system_suspend from guest PSCI system suspend instead of
> arch_domain_shutdown, reducing the complexity of the new code
> - update some comments
>
> Changes introduced in V4:
> - drop code for saving and restoring VCPU context (for more information
> refer part 1 of patch series)
> - remove IOMMU suspend and resume calls until they will be implemented
> - move system suspend logic to arch_domain_shutdown, invoked from
> domain_shutdown
> - apply minor fixes such as renaming functions, updating comments, and
> modifying the commit message to reflect these changes
> - add console_end_sync to resume path after system suspend
>
> Changes introduced in V3:
> - merge changes from other commits into this patch (previously stashed):
> 1) disable/enable non-boot physical CPUs and freeze/thaw domains during
> suspend/resume
> 2) suspend/resume the timer, GIC, console, IOMMU, and hardware domain
> ---
> xen/arch/arm/Kconfig | 1 +
> xen/arch/arm/include/asm/mm.h | 2 +
> xen/arch/arm/include/asm/suspend.h | 2 +
> xen/arch/arm/mmu/smpboot.c | 2 +-
> xen/arch/arm/suspend.c | 150 +++++++++++++++++++++++++++++
> xen/arch/arm/vpsci.c | 9 +-
> xen/common/domain.c | 4 +
> 7 files changed, 168 insertions(+), 2 deletions(-)
>
> diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig
> index 5355534f3d..fdad53fd68 100644
> --- a/xen/arch/arm/Kconfig
> +++ b/xen/arch/arm/Kconfig
> @@ -8,6 +8,7 @@ config ARM_64
> depends on !ARM_32
> select 64BIT
> select HAS_FAST_MULTIPLY
> + select HAS_SYSTEM_SUSPEND if UNSUPPORTED
> select HAS_VPCI_GUEST_SUPPORT if PCI_PASSTHROUGH
>
> config ARM
> diff --git a/xen/arch/arm/include/asm/mm.h b/xen/arch/arm/include/asm/mm.h
> index 7a93dad2ed..61e211d087 100644
> --- a/xen/arch/arm/include/asm/mm.h
> +++ b/xen/arch/arm/include/asm/mm.h
> @@ -365,6 +365,8 @@ static inline void page_set_xenheap_gfn(struct page_info
> *p, gfn_t gfn)
> } while ( (y = cmpxchg(&p->u.inuse.type_info, x, nx)) != x );
> }
>
> +void set_init_ttbr(lpae_t *root);
> +
> #endif /* __ARCH_ARM_MM__ */
> /*
> * Local variables:
> diff --git a/xen/arch/arm/include/asm/suspend.h
> b/xen/arch/arm/include/asm/suspend.h
> index 29eed4ee7f..8d30b01b7c 100644
> --- a/xen/arch/arm/include/asm/suspend.h
> +++ b/xen/arch/arm/include/asm/suspend.h
> @@ -29,6 +29,8 @@ extern struct cpu_context cpu_context;
> void hyp_resume(void);
> int prepare_resume_ctx(struct cpu_context *ptr);
>
> +void host_system_suspend(void);
> +
> #endif /* CONFIG_SYSTEM_SUSPEND */
>
> #endif /* __ASM_ARM_SUSPEND_H__ */
> diff --git a/xen/arch/arm/mmu/smpboot.c b/xen/arch/arm/mmu/smpboot.c
> index 37e91d72b7..ff508ecf40 100644
> --- a/xen/arch/arm/mmu/smpboot.c
> +++ b/xen/arch/arm/mmu/smpboot.c
> @@ -72,7 +72,7 @@ static void clear_boot_pagetables(void)
> clear_table(boot_third);
> }
>
> -static void set_init_ttbr(lpae_t *root)
> +void set_init_ttbr(lpae_t *root)
> {
> /*
> * init_ttbr is part of the identity mapping which is read-only. So
> diff --git a/xen/arch/arm/suspend.c b/xen/arch/arm/suspend.c
> index 5093f1bf3d..35b20581f1 100644
> --- a/xen/arch/arm/suspend.c
> +++ b/xen/arch/arm/suspend.c
> @@ -1,9 +1,159 @@
> /* SPDX-License-Identifier: GPL-2.0-only */
>
> +#include <asm/psci.h>
> #include <asm/suspend.h>
>
> +#include <xen/console.h>
> +#include <xen/cpu.h>
> +#include <xen/llc-coloring.h>
> +#include <xen/sched.h>
> +#include <xen/tasklet.h>
> +
> +/*
> + * TODO list:
> + * - Decide which domain will trigger system suspend ctl or hw ?
> + * - Test system suspend with LLC_COLORING enabled and verify functionality
> + * - Implement IOMMU suspend/resume handlers and integrate them
> + * into the suspend/resume path (SMMU)
> + * - Enable "xl suspend" support on ARM architecture
> + * - Properly disable Xen timer watchdog from relevant services (init.d
> left)
> + * - Add suspend/resume CI test for ARM (QEMU if feasible)
> + * - Investigate feasibility and need for implementing system suspend on
> ARM32
> + */
> +
> struct cpu_context cpu_context;
>
> +/* Xen suspend. Note: data is not used (suspend is the suspend to RAM) */
> +static void cf_check system_suspend(void *data)
> +{
> + int status;
> + unsigned long flags;
> + /* TODO: drop check after verification that features can work together */
> + if ( llc_coloring_enabled )
> + {
> + dprintk(XENLOG_ERR,
> + "System suspend is not supported with LLC_COLORING
> enabled\n");
> + status = -ENOSYS;
> + goto dom_resume;
> + }
> +
> + BUG_ON(system_state != SYS_STATE_active);
> +
> + system_state = SYS_STATE_suspend;
> +
> + printk("Xen suspending...\n");
> +
> + freeze_domains();
> + scheduler_disable();
> +
> + /*
> + * Non-boot CPUs have to be disabled on suspend and enabled on resume
> + * (hotplug-based mechanism). Disabling non-boot CPUs will lead to PSCI
> + * CPU_OFF to be called by each non-boot CPU. Depending on the underlying
> + * platform capabilities, this may lead to the physical powering down of
> + * CPUs.
> + */
> + status = disable_nonboot_cpus();
> + if ( status )
> + {
> + system_state = SYS_STATE_resume;
> + goto resume_nonboot_cpus;
> + }
> +
> + time_suspend();
> +
> + console_start_sync();
> + status = console_suspend();
> + if ( status )
> + {
> + dprintk(XENLOG_ERR, "Failed to suspend the console, err=%d\n",
> status);
> + system_state = SYS_STATE_resume;
> + goto resume_console;
> + }
> +
> + local_irq_save(flags);
> + status = gic_suspend();
> + if ( status )
> + {
> + system_state = SYS_STATE_resume;
> + goto resume_irqs;
> + }
> +
> + set_init_ttbr(xen_pgtable);
> +
> + /*
> + * Enable identity mapping before entering suspend to simplify
> + * the resume path
> + */
> + update_boot_mapping(true);
> +
> + if ( prepare_resume_ctx(&cpu_context) )
> + {
> + status = call_psci_system_suspend();
> + /*
> + * If suspend is finalized properly by above system suspend PSCI
> call,
> + * the code below in this 'if' branch will never execute. Execution
> + * will continue from hyp_resume which is the hypervisor's resume
> point.
> + * In hyp_resume CPU context will be restored and since
> link-register is
> + * restored as well, it will appear to return from
> prepare_resume_ctx.
> + * The difference in returning from prepare_resume_ctx on system
> suspend
> + * versus resume is in function's return value: on suspend, the
> return
> + * value is a non-zero value, on resume it is zero. That is why the
> + * control flow will not re-enter this 'if' branch on resume.
> + */
> + if ( status )
> + dprintk(XENLOG_WARNING, "PSCI system suspend failed, err=%d\n",
> + status);
> + }
> +
> + system_state = SYS_STATE_resume;
> + update_boot_mapping(false);
> +
> + gic_resume();
> +
> + resume_irqs:
> + local_irq_restore(flags);
> +
> + resume_console:
> + console_resume();
> + console_end_sync();
> +
> + time_resume();
> +
> + resume_nonboot_cpus:
> + /*
> + * The rcu_barrier() has to be added to ensure that the per cpu area is
> + * freed before a non-boot CPU tries to initialize it
> (_free_percpu_area()
> + * has to be called before the init_percpu_area()). This scenario occurs
> + * when non-boot CPUs are hot-unplugged on suspend and hotplugged on
> resume.
> + */
> + rcu_barrier();
> + enable_nonboot_cpus();
> + scheduler_enable();
> + thaw_domains();
> +
> + system_state = SYS_STATE_active;
> +
> + printk("Resume (status %d)\n", status);
> +
> + dom_resume:
> + /* The resume of hardware domain should always follow Xen's resume. */
> + domain_resume(hardware_domain);
> +}
> +
> +static DECLARE_TASKLET(system_suspend_tasklet, system_suspend, NULL);
> +
> +void host_system_suspend(void)
> +{
> + /*
> + * system_suspend should be called when hardware domain finalizes the
> + * suspend procedure from its boot core (VCPU#0). However, Dom0's vCPU#0
> + * could be mapped to any pCPU. The suspend procedure has to be finalized
> + * by the pCPU#0 (non-boot pCPUs will be disabled during the suspend).
> + */
> + tasklet_schedule_on_cpu(&system_suspend_tasklet, 0);
> +}
> +
> /*
> * Local variables:
> * mode: C
> diff --git a/xen/arch/arm/vpsci.c b/xen/arch/arm/vpsci.c
> index 22c3a5f544..2f52aba48d 100644
> --- a/xen/arch/arm/vpsci.c
> +++ b/xen/arch/arm/vpsci.c
> @@ -4,6 +4,7 @@
> #include <xen/types.h>
>
> #include <asm/current.h>
> +#include <asm/suspend.h>
> #include <asm/vgic.h>
> #include <asm/vpsci.h>
> #include <asm/event.h>
> @@ -221,9 +222,10 @@ static int32_t do_psci_1_0_system_suspend(register_t
> epoint, register_t cid)
> if ( is_64bit_domain(d) && is_thumb )
> return PSCI_INVALID_ADDRESS;
>
> - /* SYSTEM_SUSPEND is not supported for the hardware domain yet */
> +#ifndef CONFIG_SYSTEM_SUSPEND
> if ( is_hardware_domain(d) )
> return PSCI_NOT_SUPPORTED;
> +#endif
>
> /* Ensure that all CPUs other than the calling one are offline */
> domain_lock(d);
> @@ -249,6 +251,11 @@ static int32_t do_psci_1_0_system_suspend(register_t
> epoint, register_t cid)
> "SYSTEM_SUSPEND requested, epoint=0x%"PRIregister",
> cid=0x%"PRIregister"\n",
> epoint, cid);
>
> +#ifdef CONFIG_SYSTEM_SUSPEND
> + if ( is_hardware_domain(d) )
> + host_system_suspend();
> +#endif
> +
> return rc;
> }
>
> diff --git a/xen/common/domain.c b/xen/common/domain.c
> index 667017c5e1..5e224740d3 100644
> --- a/xen/common/domain.c
> +++ b/xen/common/domain.c
> @@ -1317,7 +1317,11 @@ int domain_shutdown(struct domain *d, u8 reason)
> d->shutdown_code = reason;
> reason = d->shutdown_code;
>
> +#if defined(CONFIG_SYSTEM_SUSPEND) && defined(CONFIG_ARM)
> + if ( reason != SHUTDOWN_suspend && is_hardware_domain(d) )
> +#else
> if ( is_hardware_domain(d) )
> +#endif
> hwdom_shutdown(reason);
>
> if ( d->is_shutting_down )
> --
> 2.48.1
>
Best regards,
Mykola
