Hi Oleksandr,
On Tue, Sep 2, 2025 at 7:08 PM Oleksandr Tyshchenko <olekst...@gmail.com> wrote:
>
>
>
> On 02.09.25 01:10, Mykola Kvach wrote:
>
> Hello Mykola
>
>
> > From: Mykola Kvach <mykola_kv...@epam.com>
> >
> > System suspend may lead to a state where GIC would be powered down.
> > Therefore, Xen should save/restore the context of GIC on suspend/resume.
> >
> > Note that the context consists of states of registers which are
> > controlled by the hypervisor. Other GIC registers which are accessible
> > by guests are saved/restored on context switch.
> >
> > Signed-off-by: Mykola Kvach <mykola_kv...@epam.com>
> > ---
> > Changes in V6:
> > - Drop gicv3_save/restore_state since it is already handled during vCPU
> > context switch.
> > - The comment about systems without SPIs is clarified for readability.
> > - Error and warning messages related to suspend context allocation are
> > unified
> > and now use printk() with XENLOG_ERR for consistency.
> > - The check for suspend context allocation in gicv3_resume() is removed,
> > as it is handled earlier in the suspend path.
> > - The loop for saving and restoring PPI/SGI priorities is corrected to use
> > the proper increment.
> > - The gicv3_suspend() function now prints an explicit error if ITS suspend
> > support is not implemented, and returns ENOSYS in this case.
> > - The GICD_CTLR_DS bit definition is added to gic_v3_defs.h.
> > - The comment for GICR_WAKER access is expanded to reference the relevant
> > ARM specification section and clarify the RAZ/WI behavior for Non-secure
> > accesses.
> > - Cleanup active and enable registers before restoring.
> > ---
> > xen/arch/arm/gic-v3-lpi.c | 3 +
> > xen/arch/arm/gic-v3.c | 235 +++++++++++++++++++++++++
> > xen/arch/arm/include/asm/gic_v3_defs.h | 1 +
> > 3 files changed, 239 insertions(+)
> >
> > diff --git a/xen/arch/arm/gic-v3-lpi.c b/xen/arch/arm/gic-v3-lpi.c
> > index de5052e5cf..61a6e18303 100644
> > --- a/xen/arch/arm/gic-v3-lpi.c
> > +++ b/xen/arch/arm/gic-v3-lpi.c
> > @@ -391,6 +391,9 @@ static int cpu_callback(struct notifier_block *nfb,
> > unsigned long action,
> > switch ( action )
> > {
> > case CPU_UP_PREPARE:
> > + if ( system_state == SYS_STATE_resume )
> > + break;
> > +
> > rc = gicv3_lpi_allocate_pendtable(cpu);
> > if ( rc )
> > printk(XENLOG_ERR "Unable to allocate the pendtable for
> > CPU%lu\n",
> > diff --git a/xen/arch/arm/gic-v3.c b/xen/arch/arm/gic-v3.c
> > index cd3e1acf79..9f1be7e905 100644
> > --- a/xen/arch/arm/gic-v3.c
> > +++ b/xen/arch/arm/gic-v3.c
> > @@ -1776,6 +1776,233 @@ static bool gic_dist_supports_lpis(void)
> > return (readl_relaxed(GICD + GICD_TYPER) & GICD_TYPE_LPIS);
> > }
> >
> > +#ifdef CONFIG_SYSTEM_SUSPEND
> > +
> > +/* GICv3 registers to be saved/restored on system suspend/resume */
> > +struct gicv3_ctx {
> > + struct dist_ctx {
> > + uint32_t ctlr;
> > + /*
> > + * This struct represent block of 32 IRQs
> > + * TODO: store extended SPI configuration (GICv3.1+)
> > + */
> > + struct irq_regs {
> > + uint32_t icfgr[2];
> > + uint32_t ipriorityr[8];
> > + uint64_t irouter[32];
> > + uint32_t isactiver;
> > + uint32_t isenabler;
> > + } *irqs;
> > + } dist;
> > +
> > + /* have only one rdist structure for last running CPU during suspend */
> > + struct redist_ctx {
> > + uint32_t ctlr;
> > + /* TODO: handle case when we have more than 16 PPIs (GICv3.1+) */
> > + uint32_t icfgr[2];
> > + uint32_t igroupr;
> > + uint32_t ipriorityr[8];
> > + uint32_t isactiver;
> > + uint32_t isenabler;
> > + } rdist;
> > +
> > + struct cpu_ctx {
> > + uint32_t ctlr;
> > + uint32_t pmr;
> > + uint32_t bpr;
> > + uint32_t sre_el2;
> > + uint32_t grpen;
> > + } cpu;
> > +};
> > +
> > +static struct gicv3_ctx gicv3_ctx;
> > +
> > +static void __init gicv3_alloc_context(void)
> > +{
> > + uint32_t blocks = DIV_ROUND_UP(gicv3_info.nr_lines, 32);
> > +
> > + /* We don't have ITS support for suspend */
> > + if ( gicv3_its_host_has_its() )
> > + return;
> > +
> > + /* The spec allows for systems without any SPIs */
> > + if ( blocks > 1 )
> > + {
> > + gicv3_ctx.dist.irqs = xzalloc_array(typeof(*gicv3_ctx.dist.irqs),
> > + blocks - 1);
> > + if ( !gicv3_ctx.dist.irqs )
> > + printk(XENLOG_ERR "Failed to allocate memory for GICv3 suspend
> > context\n");
> > + }
> > +}
> > +
> > +static void gicv3_disable_redist(void)
> > +{
> > + void __iomem* waker = GICD_RDIST_BASE + GICR_WAKER;
> > +
> > + /*
> > + * Avoid infinite loop if Non-secure does not have access to
> > GICR_WAKER.
> > + * See Arm IHI 0069H.b, 12.11.42 GICR_WAKER:
> > + * When GICD_CTLR.DS == 0 and an access is Non-secure accesses to
> > this
> > + * register are RAZ/WI.
> > + */
> > + if ( !(readl_relaxed(GICD + GICD_CTLR) & GICD_CTLR_DS) )
> > + return;
> > +
> > + writel_relaxed(readl_relaxed(waker) | GICR_WAKER_ProcessorSleep,
> > waker);
> > + while ( (readl_relaxed(waker) & GICR_WAKER_ChildrenAsleep) == 0 );
> > +}
> > +
> > +static int gicv3_suspend(void)
> > +{
> > + unsigned int i;
> > + void __iomem *base;
> > + typeof(gicv3_ctx.rdist)* rdist = &gicv3_ctx.rdist;
> > +
> > + /* TODO: implement support for ITS */
> > + if ( gicv3_its_host_has_its() )
> > + {
> > + printk(XENLOG_ERR "GICv3: ITS suspend support is not
> > implemented\n");
> > + return -ENOSYS;
> > + }
> > +
> > + if ( !gicv3_ctx.dist.irqs && gicv3_info.nr_lines > NR_GIC_LOCAL_IRQS )
> > + {
> > + printk(XENLOG_ERR "GICv3: suspend context is not allocated!\n");
> > + return -ENOMEM;
> > + }
> > +
> > + /* Save GICC configuration */
> > + gicv3_ctx.cpu.ctlr = READ_SYSREG(ICC_CTLR_EL1);
> > + gicv3_ctx.cpu.pmr = READ_SYSREG(ICC_PMR_EL1);
> > + gicv3_ctx.cpu.bpr = READ_SYSREG(ICC_BPR1_EL1);
> > + gicv3_ctx.cpu.sre_el2 = READ_SYSREG(ICC_SRE_EL2);
> > + gicv3_ctx.cpu.grpen = READ_SYSREG(ICC_IGRPEN1_EL1);
> > +
> > + gicv3_disable_interface();
> > + gicv3_disable_redist();
> > +
> > + /* Save GICR configuration */
> > + gicv3_redist_wait_for_rwp();
> > +
> > + base = GICD_RDIST_SGI_BASE;
> > +
> > + rdist->ctlr = readl_relaxed(base + GICR_CTLR);
> > +
> > + /* Save priority on PPI and SGI interrupts */
> > + for ( i = 0; i < NR_GIC_LOCAL_IRQS / 4; i++ )
> > + rdist->ipriorityr[i] = readl_relaxed(base + GICR_IPRIORITYR0 + 4 *
> > i);
> > +
> > + rdist->isactiver = readl_relaxed(base + GICR_ISACTIVER0);
> > + rdist->isenabler = readl_relaxed(base + GICR_ISENABLER0);
> > + rdist->igroupr = readl_relaxed(base + GICR_IGROUPR0);
> > + rdist->icfgr[0] = readl_relaxed(base + GICR_ICFGR0);
>
> GICR_ICFGR0 is for SGIs, which are always edge-triggered, so I am not
> sure that we need to save it here ...
Looks like I didn’t read the spec carefully and only paid attention to:
12.11.7 GICR_ICFGR0, Interrupt Configuration Register 0
Determines whether the corresponding SGI is edge-triggered or level-sensitive.
But a few lines below it states:
but a few lines below
Int_config<x>, bits [2x+1:2x], for x = 15 to 0
Indicates whether the is level-sensitive or edge-triggered.
0b00 Corresponding interrupt is level-sensitive.
0b10 Corresponding interrupt is edge-triggered.
SGIs are always edge-triggered.
>
>
> > + rdist->icfgr[1] = readl_relaxed(base + GICR_ICFGR1);
> > +
> > + /* Save GICD configuration */
> > + gicv3_dist_wait_for_rwp();
> > + gicv3_ctx.dist.ctlr = readl_relaxed(GICD + GICD_CTLR);
> > +
> > + for ( i = 1; i < DIV_ROUND_UP(gicv3_info.nr_lines, 32); i++ )
> > + {
> > + typeof(gicv3_ctx.dist.irqs) irqs = gicv3_ctx.dist.irqs + i - 1;
> > + unsigned int irq;
> > +
> > + base = GICD + GICD_ICFGR + 8 * i;
> > + irqs->icfgr[0] = readl_relaxed(base);
> > + irqs->icfgr[1] = readl_relaxed(base + 4);
> > +
> > + base = GICD + GICD_IPRIORITYR + 32 * i;
> > + for ( irq = 0; irq < 8; irq++ )
> > + irqs->ipriorityr[irq] = readl_relaxed(base + 4 * irq);
> > +
> > + base = GICD + GICD_IROUTER + 32 * i;
> > + for ( irq = 0; irq < 32; irq++ )
> > + irqs->irouter[irq] = readq_relaxed_non_atomic(base + 8 * irq);
> > +
> > + irqs->isactiver = readl_relaxed(GICD + GICD_ISACTIVER + 4 * i);
> > + irqs->isenabler = readl_relaxed(GICD + GICD_ISENABLER + 4 * i);
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static void gicv3_resume(void)
> > +{
> > + unsigned int i;
> > + void __iomem *base;
> > + typeof(gicv3_ctx.rdist)* rdist = &gicv3_ctx.rdist;
> > +
> > + writel_relaxed(0, GICD + GICD_CTLR);
> > +
> > + for ( i = NR_GIC_LOCAL_IRQS; i < gicv3_info.nr_lines; i += 32 )
> > + writel_relaxed(GENMASK(31, 0), GICD + GICD_IGROUPR + (i / 32) * 4);
> > +
> > + for ( i = 1; i < DIV_ROUND_UP(gicv3_info.nr_lines, 32); i++ )
> > + {
> > + typeof(gicv3_ctx.dist.irqs) irqs = gicv3_ctx.dist.irqs + i - 1;
> > + unsigned int irq;
> > +
> > + base = GICD + GICD_ICFGR + 8 * i;
> > + writel_relaxed(irqs->icfgr[0], base);
> > + writel_relaxed(irqs->icfgr[1], base + 4);
> > +
> > + base = GICD + GICD_IPRIORITYR + 32 * i;
> > + for ( irq = 0; irq < 8; irq++ )
> > + writel_relaxed(irqs->ipriorityr[irq], base + 4 * irq);
> > +
> > + base = GICD + GICD_IROUTER + 32 * i;
> > + for ( irq = 0; irq < 32; irq++ )
> > + writeq_relaxed_non_atomic(irqs->irouter[irq], base + 8 * irq);
> > +
> > + writel_relaxed(GENMASK(31, 0), GICD + GICD_ICENABLER + i * 4);
> > + writel_relaxed(irqs->isenabler, GICD + GICD_ISENABLER + i * 4);
> > +
> > + writel_relaxed(GENMASK(31, 0), GICD + GICD_ICACTIVER + i * 4);
> > + writel_relaxed(irqs->isactiver, GICD + GICD_ISACTIVER + i * 4);
> > + }
> > +
> > + writel_relaxed(gicv3_ctx.dist.ctlr, GICD + GICD_CTLR);
> > + gicv3_dist_wait_for_rwp();
> > +
> > + /* Restore GICR (Redistributor) configuration */
> > + gicv3_enable_redist();
> > +
> > + base = GICD_RDIST_SGI_BASE;
> > +
> > + writel_relaxed(0xffffffff, base + GICR_ICENABLER0);
> > + gicv3_redist_wait_for_rwp();
> > +
> > + for ( i = 0; i < NR_GIC_LOCAL_IRQS / 4; i++ )
> > + writel_relaxed(rdist->ipriorityr[i], base + GICR_IPRIORITYR0 + i *
> > 4);
> > +
> > + writel_relaxed(rdist->isactiver, base + GICR_ISACTIVER0);
> > +
> > + writel_relaxed(rdist->igroupr, base + GICR_IGROUPR0);
> > + writel_relaxed(rdist->icfgr[0], base + GICR_ICFGR0);
>
> ... and restore it here.
Thank you for pointing that out.
I will remove it in the next version of the patch series.
>
>
> > + writel_relaxed(rdist->icfgr[1], base + GICR_ICFGR1);
> > +
>
> [snip]
Best regards,
Mykola
