Module Name: src
Committed By: jmcneill
Date: Fri Nov 9 23:36:24 UTC 2018
Modified Files:
src/sys/arch/arm/cortex: files.cortex gic_reg.h gicv3.c gicv3.h
Added Files:
src/sys/arch/arm/cortex: gicv3_its.c gicv3_its.h
Log Message:
Add support for GICv3 Locality-specific Periphal Interrupts (LPIs) and the
Interrupt Translation Service (ITS).
To generate a diff of this commit:
cvs rdiff -u -r1.9 -r1.10 src/sys/arch/arm/cortex/files.cortex
cvs rdiff -u -r1.7 -r1.8 src/sys/arch/arm/cortex/gic_reg.h
cvs rdiff -u -r1.4 -r1.5 src/sys/arch/arm/cortex/gicv3.c
cvs rdiff -u -r1.1 -r1.2 src/sys/arch/arm/cortex/gicv3.h
cvs rdiff -u -r0 -r1.1 src/sys/arch/arm/cortex/gicv3_its.c \
src/sys/arch/arm/cortex/gicv3_its.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/arm/cortex/files.cortex
diff -u src/sys/arch/arm/cortex/files.cortex:1.9 src/sys/arch/arm/cortex/files.cortex:1.10
--- src/sys/arch/arm/cortex/files.cortex:1.9 Sun Oct 21 00:42:05 2018
+++ src/sys/arch/arm/cortex/files.cortex Fri Nov 9 23:36:24 2018
@@ -1,4 +1,4 @@
-# $NetBSD: files.cortex,v 1.9 2018/10/21 00:42:05 jmcneill Exp $
+# $NetBSD: files.cortex,v 1.10 2018/11/09 23:36:24 jmcneill Exp $
defflag opt_cpu_in_cksum.h NEON_IN_CKSUM
@@ -20,6 +20,7 @@ file arch/arm/cortex/gic_v2m.c armgic &
# ARM Generic Interrupt Controller v3+
device gicvthree: pic, pic_splfuncs
file arch/arm/cortex/gicv3.c gicvthree
+file arch/arm/cortex/gicv3_its.c gicvthree
# ARM PL310 L2 Cache Controller(initially on Cortex-A9)
device arml2cc
Index: src/sys/arch/arm/cortex/gic_reg.h
diff -u src/sys/arch/arm/cortex/gic_reg.h:1.7 src/sys/arch/arm/cortex/gic_reg.h:1.8
--- src/sys/arch/arm/cortex/gic_reg.h:1.7 Wed Aug 8 19:01:54 2018
+++ src/sys/arch/arm/cortex/gic_reg.h Fri Nov 9 23:36:24 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: gic_reg.h,v 1.7 2018/08/08 19:01:54 jmcneill Exp $ */
+/* $NetBSD: gic_reg.h,v 1.8 2018/11/09 23:36:24 jmcneill Exp $ */
/*-
* Copyright (c) 2012 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -149,6 +149,12 @@
#define GICD_CTRL_EnableGrp1NS __BIT(1) // GICv3
#define GICD_CTRL_Enable __BIT(0)
+#define GICD_TYPER_No1N __BIT(25) // GICv3
+#define GICD_TYPER_A3V __BIT(24) // GICv3
+#define GICD_TYPER_IDbits __BITS(23,19) // GICv3
+#define GICD_TYPER_DVIS __BIT(18) // GICv3
+#define GICD_TYPER_LPIS __BIT(17) // GICv3
+#define GICD_TYPER_MBIS __BIT(16) // GICv3
#define GICD_TYPER_LSPI __BITS(15,11)
#define GICD_TYPER_SecurityExtn __BIT(10)
#define GICD_TYPER_CPUNumber __BITS(7,5)
@@ -190,7 +196,7 @@
#define GICD_IROUTER_Aff1 __BITS(15,8)
#define GICD_IROUTER_Aff0 __BITS(7,0)
-#define GICR_CTRL 0x0000 // Redistributor Control Register
+#define GICR_CTLR 0x0000 // Redistributor Control Register
#define GICR_IIDR 0x0004 // Implementor Identification Register
#define GICR_TYPER 0x0008 // Redistributor Type Register
#define GICR_STATUSR 0x0010 // Error Reporting Status Register, optional
@@ -215,12 +221,12 @@
#define GICR_IGRPMODR0 0x10D00 // Interrupt Group Modifier Register 0
#define GICR_NSACR 0x10E00 // Non-Secure Access Control Register
-#define GICR_CTRL_UWP __BIT(31)
-#define GICR_CTRL_DPG1S __BIT(26)
-#define GICR_CTRL_DPG1NS __BIT(25)
-#define GICR_CTRL_DPG0 __BIT(24)
-#define GICR_CTRL_RWP __BIT(3)
-#define GICR_CTRL_Enable_LPIs __BIT(0)
+#define GICR_CTLR_UWP __BIT(31)
+#define GICR_CTLR_DPG1S __BIT(26)
+#define GICR_CTLR_DPG1NS __BIT(25)
+#define GICR_CTLR_DPG0 __BIT(24)
+#define GICR_CTLR_RWP __BIT(3)
+#define GICR_CTLR_Enable_LPIs __BIT(0)
#define GICR_TYPER_Affinity_Value __BITS(63,32)
#define GICR_TYPER_Affinity_Value_Aff3 __BITS(63,56)
@@ -238,6 +244,40 @@
#define GICR_WAKER_ChildrenAsleep __BIT(2)
#define GICR_WAKER_ProcessorSleep __BIT(1)
+#define GICR_PROPBASER_OuterCache __BITS(58,56)
+#define GICR_PROPBASER_Physical_Address __BITS(51,12)
+#define GICR_PROPBASER_Shareability __BITS(11,10)
+#define GICR_PROPBASER_InnerCache __BITS(9,7)
+#define GICR_PROPBASER_IDbits __BITS(4,0)
+
+#define GICR_PENDBASER_PTZ __BIT(62)
+#define GICR_PENDBASER_OuterCache __BITS(58,56)
+#define GICR_PENDBASER_Physical_Address __BITS(51,16)
+#define GICR_PENDBASER_Shareability __BITS(11,10)
+#define GICR_PENDBASER_InnerCache __BITS(9,7)
+
+#define GICR_Shareability_NS 0 // Non-shareable
+#define GICR_Shareability_IS 1 // Inner Shareable
+#define GICR_Shareability_OS 2 // Outer Shareable
+
+#define GICR_Cache_DEVICE_nGnRnE 0 // Device-nGnRnE
+#define GICR_Cache_NORMAL_NC 1 // Non-cacheable
+#define GICR_Cache_NORMAL_RA_WT 2 // Cacheable Read-allocate, Write-through
+#define GICR_Cache_NORMAL_RA_WB 3 // Cacheable Read-allocate, Write-back
+#define GICR_Cache_NORMAL_WA_WT 4 // Cacheable Write-allocate, Write-through
+#define GICR_Cache_NORMAL_WA_WB 5 // Cacheable Write-allocate, Write-back
+#define GICR_Cache_NORMAL_RA_WA_WT 6 // Cacheable Read-allocate, Write-allocate, Write-through
+#define GICR_Cache_NORMAL_RA_WA_WB 7 // Cacheable Read-allocate, Write-allocate, Write-back
+
+/*
+ * GICv3 Locality-specific Peripheral Interrupts
+ */
+
+#define GIC_LPI_BASE 0x2000 // Base LPI INTID
+
+#define GIC_LPICONF_Priority __BITS(7,2)
+#define GIC_LPICONF_Res1 __BIT(1)
+#define GIC_LPICONF_Enable __BIT(0)
/*
* GICv1 names
@@ -266,7 +306,9 @@
#define GICv1_ICCHPIR GICC_HPPIR
#define GICv1_ICCIIDR GICC_IIDR
-/* GICv2m (MSI) */
+/*
+ * GICv2m (MSI)
+ */
#define GIC_MSI_TYPER 0x0008
#define GIC_MSI_SETSPI 0x0040
@@ -276,4 +318,106 @@
#define GIC_MSI_TYPER_BASE __BITS(25,16) // Starting SPI of MSIs
#define GIC_MSI_TYPER_NUMBER __BITS(9,0) // Count of MSIs
+/*
+ * GICv3 Interrupt Translation Service (ITS)
+ */
+
+#define GITS_CTLR 0x00000 // ITS control register
+#define GITS_IIDR 0x00004 // ITS Identification register
+#define GITS_TYPER 0x00008 // ITS Type register
+#define GITS_CBASER 0x00080 // ITS Command Queue Descriptor
+#define GITS_CWRITER 0x00088 // ITS Write register
+#define GITS_CREADR 0x00090 // ITS Read register
+#define GITS_BASERn(n) (0x00100+8*(n)) // ITS Translation Table Descriptors
+#define GITS_PIDR2 0x0FFE8 // ITS Peripheral ID2 Register
+#define GITS_TRANSLATER 0x10040 // ITS Translation register
+
+#define GITS_CTLR_Quiescent __BIT(31)
+#define GITS_CTLR_ITS_Number __BITS(7,4)
+#define GITS_CTLR_ImDe __BIT(1)
+#define GITS_CTLR_Enabled __BIT(0)
+
+#define GITS_TYPER_VMOVP __BIT(37)
+#define GITS_TYPER_CIL __BIT(36)
+#define GITS_TYPER_CIDbits __BITS(35,32)
+#define GITS_TYPER_HCC __BITS(31,24)
+#define GITS_TYPER_PTA __BIT(19)
+#define GITS_TYPER_SEIS __BIT(18)
+#define GITS_TYPER_Devbits __BITS(17,13)
+#define GITS_TYPER_ID_bits __BITS(12,8)
+#define GITS_TYPER_ITT_entry_size __BITS(7,4)
+#define GITS_TYPER_CCT __BIT(2)
+#define GITS_TYPER_Virtual __BIT(1)
+#define GITS_TYPER_Physical __BIT(0)
+
+#define GITS_CBASER_Valid __BIT(63)
+#define GITS_CBASER_InnerCache __BITS(61,59)
+#define GITS_CBASER_OuterCache __BITS(55,53)
+#define GITS_CBASER_Physical_Address __BITS(51,12)
+#define GITS_CBASER_Shareability __BITS(11,10)
+#define GITS_CBASER_Size __BITS(7,0)
+
+#define GITS_CWRITER_Offset __BITS(19,5)
+#define GITS_CWRITER_Retry __BIT(0)
+
+#define GITS_CREADR_Offset __BITS(19,5)
+#define GITS_CREADR_Stalled __BIT(0)
+
+#define GITS_BASER_Valid __BIT(63)
+#define GITS_BASER_Indirect __BIT(62)
+#define GITS_BASER_InnerCache __BITS(61,59)
+#define GITS_BASER_Type __BITS(58,56)
+#define GITS_BASER_OuterCache __BITS(55,53)
+#define GITS_BASER_Entry_Size __BITS(52,48)
+#define GITS_BASER_Physical_Address __BITS(47,12)
+#define GITS_BASER_Shareability __BITS(11,10)
+#define GITS_BASER_Page_Size __BITS(9,8)
+#define GITS_BASER_Size __BITS(7,0)
+
+#define GITS_Shareability_NS 0 // Non-shareable
+#define GITS_Shareability_IS 1 // Inner Shareable
+#define GITS_Shareability_OS 2 // Outer Shareable
+
+#define GITS_Cache_DEVICE_nGnRnE 0 // Device-nGnRnE
+#define GITS_Cache_NORMAL_NC 1 // Non-cacheable
+#define GITS_Cache_NORMAL_RA_WT 2 // Cacheable Read-allocate, Write-through
+#define GITS_Cache_NORMAL_RA_WB 3 // Cacheable Read-allocate, Write-back
+#define GITS_Cache_NORMAL_WA_WT 4 // Cacheable Write-allocate, Write-through
+#define GITS_Cache_NORMAL_WA_WB 5 // Cacheable Write-allocate, Write-back
+#define GITS_Cache_NORMAL_RA_WA_WT 6 // Cacheable Read-allocate, Write-allocate, Write-through
+#define GITS_Cache_NORMAL_RA_WA_WB 7 // Cacheable Read-allocate, Write-allocate, Write-back
+
+#define GITS_Type_Unimplemented 0 // Unimplemented
+#define GITS_Type_Devices 1 // Devices table
+#define GITS_Type_vPEs 2 // vPEs table
+#define GITS_Type_InterruptCollections 4 // Interrupt collections table
+
+#define GITS_Page_Size_4KB 0
+#define GITS_Page_Size_16KB 1
+#define GITS_Page_Size_64KB 2
+
+struct gicv3_its_command {
+ uint64_t dw[4];
+};
+
+#define GITS_CMD_MOVI 0x01
+#define GITS_CMD_INT 0x03
+#define GITS_CMD_CLEAR 0x04
+#define GITS_CMD_SYNC 0x05
+#define GITS_CMD_MAPD 0x08
+#define GITS_CMD_MAPC 0x09
+#define GITS_CMD_MAPTI 0x0A
+#define GITS_CMD_MAPI 0x0B
+#define GITS_CMD_INV 0x0C
+#define GITS_CMD_INVALL 0x0D
+#define GITS_CMD_MOVALL 0x0E
+#define GITS_CMD_DISCARD 0x0F
+#define GITS_CMD_VMOVI 0x21
+#define GITS_CMD_VMOVP 0x22
+#define GITS_CMD_VSYNC 0x25
+#define GITS_CMD_VMAPP 0x29
+#define GITS_CMD_VMAPTI 0x2A
+#define GITS_CMD_VMAPI 0x2B
+#define GITS_CMD_VINVALL 0x2D
+
#endif /* !_ARM_CORTEX_GICREG_H_ */
Index: src/sys/arch/arm/cortex/gicv3.c
diff -u src/sys/arch/arm/cortex/gicv3.c:1.4 src/sys/arch/arm/cortex/gicv3.c:1.5
--- src/sys/arch/arm/cortex/gicv3.c:1.4 Mon Nov 5 11:50:15 2018
+++ src/sys/arch/arm/cortex/gicv3.c Fri Nov 9 23:36:24 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: gicv3.c,v 1.4 2018/11/05 11:50:15 jmcneill Exp $ */
+/* $NetBSD: gicv3.c,v 1.5 2018/11/09 23:36:24 jmcneill Exp $ */
/*-
* Copyright (c) 2018 Jared McNeill <[email protected]>
@@ -31,7 +31,7 @@
#define _INTR_PRIVATE
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.4 2018/11/05 11:50:15 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.5 2018/11/09 23:36:24 jmcneill Exp $");
#include <sys/param.h>
#include <sys/kernel.h>
@@ -49,6 +49,8 @@ __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.
#define PICTOSOFTC(pic) \
((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic)))
+#define LPITOSOFTC(lpi) \
+ ((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi)))
#define IPL_TO_PRIORITY(ipl) ((IPL_HIGH - (ipl)) << 4)
@@ -110,7 +112,7 @@ gicv3_unblock_irqs(struct pic_softc *pic
if (group == 0) {
sc->sc_enabled_sgippi |= mask;
gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask);
- while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTRL) & GICR_CTRL_RWP)
+ while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
;
} else {
gicd_write_4(sc, GICD_ISENABLERn(group), mask);
@@ -129,7 +131,7 @@ gicv3_block_irqs(struct pic_softc *pic,
if (group == 0) {
sc->sc_enabled_sgippi &= ~mask;
gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask);
- while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTRL) & GICR_CTRL_RWP)
+ while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
;
} else {
gicd_write_4(sc, GICD_ICENABLERn(group), mask);
@@ -251,7 +253,7 @@ gicv3_redist_enable(struct gicv3_softc *
gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0);
/* Wait for register write to complete */
- while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTRL) & GICR_CTRL_RWP)
+ while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
;
/* Set default priorities */
@@ -286,7 +288,7 @@ gicv3_redist_enable(struct gicv3_softc *
gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi);
/* Wait for register write to complete */
- while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTRL) & GICR_CTRL_RWP)
+ while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
;
}
@@ -457,11 +459,148 @@ static const struct pic_ops gicv3_picops
#endif
};
+static void
+gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
+{
+ struct gicv3_softc * const sc = LPITOSOFTC(pic);
+ int bit;
+
+ while ((bit = ffs(mask)) != 0) {
+ sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable;
+ mask &= ~__BIT(bit - 1);
+ }
+
+ bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, irqbase, 32, BUS_DMASYNC_PREWRITE);
+}
+
+static void
+gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
+{
+ struct gicv3_softc * const sc = LPITOSOFTC(pic);
+ const u_int off = irqbase - pic->pic_irqbase;
+ int bit;
+
+ while ((bit = ffs(mask)) != 0) {
+ sc->sc_lpiconf.base[off + bit - 1] &= ~GIC_LPICONF_Enable;
+ mask &= ~__BIT(bit - 1);
+ }
+
+ bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, off, 32, BUS_DMASYNC_PREWRITE);
+}
+
+static void
+gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is)
+{
+ struct gicv3_softc * const sc = LPITOSOFTC(pic);
+
+ sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(is->is_ipl) | GIC_LPICONF_Res1;
+
+ bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, is->is_irq, 1, BUS_DMASYNC_PREWRITE);
+}
+
+static void
+gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
+{
+ struct gicv3_softc * const sc = LPITOSOFTC(pic);
+ struct gicv3_cpu_init *cpu_init;
+ uint32_t ctlr;
+
+ /* If physical LPIs are not supported on this redistributor, just return. */
+ const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER);
+ if ((typer & GICR_TYPER_PLPIS) == 0)
+ return;
+
+ /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */
+ sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number);
+
+ /* Disable LPIs before making changes */
+ ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
+ ctlr &= ~GICR_CTLR_Enable_LPIs;
+ gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
+ arm_dsb();
+
+ /* Setup the LPI configuration table */
+ const uint64_t propbase = sc->sc_lpiconf.segs[0].ds_addr |
+ __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) |
+ __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability) |
+ __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache);
+ gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase);
+
+ /* Setup the LPI pending table */
+ const uint64_t pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr |
+ __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability) |
+ __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache) |
+ GICR_PENDBASER_PTZ;
+ gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase);
+
+ /* Enable LPIs */
+ ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
+ ctlr |= GICR_CTLR_Enable_LPIs;
+ gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
+ arm_dsb();
+
+ /* Setup ITS if present */
+ LIST_FOREACH(cpu_init, &sc->sc_cpu_init, list)
+ cpu_init->func(cpu_init->arg, ci);
+}
+
+static const struct pic_ops gicv3_lpiops = {
+ .pic_unblock_irqs = gicv3_lpi_unblock_irqs,
+ .pic_block_irqs = gicv3_lpi_block_irqs,
+ .pic_establish_irq = gicv3_lpi_establish_irq,
+#ifdef MULTIPROCESSOR
+ .pic_cpu_init = gicv3_lpi_cpu_init,
+#endif
+};
+
+void
+gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align)
+{
+ int nsegs, error;
+
+ dma->len = len;
+ error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK);
+ if (error)
+ panic("bus_dmamem_alloc failed: %d", error);
+ error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK);
+ if (error)
+ panic("bus_dmamem_map failed: %d", error);
+ error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map);
+ if (error)
+ panic("bus_dmamap_create failed: %d", error);
+ error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK);
+ if (error)
+ panic("bus_dmamap_load failed: %d", error);
+
+ memset(dma->base, 0, dma->len);
+ bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE);
+}
+
+static void
+gicv3_lpi_init(struct gicv3_softc *sc)
+{
+ /*
+ * Allocate LPI configuration table
+ */
+ gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000);
+ KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0);
+
+ /*
+ * Allocate LPI pending tables
+ */
+ const bus_size_t lpipend_sz = (sc->sc_lpi.pic_maxsources + sc->sc_lpi.pic_irqbase) / NBBY;
+ for (int cpuindex = 0; cpuindex < MAXCPUS; cpuindex++) {
+ gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000);
+ KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0);
+ }
+}
+
void
gicv3_irq_handler(void *frame)
{
struct cpu_info * const ci = curcpu();
struct gicv3_softc * const sc = gicv3_softc;
+ struct pic_softc *pic;
const int oldipl = ci->ci_cpl;
ci->ci_data.cpu_nintr++;
@@ -472,10 +611,11 @@ gicv3_irq_handler(void *frame)
if (irq == ICC_IAR_INTID_SPURIOUS)
break;
- if (irq >= sc->sc_pic.pic_maxsources)
+ pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic;
+ if (irq - pic->pic_irqbase >= pic->pic_maxsources)
continue;
- struct intrsource * const is = sc->sc_pic.pic_sources[irq];
+ struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase];
KASSERT(is != NULL);
const int ipl = is->is_ipl;
@@ -500,6 +640,8 @@ gicv3_init(struct gicv3_softc *sc)
KASSERT(CPU_IS_PRIMARY(curcpu()));
+ LIST_INIT(&sc->sc_cpu_init);
+
sc->sc_pic.pic_ops = &gicv3_picops;
sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(gicd_typer);
snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3");
@@ -508,6 +650,15 @@ gicv3_init(struct gicv3_softc *sc)
#endif
pic_add(&sc->sc_pic, 0);
+ if ((gicd_typer & GICD_TYPER_LPIS) != 0) {
+ sc->sc_lpi.pic_ops = &gicv3_lpiops;
+ sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */
+ snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi");
+ pic_add(&sc->sc_lpi, GIC_LPI_BASE);
+
+ gicv3_lpi_init(sc);
+ }
+
KASSERT(gicv3_softc == NULL);
gicv3_softc = sc;
@@ -525,6 +676,8 @@ gicv3_init(struct gicv3_softc *sc)
gicv3_dist_enable(sc);
gicv3_cpu_init(&sc->sc_pic, curcpu());
+ if ((gicd_typer & GICD_TYPER_LPIS) != 0)
+ gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu());
#ifdef __HAVE_PIC_FAST_SOFTINTS
intr_establish(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_BIO);
Index: src/sys/arch/arm/cortex/gicv3.h
diff -u src/sys/arch/arm/cortex/gicv3.h:1.1 src/sys/arch/arm/cortex/gicv3.h:1.2
--- src/sys/arch/arm/cortex/gicv3.h:1.1 Wed Aug 8 19:02:28 2018
+++ src/sys/arch/arm/cortex/gicv3.h Fri Nov 9 23:36:24 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: gicv3.h,v 1.1 2018/08/08 19:02:28 jmcneill Exp $ */
+/* $NetBSD: gicv3.h,v 1.2 2018/11/09 23:36:24 jmcneill Exp $ */
/*-
* Copyright (c) 2018 Jared McNeill <[email protected]>
@@ -31,11 +31,27 @@
#include <sys/intr.h>
+struct gicv3_dma {
+ bus_dma_segment_t segs[1];
+ bus_dmamap_t map;
+ uint8_t *base;
+ bus_size_t len;
+};
+
+struct gicv3_cpu_init {
+ void (*func)(void *, struct cpu_info *ci);
+ void *arg;
+
+ LIST_ENTRY(gicv3_cpu_init) list;
+};
+
struct gicv3_softc {
- struct pic_softc sc_pic;
+ struct pic_softc sc_pic; /* SGI/PPI/SGIs */
+ struct pic_softc sc_lpi; /* LPIs */
device_t sc_dev;
bus_space_tag_t sc_bst;
+ bus_dma_tag_t sc_dmat;
bus_space_handle_t sc_bsh_d; /* GICD */
bus_space_handle_t *sc_bsh_r; /* GICR */
@@ -43,9 +59,22 @@ struct gicv3_softc {
uint32_t sc_enabled_sgippi;
uint64_t sc_default_irouter;
+
+ /* LPI configuration table */
+ struct gicv3_dma sc_lpiconf;
+
+ /* LPI pending tables */
+ struct gicv3_dma sc_lpipend[MAXCPUS];
+
+ /* Unique identifier for PEs */
+ u_int sc_processor_id[MAXCPUS];
+
+ /* CPU init callbacks */
+ LIST_HEAD(, gicv3_cpu_init) sc_cpu_init;
};
int gicv3_init(struct gicv3_softc *);
+void gicv3_dma_alloc(struct gicv3_softc *, struct gicv3_dma *, bus_size_t, bus_size_t);
void gicv3_irq_handler(void *);
#endif /* _ARM_CORTEX_GICV3_H */
Added files:
Index: src/sys/arch/arm/cortex/gicv3_its.c
diff -u /dev/null src/sys/arch/arm/cortex/gicv3_its.c:1.1
--- /dev/null Fri Nov 9 23:36:24 2018
+++ src/sys/arch/arm/cortex/gicv3_its.c Fri Nov 9 23:36:24 2018
@@ -0,0 +1,703 @@
+/* $NetBSD: gicv3_its.c,v 1.1 2018/11/09 23:36:24 jmcneill Exp $ */
+
+/*-
+ * Copyright (c) 2018 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jared McNeill <[email protected]>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define _INTR_PRIVATE
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: gicv3_its.c,v 1.1 2018/11/09 23:36:24 jmcneill Exp $");
+
+#include <sys/param.h>
+#include <sys/kmem.h>
+#include <sys/bus.h>
+#include <sys/cpu.h>
+
+#include <uvm/uvm.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include <arm/pic/picvar.h>
+#include <arm/cortex/gicv3_its.h>
+
+/*
+ * ITS translation table sizes
+ */
+#define GITS_COMMANDS_SIZE 0x1000
+#define GITS_COMMANDS_ALIGN 0x10000
+
+#define GITS_ITT_ALIGN 0x100
+
+static inline uint32_t
+gits_read_4(struct gicv3_its *its, bus_size_t reg)
+{
+ return bus_space_read_4(its->its_bst, its->its_bsh, reg);
+}
+
+static inline void
+gits_write_4(struct gicv3_its *its, bus_size_t reg, uint32_t val)
+{
+ bus_space_write_4(its->its_bst, its->its_bsh, reg, val);
+}
+
+static inline uint64_t
+gits_read_8(struct gicv3_its *its, bus_size_t reg)
+{
+ return bus_space_read_8(its->its_bst, its->its_bsh, reg);
+}
+
+static inline void
+gits_write_8(struct gicv3_its *its, bus_size_t reg, uint64_t val)
+{
+ bus_space_write_8(its->its_bst, its->its_bsh, reg, val);
+}
+
+static inline void
+gits_command(struct gicv3_its *its, const struct gicv3_its_command *cmd)
+{
+ uint64_t cwriter;
+ u_int woff;
+
+ cwriter = gits_read_8(its, GITS_CWRITER);
+ woff = cwriter & GITS_CWRITER_Offset;
+
+ memcpy(its->its_cmd.base + woff, cmd->dw, sizeof(cmd->dw));
+ bus_dmamap_sync(its->its_dmat, its->its_cmd.map, woff, sizeof(cmd->dw), BUS_DMASYNC_PREWRITE);
+
+ woff += sizeof(cmd->dw);
+ if (woff == its->its_cmd.len)
+ woff = 0;
+
+ gits_write_8(its, GITS_CWRITER, woff);
+}
+
+static inline void
+gits_command_mapc(struct gicv3_its *its, uint16_t icid, uint64_t rdbase, bool v)
+{
+ struct gicv3_its_command cmd;
+
+ KASSERT((rdbase & 0xffff) == 0);
+
+ /*
+ * Map a collection table entry (ICID) to the target redistributor (RDbase).
+ */
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.dw[0] = GITS_CMD_MAPC;
+ cmd.dw[2] = icid;
+ if (v) {
+ cmd.dw[2] |= rdbase;
+ cmd.dw[2] |= __BIT(63);
+ }
+
+ gits_command(its, &cmd);
+}
+
+static inline void
+gits_command_mapd(struct gicv3_its *its, uint32_t deviceid, uint64_t itt_addr, u_int size, bool v)
+{
+ struct gicv3_its_command cmd;
+
+ KASSERT((itt_addr & 0xff) == 0);
+
+ /*
+ * Map a device table entry (DeviceID) to its associated ITT (ITT_addr).
+ */
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.dw[0] = GITS_CMD_MAPD | ((uint64_t)deviceid << 32);
+ cmd.dw[1] = size;
+ if (v) {
+ cmd.dw[2] = itt_addr | __BIT(63);
+ }
+
+ gits_command(its, &cmd);
+}
+
+static inline void
+gits_command_mapi(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint16_t icid)
+{
+ struct gicv3_its_command cmd;
+
+ /*
+ * Map the event defined by EventID and DeviceID into an ITT entry with ICID and pINTID = EventID
+ */
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.dw[0] = GITS_CMD_MAPI | ((uint64_t)deviceid << 32);
+ cmd.dw[1] = eventid;
+ cmd.dw[2] = icid;
+
+ gits_command(its, &cmd);
+}
+
+static inline void
+gits_command_inv(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid)
+{
+ struct gicv3_its_command cmd;
+
+ /*
+ * Ensure any caching in the redistributors associated with the specified
+ * EventID is consistent with the LPI configuration tables.
+ */
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.dw[0] = GITS_CMD_INV | ((uint64_t)deviceid << 32);
+ cmd.dw[1] = eventid;
+
+ gits_command(its, &cmd);
+}
+
+static inline void
+gits_command_invall(struct gicv3_its *its, uint16_t icid)
+{
+ struct gicv3_its_command cmd;
+
+ /*
+ * Ensure any caching associated with this ICID is consistent with LPI
+ * configuration tables for all redistributors.
+ */
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.dw[0] = GITS_CMD_INVALL;
+ cmd.dw[2] = icid;
+
+ gits_command(its, &cmd);
+}
+
+static inline void
+gits_command_sync(struct gicv3_its *its, uint64_t rdbase)
+{
+ struct gicv3_its_command cmd;
+
+ KASSERT((rdbase & 0xffff) == 0);
+
+ /*
+ * Ensure all outstanding ITS operations associated with physical interrupts
+ * for the specified redistributor (RDbase) are globally observed before
+ * further ITS commands are executed.
+ */
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.dw[0] = GITS_CMD_SYNC;
+ cmd.dw[2] = rdbase;
+
+ gits_command(its, &cmd);
+}
+
+static inline int
+gits_wait(struct gicv3_its *its)
+{
+ u_int woff, roff;
+ int retry = 100000;
+
+ /*
+ * The ITS command queue is empty when CWRITER and CREADR specify the
+ * same base address offset value.
+ */
+ for (retry = 1000; retry > 0; retry--) {
+ woff = gits_read_8(its, GITS_CWRITER) & GITS_CWRITER_Offset;
+ roff = gits_read_8(its, GITS_CREADR) & GITS_CREADR_Offset;
+ if (woff == roff)
+ break;
+ delay(100);
+ }
+ if (retry == 0) {
+ device_printf(its->its_gic->sc_dev, "ITS command queue timeout\n");
+ return ETIMEDOUT;
+ }
+
+ return 0;
+}
+
+static int
+gicv3_its_msi_alloc_lpi(struct gicv3_its *its,
+ const struct pci_attach_args *pa)
+{
+ int n;
+
+ for (n = 0; n < its->its_pic->pic_maxsources; n++) {
+ if (its->its_pa[n] == NULL) {
+ its->its_pa[n] = pa;
+ return n + its->its_pic->pic_irqbase;
+ }
+ }
+
+ return -1;
+}
+
+static void
+gicv3_its_msi_free_lpi(struct gicv3_its *its, int lpi)
+{
+ KASSERT(lpi >= its->its_pic->pic_irqbase);
+ its->its_pa[lpi - its->its_pic->pic_irqbase] = NULL;
+}
+
+static uint32_t
+gicv3_its_devid(pci_chipset_tag_t pc, pcitag_t tag)
+{
+ int b, d, f;
+
+ pci_decompose_tag(pc, tag, &b, &d, &f);
+
+ return (b << 8) | (d << 3) | f;
+}
+
+static struct gicv3_its_device *
+gicv3_its_device_lookup(struct gicv3_its *its, uint32_t devid)
+{
+ struct gicv3_its_device *dev;
+
+ LIST_FOREACH(dev, &its->its_devices, dev_list)
+ if (dev->dev_id == devid)
+ return dev;
+
+ const uint64_t typer = gits_read_8(its, GITS_TYPER);
+ const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1;
+ const u_int itt_entry_size = __SHIFTOUT(typer, GITS_TYPER_ITT_entry_size) + 1;
+ const u_int itt_size = roundup2(itt_entry_size * (1 << id_bits), GITS_ITT_ALIGN);
+
+ dev = kmem_alloc(sizeof(*dev), KM_SLEEP);
+ dev->dev_id = devid;
+ gicv3_dma_alloc(its->its_gic, &dev->dev_itt, itt_size, GITS_ITT_ALIGN);
+ LIST_INSERT_HEAD(&its->its_devices, dev, dev_list);
+
+ return dev;
+}
+
+static void
+gicv3_its_msi_enable(struct gicv3_its *its, int lpi)
+{
+ const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
+ pci_chipset_tag_t pc = pa->pa_pc;
+ pcitag_t tag = pa->pa_tag;
+ pcireg_t ctl;
+ int off;
+
+ if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL))
+ panic("gicv3_its_msi_enable: device is not MSI-capable");
+
+ const uint64_t addr = its->its_base + GITS_TRANSLATER;
+ ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
+ if (ctl & PCI_MSI_CTL_64BIT_ADDR) {
+ pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO,
+ addr & 0xffffffff);
+ pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI,
+ (addr >> 32) & 0xffffffff);
+ pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, lpi);
+ } else {
+ pci_conf_write(pc, tag, off + PCI_MSI_MADDR,
+ addr & 0xffffffff);
+ pci_conf_write(pc, tag, off + PCI_MSI_MDATA, lpi);
+ }
+ ctl |= PCI_MSI_CTL_MSI_ENABLE;
+ pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
+}
+
+static void
+gicv3_its_msi_disable(struct gicv3_its *its, int lpi)
+{
+ const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
+ pci_chipset_tag_t pc = pa->pa_pc;
+ pcitag_t tag = pa->pa_tag;
+ pcireg_t ctl;
+ int off;
+
+ if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL))
+ panic("gicv3_its_msi_enable: device is not MSI-capable");
+
+ ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
+ ctl &= ~PCI_MSI_CTL_MSI_ENABLE;
+ pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
+}
+
+static void
+gicv3_its_msix_enable(struct gicv3_its *its, int lpi, int msix_vec,
+ bus_space_tag_t bst, bus_space_handle_t bsh)
+{
+ const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
+ pci_chipset_tag_t pc = pa->pa_pc;
+ pcitag_t tag = pa->pa_tag;
+ pcireg_t ctl;
+ int off;
+
+ if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL))
+ panic("gicv3_its_msix_enable: device is not MSI-X-capable");
+
+ const uint64_t addr = its->its_base + GITS_TRANSLATER;
+ const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;
+ bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, (uint32_t)addr);
+ bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, (uint32_t)(addr >> 32));
+ bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA, lpi);
+ bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 0);
+
+ ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
+ ctl |= PCI_MSIX_CTL_ENABLE;
+ pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
+}
+
+static void
+gicv3_its_msix_disable(struct gicv3_its *its, int lpi)
+{
+ const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
+ pci_chipset_tag_t pc = pa->pa_pc;
+ pcitag_t tag = pa->pa_tag;
+ pcireg_t ctl;
+ int off;
+
+ if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL))
+ panic("gicv3_its_msix_disable: device is not MSI-X-capable");
+
+ ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
+ ctl &= ~PCI_MSIX_CTL_ENABLE;
+ pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
+}
+
+static pci_intr_handle_t *
+gicv3_its_msi_alloc(struct arm_pci_msi *msi, int *count,
+ const struct pci_attach_args *pa, bool exact)
+{
+ struct gicv3_its * const its = msi->msi_priv;
+ struct gicv3_its_device *dev;
+ struct cpu_info *ci = curcpu(); /* XXX */
+ pci_intr_handle_t *vectors;
+ int n, off;
+
+ if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL))
+ return NULL;
+
+ const uint64_t typer = gits_read_8(its, GITS_TYPER);
+ const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1;
+ if (*count == 0 || *count > (1 << id_bits))
+ return NULL;
+
+ const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag);
+
+ /*
+ * Map device
+ */
+ dev = gicv3_its_device_lookup(its, devid);
+ gits_command_mapd(its, devid, dev->dev_itt.segs[0].ds_addr, id_bits - 1, true);
+ gits_wait(its);
+
+ vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP);
+ for (n = 0; n < *count; n++) {
+ const int lpi = gicv3_its_msi_alloc_lpi(its, pa);
+ vectors[n] = ARM_PCI_INTR_MSI |
+ __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) |
+ __SHIFTIN(n, ARM_PCI_INTR_MSI_VEC) |
+ __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME);
+
+ gicv3_its_msi_enable(its, lpi);
+
+ /*
+ * Map event
+ */
+ gits_command_mapi(its, devid, lpi, cpu_index(ci));
+ gits_command_sync(its, its->its_rdbase[cpu_index(ci)]);
+ }
+ gits_wait(its);
+
+ return vectors;
+}
+
+static pci_intr_handle_t *
+gicv3_its_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes, int *count,
+ const struct pci_attach_args *pa, bool exact)
+{
+ struct gicv3_its * const its = msi->msi_priv;
+ struct gicv3_its_device *dev;
+ struct cpu_info *ci = curcpu(); /* XXX */
+ pci_intr_handle_t *vectors;
+ bus_space_tag_t bst;
+ bus_space_handle_t bsh;
+ bus_size_t bsz;
+ uint32_t table_offset, table_size;
+ int n, off, bar, error;
+ pcireg_t tbl;
+
+ if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL))
+ return NULL;
+
+ const uint64_t typer = gits_read_8(its, GITS_TYPER);
+ const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1;
+ if (*count == 0 || *count > (1 << id_bits))
+ return NULL;
+
+ tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET);
+ bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_PBABIR_MASK));
+ table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK;
+ table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE;
+ if (table_size == 0)
+ return NULL;
+
+ error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar),
+ BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset,
+ &bst, &bsh, NULL, &bsz);
+ if (error)
+ return NULL;
+
+ const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag);
+
+ /*
+ * Map device
+ */
+ dev = gicv3_its_device_lookup(its, devid);
+ gits_command_mapd(its, devid, dev->dev_itt.segs[0].ds_addr, id_bits - 1, true);
+ gits_wait(its);
+
+ vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP);
+ for (n = 0; n < *count; n++) {
+ const int lpi = gicv3_its_msi_alloc_lpi(its, pa);
+ const int msix_vec = table_indexes ? table_indexes[n] : n;
+ vectors[msix_vec] = ARM_PCI_INTR_MSIX |
+ __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) |
+ __SHIFTIN(msix_vec, ARM_PCI_INTR_MSI_VEC) |
+ __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME);
+
+ gicv3_its_msix_enable(its, lpi, msix_vec, bst, bsh);
+
+ /*
+ * Map event
+ */
+ gits_command_mapi(its, devid, lpi, cpu_index(ci));
+ gits_command_sync(its, its->its_rdbase[cpu_index(ci)]);
+ }
+ gits_wait(its);
+
+ bus_space_unmap(bst, bsh, bsz);
+
+ return vectors;
+}
+
+static void *
+gicv3_its_msi_intr_establish(struct arm_pci_msi *msi,
+ pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg)
+{
+ struct gicv3_its * const its = msi->msi_priv;
+ const struct pci_attach_args *pa;
+ void *intrh;
+
+ const int lpi = __SHIFTOUT(ih, ARM_PCI_INTR_IRQ);
+ const int mpsafe = (ih & ARM_PCI_INTR_MPSAFE) ? IST_MPSAFE : 0;
+
+ intrh = pic_establish_intr(its->its_pic, lpi - its->its_pic->pic_irqbase, ipl,
+ IST_EDGE | mpsafe, func, arg);
+ if (intrh == NULL)
+ return NULL;
+
+ /* Invalidate LPI configuration tables */
+ pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
+ KASSERT(pa != NULL);
+ const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag);
+ gits_command_inv(its, devid, lpi);
+
+ return intrh;
+}
+
+static void
+gicv3_its_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih,
+ int count)
+{
+ struct gicv3_its * const its = msi->msi_priv;
+ int n;
+
+ for (n = 0; n < count; n++) {
+ const int lpi = __SHIFTOUT(pih[n], ARM_PCI_INTR_IRQ);
+ KASSERT(lpi >= its->its_pic->pic_irqbase);
+ if (pih[n] & ARM_PCI_INTR_MSIX)
+ gicv3_its_msix_disable(its, lpi);
+ if (pih[n] & ARM_PCI_INTR_MSI)
+ gicv3_its_msi_disable(its, lpi);
+ gicv3_its_msi_free_lpi(its, lpi);
+ struct intrsource * const is =
+ its->its_pic->pic_sources[lpi - its->its_pic->pic_irqbase];
+ if (is != NULL)
+ pic_disestablish_source(is);
+ }
+}
+
+static void
+gicv3_its_command_init(struct gicv3_softc *sc, struct gicv3_its *its)
+{
+ uint64_t cbaser;
+
+ gicv3_dma_alloc(sc, &its->its_cmd, GITS_COMMANDS_SIZE, GITS_COMMANDS_ALIGN);
+
+ cbaser = its->its_cmd.segs[0].ds_addr;
+ cbaser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_CBASER_InnerCache);
+ cbaser |= __SHIFTIN(GITS_Shareability_NS, GITS_CBASER_Shareability);
+ cbaser |= __SHIFTIN((its->its_cmd.len / 4096) - 1, GITS_CBASER_Size);
+ cbaser |= GITS_CBASER_Valid;
+
+ gits_write_8(its, GITS_CBASER, cbaser);
+ gits_write_8(its, GITS_CWRITER, 0);
+}
+
+static void
+gicv3_its_table_init(struct gicv3_softc *sc, struct gicv3_its *its)
+{
+ u_int table_size, page_size, table_align;
+ uint64_t baser;
+ int tab;
+
+ const uint64_t typer = gits_read_8(its, GITS_TYPER);
+ const u_int devbits = __SHIFTOUT(typer, GITS_TYPER_Devbits) + 1;
+
+ for (tab = 0; tab < 8; tab++) {
+ baser = gits_read_8(its, GITS_BASERn(tab));
+
+ const u_int entry_size = __SHIFTOUT(baser, GITS_BASER_Entry_Size) + 1;
+
+ switch (__SHIFTOUT(baser, GITS_BASER_Page_Size)) {
+ case GITS_Page_Size_4KB:
+ page_size = 4096;
+ table_align = 4096;
+ break;
+ case GITS_Page_Size_16KB:
+ page_size = 16384;
+ table_align = 4096;
+ break;
+ case GITS_Page_Size_64KB:
+ default:
+ page_size = 65536;
+ table_align = 65536;
+ break;
+ }
+
+ switch (__SHIFTOUT(baser, GITS_BASER_Type)) {
+ case GITS_Type_Devices:
+ /*
+ * Table size scales with the width of the DeviceID.
+ */
+ table_size = roundup(entry_size * (1 << devbits), page_size);
+ break;
+ case GITS_Type_InterruptCollections:
+ /*
+ * Allocate space for one interrupt collection per CPU.
+ */
+ table_size = roundup(entry_size * MAXCPUS, page_size);
+ break;
+ default:
+ table_size = 0;
+ break;
+ }
+
+ if (table_size == 0)
+ continue;
+
+ aprint_normal_dev(sc->sc_dev, "ITS TT%u type %#x size %#x\n", tab, (u_int)__SHIFTOUT(baser, GITS_BASER_Type), table_size);
+ gicv3_dma_alloc(sc, &its->its_tab[tab], table_size, table_align);
+
+ baser &= ~GITS_BASER_Size;
+ baser |= __SHIFTIN(table_size / page_size - 1, GITS_BASER_Size);
+ baser &= ~GITS_BASER_Physical_Address;
+ baser |= its->its_tab[tab].segs[0].ds_addr;
+ baser &= ~GITS_BASER_InnerCache;
+ baser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_BASER_InnerCache);
+ baser &= ~GITS_BASER_Shareability;
+ baser |= __SHIFTIN(GITS_Shareability_NS, GITS_BASER_Shareability);
+ baser |= GITS_BASER_Valid;
+
+ gits_write_8(its, GITS_BASERn(tab), baser);
+ }
+}
+
+static void
+gicv3_its_enable(struct gicv3_softc *sc, struct gicv3_its *its)
+{
+ uint32_t ctlr;
+
+ ctlr = gits_read_4(its, GITS_CTLR);
+ ctlr |= GITS_CTLR_Enabled;
+ gits_write_4(its, GITS_CTLR, ctlr);
+}
+
+static void
+gicv3_its_cpu_init(void *priv, struct cpu_info *ci)
+{
+ struct gicv3_its * const its = priv;
+ struct gicv3_softc * const sc = its->its_gic;
+ uint64_t rdbase;
+
+ const uint64_t typer = bus_space_read_8(sc->sc_bst, its->its_bsh, GITS_TYPER);
+ if (typer & GITS_TYPER_PTA) {
+ void *va = bus_space_vaddr(sc->sc_bst, sc->sc_bsh_r[ci->ci_gic_redist]);
+ rdbase = vtophys((vaddr_t)va);
+ } else {
+ rdbase = (uint64_t)sc->sc_processor_id[cpu_index(ci)] << 16;
+ }
+ its->its_rdbase[cpu_index(ci)] = rdbase;
+
+ /*
+ * Map collection ID of this CPU's index to this CPU's redistributor.
+ */
+ gits_command_mapc(its, cpu_index(ci), rdbase, true);
+ gits_command_invall(its, cpu_index(ci));
+ gits_wait(its);
+}
+
+int
+gicv3_its_init(struct gicv3_softc *sc, bus_space_handle_t bsh,
+ uint64_t its_base, uint32_t its_id)
+{
+ struct gicv3_its *its;
+ struct arm_pci_msi *msi;
+
+ const uint64_t typer = bus_space_read_8(sc->sc_bst, bsh, GITS_TYPER);
+ if ((typer & GITS_TYPER_Physical) == 0)
+ return ENXIO;
+
+ its = kmem_alloc(sizeof(*its), KM_SLEEP);
+ its->its_id = its_id;
+ its->its_bst = sc->sc_bst;
+ its->its_bsh = bsh;
+ its->its_dmat = sc->sc_dmat;
+ its->its_base = its_base;
+ its->its_pic = &sc->sc_lpi;
+ KASSERT(its->its_pic->pic_maxsources > 0);
+ its->its_pa = kmem_zalloc(sizeof(struct pci_attach_args *) * its->its_pic->pic_maxsources, KM_SLEEP);
+ its->its_gic = sc;
+ its->its_init.func = gicv3_its_cpu_init;
+ its->its_init.arg = its;
+ LIST_INIT(&its->its_devices);
+ LIST_INSERT_HEAD(&sc->sc_cpu_init, &its->its_init, list);
+
+ gicv3_its_command_init(sc, its);
+ gicv3_its_table_init(sc, its);
+
+ gicv3_its_enable(sc, its);
+
+ gicv3_its_cpu_init(its, curcpu());
+
+ msi = &its->its_msi;
+ msi->msi_dev = sc->sc_dev;
+ msi->msi_priv = its;
+ msi->msi_alloc = gicv3_its_msi_alloc;
+ msi->msix_alloc = gicv3_its_msix_alloc;
+ msi->msi_intr_establish = gicv3_its_msi_intr_establish;
+ msi->msi_intr_release = gicv3_its_msi_intr_release;
+
+ return arm_pci_msi_add(msi);
+}
Index: src/sys/arch/arm/cortex/gicv3_its.h
diff -u /dev/null src/sys/arch/arm/cortex/gicv3_its.h:1.1
--- /dev/null Fri Nov 9 23:36:24 2018
+++ src/sys/arch/arm/cortex/gicv3_its.h Fri Nov 9 23:36:24 2018
@@ -0,0 +1,70 @@
+/* $NetBSD: gicv3_its.h,v 1.1 2018/11/09 23:36:24 jmcneill Exp $ */
+
+/*-
+ * Copyright (c) 2018 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jared McNeill <[email protected]>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _ARM_CORTEX_GICV3_ITS_H
+#define _ARM_CORTEX_GICV3_ITS_H
+
+#include <arm/pci/pci_msi_machdep.h>
+#include <arm/cortex/gic_reg.h>
+#include <arm/cortex/gicv3.h>
+
+struct gicv3_its_device {
+ uint32_t dev_id;
+ struct gicv3_dma dev_itt;
+
+ LIST_ENTRY(gicv3_its_device) dev_list;
+};
+
+struct gicv3_its {
+ bus_space_tag_t its_bst;
+ bus_space_handle_t its_bsh;
+ bus_dma_tag_t its_dmat;
+ uint32_t its_id;
+ uint64_t its_base;
+ uint64_t its_rdbase[MAXCPUS];
+
+ struct gicv3_softc *its_gic;
+ struct gicv3_cpu_init its_init;
+
+ struct pic_softc *its_pic;
+ const struct pci_attach_args **its_pa;
+
+ LIST_HEAD(, gicv3_its_device) its_devices;
+
+ struct gicv3_dma its_cmd; /* Command queue */
+ struct gicv3_dma its_tab[8]; /* ITS tables */
+
+ struct arm_pci_msi its_msi;
+};
+
+int gicv3_its_init(struct gicv3_softc *, bus_space_handle_t, uint64_t, uint32_t);
+
+#endif /* !_ARM_CORTEX_GICV3_ITS_H */
