Author: markj
Date: Sun Nov 22 18:54:14 2020
New Revision: 367939
URL: https://svnweb.freebsd.org/changeset/base/367939
Log:
MFC r366969, r366973:
ntb: Add Intel Xeon Gen3 support
Modified:
stable/12/share/man/man4/ntb_hw_intel.4
stable/12/sys/dev/ntb/ntb_hw/ntb_hw_intel.c
stable/12/sys/dev/ntb/ntb_hw/ntb_hw_intel.h
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/share/man/man4/ntb_hw_intel.4
==============================================================================
--- stable/12/share/man/man4/ntb_hw_intel.4 Sun Nov 22 18:42:08 2020
(r367938)
+++ stable/12/share/man/man4/ntb_hw_intel.4 Sun Nov 22 18:54:14 2020
(r367939)
@@ -25,7 +25,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd August 30, 2017
+.Dd October 11, 2020
.Dt NTB_HW_INTEL 4
.Os
.Sh NAME
@@ -50,16 +50,18 @@ The
driver provides support for the Non-Transparent Bridge (NTB) hardware in
Intel Xeon E3/E5 and S1200 processor families, which allow one of their PCIe
ports to be switched from transparent to non-transparent bridge mode.
-In this mode bridge looks not as a PCI bridge, but as PCI endpoint device.
+In this mode the bridge looks not like a PCI bridge, but like a PCI endpoint
+device.
The driver hides hardware details, exposing memory windows, scratchpads
-and doorbells of the other side via hardware independent KPI to
+and doorbells of the other side via a hardware independent KPI to the
.Xr ntb 4
subsystem.
.Pp
The hardware provides 2 or 3 memory windows to the other system's memory,
-16 scratchpad registers and 14 or 34 doorbells to interrupt the other system.
-On Xeon processors one of memory windows is typically consumed by the driver
-itself to workaround multiple hardware erratas.
+16 scratchpad registers and 14, 31 or 34 doorbells to interrupt the other
+system, depending on the platform.
+On Xeon processors one of the memory windows is typically consumed by the
driver
+itself to work around multiple hardware errata.
.Sh CONFIGURATION
The NTB configuration should be set by BIOS.
It includes enabling NTB, choosing between NTB-to-NTB (back-to-back) or
@@ -67,9 +69,10 @@ NTB-to-Root Port mode,
enabling split BAR mode (one of two 64-bit BARs can be split into two 32-bit
ones) and configuring BAR sizes in bits (from 12 to 29/39) for both NTB sides.
.Pp
-The recommended configuration is NTB-to-NTB mode, split bar is enabled and
-all BAR sizes are set to 20 (1 MiB).
+The recommended configuration is NTB-to-NTB mode, split bar enabled and
+all BAR sizes set to 20 (1 MiB).
This needs to be done on both systems.
+Note, on Xeon SkyLake and newer platforms, split bar mode is not available.
.Sh SEE ALSO
.Xr if_ntb 4 ,
.Xr ntb_transport 4 ,
Modified: stable/12/sys/dev/ntb/ntb_hw/ntb_hw_intel.c
==============================================================================
--- stable/12/sys/dev/ntb/ntb_hw/ntb_hw_intel.c Sun Nov 22 18:42:08 2020
(r367938)
+++ stable/12/sys/dev/ntb/ntb_hw/ntb_hw_intel.c Sun Nov 22 18:54:14 2020
(r367939)
@@ -64,7 +64,8 @@ __FBSDID("$FreeBSD$");
#include "ntb_hw_intel.h"
#include "../ntb.h"
-#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT)
+#define MAX_MSIX_INTERRUPTS \
+ MAX(MAX(XEON_DB_COUNT, ATOM_DB_COUNT), XEON_GEN3_DB_COUNT)
#define NTB_HB_TIMEOUT 1 /* second */
#define ATOM_LINK_RECOVERY_TIME 500 /* ms */
@@ -83,7 +84,8 @@ __FBSDID("$FreeBSD$");
#define PCI_MSIX_ENTRY_DATA 8
enum ntb_device_type {
- NTB_XEON,
+ NTB_XEON_GEN1,
+ NTB_XEON_GEN3,
NTB_ATOM
};
@@ -334,6 +336,7 @@ static int map_memory_window_bar(struct ntb_softc *ntb
static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb);
static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
static int intel_ntb_init_isr(struct ntb_softc *ntb);
+static int intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb);
static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb);
@@ -351,8 +354,10 @@ static void intel_ntb_exchange_msix(void *);
static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id);
static void intel_ntb_detect_max_mw(struct ntb_softc *ntb);
static int intel_ntb_detect_xeon(struct ntb_softc *ntb);
+static int intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb);
static int intel_ntb_detect_atom(struct ntb_softc *ntb);
static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb);
+static int intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb);
static int intel_ntb_atom_init_dev(struct ntb_softc *ntb);
static void intel_ntb_teardown_xeon(struct ntb_softc *ntb);
static void configure_atom_secondary_side_bars(struct ntb_softc *ntb);
@@ -364,6 +369,9 @@ static void xeon_set_pbar_xlat(struct ntb_softc *, uin
enum ntb_bar idx);
static int xeon_setup_b2b_mw(struct ntb_softc *,
const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
+static int xeon_gen3_setup_b2b_mw(struct ntb_softc *);
+static int intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr,
+ size_t size);
static inline bool link_is_up(struct ntb_softc *ntb);
static inline bool _xeon_link_is_up(struct ntb_softc *ntb);
static inline bool atom_link_is_err(struct ntb_softc *ntb);
@@ -475,6 +483,7 @@ SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTU
#define NTB_B2BDOORBELL_BIT14 (1 << 3)
/* Software/configuration owns the top 16 bits. */
#define NTB_SPLIT_BAR (1ull << 16)
+#define NTB_ONE_MSIX (1ull << 17)
#define NTB_FEATURES_STR \
"\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \
@@ -486,18 +495,21 @@ static struct ntb_hw_info pci_ids[] = {
NTB_ATOM, 0 },
{ 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",
- NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
+ NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
{ 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",
- NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
- { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
- NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
+ NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
+ { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B",
+ NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K },
- { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON,
- NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
+ { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B",
+ NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
NTB_SB01BASE_LOCKUP },
- { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON,
- NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
+ { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B",
+ NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
NTB_SB01BASE_LOCKUP },
+
+ { 0x201C8086, "SKL Xeon E5 V5 Non-Transparent Bridge B2B",
+ NTB_XEON_GEN3, 0 },
};
static const struct ntb_reg atom_reg = {
@@ -582,7 +594,38 @@ static struct ntb_b2b_addr xeon_b2b_dsd_addr = {
.bar5_addr32 = XEON_B2B_BAR5_ADDR32,
};
-SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0,
+static const struct ntb_reg xeon_gen3_reg = {
+ .ntb_ctl = XEON_GEN3_REG_IMNTB_CTRL,
+ .lnk_sta = XEON_GEN3_INT_LNK_STS_OFFSET,
+ .db_size = sizeof(uint32_t),
+ .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
+};
+
+static const struct ntb_alt_reg xeon_gen3_pri_reg = {
+ .db_bell = XEON_GEN3_REG_EMDOORBELL,
+ .db_mask = XEON_GEN3_REG_IMINT_DISABLE,
+ .spad = XEON_GEN3_REG_IMSPAD,
+};
+
+static const struct ntb_alt_reg xeon_gen3_b2b_reg = {
+ .db_bell = XEON_GEN3_REG_IMDOORBELL,
+ .db_mask = XEON_GEN3_REG_EMINT_DISABLE,
+ .spad = XEON_GEN3_REG_IMB2B_SSPAD,
+};
+
+static const struct ntb_xlat_reg xeon_gen3_sec_xlat = {
+ .bar0_base = XEON_GEN3_EXT_REG_BAR0BASE,
+ .bar2_base = XEON_GEN3_EXT_REG_BAR1BASE,
+ .bar4_base = XEON_GEN3_EXT_REG_BAR2BASE,
+
+ .bar2_limit = XEON_GEN3_REG_IMBAR1XLIMIT,
+ .bar4_limit = XEON_GEN3_REG_IMBAR2XLIMIT,
+
+ .bar2_xlat = XEON_GEN3_REG_IMBAR1XBASE,
+ .bar4_xlat = XEON_GEN3_REG_IMBAR2XBASE,
+};
+
+SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"B2B MW segment overrides -- MUST be the same on both sides");
SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN,
@@ -658,6 +701,8 @@ intel_ntb_attach(device_t device)
if (ntb->type == NTB_ATOM)
error = intel_ntb_detect_atom(ntb);
+ else if (ntb->type == NTB_XEON_GEN3)
+ error = intel_ntb_detect_xeon_gen3(ntb);
else
error = intel_ntb_detect_xeon(ntb);
if (error != 0)
@@ -672,6 +717,8 @@ intel_ntb_attach(device_t device)
goto out;
if (ntb->type == NTB_ATOM)
error = intel_ntb_atom_init_dev(ntb);
+ else if (ntb->type == NTB_XEON_GEN3)
+ error = intel_ntb_xeon_gen3_init_dev(ntb);
else
error = intel_ntb_xeon_init_dev(ntb);
if (error != 0)
@@ -711,7 +758,7 @@ intel_ntb_detach(device_t device)
callout_drain(&ntb->lr_timer);
callout_drain(&ntb->peer_msix_work);
pci_disable_busmaster(ntb->device);
- if (ntb->type == NTB_XEON)
+ if (ntb->type == NTB_XEON_GEN1)
intel_ntb_teardown_xeon(ntb);
intel_ntb_teardown_interrupts(ntb);
@@ -821,22 +868,39 @@ intel_ntb_map_pci_bars(struct ntb_softc *ntb)
rc = map_memory_window_bar(ntb, bar);
if (rc != 0)
goto out;
- bar->psz_off = XEON_PBAR23SZ_OFFSET;
- bar->ssz_off = XEON_SBAR23SZ_OFFSET;
- bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
+ if (ntb->type == NTB_XEON_GEN3) {
+ bar->psz_off = XEON_GEN3_INT_REG_IMBAR1SZ;
+ bar->ssz_off = XEON_GEN3_INT_REG_EMBAR1SZ;
+ bar->pbarxlat_off = XEON_GEN3_REG_EMBAR1XBASE;
+ } else {
+ bar->psz_off = XEON_PBAR23SZ_OFFSET;
+ bar->ssz_off = XEON_SBAR23SZ_OFFSET;
+ bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
+ }
bar = &ntb->bar_info[NTB_B2B_BAR_2];
bar->pci_resource_id = PCIR_BAR(4);
rc = map_memory_window_bar(ntb, bar);
if (rc != 0)
goto out;
- bar->psz_off = XEON_PBAR4SZ_OFFSET;
- bar->ssz_off = XEON_SBAR4SZ_OFFSET;
- bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
+ if (ntb->type == NTB_XEON_GEN3) {
+ bar->psz_off = XEON_GEN3_INT_REG_IMBAR2SZ;
+ bar->ssz_off = XEON_GEN3_INT_REG_EMBAR2SZ;
+ bar->pbarxlat_off = XEON_GEN3_REG_EMBAR2XBASE;
+ } else {
+ bar->psz_off = XEON_PBAR4SZ_OFFSET;
+ bar->ssz_off = XEON_SBAR4SZ_OFFSET;
+ bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
+ }
if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR))
goto out;
+ if (ntb->type == NTB_XEON_GEN3) {
+ device_printf(ntb->device, "no split bar support\n");
+ return (ENXIO);
+ }
+
bar = &ntb->bar_info[NTB_B2B_BAR_3];
bar->pci_resource_id = PCIR_BAR(5);
rc = map_memory_window_bar(ntb, bar);
@@ -1056,6 +1120,63 @@ intel_ntb_remap_msix(device_t dev, uint32_t desired, u
}
static int
+intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb)
+{
+ uint64_t i, reg;
+ uint32_t desired_vectors, num_vectors;
+ int rc;
+
+ ntb->allocated_interrupts = 0;
+ ntb->last_ts = ticks;
+
+ /* Mask all the interrupts, including hardware interrupt */
+ intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, ~0ULL);
+
+ /* Clear Interrupt Status */
+ reg = intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS);
+ intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS, reg);
+
+ num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
+ XEON_GEN3_DB_MSIX_VECTOR_COUNT);
+
+ rc = pci_alloc_msix(ntb->device, &num_vectors);
+ if (rc != 0) {
+ device_printf(ntb->device,
+ "Interrupt allocation failed %d\n", rc);
+ return (rc);
+ }
+ if (desired_vectors != num_vectors) {
+ device_printf(ntb->device, "Couldn't get %d vectors\n",
+ XEON_GEN3_DB_MSIX_VECTOR_COUNT);
+ return (ENXIO);
+ }
+ /* 32 db + 1 hardware */
+ if (num_vectors == XEON_GEN3_DB_MSIX_VECTOR_COUNT) {
+ /* Program INTVECXX source register */
+ for (i = 0; i < XEON_GEN3_DB_MSIX_VECTOR_COUNT; i++) {
+ /* interrupt source i for vector i */
+ intel_ntb_reg_write(1, XEON_GEN3_REG_IMINTVEC00 + i, i);
+ if (i == (XEON_GEN3_DB_MSIX_VECTOR_COUNT - 1)) {
+ intel_ntb_reg_write(1,
+ XEON_GEN3_REG_IMINTVEC00 + i,
+ XEON_GEN3_LINK_VECTOR_INDEX);
+ }
+ }
+
+ intel_ntb_create_msix_vec(ntb, num_vectors);
+ rc = intel_ntb_setup_msix(ntb, num_vectors);
+
+ /* enable all interrupts */
+ intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, 0ULL);
+ } else {
+ device_printf(ntb->device, "need to remap interrupts, giving
up.\n");
+ return (ENXIO);
+ }
+
+ return (0);
+}
+
+static int
intel_ntb_init_isr(struct ntb_softc *ntb)
{
uint32_t desired_vectors, num_vectors;
@@ -1094,7 +1215,7 @@ intel_ntb_init_isr(struct ntb_softc *ntb)
} else
num_vectors = 1;
- if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {
+ if (ntb->type == NTB_XEON_GEN1 && num_vectors < ntb->db_vec_count) {
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
device_printf(ntb->device,
"Errata workaround does not support MSI or INTX\n");
@@ -1173,20 +1294,17 @@ intel_ntb_teardown_interrupts(struct ntb_softc *ntb)
pci_release_msi(ntb->device);
}
-/*
- * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it
- * out to make code clearer.
- */
static inline uint64_t
db_ioread(struct ntb_softc *ntb, uint64_t regoff)
{
- if (ntb->type == NTB_ATOM)
+ switch (ntb->type) {
+ case NTB_ATOM:
+ case NTB_XEON_GEN3:
return (intel_ntb_reg_read(8, regoff));
-
- KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
-
- return (intel_ntb_reg_read(2, regoff));
+ case NTB_XEON_GEN1:
+ return (intel_ntb_reg_read(2, regoff));
+ }
}
static inline void
@@ -1207,13 +1325,15 @@ static inline void
db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
{
- if (ntb->type == NTB_ATOM) {
+ switch (ntb->type) {
+ case NTB_ATOM:
+ case NTB_XEON_GEN3:
intel_ntb_reg_write(8, regoff, val);
- return;
+ break;
+ case NTB_XEON_GEN1:
+ intel_ntb_reg_write(2, regoff, (uint16_t)val);
+ break;
}
-
- KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
- intel_ntb_reg_write(2, regoff, (uint16_t)val);
}
static void
@@ -1263,8 +1383,10 @@ intel_ntb_db_read(device_t dev)
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))
return (ntb->fake_db);
-
- return (db_ioread(ntb, ntb->self_reg->db_bell));
+ if (ntb->type == NTB_XEON_GEN3)
+ return (intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS));
+ else
+ return (db_ioread(ntb, ntb->self_reg->db_bell));
}
static void
@@ -1284,7 +1406,11 @@ intel_ntb_db_clear(device_t dev, uint64_t bits)
return;
}
- db_iowrite(ntb, ntb->self_reg->db_bell, bits);
+ if (ntb->type == NTB_XEON_GEN3)
+ intel_ntb_reg_write(4, XEON_GEN3_REG_IMINT_STATUS,
+ (uint32_t)bits);
+ else
+ db_iowrite(ntb, ntb->self_reg->db_bell, bits);
}
static inline uint64_t
@@ -1318,9 +1444,14 @@ intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t ve
ntb->last_ts = ticks;
vec_mask = intel_ntb_vec_mask(ntb, vec);
+ if (ntb->type == NTB_XEON_GEN3 && vec == XEON_GEN3_LINK_VECTOR_INDEX)
+ vec_mask |= ntb->db_link_mask;
if ((vec_mask & ntb->db_link_mask) != 0) {
if (intel_ntb_poll_link(ntb))
ntb_link_event(ntb->device);
+ if (ntb->type == NTB_XEON_GEN3)
+ intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS,
+ intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS));
}
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) &&
@@ -1445,15 +1576,23 @@ static void
intel_ntb_detect_max_mw(struct ntb_softc *ntb)
{
- if (ntb->type == NTB_ATOM) {
+ switch (ntb->type) {
+ case NTB_ATOM:
ntb->mw_count = ATOM_MW_COUNT;
- return;
+ break;
+ case NTB_XEON_GEN1:
+ if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
+ ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
+ else
+ ntb->mw_count = XEON_SNB_MW_COUNT;
+ break;
+ case NTB_XEON_GEN3:
+ if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
+ ntb->mw_count = XEON_GEN3_SPLIT_MW_COUNT;
+ else
+ ntb->mw_count = XEON_GEN3_MW_COUNT;
+ break;
}
-
- if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
- ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
- else
- ntb->mw_count = XEON_SNB_MW_COUNT;
}
static int
@@ -1530,6 +1669,54 @@ intel_ntb_detect_atom(struct ntb_softc *ntb)
}
static int
+intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb)
+{
+ uint8_t ppd, conn_type;
+
+ ppd = pci_read_config(ntb->device, XEON_GEN3_INT_REG_PPD, 1);
+ ntb->ppd = ppd;
+
+ /* check port definition */
+ conn_type = XEON_GEN3_REG_PPD_PORT_DEF_F(ppd);
+ switch (conn_type) {
+ case NTB_CONN_B2B:
+ ntb->conn_type = conn_type;
+ break;
+ default:
+ device_printf(ntb->device, "Unsupported connection type: %u\n",
+ conn_type);
+ return (ENXIO);
+ }
+
+ /* check cross link configuration status */
+ if (XEON_GEN3_REG_PPD_CONF_STS_F(ppd)) {
+ /* NTB Port is configured as DSD/USP */
+ ntb->dev_type = NTB_DEV_DSD;
+ } else {
+ /* NTB Port is configured as USD/DSP */
+ ntb->dev_type = NTB_DEV_USD;
+ }
+
+ if (XEON_GEN3_REG_PPD_ONE_MSIX_F(ppd)) {
+ /*
+ * This bit when set, causes only a single MSI-X message to be
+ * generated if MSI-X is enabled.
+ */
+ ntb->features |= NTB_ONE_MSIX;
+ }
+
+ if (XEON_GEN3_REG_PPD_BAR45_SPL_F(ppd)) {
+ /* BARs 4 and 5 are presented as two 32b non-prefetchable BARs
*/
+ ntb->features |= NTB_SPLIT_BAR;
+ }
+
+ device_printf(ntb->device, "conn type 0x%02x, dev type 0x%02x,"
+ "features 0x%02x\n", ntb->conn_type, ntb->dev_type, ntb->features);
+
+ return (0);
+}
+
+static int
intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
{
int rc;
@@ -1614,6 +1801,42 @@ intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
}
static int
+intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb)
+{
+ int rc;
+
+ ntb->spad_count = XEON_GEN3_SPAD_COUNT;
+ ntb->db_count = XEON_GEN3_DB_COUNT;
+ ntb->db_link_mask = XEON_GEN3_DB_LINK_BIT;
+ ntb->db_vec_count = XEON_GEN3_DB_MSIX_VECTOR_COUNT;
+ ntb->db_vec_shift = XEON_GEN3_DB_MSIX_VECTOR_SHIFT;
+
+ if (ntb->conn_type != NTB_CONN_B2B) {
+ device_printf(ntb->device, "Connection type %d not supported\n",
+ ntb->conn_type);
+ return (ENXIO);
+ }
+
+ ntb->reg = &xeon_gen3_reg;
+ ntb->self_reg = &xeon_gen3_pri_reg;
+ ntb->peer_reg = &xeon_gen3_b2b_reg;
+ ntb->xlat_reg = &xeon_gen3_sec_xlat;
+
+ ntb->db_valid_mask = (1ULL << ntb->db_count) - 1;
+
+ xeon_gen3_setup_b2b_mw(ntb);
+
+ /* Enable Bus Master and Memory Space on the External Side */
+ intel_ntb_reg_write(2, XEON_GEN3_EXT_REG_PCI_CMD,
+ PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
+
+ /* Setup Interrupt */
+ rc = intel_ntb_xeon_gen3_init_isr(ntb);
+
+ return (rc);
+}
+
+static int
intel_ntb_atom_init_dev(struct ntb_softc *ntb)
{
int error;
@@ -1905,6 +2128,50 @@ xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct
return (0);
}
+static int
+xeon_gen3_setup_b2b_mw(struct ntb_softc *ntb)
+{
+ uint64_t reg;
+ uint32_t embarsz, imbarsz;
+
+ /* IMBAR1SZ should be equal to EMBAR1SZ */
+ embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR1SZ, 1);
+ imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR1SZ, 1);
+ if (embarsz != imbarsz) {
+ device_printf(ntb->device,
+ "IMBAR1SZ (%u) should be equal to EMBAR1SZ (%u)\n",
+ imbarsz, embarsz);
+ return (EIO);
+ }
+
+ /* IMBAR2SZ should be equal to EMBAR2SZ */
+ embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR2SZ, 1);
+ imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR2SZ, 1);
+ if (embarsz != imbarsz) {
+ device_printf(ntb->device,
+ "IMBAR2SZ (%u) should be equal to EMBAR2SZ (%u)\n",
+ imbarsz, embarsz);
+ return (EIO);
+ }
+
+ /* Client will provide the incoming IMBAR1/2XBASE, zero it for now */
+ intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XBASE, 0);
+ intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XBASE, 0);
+
+ /*
+ * If the value in EMBAR1LIMIT is set equal to the value in EMBAR1,
+ * the memory window for EMBAR1 is disabled.
+ * Note: It is needed to avoid malacious access.
+ */
+ reg = pci_read_config(ntb->device, XEON_GEN3_EXT_REG_BAR1BASE, 8);
+ intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XLIMIT, reg);
+
+ reg = pci_read_config(ntb->device, XEON_GEN3_EXT_REG_BAR2BASE, 8);
+ intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XLIMIT, reg);
+
+ return (0);
+}
+
static inline bool
_xeon_link_is_up(struct ntb_softc *ntb)
{
@@ -1918,7 +2185,7 @@ static inline bool
link_is_up(struct ntb_softc *ntb)
{
- if (ntb->type == NTB_XEON)
+ if (ntb->type == NTB_XEON_GEN1 || ntb->type == NTB_XEON_GEN3)
return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good ||
!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)));
@@ -2182,7 +2449,9 @@ intel_ntb_poll_link(struct ntb_softc *ntb)
ntb->ntb_ctl = ntb_cntl;
ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta);
} else {
- db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask);
+ if (ntb->type == NTB_XEON_GEN1)
+ db_iowrite_raw(ntb, ntb->self_reg->db_bell,
+ ntb->db_link_mask);
reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
if (reg_val == ntb->lnk_sta)
@@ -2953,6 +3222,8 @@ intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus
if (limit_reg != 0 && size != mw_size)
limit = base + size;
+ else
+ limit = base + mw_size;
/* Set and verify translation address */
intel_ntb_reg_write(8, xlat_reg, addr);
@@ -2970,8 +3241,22 @@ intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus
intel_ntb_reg_write(8, xlat_reg, 0);
return (EIO);
}
+
+ if (ntb->type == NTB_XEON_GEN3) {
+ limit = base + size;
+
+ /* set EMBAR1/2XLIMIT */
+ if (!idx)
+ intel_ntb_reg_write(8,
+ XEON_GEN3_REG_EMBAR1XLIMIT, limit);
+ else
+ intel_ntb_reg_write(8,
+ XEON_GEN3_REG_EMBAR2XLIMIT, limit);
+ }
} else {
/* Configure 32-bit (split) BAR MW */
+ if (ntb->type == NTB_XEON_GEN3)
+ return (EIO);
if ((addr & UINT32_MAX) != addr)
return (ERANGE);
@@ -3055,10 +3340,17 @@ intel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, un
}
static void
-intel_ntb_peer_db_set(device_t dev, uint64_t bit)
+intel_ntb_peer_db_set(device_t dev, uint64_t bits)
{
struct ntb_softc *ntb = device_get_softc(dev);
+ uint64_t db;
+ if ((bits & ~ntb->db_valid_mask) != 0) {
+ device_printf(ntb->device, "Invalid doorbell bits %#jx\n",
+ (uintmax_t)bits);
+ return;
+ }
+
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
struct ntb_pci_bar_info *lapic;
unsigned i;
@@ -3066,7 +3358,7 @@ intel_ntb_peer_db_set(device_t dev, uint64_t bit)
lapic = ntb->peer_lapic_bar;
for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
- if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0)
+ if ((bits & intel_ntb_db_vector_mask(dev, i)) != 0)
bus_space_write_4(lapic->pci_bus_tag,
lapic->pci_bus_handle,
ntb->peer_msix_data[i].nmd_ofs,
@@ -3076,11 +3368,22 @@ intel_ntb_peer_db_set(device_t dev, uint64_t bit)
}
if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {
- intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit);
+ intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bits);
return;
}
- db_iowrite(ntb, ntb->peer_reg->db_bell, bit);
+ if (ntb->type == NTB_XEON_GEN3) {
+ while (bits != 0) {
+ db = ffsll(bits);
+
+ intel_ntb_reg_write(1,
+ ntb->peer_reg->db_bell + (db - 1) * 4, 0x1);
+
+ bits = bits & (bits - 1);
+ }
+ } else {
+ db_iowrite(ntb, ntb->peer_reg->db_bell, bits);
+ }
}
static int
Modified: stable/12/sys/dev/ntb/ntb_hw/ntb_hw_intel.h
==============================================================================
--- stable/12/sys/dev/ntb/ntb_hw/ntb_hw_intel.h Sun Nov 22 18:42:08 2020
(r367938)
+++ stable/12/sys/dev/ntb/ntb_hw/ntb_hw_intel.h Sun Nov 22 18:54:14 2020
(r367939)
@@ -30,7 +30,26 @@
#ifndef _NTB_REGS_H_
#define _NTB_REGS_H_
+#include <sys/types.h>
+#include <sys/stdint.h>
+/*---------------------------------------------------------------------------
+ * Macro: M*_M : Create a mask to isolate a bit field of a data word.
+ * M*_F : Extract value from a bit field of a data word.
+ * M*_I : Insert value into a bit field of a data word.
+ *
+ * Purpose: Bit field manipulation macros for mask, insert and extract for
+ * 8-bit, 16-bit, 32-bit and 64-bit data words.
+ *
+ * Params: [in] P = Bit position of start of the bit field (lsb is 0).
+ * [in] N = Size of the bit field in bits.
+ * [in] X = Value to insert or remove from the bit field.
+ *---------------------------------------------------------------------------
+ */
+#define M8_M(P, N) ((UINT8_MAX >> (8 - (N))) << (P))
+#define M8_F(X, P, N) (((uint8_t)(X) & M8_M(P, N)) >> (P))
+#define M8_I(X, P, N) (((uint8_t)(X) << (P)) & M8_M(P, N))
+
#define NTB_LINK_STATUS_ACTIVE 0x2000
#define NTB_LINK_SPEED_MASK 0x000f
#define NTB_LINK_WIDTH_MASK 0x03f0
@@ -164,5 +183,73 @@
/* The peer ntb secondary config space is 32KB fixed size */
#define XEON_B2B_MIN_SIZE 0x8000
+#define XEON_GEN3_MW_COUNT 2
+#define XEON_GEN3_SPLIT_MW_COUNT 3
+#define XEON_GEN3_SPAD_COUNT 16
+#define XEON_GEN3_DB_COUNT 32
+#define XEON_GEN3_DB_LINK 32
+#define XEON_GEN3_DB_LINK_BIT (1ULL << XEON_GEN3_DB_LINK)
+#define XEON_GEN3_DB_MSIX_VECTOR_COUNT 33
+#define XEON_GEN3_DB_MSIX_VECTOR_SHIFT 1
+
+#define XEON_GEN3_LINK_VECTOR_INDEX 31
+
+/* Xeon Skylake NTB register definitions */
+
+/*
+ * Internal EndPoint Configuration Registers
+ */
+#define XEON_GEN3_INT_REG_BAR0BASE 0x10
+#define XEON_GEN3_INT_REG_BAR1BASE 0x18
+#define XEON_GEN3_INT_REG_BAR2BASE 0x20
+#define XEON_GEN3_INT_REG_IMBAR1SZ 0xd0
+#define XEON_GEN3_INT_REG_IMBAR2SZ 0xd1
+#define XEON_GEN3_INT_REG_EMBAR1SZ 0xd2
+#define XEON_GEN3_INT_REG_EMBAR2SZ 0xd3
+#define XEON_GEN3_INT_REG_PPD 0xd4
+#define XEON_GEN3_INT_LNK_STS_OFFSET 0x01a2
+
+/*
+ * External EndPoint Configuration Registers
+ * These are located within BAR0 of the internal endpoint.
+ */
+#define XEON_GEN3_EXT_REG_PCI_CMD 0x4504
+#define XEON_GEN3_EXT_REG_BAR0BASE 0x4510
+#define XEON_GEN3_EXT_REG_BAR1BASE 0x4518
+#define XEON_GEN3_EXT_REG_BAR2BASE 0x4520
+
+/*
+ * Internal Endpoint Memory Mapped Registers
+ */
+#define XEON_GEN3_REG_IMNTB_CTRL 0x0000
+#define XEON_GEN3_REG_IMBAR1XBASE 0x0010
+#define XEON_GEN3_REG_IMBAR1XLIMIT 0x0018
+#define XEON_GEN3_REG_IMBAR2XBASE 0x0020
+#define XEON_GEN3_REG_IMBAR2XLIMIT 0x0028
+#define XEON_GEN3_REG_IMINT_STATUS 0x0040
+#define XEON_GEN3_REG_IMINT_DISABLE 0x0048
+#define XEON_GEN3_REG_IMSPAD 0x0080
+#define XEON_GEN3_REG_IMINTVEC00 0x00d0
+#define XEON_GEN3_REG_IMDOORBELL 0x0100
+#define XEON_GEN3_REG_IMB2B_SSPAD 0x0180 /* Pseudo SP registers */
+
+/*
+ * External Endpoint Memory Mapped Registers
+ */
+#define XEON_GEN3_REG_EMBAR0XBASE 0x4008
+#define XEON_GEN3_REG_EMBAR1XBASE 0x4010
+#define XEON_GEN3_REG_EMBAR1XLIMIT 0x4018
+#define XEON_GEN3_REG_EMBAR2XBASE 0x4020
+#define XEON_GEN3_REG_EMBAR2XLIMIT 0x4028
+#define XEON_GEN3_REG_EMINT_STATUS 0x4040
+#define XEON_GEN3_REG_EMINT_DISABLE 0x4048
+#define XEON_GEN3_REG_EMSPAD 0x4080
+#define XEON_GEN3_REG_EMDOORBELL 0x4100
+
+/* XEON_GEN3_INT_REG_PPD: PPD register */
+#define XEON_GEN3_REG_PPD_PORT_DEF_F(X) M8_F(X, 0, 2)
+#define XEON_GEN3_REG_PPD_CONF_STS_F(X) M8_F(X, 4, 1)
+#define XEON_GEN3_REG_PPD_ONE_MSIX_F(X) M8_F(X, 5, 1)
+#define XEON_GEN3_REG_PPD_BAR45_SPL_F(X) M8_F(X, 6, 1)
#endif /* _NTB_REGS_H_ */
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
