From 68d11138bec63783d24584c1f7e029174df3ee2b Mon Sep 17 00:00:00 2001 From: Jiqian Chen Date: Thu, 21 Nov 2024 10:56:17 +0800 Subject: [PATCH 1/1] combine two method --- xen/drivers/vpci/Makefile | 2 +- xen/drivers/vpci/header.c | 39 +++++++++++++++++- xen/drivers/vpci/rebar.c | 84 ++++++++++++++++++++++++++++++++++++++ xen/include/xen/pci_regs.h | 11 +++++ xen/include/xen/vpci.h | 2 + 5 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 xen/drivers/vpci/rebar.c diff --git a/xen/drivers/vpci/Makefile b/xen/drivers/vpci/Makefile index 1a1413b93e76..a7c8a30a8956 100644 --- a/xen/drivers/vpci/Makefile +++ b/xen/drivers/vpci/Makefile @@ -1,2 +1,2 @@ -obj-y += vpci.o header.o +obj-y += vpci.o header.o rebar.o obj-$(CONFIG_HAS_PCI_MSI) += msi.o msix.o diff --git a/xen/drivers/vpci/header.c b/xen/drivers/vpci/header.c index ef6c965c081c..dda42ef0b7ff 100644 --- a/xen/drivers/vpci/header.c +++ b/xen/drivers/vpci/header.c @@ -316,6 +316,9 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t cmd, bool rom_only) ASSERT(rw_is_write_locked(&pdev->domain->pci_lock)); + printk("%pp: modify bars cmd: %x rom_only: %d\n", + &pdev->sbdf, cmd, rom_only); + /* * Create a rangeset per BAR that represents the current device memory * region and compare it against all the currently active BAR memory @@ -346,6 +349,33 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t cmd, bool rom_only) bar->enabled == !!(cmd & PCI_COMMAND_MEMORY) ) continue; + if ( bar->type != VPCI_BAR_ROM && header->bars_resizable && + (cmd & PCI_COMMAND_MEMORY) ) + { + uint64_t addr, size; + + pci_size_mem_bar(pdev->sbdf, PCI_BASE_ADDRESS_0 + i * 4, + &addr, &size, 0); + + printk("%pp: BAR%u ReBAR supported addr %#lx -> %#lx size %#lx -> %#lx\n", + &pdev->sbdf, i, bar->addr, addr, bar->size, size); + + if ( bar->addr != addr ) + printk(XENLOG_G_ERR + "%pp: BAR#%u address mismatch %#lx vs %#lx\n", + &pdev->sbdf, i, bar->addr, addr); + + if ( bar->size != size ) + { + printk(XENLOG_G_DEBUG + "%pp: detected BAR#%u size change (%#lx -> %#lx)\n", + &pdev->sbdf, i, bar->size, size); + bar->size = size; + end = PFN_DOWN(bar->addr + size - 1); + end_guest = PFN_DOWN(bar->guest_addr + size - 1); + } + } + if ( !pci_check_bar(pdev, _mfn(start), _mfn(end)) ) { printk(XENLOG_G_WARNING @@ -583,8 +613,6 @@ static void cf_check bar_write( */ if ( bar->enabled ) { - /* If the value written is the current one avoid printing a warning. */ - if ( val != (uint32_t)(bar->addr >> (hi ? 32 : 0)) ) gprintk(XENLOG_WARNING, "%pp: ignored BAR %zu write while mapped\n", &pdev->sbdf, bar - pdev->vpci->header.bars + hi); @@ -601,6 +629,10 @@ static void cf_check bar_write( /* Update guest address, so hardware domain BAR is identity mapped. */ bar->guest_addr = bar->addr; + printk("%pp: write BAR%zu val: %#x BAR%zu address: %#lx\n", + &pdev->sbdf, bar - pdev->vpci->header.bars, val, + bar - pdev->vpci->header.bars + hi, bar->addr); + /* Make sure Xen writes back the same value for the BAR RO bits. */ if ( !hi ) { @@ -870,6 +902,9 @@ static int cf_check init_header(struct pci_dev *pdev) if ( pdev->ignore_bars ) return 0; + header->bars_resizable = pci_find_ext_capability(pdev->sbdf, + PCI_EXT_CAP_ID_REBAR); + cmd = pci_conf_read16(pdev->sbdf, PCI_COMMAND); /* diff --git a/xen/drivers/vpci/rebar.c b/xen/drivers/vpci/rebar.c new file mode 100644 index 000000000000..fe0d02463dd6 --- /dev/null +++ b/xen/drivers/vpci/rebar.c @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2024 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Author: Jiqian Chen + */ + +#include +#include + +static void cf_check rebar_ctrl_write(const struct pci_dev *pdev, + unsigned int reg, + uint32_t val, + void *data) +{ + uint64_t size; + unsigned int index; + struct vpci_bar *bars = data; + + if ( pci_conf_read16(pdev->sbdf, PCI_COMMAND) & PCI_COMMAND_MEMORY ) + return; + + index = pci_conf_read32(pdev->sbdf, reg) & PCI_REBAR_CTRL_BAR_IDX; + if ( index >= PCI_HEADER_NORMAL_NR_BARS ) + return; + + if ( bars[index].type != VPCI_BAR_MEM64_LO && + bars[index].type != VPCI_BAR_MEM32 ) + return; + + size = PCI_REBAR_CTRL_SIZE(val); + if ( !((size >> 20) & + MASK_EXTR(pci_conf_read32(pdev->sbdf, reg - 4), PCI_REBAR_CAP_SIZES)) ) + gprintk(XENLOG_WARNING, + "%pp: new size %#lx for BAR%u isn't supported\n", + &pdev->sbdf, size, index); + + // bars[index].size = size; + // bars[index].addr = 0; + // bars[index].guest_addr = 0; + pci_conf_write32(pdev->sbdf, reg, val); +} + +static int cf_check init_rebar(struct pci_dev *pdev) +{ + uint32_t ctrl; + unsigned int rebar_offset, nbars; + + rebar_offset = pci_find_ext_capability(pdev->sbdf, PCI_EXT_CAP_ID_REBAR); + + if ( !rebar_offset ) + return 0; + + ctrl = pci_conf_read32(pdev->sbdf, rebar_offset + PCI_REBAR_CTRL); + nbars = MASK_EXTR(ctrl, PCI_REBAR_CTRL_NBAR_MASK); + + for ( unsigned int i = 0; i < nbars; i++, rebar_offset += PCI_REBAR_CTRL ) + { + int rc; + + rc = vpci_add_register(pdev->vpci, vpci_hw_read32, rebar_ctrl_write, + rebar_offset + PCI_REBAR_CTRL, 4, + pdev->vpci->header.bars); + if ( rc ) + { + printk("%pp: add register for PCI_REBAR_CTRL failed %d\n", + &pdev->sbdf, rc); + break; + } + } + + return 0; +} +REGISTER_VPCI_INIT(init_rebar, VPCI_PRIORITY_LOW); + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * tab-width: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/include/xen/pci_regs.h b/xen/include/xen/pci_regs.h index 250ba106dbd3..d2dece63409f 100644 --- a/xen/include/xen/pci_regs.h +++ b/xen/include/xen/pci_regs.h @@ -459,6 +459,7 @@ #define PCI_EXT_CAP_ID_ARI 14 #define PCI_EXT_CAP_ID_ATS 15 #define PCI_EXT_CAP_ID_SRIOV 16 +#define PCI_EXT_CAP_ID_REBAR 21 /* Advanced Error Reporting */ #define PCI_ERR_UNCOR_STATUS 4 /* Uncorrectable Error Status */ @@ -541,6 +542,16 @@ #define PCI_VNDR_HEADER_REV(x) (((x) >> 16) & 0xf) #define PCI_VNDR_HEADER_LEN(x) (((x) >> 20) & 0xfff) +/* Resizable BARs */ +#define PCI_REBAR_CAP 4 /* capability register */ +#define PCI_REBAR_CAP_SIZES 0xFFFFFFF0 /* supported BAR sizes */ +#define PCI_REBAR_CTRL 8 /* control register */ +#define PCI_REBAR_CTRL_BAR_IDX 0x00000007 /* BAR index */ +#define PCI_REBAR_CTRL_NBAR_MASK 0x000000E0 /* # of resizable BARs */ +#define PCI_REBAR_CTRL_BAR_SIZE 0x00001F00 /* BAR size */ +#define PCI_REBAR_CTRL_SIZE(v) \ + (1UL << (MASK_EXTR(v, PCI_REBAR_CTRL_BAR_SIZE) + 20)) + /* * Hypertransport sub capability types * diff --git a/xen/include/xen/vpci.h b/xen/include/xen/vpci.h index 41e7c3bc2791..45ebc1bb3356 100644 --- a/xen/include/xen/vpci.h +++ b/xen/include/xen/vpci.h @@ -129,6 +129,8 @@ struct vpci { * upon to know whether BARs are mapped into the guest p2m. */ bool bars_mapped : 1; + /* Device has the Resizable BARs capability. */ + bool bars_resizable : 1; /* FIXME: currently there's no support for SR-IOV. */ } header; -- 2.34.1