I've tried everything you mentioned and I still get the same problem. The only
thing that seems to avoid that issue is if I reduce the aperture size from
0x2000000000ull to 0x2000000ull.
Here is the relevant section of code:
static const unsigned long long BAR_Regions[6][2] =
{
// len , type
{ 0x2000000ull, PCI_BASE_ADDRESS_SPACE_MEMORY |
PCI_BASE_ADDRESS_MEM_TYPE_64} , //BAR0,
{ 0, 0} , // BAR1
{ 0x2000000ull, PCI_BASE_ADDRESS_SPACE_IO } , //BAR2,
{ 0, 0} , // BAR3 for MSI-X
{ 0, 0} , // BAR4
{ 0, 0} , // BAR5
};
static int pcie_msix_initfn(PCIDevice *pci_dev)
{
PCIE_MSIX_DEVState *d = DO_UPCAST(PCIE_MSIX_DEVState, dev, pci_dev);
PCIBridge *br = DO_UPCAST(PCIBridge, dev, pci_dev);
PCIEPort *p = DO_UPCAST(PCIEPort, br, br);
int rc, i;
PRINT_DEBUG("%s: PCIE MSIX Device init...\n", __FUNCTION__);
pci_config_set_vendor_id(d->dev.config, PCIE_MSIX_VID);
pci_config_set_device_id(d->dev.config, PCIE_MSIX_DID);
memcpy(d->dev.config, g_cfg_init, sizeof(g_cfg_init[0x20]));
d->mmio_index = cpu_register_io_memory(pcie_msix_mem_read_fn,
pcie_msix_mem_write_fn, d);
int msix_mem_bar = 0; // Since its a 64bit BAR, we take up BAR0 & BAR1
int msix_io_bar = 2;
int msix_mmio_bar = 3;
pci_register_bar(&d->dev, msix_mem_bar, BAR_Regions[msix_mem_bar][0],
BAR_Regions[msix_mem_bar][1], pcie_msix_mem_map);
pci_register_bar(&d->dev, msix_io_bar, BAR_Regions[msix_io_bar][0],
BAR_Regions[msix_io_bar][1], pcie_msix_io_map);
rc = msix_init(&d->dev, d->vectors, msix_mmio_bar, 0);
if (!rc) {
PRINT_DEBUG("%s: Registering Bar %i as I/O BAR\n", __FUNCTION__,
msix_mmio_bar);
pci_register_bar(&d->dev, msix_mmio_bar, msix_bar_size(&d->dev),
PCI_BASE_ADDRESS_SPACE_MEMORY, msix_mmio_map);
PRINT_DEBUG("%s: MSI-X initialized (%d vectors)\n", __FUNCTION__,
d->vectors);
}
else {
PRINT_DEBUG("%s: MSI-X initialization failed!\n", __FUNCTION__);
return rc;
}
// Activate the vectors
for (i = 0; i < d->vectors; i++) {
msix_vector_use(&d->dev, i);
}
rc = pci_pcie_cap_init(&d->dev, PCIE_MSIX_EXP_OFFSET,
PCI_EXP_TYPE_ENDPOINT, p->port);
if (rc < 0) {
return rc;
}
pcie_cap_flr_init(&d->dev, &pcie_msix_flr);
pcie_cap_deverr_init(&d->dev);
pcie_cap_ari_init(&d->dev);
rc = pcie_aer_init(&d->dev, PCIE_MSIX_AER_OFFSET);
if (rc < 0) {
return rc;
}
PRINT_DEBUG("%s: Init done\n", __FUNCTION__);
return 0;
}
Another question I have is why doesn't the device show up when I try a cat
/proc/interrupts.
linux-an84:~/AriesKernelModules/gni/aries/ghal # cat /proc/interrupts
CPU0
0: 694 IO-APIC-edge timer
1: 6 IO-APIC-edge i8042
4: 753 IO-APIC-edge serial
8: 1 IO-APIC-edge rtc0
9: 0 IO-APIC-fasteoi acpi
12: 89 IO-APIC-edge i8042
14: 3522 IO-APIC-edge ata_piix
15: 785 IO-APIC-edge ata_piix
16: 162 IO-APIC-fasteoi eth0
4344: 0 PCI-MSI-edge aerdrv
4345: 0 PCI-MSI-edge aerdrv
4346: 0 PCI-MSI-edge aerdrv
4347: 0 PCI-MSI-edge aerdrv
4348: 0 PCI-MSI-edge aerdrv
4349: 0 PCI-MSI-edge aerdrv
4350: 0 PCI-MSI-edge aerdrv
4351: 0 PCI-MSI-edge aerdrv
NMI: 0 Non-maskable interrupts
LOC: 107095 Local timer interrupts
RES: 0 Rescheduling interrupts
CAL: 0 function call interrupts
TLB: 0 TLB shootdowns
TRM: 0 Thermal event interrupts
THR: 0 Threshold APIC interrupts
SPU: 0 Spurious interrupts
ERR: 0
Shouldn't there be an entry for the MSI-X device?
Thanks for all your input.
AK
_____
Probably what you want is something like
{ 0x2000000000ull, PCI_BASE_ADDRESS_SPACE_MEMORY |
PCI_BASE_ADDRESS_MEM_TYPE_64} , //BAR0
{ 0, 0} , //BAR1
// 64bit BAR occupies 2 BAR entries so that BAR1 can't be used.
{ 0x2000000ull, PCI_BASE_ADDRESS_SPACE_IO } , //BAR2
{ 0, 0} , //BAR3
// for MSI-X
{ 0, 0} , //BAR4
{ 0, 0} //BAR5
