Provide db_vector_count() and db_vector_mask() implementations for both ntb_hw_epf and pci-epf-vntb so that ntb_transport can map MSI vectors to doorbell bits. Without them, the upper layer cannot identify which doorbell vector fired and ends up scheduling rxc_db_work() for all queue pairs, resulting in a thundering-herd effect when multiple queue pairs (QPs) are enabled.
With this change, .peer_db_set() must honor the db_bits mask and raise all requested doorbell interrupts, so update those implementations accordingly. Signed-off-by: Koichiro Den <[email protected]> --- drivers/ntb/hw/epf/ntb_hw_epf.c | 47 ++++++++++++------- drivers/pci/endpoint/functions/pci-epf-vntb.c | 41 +++++++++++++--- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/drivers/ntb/hw/epf/ntb_hw_epf.c b/drivers/ntb/hw/epf/ntb_hw_epf.c index dbb5bebe63a5..c37ede4063dc 100644 --- a/drivers/ntb/hw/epf/ntb_hw_epf.c +++ b/drivers/ntb/hw/epf/ntb_hw_epf.c @@ -381,7 +381,7 @@ static int ntb_epf_init_isr(struct ntb_epf_dev *ndev, int msi_min, int msi_max) } } - ndev->db_count = irq; + ndev->db_count = irq - 1; ret = ntb_epf_send_command(ndev, CMD_CONFIGURE_DOORBELL, argument | irq); @@ -415,6 +415,22 @@ static u64 ntb_epf_db_valid_mask(struct ntb_dev *ntb) return ntb_ndev(ntb)->db_valid_mask; } +static int ntb_epf_db_vector_count(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->db_count; +} + +static u64 ntb_epf_db_vector_mask(struct ntb_dev *ntb, int db_vector) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + + db_vector--; /* vector 0 is reserved for link events */ + if (db_vector < 0 || db_vector >= ndev->db_count) + return 0; + + return ndev->db_valid_mask & BIT_ULL(db_vector); +} + static int ntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits) { return 0; @@ -507,26 +523,21 @@ static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx, static int ntb_epf_peer_db_set(struct ntb_dev *ntb, u64 db_bits) { struct ntb_epf_dev *ndev = ntb_ndev(ntb); - u32 interrupt_num = ffs(db_bits) + 1; - struct device *dev = ndev->dev; + u32 interrupt_num; u32 db_entry_size; u32 db_offset; u32 db_data; - - if (interrupt_num >= ndev->db_count) { - dev_err(dev, "DB interrupt %d greater than Max Supported %d\n", - interrupt_num, ndev->db_count); - return -EINVAL; - } + unsigned long i; db_entry_size = readl(ndev->ctrl_reg + NTB_EPF_DB_ENTRY_SIZE); - db_data = readl(ndev->ctrl_reg + NTB_EPF_DB_DATA(interrupt_num)); - db_offset = readl(ndev->ctrl_reg + NTB_EPF_DB_OFFSET(interrupt_num)); - - writel(db_data, ndev->db_reg + (db_entry_size * interrupt_num) + - db_offset); - + for_each_set_bit(i, (unsigned long *)&db_bits, ndev->db_count) { + interrupt_num = i + 1; + db_data = readl(ndev->ctrl_reg + NTB_EPF_DB_DATA(interrupt_num)); + db_offset = readl(ndev->ctrl_reg + NTB_EPF_DB_OFFSET(interrupt_num)); + writel(db_data, ndev->db_reg + (db_entry_size * interrupt_num) + + db_offset); + } return 0; } @@ -556,6 +567,8 @@ static const struct ntb_dev_ops ntb_epf_ops = { .spad_count = ntb_epf_spad_count, .peer_mw_count = ntb_epf_peer_mw_count, .db_valid_mask = ntb_epf_db_valid_mask, + .db_vector_count = ntb_epf_db_vector_count, + .db_vector_mask = ntb_epf_db_vector_mask, .db_set_mask = ntb_epf_db_set_mask, .mw_set_trans = ntb_epf_mw_set_trans, .mw_clear_trans = ntb_epf_mw_clear_trans, @@ -607,8 +620,8 @@ static int ntb_epf_init_dev(struct ntb_epf_dev *ndev) int ret; /* One Link interrupt and rest doorbell interrupt */ - ret = ntb_epf_init_isr(ndev, NTB_EPF_MIN_DB_COUNT + NTB_EPF_IRQ_RESERVE, - NTB_EPF_MAX_DB_COUNT + NTB_EPF_IRQ_RESERVE); + ret = ntb_epf_init_isr(ndev, NTB_EPF_MIN_DB_COUNT + 1 + NTB_EPF_IRQ_RESERVE, + NTB_EPF_MAX_DB_COUNT + 1 + NTB_EPF_IRQ_RESERVE); if (ret) { dev_err(dev, "Failed to init ISR\n"); return ret; diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c index 4927faa28255..39e784e21236 100644 --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -1384,6 +1384,22 @@ static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb) return BIT_ULL(ntb_ndev(ntb)->db_count) - 1; } +static int vntb_epf_db_vector_count(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->db_count; +} + +static u64 vntb_epf_db_vector_mask(struct ntb_dev *ntb, int db_vector) +{ + struct epf_ntb *ndev = ntb_ndev(ntb); + + db_vector--; /* vector 0 is reserved for link events */ + if (db_vector < 0 || db_vector >= ndev->db_count) + return 0; + + return BIT_ULL(db_vector); +} + static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits) { return 0; @@ -1487,20 +1503,29 @@ static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits) { - u32 interrupt_num = ffs(db_bits) + 1; struct epf_ntb *ntb = ntb_ndev(ndev); u8 func_no, vfunc_no; - int ret; + u64 failed = 0; + unsigned long i; func_no = ntb->epf->func_no; vfunc_no = ntb->epf->vfunc_no; - ret = pci_epc_raise_irq(ntb->epf->epc, func_no, vfunc_no, - PCI_IRQ_MSI, interrupt_num + 1); - if (ret) - dev_err(&ntb->ntb->dev, "Failed to raise IRQ\n"); + for_each_set_bit(i, (unsigned long *)&db_bits, ntb->db_count) { + /* + * DB bit i is MSI interrupt (i + 2). + * Vector 0 is used for link events and MSI vectors are + * 1-based for pci_epc_raise_irq(). + */ + if (pci_epc_raise_irq(ntb->epf->epc, func_no, vfunc_no, + PCI_IRQ_MSI, i + 2)) + failed |= BIT_ULL(i); + } + if (failed) + dev_err(&ntb->ntb->dev, "Failed to raise IRQ (%#llx)\n", + failed); - return ret; + return failed ? -EIO : 0; } static u64 vntb_epf_db_read(struct ntb_dev *ndev) @@ -1561,6 +1586,8 @@ static const struct ntb_dev_ops vntb_epf_ops = { .spad_count = vntb_epf_spad_count, .peer_mw_count = vntb_epf_peer_mw_count, .db_valid_mask = vntb_epf_db_valid_mask, + .db_vector_count = vntb_epf_db_vector_count, + .db_vector_mask = vntb_epf_db_vector_mask, .db_set_mask = vntb_epf_db_set_mask, .mw_set_trans = vntb_epf_mw_set_trans, .mw_clear_trans = vntb_epf_mw_clear_trans, -- 2.51.0
