From: Leon Romanovsky <[email protected]>
Some ports, such as Zhaoxin Switch Downstream Ports, redirect peer-to-peer
traffic without an ACS capability, as their device-specific ACS quirks
report. P2PDMA reads only ACS Control, so below such a port it hands out
bus addresses for traffic that goes to the host bridge.
Take Request and Completion Redirect from a bridge's quirk too. Ignore
endpoint quirks, which only describe traffic between an endpoint's own
functions. Skip bridges on a root bus: they never sit between two peers,
and the AMD southbridge quirk may sleep there during DMA mapping.
pci=disable_acs_redir= cannot override a quirk, so name such ports
separately.
Fixes: 52916982af48 ("PCI/P2PDMA: Support peer-to-peer memory")
Signed-off-by: Leon Romanovsky <[email protected]>
---
drivers/pci/p2pdma.c | 114 +++++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 96 insertions(+), 18 deletions(-)
diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index 45260423dfb0..ba94b0f33928 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -21,6 +21,8 @@
#include <linux/seq_buf.h>
#include <linux/xarray.h>
+#include "pci.h"
+
struct pci_p2pdma {
struct gen_pool *pool;
bool p2pmem_published;
@@ -490,6 +492,44 @@ static struct pci_dev *find_parent_pci_dev(struct device
*dev)
return NULL;
}
+/*
+ * Some ports send peer-to-peer traffic upstream without a standard ACS
+ * capability, such as Zhaoxin Root and Switch Downstream Ports. Their
+ * device-specific ACS quirks say so. Report whether @pdev's quirk says that
+ * @flag is in effect.
+ *
+ * Only ask bridges. The quirks for multi-function endpoints only say that
+ * their functions do not talk to each other directly, which says nothing
+ * about their traffic to other devices. Skip bridges on a root bus too. They
+ * never decide whether two peers connect directly, and for them the AMD
+ * southbridge quirk calls acpi_get_table(), which may sleep. DMA mapping may
+ * call this code where sleeping is not allowed.
+ */
+static bool pci_acs_p2pdma_quirk(struct pci_dev *pdev, u16 flag)
+{
+ if (!pci_is_bridge(pdev) || pci_is_root_bus(pdev->bus))
+ return false;
+
+ return pci_dev_specific_acs_enabled(pdev, flag) > 0;
+}
+
+/*
+ * Add the Request and Completion Redirect that @pdev's quirk reports to its
+ * ACS Control value. Only ever add them: a quirk that covers every device of
+ * a vendor says "no" for the ones it does not know, and that must not hide
+ * their own ACS controls. What a quirk says about other controls only means
+ * that the control does not apply, so ignore it.
+ */
+static u16 pci_acs_p2pdma_quirk_redirect(struct pci_dev *pdev, u16 ctrl)
+{
+ if (pci_acs_p2pdma_quirk(pdev, PCI_ACS_RR))
+ ctrl |= PCI_ACS_RR;
+ if (pci_acs_p2pdma_quirk(pdev, PCI_ACS_CR))
+ ctrl |= PCI_ACS_CR;
+
+ return ctrl;
+}
+
/*
* Check if a PCI bridge has its ACS redirection bits set to redirect P2P
* TLPs upstream via ACS. Returns 1 if the packets will be redirected
@@ -498,14 +538,13 @@ static struct pci_dev *find_parent_pci_dev(struct device
*dev)
static int pci_bridge_has_acs_redir(struct pci_dev *pdev)
{
int pos;
- u16 ctrl;
+ u16 ctrl = 0;
pos = pdev->acs_cap;
- if (!pos)
- return 0;
-
- pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
+ if (pos)
+ pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
+ ctrl = pci_acs_p2pdma_quirk_redirect(pdev, ctrl);
if (ctrl & (PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC))
return 1;
@@ -520,6 +559,51 @@ static void seq_buf_print_bus_devfn(struct seq_buf *buf,
struct pci_dev *pdev)
seq_buf_printf(buf, "%s;", pci_name(pdev));
}
+/*
+ * Sort a port that redirects into @acs_list, the ports to suggest for
+ * pci=disable_acs_redir=, or @quirk_list. That parameter only clears the ACS
+ * Control register, so it cannot undo a redirect that a device quirk reports.
+ */
+static void pci_p2pdma_list_redir(struct seq_buf *acs_list,
+ struct seq_buf *quirk_list,
+ struct pci_dev *pdev)
+{
+ if (pci_acs_p2pdma_quirk(pdev, PCI_ACS_RR) ||
+ pci_acs_p2pdma_quirk(pdev, PCI_ACS_CR))
+ seq_buf_print_bus_devfn(quirk_list, pdev);
+ else
+ seq_buf_print_bus_devfn(acs_list, pdev);
+}
+
+/* Drop the final semicolon. Returns false if no port was listed. */
+static bool pci_p2pdma_end_list(struct seq_buf *list)
+{
+ if (!seq_buf_used(list))
+ return false;
+
+ if (!seq_buf_has_overflowed(list))
+ list->buffer[list->len - 1] = '\0';
+ return true;
+}
+
+static void pci_p2pdma_warn_redir(struct pci_dev *client,
+ struct pci_dev *provider,
+ struct seq_buf *acs_list,
+ struct seq_buf *quirk_list)
+{
+ pci_warn(client,
+ "ACS redirect is set between the client and provider (%s)\n",
+ pci_name(provider));
+ if (pci_p2pdma_end_list(quirk_list))
+ pci_warn(client,
+ "hardware redirects peer-to-peer traffic at %s, which
no kernel parameter changes\n",
+ seq_buf_str(quirk_list));
+ if (pci_p2pdma_end_list(acs_list))
+ pci_warn(client,
+ "to disable ACS redirect for this path, add the kernel
parameter: pci=disable_acs_redir=%s\n",
+ seq_buf_str(acs_list));
+}
+
static bool cpu_supports_p2pdma(void)
{
#ifdef CONFIG_X86
@@ -722,13 +806,14 @@ calc_map_type_and_dist(struct pci_dev *provider, struct
pci_dev *client,
enum pci_p2pdma_map_type map_type = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE;
struct pci_dev *a = provider, *b = client, *bb;
struct pci_p2pdma *p2pdma;
- struct seq_buf acs_list;
+ struct seq_buf acs_list, quirk_list;
int acs_cnt = 0;
int dist_a = 0;
int dist_b = 0;
- char buf[128];
+ char buf[128], quirk_buf[64];
seq_buf_init(&acs_list, buf, sizeof(buf));
+ seq_buf_init(&quirk_list, quirk_buf, sizeof(quirk_buf));
/*
* Note, we don't need to take references to devices returned by
@@ -739,7 +824,7 @@ calc_map_type_and_dist(struct pci_dev *provider, struct
pci_dev *client,
dist_b = 0;
if (pci_bridge_has_acs_redir(a)) {
- seq_buf_print_bus_devfn(&acs_list, a);
+ pci_p2pdma_list_redir(&acs_list, &quirk_list, a);
acs_cnt++;
}
@@ -768,7 +853,7 @@ calc_map_type_and_dist(struct pci_dev *provider, struct
pci_dev *client,
break;
if (pci_bridge_has_acs_redir(bb)) {
- seq_buf_print_bus_devfn(&acs_list, bb);
+ pci_p2pdma_list_redir(&acs_list, &quirk_list, bb);
acs_cnt++;
}
@@ -782,15 +867,8 @@ calc_map_type_and_dist(struct pci_dev *provider, struct
pci_dev *client,
goto done;
}
- if (verbose) {
- /* Drop the final semicolon; the list is not empty here */
- if (!seq_buf_has_overflowed(&acs_list))
- acs_list.buffer[acs_list.len - 1] = '\0';
- pci_warn(client, "ACS redirect is set between the client and
provider (%s)\n",
- pci_name(provider));
- pci_warn(client, "to disable ACS redirect for this path, add
the kernel parameter: pci=disable_acs_redir=%s\n",
- seq_buf_str(&acs_list));
- }
+ if (verbose)
+ pci_p2pdma_warn_redir(client, provider, &acs_list, &quirk_list);
map_through_host_bridge:
if (!cpu_supports_p2pdma() &&
--
2.55.0
