RE: [RFC 2/8] cxgb4: setup pcie memory window 4 and create p2pmem region

[Steve Wise]

> 
> 
> > +static void setup_memwin_p2pmem(struct adapter *adap)
> > +{
> > +   unsigned int mem_base = t4_read_reg(adap,
> CIM_EXTMEM2_BASE_ADDR_A);
> > +   unsigned int mem_size = t4_read_reg(adap,
> CIM_EXTMEM2_ADDR_SIZE_A);
> > +
> > +   if (!use_p2pmem)
> > +   return;
> 
> This is weird, why even call this if !use_p2pmem?
> 

The use_p2pmem was added after the original change.  I'll update as you
suggest.

> > +static int init_p2pmem(struct adapter *adapter)
> > +{
> > +   unsigned int mem_size = t4_read_reg(adapter,
> CIM_EXTMEM2_ADDR_SIZE_A);
> > +   struct p2pmem_dev *p;
> > +   int rc;
> > +   struct resource res;
> > +
> > +   if (!mem_size || !use_p2pmem)
> > +   return 0;
> 
> Again, weird...

Yup.

Re: [RFC 2/8] cxgb4: setup pcie memory window 4 and create p2pmem region

[Logan Gunthorpe]

On 04/04/17 04:42 AM, Sagi Grimberg wrote:
> This is weird, why even call this if !use_p2pmem?

I personally find it cleaner than:

if (use_p2pmem)
setup_memwin_p2pmem(...)

I'm not sure why that's so weird.

Logan

Re: [RFC 2/8] cxgb4: setup pcie memory window 4 and create p2pmem region

[Sagi Grimberg]

+static void setup_memwin_p2pmem(struct adapter *adap)
+{
+   unsigned int mem_base = t4_read_reg(adap, CIM_EXTMEM2_BASE_ADDR_A);
+   unsigned int mem_size = t4_read_reg(adap, CIM_EXTMEM2_ADDR_SIZE_A);
+
+   if (!use_p2pmem)
+   return;


This is weird, why even call this if !use_p2pmem?


+static int init_p2pmem(struct adapter *adapter)
+{
+   unsigned int mem_size = t4_read_reg(adapter, CIM_EXTMEM2_ADDR_SIZE_A);
+   struct p2pmem_dev *p;
+   int rc;
+   struct resource res;
+
+   if (!mem_size || !use_p2pmem)
+   return 0;


Again, weird...

[RFC 2/8] cxgb4: setup pcie memory window 4 and create p2pmem region

[Logan Gunthorpe]

From: Steve Wise 

Some cxgb4 cards expose memory as part of BAR4. This patch registers
this memory as a p2pmem device.

Signed-off-by: Steve Wise 
Signed-off-by: Logan Gunthorpe 
Signed-off-by: Stephen Bates 
---
 drivers/net/ethernet/chelsio/cxgb4/cxgb4.h  |  3 +
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 97 -
 drivers/net/ethernet/chelsio/cxgb4/t4_regs.h|  5 ++
 3 files changed, 102 insertions(+), 3 deletions(-)

diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h 
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index 163543b..e92443b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -48,6 +48,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include "t4_chip_type.h"
 #include "cxgb4_uld.h"
@@ -859,6 +860,8 @@ struct adapter {
 
/* TC u32 offload */
struct cxgb4_tc_u32_table *tc_u32;
+
+   struct p2pmem_dev *p2pmem;
 };
 
 /* Support for "sched-class" command to allow a TX Scheduling Class to be
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c 
b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index afb0967..a33bcd1 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -172,6 +172,11 @@ module_param(select_queue, int, 0644);
 MODULE_PARM_DESC(select_queue,
 "Select between kernel provided method of selecting or driver 
method of selecting TX queue. Default is kernel method.");
 
+static bool use_p2pmem;
+module_param(use_p2pmem, bool, 0644);
+MODULE_PARM_DESC(use_p2pmem,
+"Enable registering a p2pmem device with bar space (if 
available)");
+
 static struct dentry *cxgb4_debugfs_root;
 
 LIST_HEAD(adapter_list);
@@ -2835,6 +2840,54 @@ static void setup_memwin_rdma(struct adapter *adap)
}
 }
 
+static void setup_memwin_p2pmem(struct adapter *adap)
+{
+   unsigned int mem_base = t4_read_reg(adap, CIM_EXTMEM2_BASE_ADDR_A);
+   unsigned int mem_size = t4_read_reg(adap, CIM_EXTMEM2_ADDR_SIZE_A);
+
+   if (!use_p2pmem)
+   return;
+
+   if (mem_base != 0 && mem_size != 0) {
+   unsigned int sz_kb, pcieofst;
+
+   sz_kb = roundup_pow_of_two(mem_size) >> 10;
+
+   /*
+* The start offset must be aligned to the window size.
+* Also, BAR4 has MSIX vectors using the first 8KB.
+* Further, the min allowed p2pmem region size is 1MB,
+* so set the start offset to the memory size and we're aligned
+* as well as past the 8KB vector table.
+*/
+   pcieofst = sz_kb << 10;
+
+   dev_info(adap->pdev_dev,
+"p2pmem base 0x%x, size %uB, ilog2(sk_kb) 0x%x, "
+"pcieofst 0x%X\n", mem_base, mem_size, ilog2(sz_kb),
+pcieofst);
+
+   /* Write the window offset and size */
+   t4_write_reg(adap,
+   PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A,
+   MEMWIN_RSVD4),
+   pcieofst | BIR_V(2) | WINDOW_V(ilog2(sz_kb)));
+
+   /* Write the adapter memory base/start */
+   t4_write_reg(adap,
+   PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A,
+   MEMWIN_RSVD4),
+   MEMOFST_V((mem_base >> MEMOFST_S)) | PFNUM_V(adap->pf));
+
+   /* Read it back to flush it */
+   t4_read_reg(adap,
+   PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A,
+   MEMWIN_RSVD4));
+   } else
+   dev_info(adap->pdev_dev, "p2pmem memory not reserved, "
+"base 0x%x size %uB\n", mem_base, mem_size);
+}
+
 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
 {
u32 v;
@@ -4622,6 +4675,42 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int 
num_vfs)
 }
 #endif
 
+static int init_p2pmem(struct adapter *adapter)
+{
+   unsigned int mem_size = t4_read_reg(adapter, CIM_EXTMEM2_ADDR_SIZE_A);
+   struct p2pmem_dev *p;
+   int rc;
+   struct resource res;
+
+   if (!mem_size || !use_p2pmem)
+   return 0;
+
+   mem_size = roundup_pow_of_two(mem_size);
+
+   /*
+* Create a subset of BAR4 for the p2pmem region based on the
+* exported memory size.
+*/
+   memcpy(, >pdev->resource[4], sizeof(res));
+   res.start += mem_size;
+   res.end = res.start + mem_size - 1;
+   dev_info(adapter->pdev_dev, "p2pmem resource start 0x%llx end 0x%llx 
size %lluB\n",
+res.start, res.end, resource_size());
+
+   p = p2pmem_create(>pdev->dev);
+   if