[PATCH 0/3] ntb: Asynchronous NTB devices support

[Serge Semin]

 supports, we may want to implement the traditional
linux kernel bus device-drivers matching algorithm using some new
struct ntb_device_id and id_tables.

Thanks,

=
Serge V. Semin
Leading Programmer
Embedded SW development group
T-platforms
=

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

fancer (3):
  ntb: Add asynchronous devices support to NTB-bus interface
  ntb: IDT 89HPES*NT* PCIe-switches NTB device driver
  ntb: Test client drivers for asynchronous NTB devices

 drivers/ntb/Kconfig|4 +-
 drivers/ntb/hw/Kconfig |1 +
 drivers/ntb/hw/Makefile|6 +-
 drivers/ntb/hw/amd/ntb_hw_amd.c|   49 +-
 drivers/ntb/hw/idt/Kconfig |   21 +
 drivers/ntb/hw/idt/Makefile|5 +
 drivers/ntb/hw/idt/ntb_hw_idt.c| 4050 
 drivers/ntb/hw/idt/ntb_hw_idt.h|  390 +++
 drivers/ntb/hw/idt/ntb_hw_idt_quirks.c |  163 ++
 drivers/ntb/hw/idt/ntb_hw_idt_quirks.h |  114 +
 drivers/ntb/hw/idt/ntb_hw_idt_regmap.h |  877 +++
 drivers/ntb/hw/intel/ntb_hw_intel.c|   59 +-
 drivers/ntb/ntb.c  |   86 +-
 drivers/ntb/ntb_transport.c|   19 +-
 drivers/ntb/test/Kconfig   |   32 +
 drivers/ntb/test/Makefile  |9 +-
 drivers/ntb/test/ntb_db_test.c |  677 ++
 drivers/ntb/test/ntb_msg_test.c|  736 ++
 drivers/ntb/test/ntb_mw_test.c | 1531 
 drivers/ntb/test/ntb_perf.c|   16 +-
 drivers/ntb/test/ntb_pingpong.c|5 +
 drivers/ntb/test/ntb_tool.c|   25 +-
 include/linux/ntb.h|  600 -
 23 files changed, 9309 insertions(+), 166 deletions(-)
 create mode 100644 drivers/ntb/hw/idt/Kconfig
 create mode 100644 drivers/ntb/hw/idt/Makefile
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.c
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.h
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt_quirks.c
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt_quirks.h
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt_regmap.h
 create mode 100644 drivers/ntb/test/ntb_db_test.c
 create mode 100644 drivers/ntb/test/ntb_msg_test.c
 create mode 100644 drivers/ntb/test/ntb_mw_test.c

-- 
2.6.6

[PATCH 3/3] ntb: Test client drivers for asynchronous NTB devices

[Serge Semin]

There are three drivers to independently test all interfaces implemented by
the IDT 89HPES*NT* NTB driver.

Doorbells are tested by new NTB Doorbell Pingpong client driver. It implements
the so-named algorithm. Driver starts working from setting the peer doorbell of
the last locally set doorbell bit. If there has not been locally set doorbell,
it sets the very first bit. After that the driver unmasks the events of the
just set doorbell bit and waits until the peer sets the same doorbell. When
peer does it, the local driver iterates to the next doorbell bit and starts
delayed work thread, which will set the corresponding bit and perform doorbell
bit umasking on waking up.

Messaging subsystem can be tested by the client driver implementing a simple
transmition/reception algorithm. A message can be send to a peer by writing
data to /sys/kernel/debug/ntb_msg_test/ntbA{N}/data file. The peer can read
it from the same file then.

Memory Windows test driver implements a simple write/read algorithm. The driver
allocates the predefined number of local buffers (inbound memory window -
inwndw{N}). In order to get a translated base address driver sends a
corresponding command to a peer. Then driver initialize the outbound memory
windows (outwndw{N}). The read/write operations can be performed using the
following debug nodes:
/sys/kernel/debug/ntb_mw_test/ntbA{N}/inwndw{N}
/sys/kernel/debug/ntb_mw_test/ntbA{N}/outwndw{N}

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/test/Kconfig|   32 +
 drivers/ntb/test/Makefile   |9 +-
 drivers/ntb/test/ntb_db_test.c  |  677 +
 drivers/ntb/test/ntb_msg_test.c |  736 +++
 drivers/ntb/test/ntb_mw_test.c  | 1531 +++
 5 files changed, 2983 insertions(+), 2 deletions(-)
 create mode 100644 drivers/ntb/test/ntb_db_test.c
 create mode 100644 drivers/ntb/test/ntb_msg_test.c
 create mode 100644 drivers/ntb/test/ntb_mw_test.c

diff --git a/drivers/ntb/test/Kconfig b/drivers/ntb/test/Kconfig
index a5d0eda..80f5058 100644
--- a/drivers/ntb/test/Kconfig
+++ b/drivers/ntb/test/Kconfig
@@ -25,3 +25,35 @@ config NTB_PERF
 to and from the window without additional software interaction.
 
 If unsure, say N.
+
+config NTB_DB_TEST
+   tristate "NTB Doorbell Test Client"
+   help
+This is a driver to test doorbell subsystem of NTB bus devices.
+The design is similar to the ping pong although it exchanges the
+doorbell bits one-by-one, waiting for the peer response before getting
+to a next doorbell.
+
+If unsure, say N.
+
+config NTB_MSG_TEST
+   tristate "NTB Messaging Test Client"
+   help
+This is a driver to test messaging subsystem of NTB. It just creates
+one file in the DebugFS for each NTB device of asynchronous
+architecture. In order to send a message one can just write a text to
+the file. It will be immediately sent to the peer so user can get it
+by reading from the corresponding file.
+
+If unsure, say N.
+
+config NTB_MW_TEST
+   tristate "NTB Memory Windows Test Client"
+   help
+This is a driver to test memory sharing amongst devices. It creates a
+set of files in the DebugFS, one of which are used to write a text to
+outbound memory windows and anothers can be used to read data written
+by the peer to our inbound memory window.
+
+If unsure, say N.
+
diff --git a/drivers/ntb/test/Makefile b/drivers/ntb/test/Makefile
index 9e77e0b..6ea6db4 100644
--- a/drivers/ntb/test/Makefile
+++ b/drivers/ntb/test/Makefile
@@ -1,3 +1,8 @@
+# Synchronous hardware clients (Intel/AMD)
 obj-$(CONFIG_NTB_PINGPONG) += ntb_pingpong.o
-obj-$(CONFIG_NTB_TOOL) += ntb_tool.o
-obj-$(CONFIG_NTB_PERF) += ntb_perf.o
+obj-$(CONFIG_NTB_TOOL) += ntb_tool.o
+obj-$(CONFIG_NTB_PERF) += ntb_perf.o
+# Asynchronous hardware clients (IDT)
+obj-$(CONFIG_NTB_DB_TEST)  += ntb_db_test.o
+obj-$(CONFIG_NTB_MSG_TEST) += ntb_msg_test.o
+obj-$(CONFIG_NTB_MW_TEST)  += ntb_mw_test.o
diff --git a/drivers/ntb/test/ntb_db_test.c b/drivers/ntb/test/ntb_db_test.c
new file mode 100644
index 000..e93c0c6
--- /dev/null
+++ b/drivers/ntb/test/ntb_db_test.c
@@ -0,0 +1,677 @@
+/*
+ *   This file is provided under a GPLv2 license.  When using or
+ *   redistributing this file, you may do so under that license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright (C) 2016 T-Platforms All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify it
+ *   under the terms and conditions of the GNU General Public License,
+ *   version 2, as published by the Free Software Foundation.
+ *
+ *   This program is distributed in the hope that it will be useful, but 
WITHOUT
+ *   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *   FITNESS FOR A PARTICULAR PURPOSE. See the GNU G

[PATCH 1/3] ntb: Add asynchronous devices support to NTB-bus interface

[Serge Semin]

Currently supported AMD and Intel Non-transparent PCIe-bridges are synchronous
devices, so translated base address of memory windows can be direcly written
to peer registers. But there are some IDT PCIe-switches which implement
complex interfaces using Lookup Tables of translation addresses. Due to
the way the table is accessed, it can not be done synchronously from different
RCs, that's why the asynchronous interface should be developed.

For these purpose the Memory Window related interface is correspondingly split
as it is for Doorbell and Scratchpad registers. The definition of Memory Window
is following: "It is a virtual memory region, which locally reflects a physical
memory of peer device." So to speak the "ntb_peer_mw_"-prefixed methods control
the peers memory windows, "ntb_mw_"-prefixed functions work with the local
memory windows.
Here is the description of the Memory Window related NTB-bus callback
functions:
 - ntb_mw_count() - number of local memory windows.
 - ntb_mw_get_maprsc() - get the physical address and size of the local memory
 window to map.
 - ntb_mw_set_trans() - set translation address of local memory window (this
address should be somehow retrieved from a peer).
 - ntb_mw_get_trans() - get translation address of local memory window.
 - ntb_mw_get_align() - get alignment of translated base address and size of
local memory window. Additionally one can get the
upper size limit of the memory window.
 - ntb_peer_mw_count() - number of peer memory windows (it can differ from the
 local number).
 - ntb_peer_mw_set_trans() - set translation address of peer memory window
 - ntb_peer_mw_get_trans() - get translation address of peer memory window
 - ntb_peer_mw_get_align() - get alignment of translated base address and size
 of peer memory window.Additionally one can get the
 upper size limit of the memory window.

As one can see current AMD and Intel NTB drivers mostly implement the
"ntb_peer_mw_"-prefixed methods. So this patch correspondingly renames the
driver functions. IDT NTB driver mostly expose "ntb_nw_"-prefixed methods,
since it doesn't have convenient access to the peer Lookup Table.

In order to pass information from one RC to another NTB functions of IDT
PCIe-switch implement Messaging subsystem. They currently support four message
registers to transfer DWORD sized data to a specified peer. So there are two
new callback methods are introduced:
 - ntb_msg_size() - get the number of DWORDs supported by NTB function to send
and receive messages
 - ntb_msg_post() - send message of size retrieved from ntb_msg_size()
to a peer
Additionally there is a new event function:
 - ntb_msg_event() - it is invoked when either a new message was retrieved
 (NTB_MSG_NEW), or last message was successfully sent
 (NTB_MSG_SENT), or the last message failed to be sent
 (NTB_MSG_FAIL).

The last change concerns the IDs (practically names) of NTB-devices on the
NTB-bus. It is not good to have the devices with same names in the system
and it brakes my IDT NTB driver from being loaded =) So I developed a simple
algorithm of NTB devices naming. Particulary it generates names "ntbS{N}" for
synchronous devices, "ntbA{N}" for asynchronous devices, and "ntbAS{N}" for
devices supporting both interfaces.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/Kconfig |   4 +-
 drivers/ntb/hw/amd/ntb_hw_amd.c |  49 ++-
 drivers/ntb/hw/intel/ntb_hw_intel.c |  59 +++-
 drivers/ntb/ntb.c   |  86 +-
 drivers/ntb/ntb_transport.c |  19 +-
 drivers/ntb/test/ntb_perf.c |  16 +-
 drivers/ntb/test/ntb_pingpong.c |   5 +
 drivers/ntb/test/ntb_tool.c |  25 +-
 include/linux/ntb.h | 600 +---
 9 files changed, 701 insertions(+), 162 deletions(-)

diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig
index 95944e5..67d80c4 100644
--- a/drivers/ntb/Kconfig
+++ b/drivers/ntb/Kconfig
@@ -14,8 +14,6 @@ if NTB
 
 source "drivers/ntb/hw/Kconfig"
 
-source "drivers/ntb/test/Kconfig"
-
 config NTB_TRANSPORT
tristate "NTB Transport Client"
help
@@ -25,4 +23,6 @@ config NTB_TRANSPORT
 
 If unsure, say N.
 
+source "drivers/ntb/test/Kconfig"
+
 endif # NTB
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 6ccba0d..ab6f353 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -55,6 +55,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "ntb_hw_amd.h"
@@ -84,11 +85,8 @@ static int amd_ntb_

[PATCH v2 0/3] ntb: Asynchronous NTB devices support

[Serge Semin]

Please, find the general patchset description in the cover letter of the first
patchset (see the very first message in thread).

Changes in v2:
 - Fix sparc64 compilation warning in drivers/ntb/hw/idt/ntb_hw_idt.c :
   warning: right shift count >= width of type
 - Fix sparc64 compilation warnings in drivers/ntb/test/ntb_mw_test.c :
   warning: right shift count >= width of type
   warning: cast to pointer from integer of different size

Thanks,

=
Serge V. Semin
Leading Programmer
Embedded SW development group
T-platforms
=

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

fancer (3):
  ntb: Add asynchronous devices support to NTB-bus interface
  ntb: IDT 89HPES*NT* PCIe-switches NTB device driver
  ntb: Test client drivers for asynchronous NTB devices

 drivers/ntb/Kconfig|4 +-
 drivers/ntb/hw/Kconfig |1 +
 drivers/ntb/hw/Makefile|6 +-
 drivers/ntb/hw/amd/ntb_hw_amd.c|   49 +-
 drivers/ntb/hw/idt/Kconfig |   21 +
 drivers/ntb/hw/idt/Makefile|5 +
 drivers/ntb/hw/idt/ntb_hw_idt.c| 4050 
 drivers/ntb/hw/idt/ntb_hw_idt.h|  390 +++
 drivers/ntb/hw/idt/ntb_hw_idt_quirks.c |  163 ++
 drivers/ntb/hw/idt/ntb_hw_idt_quirks.h |  114 +
 drivers/ntb/hw/idt/ntb_hw_idt_regmap.h |  877 +++
 drivers/ntb/hw/intel/ntb_hw_intel.c|   59 +-
 drivers/ntb/ntb.c  |   86 +-
 drivers/ntb/ntb_transport.c|   19 +-
 drivers/ntb/test/Kconfig   |   32 +
 drivers/ntb/test/Makefile  |9 +-
 drivers/ntb/test/ntb_db_test.c |  677 ++
 drivers/ntb/test/ntb_msg_test.c|  736 ++
 drivers/ntb/test/ntb_mw_test.c | 1539 
 drivers/ntb/test/ntb_perf.c|   16 +-
 drivers/ntb/test/ntb_pingpong.c|5 +
 drivers/ntb/test/ntb_tool.c|   25 +-
 include/linux/ntb.h|  600 -
 23 files changed, 9317 insertions(+), 166 deletions(-)
 create mode 100644 drivers/ntb/hw/idt/Kconfig
 create mode 100644 drivers/ntb/hw/idt/Makefile
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.c
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.h
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt_quirks.c
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt_quirks.h
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt_regmap.h
 create mode 100644 drivers/ntb/test/ntb_db_test.c
 create mode 100644 drivers/ntb/test/ntb_msg_test.c
 create mode 100644 drivers/ntb/test/ntb_mw_test.c

-- 
2.6.6

[PATCH v2 1/3] ntb: Add asynchronous devices support to NTB-bus interface

[Serge Semin]

Currently supported AMD and Intel Non-transparent PCIe-bridges are synchronous
devices, so translated base address of memory windows can be direcly written
to peer registers. But there are some IDT PCIe-switches which implement
complex interfaces using Lookup Tables of translation addresses. Due to
the way the table is accessed, it can not be done synchronously from different
RCs, that's why the asynchronous interface should be developed.

For these purpose the Memory Window related interface is correspondingly split
as it is for Doorbell and Scratchpad registers. The definition of Memory Window
is following: "It is a virtual memory region, which locally reflects a physical
memory of peer device." So to speak the "ntb_peer_mw_"-prefixed methods control
the peers memory windows, "ntb_mw_"-prefixed functions work with the local
memory windows.
Here is the description of the Memory Window related NTB-bus callback
functions:
 - ntb_mw_count() - number of local memory windows.
 - ntb_mw_get_maprsc() - get the physical address and size of the local memory
 window to map.
 - ntb_mw_set_trans() - set translation address of local memory window (this
address should be somehow retrieved from a peer).
 - ntb_mw_get_trans() - get translation address of local memory window.
 - ntb_mw_get_align() - get alignment of translated base address and size of
local memory window. Additionally one can get the
upper size limit of the memory window.
 - ntb_peer_mw_count() - number of peer memory windows (it can differ from the
 local number).
 - ntb_peer_mw_set_trans() - set translation address of peer memory window
 - ntb_peer_mw_get_trans() - get translation address of peer memory window
 - ntb_peer_mw_get_align() - get alignment of translated base address and size
 of peer memory window.Additionally one can get the
 upper size limit of the memory window.

As one can see current AMD and Intel NTB drivers mostly implement the
"ntb_peer_mw_"-prefixed methods. So this patch correspondingly renames the
driver functions. IDT NTB driver mostly expose "ntb_nw_"-prefixed methods,
since it doesn't have convenient access to the peer Lookup Table.

In order to pass information from one RC to another NTB functions of IDT
PCIe-switch implement Messaging subsystem. They currently support four message
registers to transfer DWORD sized data to a specified peer. So there are two
new callback methods are introduced:
 - ntb_msg_size() - get the number of DWORDs supported by NTB function to send
and receive messages
 - ntb_msg_post() - send message of size retrieved from ntb_msg_size()
to a peer
Additionally there is a new event function:
 - ntb_msg_event() - it is invoked when either a new message was retrieved
 (NTB_MSG_NEW), or last message was successfully sent
 (NTB_MSG_SENT), or the last message failed to be sent
 (NTB_MSG_FAIL).

The last change concerns the IDs (practically names) of NTB-devices on the
NTB-bus. It is not good to have the devices with same names in the system
and it brakes my IDT NTB driver from being loaded =) So I developed a simple
algorithm of NTB devices naming. Particulary it generates names "ntbS{N}" for
synchronous devices, "ntbA{N}" for asynchronous devices, and "ntbAS{N}" for
devices supporting both interfaces.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/Kconfig |   4 +-
 drivers/ntb/hw/amd/ntb_hw_amd.c |  49 ++-
 drivers/ntb/hw/intel/ntb_hw_intel.c |  59 +++-
 drivers/ntb/ntb.c   |  86 +-
 drivers/ntb/ntb_transport.c |  19 +-
 drivers/ntb/test/ntb_perf.c |  16 +-
 drivers/ntb/test/ntb_pingpong.c |   5 +
 drivers/ntb/test/ntb_tool.c |  25 +-
 include/linux/ntb.h | 600 +---
 9 files changed, 701 insertions(+), 162 deletions(-)

diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig
index 95944e5..67d80c4 100644
--- a/drivers/ntb/Kconfig
+++ b/drivers/ntb/Kconfig
@@ -14,8 +14,6 @@ if NTB
 
 source "drivers/ntb/hw/Kconfig"
 
-source "drivers/ntb/test/Kconfig"
-
 config NTB_TRANSPORT
tristate "NTB Transport Client"
help
@@ -25,4 +23,6 @@ config NTB_TRANSPORT
 
 If unsure, say N.
 
+source "drivers/ntb/test/Kconfig"
+
 endif # NTB
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 6ccba0d..ab6f353 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -55,6 +55,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "ntb_hw_amd.h"
@@ -84,11 +85,8 @@ static int amd_ntb_

[PATCH v2 3/3] ntb: Test client drivers for asynchronous NTB devices

[Serge Semin]

There are three drivers to independently test all interfaces implemented by
the IDT 89HPES*NT* NTB driver.

Doorbells are tested by new NTB Doorbell Pingpong client driver. It implements
the so-named algorithm. Driver starts working from setting the peer doorbell of
the last locally set doorbell bit. If there has not been locally set doorbell,
it sets the very first bit. After that the driver unmasks the events of the
just set doorbell bit and waits until the peer sets the same doorbell. When
peer does it, the local driver iterates to the next doorbell bit and starts
delayed work thread, which will set the corresponding bit and perform doorbell
bit umasking on waking up.

Messaging subsystem can be tested by the client driver implementing a simple
transmition/reception algorithm. A message can be send to a peer by writing
data to /sys/kernel/debug/ntb_msg_test/ntbA{N}/data file. The peer can read
it from the same file then.

Memory Windows test driver implements a simple write/read algorithm. The driver
allocates the predefined number of local buffers (inbound memory window -
inwndw{N}). In order to get a translated base address driver sends a
corresponding command to a peer. Then driver initialize the outbound memory
windows (outwndw{N}). The read/write operations can be performed using the
following debug nodes:
/sys/kernel/debug/ntb_mw_test/ntbA{N}/inwndw{N}
/sys/kernel/debug/ntb_mw_test/ntbA{N}/outwndw{N}

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/test/Kconfig|   32 +
 drivers/ntb/test/Makefile   |9 +-
 drivers/ntb/test/ntb_db_test.c  |  677 +
 drivers/ntb/test/ntb_msg_test.c |  736 +++
 drivers/ntb/test/ntb_mw_test.c  | 1539 +++
 5 files changed, 2991 insertions(+), 2 deletions(-)
 create mode 100644 drivers/ntb/test/ntb_db_test.c
 create mode 100644 drivers/ntb/test/ntb_msg_test.c
 create mode 100644 drivers/ntb/test/ntb_mw_test.c

diff --git a/drivers/ntb/test/Kconfig b/drivers/ntb/test/Kconfig
index a5d0eda..80f5058 100644
--- a/drivers/ntb/test/Kconfig
+++ b/drivers/ntb/test/Kconfig
@@ -25,3 +25,35 @@ config NTB_PERF
 to and from the window without additional software interaction.
 
 If unsure, say N.
+
+config NTB_DB_TEST
+   tristate "NTB Doorbell Test Client"
+   help
+This is a driver to test doorbell subsystem of NTB bus devices.
+The design is similar to the ping pong although it exchanges the
+doorbell bits one-by-one, waiting for the peer response before getting
+to a next doorbell.
+
+If unsure, say N.
+
+config NTB_MSG_TEST
+   tristate "NTB Messaging Test Client"
+   help
+This is a driver to test messaging subsystem of NTB. It just creates
+one file in the DebugFS for each NTB device of asynchronous
+architecture. In order to send a message one can just write a text to
+the file. It will be immediately sent to the peer so user can get it
+by reading from the corresponding file.
+
+If unsure, say N.
+
+config NTB_MW_TEST
+   tristate "NTB Memory Windows Test Client"
+   help
+This is a driver to test memory sharing amongst devices. It creates a
+set of files in the DebugFS, one of which are used to write a text to
+outbound memory windows and anothers can be used to read data written
+by the peer to our inbound memory window.
+
+If unsure, say N.
+
diff --git a/drivers/ntb/test/Makefile b/drivers/ntb/test/Makefile
index 9e77e0b..6ea6db4 100644
--- a/drivers/ntb/test/Makefile
+++ b/drivers/ntb/test/Makefile
@@ -1,3 +1,8 @@
+# Synchronous hardware clients (Intel/AMD)
 obj-$(CONFIG_NTB_PINGPONG) += ntb_pingpong.o
-obj-$(CONFIG_NTB_TOOL) += ntb_tool.o
-obj-$(CONFIG_NTB_PERF) += ntb_perf.o
+obj-$(CONFIG_NTB_TOOL) += ntb_tool.o
+obj-$(CONFIG_NTB_PERF) += ntb_perf.o
+# Asynchronous hardware clients (IDT)
+obj-$(CONFIG_NTB_DB_TEST)  += ntb_db_test.o
+obj-$(CONFIG_NTB_MSG_TEST) += ntb_msg_test.o
+obj-$(CONFIG_NTB_MW_TEST)  += ntb_mw_test.o
diff --git a/drivers/ntb/test/ntb_db_test.c b/drivers/ntb/test/ntb_db_test.c
new file mode 100644
index 000..e93c0c6
--- /dev/null
+++ b/drivers/ntb/test/ntb_db_test.c
@@ -0,0 +1,677 @@
+/*
+ *   This file is provided under a GPLv2 license.  When using or
+ *   redistributing this file, you may do so under that license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright (C) 2016 T-Platforms All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify it
+ *   under the terms and conditions of the GNU General Public License,
+ *   version 2, as published by the Free Software Foundation.
+ *
+ *   This program is distributed in the hope that it will be useful, but 
WITHOUT
+ *   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *   FITNESS FOR A PARTICULAR PURPOSE. See the GNU G

Re: [PATCH v2 0/3] ntb: Asynchronous NTB devices support

[Serge Semin]

Hello, Allen.
Thanks for the message. I see your point. Yes, I've seen a lot of cruel
threads in mailing threads in lkml.org , so it's not my intention to
argue about basic things like Coding Style. That's why I left most of
the warnings discussable. While you a digging into the Patch 1/3, I'll
do my best to fix the checkpatch warnings of the rest of the code. Regarding
the last checkpatch error message, I need to spend some more time to
find a way to set it free of the warnings. I hope I'll come up with
something good, at least I'll give it a try. Otherwise I'll have to
redesign the driver regmap subsystem.(

Regards,
-Sergey

On Thu, Jul 28, 2016 at 10:42:30AM -0400, Allen Hubbe <allen.hu...@emc.com> 
wrote:
> From: Serge Semin
> > Please, find the general patchset description in the cover letter of the 
> > first
> > patchset (see the very first message in thread).
> > 
> > Changes in v2:
> >  - Fix sparc64 compilation warning in drivers/ntb/hw/idt/ntb_hw_idt.c :
> >warning: right shift count >= width of type
> >  - Fix sparc64 compilation warnings in drivers/ntb/test/ntb_mw_test.c :
> >warning: right shift count >= width of type
> >warning: cast to pointer from integer of different size
> 
> Thanks for reacting to the test robot so quickly.  Since nobody else has 
> responded yet, I would like to assure you that the patches are not being 
> ignored.  Please be patient.  The IDT driver will be a valuable contribution 
> to the ntb subsystem.  I am working carefully through patch 1/3 first, since 
> it affects existing drivers and interface.
> 
> A word of caution regarding your statement, "There are a some types of 
> checkpatch warnings I left unfixed."  Coding style can be a touchy subject, 
> leading to some recent rants^H^H^H^H^Hdiscussion on some of the same topics 
> that are included in that list of unfixed warnings.  Be prepared to adhere to 
> the style guide, even if it is inconvenient and against your own logic, 
> because that is almost always the easier and more practical approach than 
> asking for changes or exceptions, and better for your mental health not to be 
> on the To: list of something like https://lkml.org/lkml/2016/7/8/625.
> 
> "Of course all of these warnings are discussable, except the last one."  Be 
> prepared, even if it will require significant changes to the code.  For 
> really inconvenient changes, we can talk about other more readily acceptable 
> approaches to keep the code short and elegant, as is obviously your intent.  
> Please be patient with the review.
> 

Re: [PATCH v2 1/3] ntb: Add asynchronous devices support to NTB-bus interface

[Serge Semin]

using some of a hardware mechanism. It can be anything: Scratchpad 
registers, Message registers or even "crazy" doorbells bingbanging. For 
instance, the IDT switches of the first group support:
1) Shared Memory windows. In particular local root complex can set a translated 
base address to BARs of local and peer NT-function using the cross-coupled 
PCIe/NTB configuration space, the same way as it can be done for AMD/Intel NTBs.
2) One Doorbell register.
3) Two Scratchpads.
4) Four message regietsrs.
As you can see the switches of the first group can be considered as both 
synchronous and asynchronous. All the NTB bus kernel API can be implemented for 
it including the changes introduced by this patch (I would do it if I had a 
corresponding hardware). AMD and Intel NTBs can be considered both synchronous 
and asynchronous as well, although they don't support messaging so Scratchpads 
can be used to send a data to a peer. Finally the switches of the second group 
lack of ability to initialize BARs translated base address of peers due to the 
race condition I described before.

To sum up I've spent a lot of time designing the IDT NTB driver. I've done my 
best to make the IDT driver as much compatible with current design as possible, 
nevertheless the NTB bus kernel API had to be slightly changed. You can find 
answers to the commentaries down below.

On Fri, Aug 05, 2016 at 11:31:58AM -0400, Allen Hubbe <allen.hu...@emc.com> 
wrote:
> From: Serge Semin
> > Currently supported AMD and Intel Non-transparent PCIe-bridges are 
> > synchronous
> > devices, so translated base address of memory windows can be direcly written
> > to peer registers. But there are some IDT PCIe-switches which implement
> > complex interfaces using Lookup Tables of translation addresses. Due to
> > the way the table is accessed, it can not be done synchronously from 
> > different
> > RCs, that's why the asynchronous interface should be developed.
> > 
> > For these purpose the Memory Window related interface is correspondingly 
> > split
> > as it is for Doorbell and Scratchpad registers. The definition of Memory 
> > Window
> > is following: "It is a virtual memory region, which locally reflects a 
> > physical
> > memory of peer device." So to speak the "ntb_peer_mw_"-prefixed methods 
> > control
> > the peers memory windows, "ntb_mw_"-prefixed functions work with the local
> > memory windows.
> > Here is the description of the Memory Window related NTB-bus callback
> > functions:
> >  - ntb_mw_count() - number of local memory windows.
> >  - ntb_mw_get_maprsc() - get the physical address and size of the local 
> > memory
> >  window to map.
> >  - ntb_mw_set_trans() - set translation address of local memory window (this
> > address should be somehow retrieved from a peer).
> >  - ntb_mw_get_trans() - get translation address of local memory window.
> >  - ntb_mw_get_align() - get alignment of translated base address and size of
> > local memory window. Additionally one can get the
> > upper size limit of the memory window.
> >  - ntb_peer_mw_count() - number of peer memory windows (it can differ from 
> > the
> >  local number).
> >  - ntb_peer_mw_set_trans() - set translation address of peer memory window
> >  - ntb_peer_mw_get_trans() - get translation address of peer memory window
> >  - ntb_peer_mw_get_align() - get alignment of translated base address and 
> > size
> >  of peer memory window.Additionally one can get 
> > the
> >  upper size limit of the memory window.
> > 
> > As one can see current AMD and Intel NTB drivers mostly implement the
> > "ntb_peer_mw_"-prefixed methods. So this patch correspondingly renames the
> > driver functions. IDT NTB driver mostly expose "ntb_nw_"-prefixed methods,
> > since it doesn't have convenient access to the peer Lookup Table.
> > 
> > In order to pass information from one RC to another NTB functions of IDT
> > PCIe-switch implement Messaging subsystem. They currently support four 
> > message
> > registers to transfer DWORD sized data to a specified peer. So there are two
> > new callback methods are introduced:
> >  - ntb_msg_size() - get the number of DWORDs supported by NTB function to 
> > send
> > and receive messages
> >  - ntb_msg_post() - send message of size retrieved from ntb_msg_size()
> > to a peer
> > Additionally there is a new event function:
> >  

[PATCH] NTB: Add IDT 89HPESxNTx PCIe-switches support

[Serge Semin]

IDT 89HPESxNTx device series is PCIe-switches, which support Non-Transparent
bridging between domains connected to the device ports. Since new NTB API
exposes multi-port interface and messaging API, the IDT NT-functions can
be now supported in the kernel. This driver adds the following functionality:
1) Multi-port NTB API to have information of possible NT-functions activated
in compliance with available device ports.
2) Memory windows of direct and look up table based address translation with
all possible combinations of BARs setup.
3) Traditional doorbell NTB API.
4) One-on-one messaging NTB API.

There are some IDT PCIe-switch setups, which must be done before any of
the NTB peers started. It can be performed either by system BIOS via IDT
SMBus-slave interface or by pre-initialized IDT PCIe-switch EEPROM:
1) NT-functions of corresponding ports must be activated using SWPARTxCTL
and SWPORTxCTL registers.
2) BAR0 must be configured to expose NT-function configuration registers map.
3) The rest of the BARs must have at least one memory window configured,
otherwise the driver will just return an error.
Temperature sensor of IDT PCIe-switches can be also optionally activated by
BIOS or EEPROM.
(See IDT documentations for details of how the pre-initialization can be done)

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/ntb/hw/Kconfig  |1 +
 drivers/ntb/hw/Makefile |1 +
 drivers/ntb/hw/idt/Kconfig  |   31 +
 drivers/ntb/hw/idt/Makefile |1 +
 drivers/ntb/hw/idt/ntb_hw_idt.c | 2628 +++
 drivers/ntb/hw/idt/ntb_hw_idt.h | 1160 +
 6 files changed, 3822 insertions(+)
 create mode 100644 drivers/ntb/hw/idt/Kconfig
 create mode 100644 drivers/ntb/hw/idt/Makefile
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.c
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.h

diff --git a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig
index 7116472..a89243c 100644
--- a/drivers/ntb/hw/Kconfig
+++ b/drivers/ntb/hw/Kconfig
@@ -1,2 +1,3 @@
 source "drivers/ntb/hw/amd/Kconfig"
+source "drivers/ntb/hw/idt/Kconfig"
 source "drivers/ntb/hw/intel/Kconfig"
diff --git a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile
index 532e085..87332c3 100644
--- a/drivers/ntb/hw/Makefile
+++ b/drivers/ntb/hw/Makefile
@@ -1,2 +1,3 @@
 obj-$(CONFIG_NTB_AMD)  += amd/
+obj-$(CONFIG_NTB_IDT)  += idt/
 obj-$(CONFIG_NTB_INTEL)+= intel/
diff --git a/drivers/ntb/hw/idt/Kconfig b/drivers/ntb/hw/idt/Kconfig
new file mode 100644
index 000..b360e56
--- /dev/null
+++ b/drivers/ntb/hw/idt/Kconfig
@@ -0,0 +1,31 @@
+config NTB_IDT
+   tristate "IDT PCIe-switch Non-Transparent Bridge support"
+   depends on PCI
+   help
+This driver supports NTB of cappable IDT PCIe-switches.
+
+Some of the pre-initializations must be made before IDT PCIe-switch
+exposes it NT-functions correctly. It should be done by either proper
+initialisation of EEPROM connected to master smbus of the switch or
+by BIOS using slave-SMBus interface changing corresponding registers
+value. Evidently it must be done before PCI bus enumeration is
+finished in Linux kernel.
+
+First of all partitions must be activated and properly assigned to all
+the ports with NT-functions intended to be activated (see SWPARTxCTL
+and SWPORTxCTL registers). Then all NT-function BARs must be enabled
+with chosen valid aperture. For memory windows related BARs the
+aperture settings shall determine the maximum size of memory windows
+accepted by a BAR. Note that BAR0 must map PCI configuration space
+registers.
+
+It's worth to note, that since a part of this driver relies on the
+BAR settings of peer NT-functions, the BAR setups can't be done over
+kernel PCI fixups. That's why the alternative pre-initialization
+techniques like BIOS using SMBus interface or EEPROM should be
+utilized. Additionally if one needs to have temperature sensor
+information printed to system log, the corresponding registers must
+be initialized within BIOS/EEPROM as well.
+
+If unsure, say N.
+
diff --git a/drivers/ntb/hw/idt/Makefile b/drivers/ntb/hw/idt/Makefile
new file mode 100644
index 000..a102cf1
--- /dev/null
+++ b/drivers/ntb/hw/idt/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_NTB_IDT) += ntb_hw_idt.o
diff --git a/drivers/ntb/hw/idt/ntb_hw_idt.c b/drivers/ntb/hw/idt/ntb_hw_idt.c
new file mode 100644
index 000..f9d5737
--- /dev/null
+++ b/drivers/ntb/hw/idt/ntb_hw_idt.c
@@ -0,0 +1,2628 @@
+/*
+ *   This file is provided under a GPLv2 license.  When using or
+ *   redistributing this file, you may do so under that license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright (C) 2016 T-Platforms All Rights Reserved.
+ *
+ *   This program is free software; you can redistri

[PATCH] eeprom: idt_89hpesx: Discard memory freeing allocated by devm_kmalloc

[Serge Semin]

Indeed, the data structure is allocated by device resource manager,
so the driver doesn't need to free anything on remove() callback.

Reported-by: Julia Lawall <julia.law...@lip6.fr>
Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/misc/eeprom/idt_89hpesx.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/drivers/misc/eeprom/idt_89hpesx.c 
b/drivers/misc/eeprom/idt_89hpesx.c
index 25d47d0..f73fd04 100644
--- a/drivers/misc/eeprom/idt_89hpesx.c
+++ b/drivers/misc/eeprom/idt_89hpesx.c
@@ -1227,9 +1227,6 @@ static void idt_free_pdev(struct idt_89hpesx_dev *pdev)
 {
/* Clear driver data from device private field */
i2c_set_clientdata(pdev->client, NULL);
-
-   /* Just free memory allocated for data */
-   devm_kfree(>client->dev, pdev);
 }
 
 /*
-- 
2.6.6

Re: [PATCH] eeprom: fix memory leak on buf when failed allocation of csraddr_str

[Serge Semin]

On Tue, Jan 24, 2017 at 02:38:38PM +, Colin King <colin.k...@canonical.com> 
wrote:
> From: Colin Ian King <colin.k...@canonical.com>
> 
> The error return path When csraddr_str fails to free buf, causing a
> memory leak. Fix this by returning via the free_buf label that
> performs the necessary cleanup.
> 
> Signed-off-by: Colin Ian King <colin.k...@canonical.com>
> ---
>  drivers/misc/eeprom/idt_89hpesx.c | 6 --
>  1 file changed, 4 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/misc/eeprom/idt_89hpesx.c 
> b/drivers/misc/eeprom/idt_89hpesx.c
> index 25d47d0..56bc5c2 100644
> --- a/drivers/misc/eeprom/idt_89hpesx.c
> +++ b/drivers/misc/eeprom/idt_89hpesx.c
> @@ -965,8 +965,10 @@ static ssize_t idt_dbgfs_csr_write(struct file *filep, 
> const char __user *ubuf,
>   csraddr_len = colon_ch - buf;
>   csraddr_str =
>   kmalloc(sizeof(char)*(csraddr_len + 1), GFP_KERNEL);
> - if (csraddr_str == NULL)
> - return -ENOMEM;
> + if (csraddr_str == NULL) {
> + ret = -ENOMEM;
> + goto free_buf;
> + }
>   /* Copy the register address to the substring buffer */
>   strncpy(csraddr_str, buf, csraddr_len);
>   csraddr_str[csraddr_len] = '\0';
> -- 
> 2.10.2
> 

Indeed. Thanks for the patch.

Acked-by: Serge Semin <fancer.lan...@gmail.com>

Re: [patch] eeprom/idt_89hpesx: freeing devm_ pointer with kfree()

[Serge Semin]

On Thu, Jan 26, 2017 at 12:01:46PM +0300, Dan Carpenter 
 wrote:
> "pdev->ee_file" was allocated with devm_kmalloc() so freeing it with
> kfree() is a double free.  In fact, we don't need to free it at all.
> 
> Fixes: cfad6425382e ("eeprom: Add IDT 89HPESx EEPROM/CSR driver")
> Signed-off-by: Dan Carpenter 
> 
> diff --git a/drivers/misc/eeprom/idt_89hpesx.c 
> b/drivers/misc/eeprom/idt_89hpesx.c
> index 25d47d09e1cb..f5f2c5a8d9b0 100644
> --- a/drivers/misc/eeprom/idt_89hpesx.c
> +++ b/drivers/misc/eeprom/idt_89hpesx.c
> @@ -1359,7 +1359,6 @@ static int idt_create_sysfs_files(struct 
> idt_89hpesx_dev *pdev)
>   pdev->ee_file->size = pdev->eesize;
>   ret = sysfs_create_bin_file(>kobj, pdev->ee_file);
>   if (ret != 0) {
> - kfree(pdev->ee_file);
>   dev_err(dev, "Failed to create EEPROM sysfs-node");
>   return ret;
>   }
> @@ -1381,9 +1380,6 @@ static void idt_remove_sysfs_files(struct 
> idt_89hpesx_dev *pdev)
>  
>   /* Remove EEPROM sysfs file */
>   sysfs_remove_bin_file(>kobj, pdev->ee_file);
> -
> - /* Free memory allocated for bin_attribute structure */
> - kfree(pdev->ee_file);
>  }
>  
>  /*

Just got the same patch a day earlier, but still thanks for the pointing the 
issue out.

Regards,
-Sergey

Re: [PATCH -next] eeprom: idt_89hpesx: Drop kfree for memory allocated with devm_kzalloc

[Serge Semin]

On Wed, Jan 25, 2017 at 02:09:52PM +, Wei Yongjun <weiyj...@gmail.com> 
wrote:
> From: Wei Yongjun <weiyongj...@huawei.com>
> 
> It's not necessary to free memory allocated with devm_kzalloc
> and using kfree leads to a double free.
> 
> Fixes: cfad6425382e ("eeprom: Add IDT 89HPESx EEPROM/CSR driver")
> Signed-off-by: Wei Yongjun <weiyongj...@huawei.com>
> ---
>  drivers/misc/eeprom/idt_89hpesx.c | 4 
>  1 file changed, 4 deletions(-)
> 
> diff --git a/drivers/misc/eeprom/idt_89hpesx.c 
> b/drivers/misc/eeprom/idt_89hpesx.c
> index 25d47d0..f5f2c5a 100644
> --- a/drivers/misc/eeprom/idt_89hpesx.c
> +++ b/drivers/misc/eeprom/idt_89hpesx.c
> @@ -1359,7 +1359,6 @@ static int idt_create_sysfs_files(struct 
> idt_89hpesx_dev *pdev)
>   pdev->ee_file->size = pdev->eesize;
>   ret = sysfs_create_bin_file(>kobj, pdev->ee_file);
>   if (ret != 0) {
> - kfree(pdev->ee_file);
>   dev_err(dev, "Failed to create EEPROM sysfs-node");
>   return ret;
>   }
> @@ -1381,9 +1380,6 @@ static void idt_remove_sysfs_files(struct 
> idt_89hpesx_dev *pdev)
>  
>   /* Remove EEPROM sysfs file */
>   sysfs_remove_bin_file(>kobj, pdev->ee_file);
> -
> -     /* Free memory allocated for bin_attribute structure */
> - kfree(pdev->ee_file);
>  }
>  
>  /*
> 

True point! Thanks for the patch.

Acked-by: Serge Semin <fancer.lan...@gmail.com>

Re: [PATCH v2] NTB: Add IDT 89HPESxNTx PCIe-switches support

[Serge Semin]

On Tue, Feb 21, 2017 at 05:42:37PM -0500, Allen Hubbe <allen.hu...@dell.com> 
wrote:
> From: Serge Semin
> > +/*
> > + * idt_nt_write() - PCI configuration space registers write method
> > + * @ndev:  IDT NTB hardware driver descriptor
> > + * @reg:   Register to write data to
> > + * @data:  Value to write to the register
> > + *
> > + * WARNING! IDT PCIe-switch registers are all Little endian. So 
> > corresponding
> > + * writel operations must have embedded endiannes conversion. If local
> > + * platform doesn't have it, the driver won't properly work.
> > + */
> > +static void idt_nt_write(struct idt_ntb_dev *ndev,
> > +const unsigned int reg, const u32 data)
> > +{
> > +   /*
> > +* It's obvious bug to request a register exceeding the maximum possible
> > +* value as well as to have it unaligned.
> > +*/
> > +   WARN_ON(reg > IDT_REG_PCI_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN));
> 
> If we perform the write anyway, I guess the effect of the write is unknown?
> 
> What about:
> if (WARN_ON(stuff))
>   return;
> 

Agreed. I'll alter the registers write method.

> 
> > +/*
> > + * idt_reg_set_bits() - set bits of a passed register
> > + * @ndev:  IDT NTB hardware driver descriptor
> > + * @reg:   Register to change bits of
> > + * @valid_mask:Mask of valid bits
> > + * @set_bits:  Bitmask to set
> > + *
> > + * Helper method to check whether a passed bitfield is valid and set
> > + * corresponding bits of a register.
> > + *
> > + * Return: zero on success, negative error on invalid bitmask.
> > + */
> > +static inline int idt_reg_set_bits(struct idt_ntb_dev *ndev, unsigned int 
> > reg,
> > +  u64 valid_mask, u64 set_bits)
> > +{
> > +   u32 data;
> > +
> > +   if (set_bits & ~(u64)valid_mask)
> > +   return -EINVAL;
> > +
> > +   data = idt_nt_read(ndev, reg) | (u32)set_bits;
> > +   idt_nt_write(ndev, IDT_NT_INDBELLMSK, data);
> 
> Following this function call via itd_ntb_db_set_mask(), it does not appear 
> that the register update is atomic here.  Two threads could read the same old 
> register value and modify it differently, and the second write back to the 
> register would clobber the first.
> 
> In the ntb_hw_intel driver there is a similar setting bits of the doorbell 
> mask, and clearing bits.  Instead of reading the register, modifying it, and 
> writing it back, the current value of the register is stored in memory.  With 
> a spin lock held, the value is updated in memory and then written to the 
> register.  That makes the update atomic, because the spin lock is held 
> through the update and issuing the write, and it saves a trip to read the 
> register.
> 

Great! Thanks for pointing it out. Of course, I'll make it atomic to prevent
the possible race condition.

Additional thanks for drawing my attention to this peace of code, since it's
actually got a "copy-paste" bug. The write method is called with constant
register address, but must be called with "reg" argument.

The same things are going to be fixed in the next idt_reg_clear_bits() method
as well. =)

> 
> > +/*
> > + * idt_get_mw_type() - get memory window size
> 
> Doc doesn't match the function name.
> 
> > + * @mw_type:   Memory window type
> > + *
> > + * Return: number of memory windows corresponding to the type
> 
> This is more like a "count" than a "size".
> 

Agreed.

> > + */
> > +static inline unsigned char idt_get_mw_size(enum idt_mw_type mw_type)
> > +{
> 
> 
> > +/*
> > + * idt_ntb_db_set_mask() - set bits in the local doorbell mask
> > + *(NTB API callback)
> > + * @ntb:   NTB device context.
> > + * @db_bits:   Doorbell mask bits to set.
> > + *
> > + * The inbound doorbell register mask value must be read, then OR'ed with
> > + * passed field and only then set back.
> > + *
> > + * Return: zero on success, negative error if invalid argument passed.
> > + */
> > +static int idt_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
> > +{
> > +   struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
> > +
> > +   return idt_reg_set_bits(ndev, IDT_NT_INDBELLMSK, IDT_DBELL_MASK,
> > +   db_bits);
> 
> As noted above, this does not appear to be atomic.
> 

Ok.

Thanks for the careful review. The issues will be fixed within the next
patch.

Regards,
-Sergey

[PATCH v2] NTB: Add IDT 89HPESxNTx PCIe-switches support

[Serge Semin]

IDT 89HPESxNTx device series is PCIe-switches, which support
Non-Transparent bridging between domains connected to the device ports.
Since new NTB API exposes multi-port interface and messaging API, the IDT
NT-functions can be now supported in the kernel. This driver adds the
following functionality:
1) Multi-port NTB API to have information of possible NT-functions
activated in compliance with available device ports.
2) Memory windows of direct and look up table based address translation
with all possible combinations of BARs setup.
3) Traditional doorbell NTB API.
4) One-on-one messaging NTB API.

There are some IDT PCIe-switch setups, which must be done before any of
the NTB peers started. It can be performed either by system BIOS via IDT
SMBus-slave interface or by pre-initialized IDT PCIe-switch EEPROM:
1) NT-functions of corresponding ports must be activated using SWPARTxCTL
and SWPORTxCTL registers.
2) BAR0 must be configured to expose NT-function configuration registers
map.
3) The rest of the BARs must have at least one memory window configured,
otherwise the driver will just return an error.
Temperature sensor of IDT PCIe-switches can be also optionally activated
by BIOS or EEPROM.
(See IDT documentations for details of how the pre-initialization can be
done)

Changelog v2:
- Fix minor checkpatch.pl issues
- Get rid of obfuscating macros

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/ntb/hw/Kconfig  |1 +
 drivers/ntb/hw/Makefile |1 +
 drivers/ntb/hw/idt/Kconfig  |   31 +
 drivers/ntb/hw/idt/Makefile |1 +
 drivers/ntb/hw/idt/ntb_hw_idt.c | 2628 +++
 drivers/ntb/hw/idt/ntb_hw_idt.h | 1158 +
 6 files changed, 3820 insertions(+)
 create mode 100644 drivers/ntb/hw/idt/Kconfig
 create mode 100644 drivers/ntb/hw/idt/Makefile
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.c
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.h

diff --git a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig
index 7116472..a89243c 100644
--- a/drivers/ntb/hw/Kconfig
+++ b/drivers/ntb/hw/Kconfig
@@ -1,2 +1,3 @@
 source "drivers/ntb/hw/amd/Kconfig"
+source "drivers/ntb/hw/idt/Kconfig"
 source "drivers/ntb/hw/intel/Kconfig"
diff --git a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile
index 532e085..87332c3 100644
--- a/drivers/ntb/hw/Makefile
+++ b/drivers/ntb/hw/Makefile
@@ -1,2 +1,3 @@
 obj-$(CONFIG_NTB_AMD)  += amd/
+obj-$(CONFIG_NTB_IDT)  += idt/
 obj-$(CONFIG_NTB_INTEL)+= intel/
diff --git a/drivers/ntb/hw/idt/Kconfig b/drivers/ntb/hw/idt/Kconfig
new file mode 100644
index 000..b360e56
--- /dev/null
+++ b/drivers/ntb/hw/idt/Kconfig
@@ -0,0 +1,31 @@
+config NTB_IDT
+   tristate "IDT PCIe-switch Non-Transparent Bridge support"
+   depends on PCI
+   help
+This driver supports NTB of cappable IDT PCIe-switches.
+
+Some of the pre-initializations must be made before IDT PCIe-switch
+exposes it NT-functions correctly. It should be done by either proper
+initialisation of EEPROM connected to master smbus of the switch or
+by BIOS using slave-SMBus interface changing corresponding registers
+value. Evidently it must be done before PCI bus enumeration is
+finished in Linux kernel.
+
+First of all partitions must be activated and properly assigned to all
+the ports with NT-functions intended to be activated (see SWPARTxCTL
+and SWPORTxCTL registers). Then all NT-function BARs must be enabled
+with chosen valid aperture. For memory windows related BARs the
+aperture settings shall determine the maximum size of memory windows
+accepted by a BAR. Note that BAR0 must map PCI configuration space
+registers.
+
+It's worth to note, that since a part of this driver relies on the
+BAR settings of peer NT-functions, the BAR setups can't be done over
+kernel PCI fixups. That's why the alternative pre-initialization
+techniques like BIOS using SMBus interface or EEPROM should be
+utilized. Additionally if one needs to have temperature sensor
+information printed to system log, the corresponding registers must
+be initialized within BIOS/EEPROM as well.
+
+If unsure, say N.
+
diff --git a/drivers/ntb/hw/idt/Makefile b/drivers/ntb/hw/idt/Makefile
new file mode 100644
index 000..a102cf1
--- /dev/null
+++ b/drivers/ntb/hw/idt/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_NTB_IDT) += ntb_hw_idt.o
diff --git a/drivers/ntb/hw/idt/ntb_hw_idt.c b/drivers/ntb/hw/idt/ntb_hw_idt.c
new file mode 100644
index 000..68ece5a
--- /dev/null
+++ b/drivers/ntb/hw/idt/ntb_hw_idt.c
@@ -0,0 +1,2628 @@
+/*
+ *   This file is provided under a GPLv2 license.  When using or
+ *   redistributing this file, you may do so under that license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright (C) 2016 T-Platforms All R

[PATCH v3] NTB: Add IDT 89HPESxNTx PCIe-switches support

[Serge Semin]

IDT 89HPESxNTx device series is PCIe-switches, which support
Non-Transparent bridging between domains connected to the device ports.
Since new NTB API exposes multi-port interface and messaging API, the IDT
NT-functions can be now supported in the kernel. This driver adds the
following functionality:
1) Multi-port NTB API to have information of possible NT-functions
activated in compliance with available device ports.
2) Memory windows of direct and look up table based address translation
with all possible combinations of BARs setup.
3) Traditional doorbell NTB API.
4) One-on-one messaging NTB API.

There are some IDT PCIe-switch setups, which must be done before any of
the NTB peers started. It can be performed either by system BIOS via IDT
SMBus-slave interface or by pre-initialized IDT PCIe-switch EEPROM:
1) NT-functions of corresponding ports must be activated using SWPARTxCTL
and SWPORTxCTL registers.
2) BAR0 must be configured to expose NT-function configuration registers
map.
3) The rest of the BARs must have at least one memory window configured,
otherwise the driver will just return an error.
Temperature sensor of IDT PCIe-switches can be also optionally activated
by BIOS or EEPROM.
(See IDT documentations for details of how the pre-initialization can be
done)

Changelog v2:
- Fix minor checkpatch.pl issues
- Get rid of obfuscating macros

Changelog V3:
- No write to registers if address is either out of bound or unaligned
- Fix idt_reg_set_bits()/idt_reg_clear_bits() methods race condition
- Fix invalid argument of write method called from
idt_reg_set_bits()/idt_reg_clear_bits() functions
- Add appropriate naming of function idt_get_mw_size()
- Fix some documentation notes
- Replace symbolic permission S_IRUSR with octal 0400

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/ntb/hw/Kconfig  |1 +
 drivers/ntb/hw/Makefile |1 +
 drivers/ntb/hw/idt/Kconfig  |   31 +
 drivers/ntb/hw/idt/Makefile |1 +
 drivers/ntb/hw/idt/ntb_hw_idt.c | 2653 +++
 drivers/ntb/hw/idt/ntb_hw_idt.h | 1162 +
 6 files changed, 3849 insertions(+)
 create mode 100644 drivers/ntb/hw/idt/Kconfig
 create mode 100644 drivers/ntb/hw/idt/Makefile
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.c
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.h

diff --git a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig
index 7116472..a89243c 100644
--- a/drivers/ntb/hw/Kconfig
+++ b/drivers/ntb/hw/Kconfig
@@ -1,2 +1,3 @@
 source "drivers/ntb/hw/amd/Kconfig"
+source "drivers/ntb/hw/idt/Kconfig"
 source "drivers/ntb/hw/intel/Kconfig"
diff --git a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile
index 532e085..87332c3 100644
--- a/drivers/ntb/hw/Makefile
+++ b/drivers/ntb/hw/Makefile
@@ -1,2 +1,3 @@
 obj-$(CONFIG_NTB_AMD)  += amd/
+obj-$(CONFIG_NTB_IDT)  += idt/
 obj-$(CONFIG_NTB_INTEL)+= intel/
diff --git a/drivers/ntb/hw/idt/Kconfig b/drivers/ntb/hw/idt/Kconfig
new file mode 100644
index 000..b360e56
--- /dev/null
+++ b/drivers/ntb/hw/idt/Kconfig
@@ -0,0 +1,31 @@
+config NTB_IDT
+   tristate "IDT PCIe-switch Non-Transparent Bridge support"
+   depends on PCI
+   help
+This driver supports NTB of cappable IDT PCIe-switches.
+
+Some of the pre-initializations must be made before IDT PCIe-switch
+exposes it NT-functions correctly. It should be done by either proper
+initialisation of EEPROM connected to master smbus of the switch or
+by BIOS using slave-SMBus interface changing corresponding registers
+value. Evidently it must be done before PCI bus enumeration is
+finished in Linux kernel.
+
+First of all partitions must be activated and properly assigned to all
+the ports with NT-functions intended to be activated (see SWPARTxCTL
+and SWPORTxCTL registers). Then all NT-function BARs must be enabled
+with chosen valid aperture. For memory windows related BARs the
+aperture settings shall determine the maximum size of memory windows
+accepted by a BAR. Note that BAR0 must map PCI configuration space
+registers.
+
+It's worth to note, that since a part of this driver relies on the
+BAR settings of peer NT-functions, the BAR setups can't be done over
+kernel PCI fixups. That's why the alternative pre-initialization
+techniques like BIOS using SMBus interface or EEPROM should be
+utilized. Additionally if one needs to have temperature sensor
+information printed to system log, the corresponding registers must
+be initialized within BIOS/EEPROM as well.
+
+If unsure, say N.
+
diff --git a/drivers/ntb/hw/idt/Makefile b/drivers/ntb/hw/idt/Makefile
new file mode 100644
index 000..a102cf1
--- /dev/null
+++ b/drivers/ntb/hw/idt/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_NTB_IDT) += ntb_hw_idt.o
diff --git a/dr

Re: [PATCH v2 1/3] ntb: Add asynchronous devices support to NTB-bus interface

[Serge Semin]

 of MSGSTSMSK.
ntb_msg_count(ntb); - number of message registers
ntb_msg_recv(ntb, idx, msg, src_port); - read a message register of the 
corresponding
index and the source port of data it retrieved.
ntb_msg_send(ntb, idx, msg, target_port); - send a message to the corresponding 
port.

3) IDT driver redevelopment will take a lot of time, since I don't have much 
free time to do
it. It may be half of year or even more.

>From my side, such an improvement will significantly complicate the NTB Kernel 
>API. Since you
are the subsystem maintainer it's your decision which design to choose, but I 
don't think I'll do
the IDT driver suitable for this design anytime soon.

Regards,
-Sergey


On Fri, Aug 19, 2016 at 12:56:04AM +0300, Serge Semin <fancer.lan...@gmail.com> 
wrote:
> Hello Allen,
> Sorry for the delay with response and thanks for thoughtful review.
> 
> On Mon, Aug 08, 2016 at 05:48:42PM -0400, Allen Hubbe <allen.hu...@emc.com> 
> wrote:
> > From: Serge Semin
> > > Hello Allen.
> > > 
> > > Thanks for your careful review. Going through this mailing thread I hope 
> > > we'll come up
> > > with solutions, which improve the driver code as well as extend the Linux 
> > > kernel support
> > > of new devices like IDT PCIe-swtiches.
> > > 
> > > Before getting to the inline commentaries I need to give some 
> > > introduction to the IDT NTB-
> > > related hardware so we could speak on the same language. Additionally 
> > > I'll give a brief
> > > explanation how the setup of memory windows works in IDT PCIe-switches.
> > 
> > I found this to use as a reference for IDT:
> > https://www.idt.com/document/man/89hpes24nt24g2-device-user-manual
> 
> Yes, it's supported by the IDT driver, although I am using a device with 
> lesser number of ports:
> https://www.idt.com/document/man/89hpes32nt8ag2-device-user-manual
> 
> > 
> > > First of all, before getting into the IDT NTB driver development I had 
> > > made a research of
> > > the currently developed NTB kernel API and AMD/Intel hardware drivers. 
> > > Due to lack of the
> > > hardware manuals It might be not in deep details, but I understand how 
> > > the AMD/Intel NTB-
> > > hardware drivers work. At least I understand the concept of memory 
> > > windowing, which led to
> > > the current NTB bus kernel API.
> > > 
> > > So lets get to IDT PCIe-switches. There is a whole series of NTB-related 
> > > switches IDT
> > > produces. All of them I split into two distinct groups:
> > > 1) Two NTB-ported switches (models 89PES8NT2, 89PES16NT2, 89PES12NT3, 
> > > 89PES124NT3),
> > > 2) Multi NTB-ported switches (models 89HPES24NT6AG2, 89HPES32NT8AG2, 
> > > 89HPES32NT8BG2,
> > > 89HPES12NT12G2, 89HPES16NT16G2, 89HPES24NT24G2, 89HPES32NT24AG2, 
> > > 89HPES32NT24BG2).
> > > Just to note all of these switches are a part of IDT PRECISE(TM) family 
> > > of PCI Express®
> > > switching solutions. Why do I split them up? Because of the next reasons:
> > > 1) Number of upstream ports, which have access to NTB functions 
> > > (obviously, yeah? =)). So
> > > the switches of the first group can connect just two domains over NTB. 
> > > Unlike the second
> > > group of switches, which expose a way to setup an interaction between 
> > > several PCIe-switch
> > > ports, which have NT-function activated.
> > > 2) The groups are significantly distinct by the way of NT-functions 
> > > configuration.
> > > 
> > > Before getting further, I should note, that the uploaded driver supports 
> > > the second group
> > > of devices only. But still I'll give a comparative explanation, since the 
> > > first group of
> > > switches is very similar to the AMD/Intel NTBs.
> > > 
> > > Lets dive into the configurations a bit deeper. Particularly NT-functions 
> > > of the first
> > > group of switches can be configured the same way as AMD/Intel 
> > > NTB-functions are. There is
> > > an PCIe end-point configuration space, which fully reflects the 
> > > cross-coupled local and
> > > peer PCIe/NTB settings. So local Root complex can set any of the peer 
> > > registers by direct
> > > writing to mapped memory. Here is the image, which perfectly explains the 
> > > configuration
> > > registers mapping:
> > > https://s8.postimg.org/3nhkzqfxx/IDT_NTB_old_configspace.png
> > > Since the first group switches connect only two root complexes, the 

Re: [PATCH v2 1/3] ntb: Add asynchronous devices support to NTB-bus interface

[Serge Semin]

Hello Allen,
Sorry for the delay with response and thanks for thoughtful review.

On Mon, Aug 08, 2016 at 05:48:42PM -0400, Allen Hubbe <allen.hu...@emc.com> 
wrote:
> From: Serge Semin
> > Hello Allen.
> > 
> > Thanks for your careful review. Going through this mailing thread I hope 
> > we'll come up
> > with solutions, which improve the driver code as well as extend the Linux 
> > kernel support
> > of new devices like IDT PCIe-swtiches.
> > 
> > Before getting to the inline commentaries I need to give some introduction 
> > to the IDT NTB-
> > related hardware so we could speak on the same language. Additionally I'll 
> > give a brief
> > explanation how the setup of memory windows works in IDT PCIe-switches.
> 
> I found this to use as a reference for IDT:
> https://www.idt.com/document/man/89hpes24nt24g2-device-user-manual

Yes, it's supported by the IDT driver, although I am using a device with lesser 
number of ports:
https://www.idt.com/document/man/89hpes32nt8ag2-device-user-manual

> 
> > First of all, before getting into the IDT NTB driver development I had made 
> > a research of
> > the currently developed NTB kernel API and AMD/Intel hardware drivers. Due 
> > to lack of the
> > hardware manuals It might be not in deep details, but I understand how the 
> > AMD/Intel NTB-
> > hardware drivers work. At least I understand the concept of memory 
> > windowing, which led to
> > the current NTB bus kernel API.
> > 
> > So lets get to IDT PCIe-switches. There is a whole series of NTB-related 
> > switches IDT
> > produces. All of them I split into two distinct groups:
> > 1) Two NTB-ported switches (models 89PES8NT2, 89PES16NT2, 89PES12NT3, 
> > 89PES124NT3),
> > 2) Multi NTB-ported switches (models 89HPES24NT6AG2, 89HPES32NT8AG2, 
> > 89HPES32NT8BG2,
> > 89HPES12NT12G2, 89HPES16NT16G2, 89HPES24NT24G2, 89HPES32NT24AG2, 
> > 89HPES32NT24BG2).
> > Just to note all of these switches are a part of IDT PRECISE(TM) family of 
> > PCI Express®
> > switching solutions. Why do I split them up? Because of the next reasons:
> > 1) Number of upstream ports, which have access to NTB functions (obviously, 
> > yeah? =)). So
> > the switches of the first group can connect just two domains over NTB. 
> > Unlike the second
> > group of switches, which expose a way to setup an interaction between 
> > several PCIe-switch
> > ports, which have NT-function activated.
> > 2) The groups are significantly distinct by the way of NT-functions 
> > configuration.
> > 
> > Before getting further, I should note, that the uploaded driver supports 
> > the second group
> > of devices only. But still I'll give a comparative explanation, since the 
> > first group of
> > switches is very similar to the AMD/Intel NTBs.
> > 
> > Lets dive into the configurations a bit deeper. Particularly NT-functions 
> > of the first
> > group of switches can be configured the same way as AMD/Intel NTB-functions 
> > are. There is
> > an PCIe end-point configuration space, which fully reflects the 
> > cross-coupled local and
> > peer PCIe/NTB settings. So local Root complex can set any of the peer 
> > registers by direct
> > writing to mapped memory. Here is the image, which perfectly explains the 
> > configuration
> > registers mapping:
> > https://s8.postimg.org/3nhkzqfxx/IDT_NTB_old_configspace.png
> > Since the first group switches connect only two root complexes, the race 
> > condition of
> > read/write operations to cross-coupled registers can be easily resolved 
> > just by roles
> > distribution. So local root complex sets the translated base address 
> > directly to a peer
> > configuration space registers, which correspond to BAR0-BAR3 locally mapped 
> > memory
> > windows. Of course 2-4 memory windows is enough to connect just two 
> > domains. That's why
> > you made the NTB bus kernel API the way it is.
> > 
> > The things get different when one wants to have an access from one domain 
> > to multiple
> > coupling up to eight root complexes in the second group of switches. First 
> > of all the
> > hardware doesn't support the configuration space cross-coupling anymore. 
> > Instead there are
> > two Global Address Space Access registers provided to have an access to a 
> > peers
> > configuration space. In fact it is not a big problem, since there are no 
> > much differences
> > in accessing registers over a memory mapped space or a pair of fixed 
> > Address/Data
> > registers. Th

[PATCH v4] NTB: Add IDT 89HPESxNTx PCIe-switches support

[Serge Semin]

IDT 89HPESxNTx device series is PCIe-switches, which support
Non-Transparent bridging between domains connected to the device ports.
Since new NTB API exposes multi-port interface and messaging API, the IDT
NT-functions can be now supported in the kernel. This driver adds the
following functionality:
1) Multi-port NTB API to have information of possible NT-functions
activated in compliance with available device ports.
2) Memory windows of direct and look up table based address translation
with all possible combinations of BARs setup.
3) Traditional doorbell NTB API.
4) One-on-one messaging NTB API.

There are some IDT PCIe-switch setups, which must be done before any of
the NTB peers started. It can be performed either by system BIOS via IDT
SMBus-slave interface or by pre-initialized IDT PCIe-switch EEPROM:
1) NT-functions of corresponding ports must be activated using SWPARTxCTL
and SWPORTxCTL registers.
2) BAR0 must be configured to expose NT-function configuration registers
map.
3) The rest of the BARs must have at least one memory window configured,
otherwise the driver will just return an error.
Temperature sensor of IDT PCIe-switches can be also optionally activated
by BIOS or EEPROM.
(See IDT documentations for details of how the pre-initialization can be
done)

Changelog v2:
- Fix minor checkpatch.pl issues
- Get rid of obfuscating macros

Changelog v3:
- No write to registers if address is either out of bound or unaligned
- Fix idt_reg_set_bits()/idt_reg_clear_bits() methods race condition
- Fix invalid argument of write method called from
idt_reg_set_bits()/idt_reg_clear_bits() functions
- Add appropriate naming of function idt_get_mw_size()
- Fix some documentation notes
- Replace symbolic permission S_IRUSR with octal 0400

Changelog v4:
- Return ~0 on read from registers with invalid address
- Don't check bits validity on registers bits clearing
- Keep up driver loading (just print a warning) if there is no any peer
NTBs found
- Fix unnecessary branching logic
- Fix some documentation notes

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
Acked-by: Allen Hubbe <allen.hu...@dell.com>

---
 drivers/ntb/hw/Kconfig  |1 +
 drivers/ntb/hw/Makefile |1 +
 drivers/ntb/hw/idt/Kconfig  |   31 +
 drivers/ntb/hw/idt/Makefile |1 +
 drivers/ntb/hw/idt/ntb_hw_idt.c | 2647 +++
 drivers/ntb/hw/idt/ntb_hw_idt.h | 1162 +
 6 files changed, 3843 insertions(+)
 create mode 100644 drivers/ntb/hw/idt/Kconfig
 create mode 100644 drivers/ntb/hw/idt/Makefile
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.c
 create mode 100644 drivers/ntb/hw/idt/ntb_hw_idt.h

diff --git a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig
index 7116472..a89243c 100644
--- a/drivers/ntb/hw/Kconfig
+++ b/drivers/ntb/hw/Kconfig
@@ -1,2 +1,3 @@
 source "drivers/ntb/hw/amd/Kconfig"
+source "drivers/ntb/hw/idt/Kconfig"
 source "drivers/ntb/hw/intel/Kconfig"
diff --git a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile
index 532e085..87332c3 100644
--- a/drivers/ntb/hw/Makefile
+++ b/drivers/ntb/hw/Makefile
@@ -1,2 +1,3 @@
 obj-$(CONFIG_NTB_AMD)  += amd/
+obj-$(CONFIG_NTB_IDT)  += idt/
 obj-$(CONFIG_NTB_INTEL)+= intel/
diff --git a/drivers/ntb/hw/idt/Kconfig b/drivers/ntb/hw/idt/Kconfig
new file mode 100644
index 000..b360e56
--- /dev/null
+++ b/drivers/ntb/hw/idt/Kconfig
@@ -0,0 +1,31 @@
+config NTB_IDT
+   tristate "IDT PCIe-switch Non-Transparent Bridge support"
+   depends on PCI
+   help
+This driver supports NTB of cappable IDT PCIe-switches.
+
+Some of the pre-initializations must be made before IDT PCIe-switch
+exposes it NT-functions correctly. It should be done by either proper
+initialisation of EEPROM connected to master smbus of the switch or
+by BIOS using slave-SMBus interface changing corresponding registers
+value. Evidently it must be done before PCI bus enumeration is
+finished in Linux kernel.
+
+First of all partitions must be activated and properly assigned to all
+the ports with NT-functions intended to be activated (see SWPARTxCTL
+and SWPORTxCTL registers). Then all NT-function BARs must be enabled
+with chosen valid aperture. For memory windows related BARs the
+aperture settings shall determine the maximum size of memory windows
+accepted by a BAR. Note that BAR0 must map PCI configuration space
+registers.
+
+It's worth to note, that since a part of this driver relies on the
+BAR settings of peer NT-functions, the BAR setups can't be done over
+kernel PCI fixups. That's why the alternative pre-initialization
+techniques like BIOS using SMBus interface or EEPROM should be
+utilized. Additionally if one needs to have temperature sensor
+information printed to system log, the corresponding regis

[RFC] PCI: Fix kernel panic of root-port-less PCIe enum due to ASPM

[Serge Semin]

Hello linux folks,

Sometime ago I discovered a kernel panic popping up when PCI subsystem was
trying to enumerate PCI express bus with ASPM service enabled. Here it is:

[5.089667] CPU 0 Unable to handle kernel paging request at virtual
address 0060, epc == 80317004, ra == 80316ac8
[5.120952] Oops[#1]:
  ...
[5.528438] Call Trace:
[5.535640] [<80317004>] pcie_aspm_init_link_state+0x6c0/0x814
[5.552843] [<80300c44>] pci_scan_slot+0x140/0x148
[5.566957] [<80301dcc>] pci_scan_child_bus+0x50/0x1b0
[5.582096] [<80301944>] pci_scan_bridge+0x25c/0x694
[5.596724] [<80301e78>] pci_scan_child_bus+0xfc/0x1b0
[5.611862] [<80301944>] pci_scan_bridge+0x25c/0x694
[5.626488] [<80301e78>] pci_scan_child_bus+0xfc/0x1b0
[5.641628] [<8030215c>] pci_scan_root_bus+0x64/0x124
[5.656528] [<804ca298>] pcibios_scanbus+0xa8/0x188

I more than sure you are familiar with the issue, since I've found the
mailing discussion: "PCI: avoid NULL deref in alloc_pcie_link_state"
https://patchwork.kernel.org/patch/2751651/
https://bugzilla.kernel.org/show_bug.cgi?id=60111

You closed the bugzilla ticket with the next statement:
"I'm closing this as invalid because the simulated machine where the problem
occurs has an invalid PCIe topology (an Upstream Port with no Downstream Port
or Root Port above it).  As far as I know, there is no valid topology, e.g.,
a real hardware machine in the field, that would cause this failure."

I'm strongly disagree with it, since I've got at least two hardware with
PCIe-bus hierarchy as described in the mailing list. One of them is based on
Cavium Octeon III CN7020. Here is a ASCII-diagram of PCIe-bus:

-+-[:01]---00.0-[02-06]--+-02.0-[03-05]--+-00.0-[04-05]00.0-[05]--
 |   |   \-00.1  Device [111d:808f]
 |   \-04.0-[06]00.0  Device [126f:0750]
 \-[:00]-

where 01:00.0 is an Upstream port of IDT PCIe-swtich.
/ # /usr/local/sbin/lspci -v -s 01:00.0
01:00.0 Class 0604: Device 111d:8061
Flags: bus master, fast devsel, latency 0
Memory at  (32-bit, non-prefetchable) [size=2]
Memory at  (32-bit, non-prefetchable) [size=2]
Bus: primary=01, secondary=02, subordinate=06, sec-latency=0
Memory behind bridge: 0800-0dff
Expansion ROM at  [disabled] [size=2]
Capabilities: [40] Express Upstream Port, MSI 00
Capabilities: [c0] Power Management version 3
Capabilities: [100] Advanced Error Reporting
Capabilities: [200] Virtual Channel
Kernel driver in use: pcieport

As you can see PCI-bus hierarchy doesn't have root port and the very first
upstream port is directly connected to Host-PCIe bridge of MCU, which of
course is not listed by the lspci utility.

Despite of Radim Kr?má?, who suggested a fix, which would de-facto just
turned ASPM off, I found a quick solution, which disabled ASPM only in 
the first link (Host-PCIe=>Upstream port) of PCIe-bus for such hierarchy.
ASPM for other PCIe-bus topologies shall work the way it was.

I hope the fix will be helpful.
Thanks,

=
Serge V. Semin
Leading Programmer
Embedded SW development group
T-platforms
=

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/pci/pcie/aspm.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index 0ec649d..a9295f29 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -522,7 +522,8 @@ static struct pcie_link_state *alloc_pcie_link_state(struct 
pci_dev *pdev)
INIT_LIST_HEAD(>children);
INIT_LIST_HEAD(>link);
link->pdev = pdev;
-   if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) {
+   if ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) &&
+   (!pci_is_root_bus(pdev->bus->parent))) {
struct pcie_link_state *parent;
parent = pdev->bus->parent->self->link_state;
if (!parent) {
-- 
2.6.6

Re: [RFC] PCI: Fix kernel panic of root-port-less PCIe enum due to ASPM

[Serge Semin]

On Thu, Oct 06, 2016 at 08:13:58AM -0500, Bjorn Helgaas <helg...@kernel.org> 
wrote:
> Hi Serge,
> 
> On Thu, Oct 06, 2016 at 12:34:15PM +0300, Serge Semin wrote:
> > Hello linux folks,
> > 
> > Sometime ago I discovered a kernel panic popping up when PCI subsystem 
> > was
> > trying to enumerate PCI express bus with ASPM service enabled. Here it is:
> > 
> > [5.089667] CPU 0 Unable to handle kernel paging request at virtual
> > address 0060, epc == 80317004, ra == 80316ac8
> > [5.120952] Oops[#1]:
> >   ...
> > [5.528438] Call Trace:
> > [5.535640] [<80317004>] pcie_aspm_init_link_state+0x6c0/0x814
> > [5.552843] [<80300c44>] pci_scan_slot+0x140/0x148
> > [5.566957] [<80301dcc>] pci_scan_child_bus+0x50/0x1b0
> > [5.582096] [<80301944>] pci_scan_bridge+0x25c/0x694
> > [5.596724] [<80301e78>] pci_scan_child_bus+0xfc/0x1b0
> > [5.611862] [<80301944>] pci_scan_bridge+0x25c/0x694
> > [5.626488] [<80301e78>] pci_scan_child_bus+0xfc/0x1b0
> > [5.641628] [<8030215c>] pci_scan_root_bus+0x64/0x124
> > [5.656528] [<804ca298>] pcibios_scanbus+0xa8/0x188
> > 
> > I more than sure you are familiar with the issue, since I've found the
> > mailing discussion: "PCI: avoid NULL deref in alloc_pcie_link_state"
> > https://patchwork.kernel.org/patch/2751651/
> > https://bugzilla.kernel.org/show_bug.cgi?id=60111
> > 
> > You closed the bugzilla ticket with the next statement:
> > "I'm closing this as invalid because the simulated machine where the problem
> > occurs has an invalid PCIe topology (an Upstream Port with no Downstream 
> > Port
> > or Root Port above it).  As far as I know, there is no valid topology, e.g.,
> > a real hardware machine in the field, that would cause this failure."
> > 
> > I'm strongly disagree with it, since I've got at least two hardware with
> > PCIe-bus hierarchy as described in the mailing list. One of them is based on
> > Cavium Octeon III CN7020. Here is a ASCII-diagram of PCIe-bus:
> 
> Thanks for this information.  I reopened that bugzilla; can you attach
> complete dmesg logs and "lspci -vv" output for your systems?  As I
> mentioned in comment #4, I'm completely open to fixing this.  My
> objections at the time were (1) there was no known hardware that could
> trigger the problem, and (2) the proposed fix was ugly and prone to
> future breakage.  Since we now have real systems that trip over this,
> we need to revisit it.
> 
> Bjorn
> 

Done. Welcome back to the bugzilla thread.

-Serge

> > -+-[:01]---00.0-[02-06]--+-02.0-[03-05]--+-00.0-[04-05]00.0-[05]--
> >  |   |   \-00.1  Device [111d:808f]
> >  |   \-04.0-[06]00.0  Device [126f:0750]
> >  \-[:00]-
> > 
> > where 01:00.0 is an Upstream port of IDT PCIe-swtich.
> > / # /usr/local/sbin/lspci -v -s 01:00.0
> > 01:00.0 Class 0604: Device 111d:8061
> > Flags: bus master, fast devsel, latency 0
> > Memory at  (32-bit, non-prefetchable) [size=2]
> > Memory at  (32-bit, non-prefetchable) [size=2]
> > Bus: primary=01, secondary=02, subordinate=06, sec-latency=0
> > Memory behind bridge: 0800-0dff
> > Expansion ROM at  [disabled] [size=2]
> > Capabilities: [40] Express Upstream Port, MSI 00
> > Capabilities: [c0] Power Management version 3
> > Capabilities: [100] Advanced Error Reporting
> > Capabilities: [200] Virtual Channel
> > Kernel driver in use: pcieport
> > 
> > As you can see PCI-bus hierarchy doesn't have root port and the very first
> > upstream port is directly connected to Host-PCIe bridge of MCU, which of
> > course is not listed by the lspci utility.
> > 
> > Despite of Radim Kr?má?, who suggested a fix, which would de-facto just
> > turned ASPM off, I found a quick solution, which disabled ASPM only in 
> > the first link (Host-PCIe=>Upstream port) of PCIe-bus for such hierarchy.
> > ASPM for other PCIe-bus topologies shall work the way it was.
> > 
> > I hope the fix will be helpful.
> > Thanks,
> > 
> > =
> > Serge V. Semin
> > Leading Programmer
> > Embedded SW development group
> > T-platforms
> > =
> > 
> > Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
> > 
> > ---
> >  drivers/pci/pcie/aspm.c | 3 ++-
> >

[PATCH] Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

Hello linux folks,

This driver primarily is developed to give an access to EEPROM of IDT
PCIe-switches. Such switches provide a simple SMBus interface to perform
IO-operations from/to EEPROM, which is located at private (also called master)
SMBus of the switches. Using that interface this driver creates a simple
binary sysfs-file in the device directory:
/sys/bus/i2c/devices/-/eeprom
In case if read-only flag is specified in the device dts-node, user-space
applications won't be able to write to the EEPROM sysfs-node.

Additionally IDT 89HPESx SMBus interface provides an ability to write/read
data of device CSRs. This driver exposes corresponding sysfs-file to perform
simple IO operations using that facility for just basic debug purpose.
Particularly next file is created in the device specific sysfs-directory:
/sys/bus/i2c/devices/-/csr
Format of the sysfs-node is:
$ cat /sys/bus/i2c/devices/-/csr;
:
So reading the content of the sysfs-file gives current CSR address and
it value. If user-space application wishes to change current CSR address,
it can just write a proper value to the sysfs-file:
$ echo "" > /sys/bus/i2c/devices/-/csr
If it wants to change the CSR value as well, the format of the write
operation is:
$ echo ":" > \
/sys/bus/i2c/devices/-/csr;
CSR address and value can be any of hexadecimal, decimal or octal format.

The driver supports the most of the commonly available SMBus operations:
SMBus i2c block, SMBus block, smbus word and byte. The code has been tested
to be built for x32/x64 MIPS architecture and tested on the x32 MIPS machine.
The patch was applied on top of commit c6935931c1894ff857616ff8549b61236a19148f
of master branch of repository
git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git


Thanks,

=
Serge V. Semin
Leading Programmer
Embedded SW development group
T-platforms
=====

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 .../devicetree/bindings/misc/idt_89hpesx.txt   |   41 +
 drivers/misc/eeprom/Kconfig|   10 +
 drivers/misc/eeprom/Makefile   |1 +
 drivers/misc/eeprom/idt_89hpesx.c  | 1483 
 4 files changed, 1535 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt
 create mode 100644 drivers/misc/eeprom/idt_89hpesx.c

diff --git a/Documentation/devicetree/bindings/misc/idt_89hpesx.txt 
b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
new file mode 100644
index 000..469cc93
--- /dev/null
+++ b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
@@ -0,0 +1,41 @@
+EEPROM / CSR SMBus-slave interface of IDT 89HPESx devices
+
+Required properties:
+  - compatible : should be ","
+Basically there is only one manufacturer: idt, but some
+compatible devices may be produced in future. Following devices
+are supported: 89hpes8nt2, 89hpes12nt3, 89hpes24nt6ag2,
+89hpes32nt8ag2, 89hpes32nt8bg2, 89hpes12nt12g2, 89hpes16nt16g2,
+89hpes24nt24g2, 89hpes32nt24ag2, 89hpes32nt24bg2;
+89hpes12n3, 89hpes12n3a, 89hpes24n3, 89hpes24n3a;
+89hpes32h8, 89hpes32h8g2, 89hpes48h12, 89hpes48h12g2,
+89hpes48h12ag2, 89hpes16h16, 89hpes22h16, 89hpes22h16g2,
+89hpes34h16, 89hpes34h16g2, 89hpes64h16, 89hpes64h16g2,
+89hpes64h16ag2;
+89hpes12t3g2, 89hpes24t3g2, 89hpes16t4, 89hpes4t4g2,
+89hpes10t4g2, 89hpes16t4g2, 89hpes16t4ag2, 89hpes5t5,
+89hpes6t5, 89hpes8t5, 89hpes8t5a, 89hpes24t6, 89hpes6t6g2,
+89hpes24t6g2, 89hpes16t7, 89hpes32t8, 89hpes32t8g2,
+89hpes48t12, 89hpes48t12g2.
+Current implementation of the driver doesn't have any device-
+specific functionalities. But since each of them differs
+by registers mapping, CSRs read/write restrictions can be
+added in future.
+  - reg :   I2C address of the IDT 89HPES device.
+
+Optional properties:
+  - read-only : Parameterless property disables writes to the EEPROM
+  - idt,eesize : Size of EEPROM device connected to IDT 89HPES i2c-master bus
+(default value is 4096 bytes if option isn't specified)
+  - idt,eeaddr : Custom address of EEPROM device
+(If not specified IDT 89HPESx device will try to communicate
+ with EEPROM sited by default address - 0x50)
+
+Example:
+   idt_pcie_sw@60 {
+   compatible = "idt,89hpes12nt3";
+   reg = <0x60>;
+   read-only;
+   idt,eesize = <65536>;
+   idt,eeaddr = <0x50>;
+   };
diff --git a/drivers/misc/eeprom/Kconfig b/drivers/misc/eeprom/Kconfig
index c4e41c2..de58762 100644
--- a/drivers/misc/eeprom

Re: [RFC] PCI: Fix kernel panic of root-port-less PCIe enum due to ASPM

[Serge Semin]

On Tue, Nov 08, 2016 at 05:29:57PM -0600, Bjorn Helgaas <helg...@kernel.org> 
wrote:

Hello Bjorn,
Here are the answers on your questions inlined in the text.

> Hi Serge,
> 
> On Thu, Oct 06, 2016 at 12:34:15PM +0300, Serge Semin wrote:
> > Hello linux folks,
> > 
> > Sometime ago I discovered a kernel panic popping up when PCI subsystem 
> > was
> > trying to enumerate PCI express bus with ASPM service enabled. Here it is:
> > 
> > [5.089667] CPU 0 Unable to handle kernel paging request at virtual
> > address 0060, epc == 80317004, ra == 80316ac8
> > [5.120952] Oops[#1]:
> >   ...
> > [5.528438] Call Trace:
> > [5.535640] [<80317004>] pcie_aspm_init_link_state+0x6c0/0x814
> > [5.552843] [<80300c44>] pci_scan_slot+0x140/0x148
> > [5.566957] [<80301dcc>] pci_scan_child_bus+0x50/0x1b0
> > [5.582096] [<80301944>] pci_scan_bridge+0x25c/0x694
> > [5.596724] [<80301e78>] pci_scan_child_bus+0xfc/0x1b0
> > [5.611862] [<80301944>] pci_scan_bridge+0x25c/0x694
> > [5.626488] [<80301e78>] pci_scan_child_bus+0xfc/0x1b0
> > [5.641628] [<8030215c>] pci_scan_root_bus+0x64/0x124
> > [5.656528] [<804ca298>] pcibios_scanbus+0xa8/0x188
> > 
> > I more than sure you are familiar with the issue, since I've found the
> > mailing discussion: "PCI: avoid NULL deref in alloc_pcie_link_state"
> > https://patchwork.kernel.org/patch/2751651/
> > https://bugzilla.kernel.org/show_bug.cgi?id=60111
> 
> I'm trying to puzzle out a few things here.  Maybe you can help me out?
> 
> - Does this issue exist in current upstream kernels?  Your dmesg shows a
>   v3.19-based kernel.  c8fc9339409d ("PCI/ASPM: Use dev->has_secondary_link
>   to find downstream links"), which appeared in v4.2, fixes a problem very
>   similar to what you're reporting.
> 

I saw that fix, but alas it hasn't fixed the issue. I've tested kernel 4.4.24
without my patch applied and the problem with ASPM-related kernel panic still
exists (see the stack-trace above).

> - When we dereference the NULL pointer, which device did we call
>   pcie_aspm_init_link_state() for?
> 

My suggestion was that the problem arised in the framework of bus 2 enumeration.
Since there was no root bus on my architecture, the pci_link_state structure was
not created. So when the algorithm tried to enumerate the second bus, it needed
actual pci_link_state structure of parental bus, which hadn't been created.
That's how the NULL-dereference happened.

> - https://bugzilla.kernel.org/attachment.cgi?id=240981 is the failing dmesg
>   log, and it shows "vgaarb: device added: PCI::04:00.0".
>   
>   Your lspci output (https://bugzilla.kernel.org/attachment.cgi?id=241001)
>   shows 04:00.0 is a downstream port, but vga_arbiter_add_pci_device() only
>   prints that message for VGA class devices.
> 
>   https://bugzilla.kernel.org/attachment.cgi?id=240991, the successful
>   dmesg log, shows "vgaarb: device added: PCI::06:00.0".  That makes
>   more sense because 06:00.0 is class 0300, which is a VGA device.
> 
> Bjorn

I can't be sure about the reason of that strange enumeration. But I can assure
you, that that bus confusion isn't the reason of the ASPM panicing. So I can
just guess, that the misleading BDF can be caused by SMP (I've got a processor
with two cores) and ASPM panic. VGA driver initialization may happen
concurrently with PCI bus enumeration.

Regards,
-Sergey

[PATCH v2 1/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

See cover-letter for changelog

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/misc/eeprom/Kconfig   |   10 +
 drivers/misc/eeprom/Makefile  |1 +
 drivers/misc/eeprom/idt_89hpesx.c | 1577 +
 3 files changed, 1588 insertions(+)
 create mode 100644 drivers/misc/eeprom/idt_89hpesx.c

diff --git a/drivers/misc/eeprom/Kconfig b/drivers/misc/eeprom/Kconfig
index c4e41c2..de58762 100644
--- a/drivers/misc/eeprom/Kconfig
+++ b/drivers/misc/eeprom/Kconfig
@@ -100,4 +100,14 @@ config EEPROM_DIGSY_MTC_CFG
 
  If unsure, say N.
 
+config EEPROM_IDT_89HPESX
+   tristate "IDT 89HPESx PCIe-swtiches EEPROM / CSR support"
+   depends on I2C && SYSFS
+   help
+ Enable this driver to get read/write access to EEPROM / CSRs
+ over IDT PCIe-swtich i2c-slave interface.
+
+ This driver can also be built as a module. If so, the module
+ will be called idt_89hpesx.
+
 endmenu
diff --git a/drivers/misc/eeprom/Makefile b/drivers/misc/eeprom/Makefile
index fc1e81d..90a5262 100644
--- a/drivers/misc/eeprom/Makefile
+++ b/drivers/misc/eeprom/Makefile
@@ -5,3 +5,4 @@ obj-$(CONFIG_EEPROM_MAX6875)+= max6875.o
 obj-$(CONFIG_EEPROM_93CX6) += eeprom_93cx6.o
 obj-$(CONFIG_EEPROM_93XX46)+= eeprom_93xx46.o
 obj-$(CONFIG_EEPROM_DIGSY_MTC_CFG) += digsy_mtc_eeprom.o
+obj-$(CONFIG_EEPROM_IDT_89HPESX) += idt_89hpesx.o
diff --git a/drivers/misc/eeprom/idt_89hpesx.c 
b/drivers/misc/eeprom/idt_89hpesx.c
index 000..00cbbec
--- /dev/null
+++ b/drivers/misc/eeprom/idt_89hpesx.c
@@ -0,0 +1,1577 @@
+/*
+ *   This file is provided under a GPLv2 license.  When using or
+ *   redistributing this file, you may do so under that license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify it
+ *   under the terms and conditions of the GNU General Public License,
+ *   version 2, as published by the Free Software Foundation.
+ *
+ *   This program is distributed in the hope that it will be useful, but 
WITHOUT
+ *   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ *   more details.
+ *
+ *   You should have received a copy of the GNU General Public License along
+ *   with this program; if not, it can be found <http://www.gnu.org/licenses/>.
+ *
+ *   The full GNU General Public License is included in this distribution in
+ *   the file called "COPYING".
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * IDT PCIe-switch NTB Linux driver
+ *
+ * Contact Information:
+ * Serge Semin <fancer.lan...@gmail.com>, <sergey.se...@t-platforms.ru>
+ */
+/*
+ *   NOTE of the IDT 89HPESx SMBus-slave interface driver
+ *This driver primarily is developed to have an access to EEPROM device of
+ * IDT PCIe-switches. IDT provides a simple SMBus interface to perform IO-
+ * operations from/to EEPROM, which is located at private (so called Master)
+ * SMBus of switches. Using that interface this the driver creates a simple
+ * binary sysfs-file in the device directory:
+ * /sys/bus/i2c/devices/-/eeprom
+ * In case if read-only flag is specified in the dts-node of device desription,
+ * User-space applications won't be able to write to the EEPROM sysfs-node.
+ *Additionally IDT 89HPESx SMBus interface has an ability to write/read
+ * data of device CSRs. This driver exposes debugf-file to perform simple IO
+ * operations using that ability for just basic debug purpose. Particularly
+ * next file is created in the specific debugfs-directory:
+ * /sys/kernel/debug/idt_csr/
+ * Format of the debugfs-node is:
+ * $ cat /sys/kernel/debug/idt_csr/-/;
+ * :
+ * So reading the content of the file gives current CSR address and it value.
+ * If User-space application wishes to change current CSR address,
+ * it can just write a proper value to the sysfs-file:
+ * $ echo "" > /sys/kernel/debug/idt_csr/-/
+ * If it wants to change the CSR value as well, the format of the write
+ * operat

[PATCH v2 0/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

Following changes are made in accordance with Greg KH notes as well as
fixing some found issues:
- Get rid of dev_*_idt() macros
- IDT CSR debug file is moved to debugfs
- BIN_ATTR is used to declare sysfs binary attribute
- Moved bindings file to a separate patch
- Need to create a specific bin_attribute structure for each device
- Perform a few read retries with delays if EEPROM is busy

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

Serge Semin (2):
  MISC eeprom: Add IDT 89HPESx EEPROM/CSR driver
  MISC eeprom: Add IDT 89HPESx driver bindings file

 .../devicetree/bindings/misc/idt_89hpesx.txt   |   41 +
 drivers/misc/eeprom/Kconfig|   10 +
 drivers/misc/eeprom/Makefile   |1 +
 drivers/misc/eeprom/idt_89hpesx.c  | 1577 
 4 files changed, 1629 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt
 create mode 100644 drivers/misc/eeprom/idt_89hpesx.c

-- 
2.6.6

[PATCH v2 2/2] eeprom: Add IDT 89HPESx driver bindings file

[Serge Semin]

See cover-letter for changelog

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 .../devicetree/bindings/misc/idt_89hpesx.txt   | 41 ++
 1 file changed, 41 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt

diff --git a/Documentation/devicetree/bindings/misc/idt_89hpesx.txt 
b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
index 000..469cc93
--- /dev/null
+++ b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
@@ -0,0 +1,41 @@
+EEPROM / CSR SMBus-slave interface of IDT 89HPESx devices
+
+Required properties:
+  - compatible : should be ","
+Basically there is only one manufacturer: idt, but some
+compatible devices may be produced in future. Following devices
+are supported: 89hpes8nt2, 89hpes12nt3, 89hpes24nt6ag2,
+89hpes32nt8ag2, 89hpes32nt8bg2, 89hpes12nt12g2, 89hpes16nt16g2,
+89hpes24nt24g2, 89hpes32nt24ag2, 89hpes32nt24bg2;
+89hpes12n3, 89hpes12n3a, 89hpes24n3, 89hpes24n3a;
+89hpes32h8, 89hpes32h8g2, 89hpes48h12, 89hpes48h12g2,
+89hpes48h12ag2, 89hpes16h16, 89hpes22h16, 89hpes22h16g2,
+89hpes34h16, 89hpes34h16g2, 89hpes64h16, 89hpes64h16g2,
+89hpes64h16ag2;
+89hpes12t3g2, 89hpes24t3g2, 89hpes16t4, 89hpes4t4g2,
+89hpes10t4g2, 89hpes16t4g2, 89hpes16t4ag2, 89hpes5t5,
+89hpes6t5, 89hpes8t5, 89hpes8t5a, 89hpes24t6, 89hpes6t6g2,
+89hpes24t6g2, 89hpes16t7, 89hpes32t8, 89hpes32t8g2,
+89hpes48t12, 89hpes48t12g2.
+Current implementation of the driver doesn't have any device-
+specific functionalities. But since each of them differs
+by registers mapping, CSRs read/write restrictions can be
+added in future.
+  - reg :   I2C address of the IDT 89HPES device.
+
+Optional properties:
+  - read-only : Parameterless property disables writes to the EEPROM
+  - idt,eesize : Size of EEPROM device connected to IDT 89HPES i2c-master bus
+(default value is 4096 bytes if option isn't specified)
+  - idt,eeaddr : Custom address of EEPROM device
+(If not specified IDT 89HPESx device will try to communicate
+ with EEPROM sited by default address - 0x50)
+
+Example:
+   idt_pcie_sw@60 {
+   compatible = "idt,89hpes12nt3";
+   reg = <0x60>;
+   read-only;
+   idt,eesize = <65536>;
+   idt,eeaddr = <0x50>;
+   };
-- 
2.6.6

Re: [PATCH] Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

On Sun, Oct 30, 2016 at 09:53:58AM -0400, Greg KH <gre...@linuxfoundation.org> 
wrote:

> On Mon, Oct 03, 2016 at 02:13:45AM +0300, Serge Semin wrote:
> > Hello linux folks,
> > 
> > This driver primarily is developed to give an access to EEPROM of IDT
> > PCIe-switches. Such switches provide a simple SMBus interface to perform
> > IO-operations from/to EEPROM, which is located at private (also called 
> > master)
> > SMBus of the switches. Using that interface this driver creates a simple
> > binary sysfs-file in the device directory:
> > /sys/bus/i2c/devices/-/eeprom
> > In case if read-only flag is specified in the device dts-node, user-space
> > applications won't be able to write to the EEPROM sysfs-node.
> > 
> > Additionally IDT 89HPESx SMBus interface provides an ability to 
> > write/read
> > data of device CSRs. This driver exposes corresponding sysfs-file to perform
> > simple IO operations using that facility for just basic debug purpose.
> 
> If it's for debugging, please put it in debugfs, not in sysfs.  sysfs
> files for one-off drivers is usually discouraged, but at the least, you
> have to document it in a Documentation/ABI/ file.  For debugfs files, we
> don't care, you can do whatever you want in them :)
> 

Undrestood. I'll move it to Debugfs.

> > Particularly next file is created in the device specific sysfs-directory:
> > /sys/bus/i2c/devices/-/csr
> > Format of the sysfs-node is:
> > $ cat /sys/bus/i2c/devices/-/csr;
> > :
> > So reading the content of the sysfs-file gives current CSR address and
> > it value. If user-space application wishes to change current CSR address,
> > it can just write a proper value to the sysfs-file:
> > $ echo "" > /sys/bus/i2c/devices/-/csr
> > If it wants to change the CSR value as well, the format of the write
> > operation is:
> > $ echo ":" > \
> > /sys/bus/i2c/devices/-/csr;
> > CSR address and value can be any of hexadecimal, decimal or octal format.
> 
> Yeah, that all should go into debugfs.
> 

Ok.

> > The driver supports the most of the commonly available SMBus operations:
> > SMBus i2c block, SMBus block, smbus word and byte. The code has been tested
> > to be built for x32/x64 MIPS architecture and tested on the x32 MIPS 
> > machine.
> > The patch was applied on top of commit 
> > c6935931c1894ff857616ff8549b61236a19148f
> > of master branch of repository
> > git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git
> > 
> > 
> > Thanks,
> > 
> > =
> > Serge V. Semin
> > Leading Programmer
> > Embedded SW development group
> > T-platforms
> > =
> > 
> > Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
> 
> meta-comment, the "hello", and "thanks" and signature doesn't need to be
> in a changelog text, next time you can drop that.  See the many kernel
> patches on the mailing lists for examples of how to do this.
> 

Understood.

> 
> > 
> > ---
> >  .../devicetree/bindings/misc/idt_89hpesx.txt   |   41 +
> 
> 
> Can you split this out into a separate file and be sure to cc: the patch
> to the devicetree maintainers?
> 
> >  drivers/misc/eeprom/Kconfig|   10 +
> >  drivers/misc/eeprom/Makefile   |1 +
> >  drivers/misc/eeprom/idt_89hpesx.c  | 1483 
> > 
> >  4 files changed, 1535 insertions(+)
> >  create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> >  create mode 100644 drivers/misc/eeprom/idt_89hpesx.c
> > 
> > diff --git a/Documentation/devicetree/bindings/misc/idt_89hpesx.txt 
> > b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> > new file mode 100644
> > index 000..469cc93
> > --- /dev/null
> > +++ b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> > @@ -0,0 +1,41 @@
> > +EEPROM / CSR SMBus-slave interface of IDT 89HPESx devices
> > +
> > +Required properties:
> > +  - compatible : should be ","
> > +Basically there is only one manufacturer: idt, but some
> > +compatible devices may be produced in future. Following devices
> > +are supported: 89hpes8nt2, 89hpes12nt3, 89hpes24nt6ag2,
> > +89hpes32nt8ag2, 89hpes32nt8bg2, 89hpes12nt12g2, 89hpes16nt16g2,
> > +89hpes24nt24g2, 89hpes32nt24ag2, 89hpes32nt24bg2;
> > +89hpes12n3, 89hpes12n3a, 89hpes24n3, 89hpes24n3a;
> &g

[PATCH v3 1/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

  This driver provides an access to EEPROM of IDT PCIe-switches. IDT PCIe-
switches expose a simple SMBus interface to perform IO-operations from/to
EEPROM, which is located at private (so called Master) SMBus. The driver
creates a simple binary sysfs-file to have an access to the EEPROM using
the SMBus-slave interface in the i2c-device susfs-directory.
  Additionally IDT 89HPESx SMBus interface has an ability to read/write
values of device CSRs. This driver exposes debugfs-file to perform simple
IO-operations using that ability for just basic debug purpose.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/misc/eeprom/Kconfig   |   10 +
 drivers/misc/eeprom/Makefile  |1 +
 drivers/misc/eeprom/idt_89hpesx.c | 1574 +
 3 files changed, 1585 insertions(+)
 create mode 100644 drivers/misc/eeprom/idt_89hpesx.c

diff --git a/drivers/misc/eeprom/Kconfig b/drivers/misc/eeprom/Kconfig
index c4e41c2..de58762 100644
--- a/drivers/misc/eeprom/Kconfig
+++ b/drivers/misc/eeprom/Kconfig
@@ -100,4 +100,14 @@ config EEPROM_DIGSY_MTC_CFG
 
  If unsure, say N.
 
+config EEPROM_IDT_89HPESX
+   tristate "IDT 89HPESx PCIe-swtiches EEPROM / CSR support"
+   depends on I2C && SYSFS
+   help
+ Enable this driver to get read/write access to EEPROM / CSRs
+ over IDT PCIe-swtich i2c-slave interface.
+
+ This driver can also be built as a module. If so, the module
+ will be called idt_89hpesx.
+
 endmenu
diff --git a/drivers/misc/eeprom/Makefile b/drivers/misc/eeprom/Makefile
index fc1e81d..90a5262 100644
--- a/drivers/misc/eeprom/Makefile
+++ b/drivers/misc/eeprom/Makefile
@@ -5,3 +5,4 @@ obj-$(CONFIG_EEPROM_MAX6875)+= max6875.o
 obj-$(CONFIG_EEPROM_93CX6) += eeprom_93cx6.o
 obj-$(CONFIG_EEPROM_93XX46)+= eeprom_93xx46.o
 obj-$(CONFIG_EEPROM_DIGSY_MTC_CFG) += digsy_mtc_eeprom.o
+obj-$(CONFIG_EEPROM_IDT_89HPESX) += idt_89hpesx.o
diff --git a/drivers/misc/eeprom/idt_89hpesx.c 
b/drivers/misc/eeprom/idt_89hpesx.c
index 000..a54c0f4
--- /dev/null
+++ b/drivers/misc/eeprom/idt_89hpesx.c
@@ -0,0 +1,1574 @@
+/*
+ *   This file is provided under a GPLv2 license.  When using or
+ *   redistributing this file, you may do so under that license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify it
+ *   under the terms and conditions of the GNU General Public License,
+ *   version 2, as published by the Free Software Foundation.
+ *
+ *   This program is distributed in the hope that it will be useful, but 
WITHOUT
+ *   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ *   more details.
+ *
+ *   You should have received a copy of the GNU General Public License along
+ *   with this program; if not, it can be found <http://www.gnu.org/licenses/>.
+ *
+ *   The full GNU General Public License is included in this distribution in
+ *   the file called "COPYING".
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * IDT PCIe-switch NTB Linux driver
+ *
+ * Contact Information:
+ * Serge Semin <fancer.lan...@gmail.com>, <sergey.se...@t-platforms.ru>
+ */
+/*
+ *   NOTE of the IDT 89HPESx SMBus-slave interface driver
+ *This driver primarily is developed to have an access to EEPROM device of
+ * IDT PCIe-switches. IDT provides a simple SMBus interface to perform IO-
+ * operations from/to EEPROM, which is located at private (so called Master)
+ * SMBus of switches. Using that interface this the driver creates a simple
+ * binary sysfs-file in the device directory:
+ * /sys/bus/i2c/devices/-/eeprom
+ * In case if read-only flag is specified in the dts-node of device desription,
+ * User-space applications won't be able to write to the EEPROM sysfs-node.
+ *Additionally IDT 89HPESx SMBus interface has an ability to write/read
+ * data of device CSRs. This driver exposes debugf-file to perform simple IO
+ * operations using that ability for just basic debug 

[PATCH v3 0/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

Changelog v3:
- Get rid of dev_*_idt() macros
- Replace to_pdev_kobj() macro with naked dev_get_drvdata() call
- Return naked 0 instead of SUCCESS macro
- IDT CSR debug file is moved to debugfs
- BIN_ATTR_RW is used to declare sysfs binary attribute
- Moved bindings file to a separate patch
- Need to create a specific bin_attribute structure for each device
- Perform a few read retries with delays if EEPROM is busy

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

Serge Semin (2):
  eeprom: Add IDT 89HPESx EEPROM/CSR driver
  eeprom: Add IDT 89HPESx driver dts-binding file

 .../devicetree/bindings/misc/idt_89hpesx.txt   |   41 +
 drivers/misc/eeprom/Kconfig|   10 +
 drivers/misc/eeprom/Makefile   |1 +
 drivers/misc/eeprom/idt_89hpesx.c  | 1574 
 4 files changed, 1626 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt
 create mode 100644 drivers/misc/eeprom/idt_89hpesx.c

-- 
2.6.6

[PATCH v3 2/2] eeprom: Add IDT 89HPESx driver dts-binding file

[Serge Semin]

IDT 89HPESx PCIe-switches exposes SMBus interface to have an access to
the device CSRs and EEPROM. So to properly utilize the interface
functionality, developer should declare a valid dts-file node, which
would refer to the corresponding 89HPESx device.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 .../devicetree/bindings/misc/idt_89hpesx.txt   | 41 ++
 1 file changed, 41 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt

diff --git a/Documentation/devicetree/bindings/misc/idt_89hpesx.txt 
b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
index 000..469cc93
--- /dev/null
+++ b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
@@ -0,0 +1,41 @@
+EEPROM / CSR SMBus-slave interface of IDT 89HPESx devices
+
+Required properties:
+  - compatible : should be ","
+Basically there is only one manufacturer: idt, but some
+compatible devices may be produced in future. Following devices
+are supported: 89hpes8nt2, 89hpes12nt3, 89hpes24nt6ag2,
+89hpes32nt8ag2, 89hpes32nt8bg2, 89hpes12nt12g2, 89hpes16nt16g2,
+89hpes24nt24g2, 89hpes32nt24ag2, 89hpes32nt24bg2;
+89hpes12n3, 89hpes12n3a, 89hpes24n3, 89hpes24n3a;
+89hpes32h8, 89hpes32h8g2, 89hpes48h12, 89hpes48h12g2,
+89hpes48h12ag2, 89hpes16h16, 89hpes22h16, 89hpes22h16g2,
+89hpes34h16, 89hpes34h16g2, 89hpes64h16, 89hpes64h16g2,
+89hpes64h16ag2;
+89hpes12t3g2, 89hpes24t3g2, 89hpes16t4, 89hpes4t4g2,
+89hpes10t4g2, 89hpes16t4g2, 89hpes16t4ag2, 89hpes5t5,
+89hpes6t5, 89hpes8t5, 89hpes8t5a, 89hpes24t6, 89hpes6t6g2,
+89hpes24t6g2, 89hpes16t7, 89hpes32t8, 89hpes32t8g2,
+89hpes48t12, 89hpes48t12g2.
+Current implementation of the driver doesn't have any device-
+specific functionalities. But since each of them differs
+by registers mapping, CSRs read/write restrictions can be
+added in future.
+  - reg :   I2C address of the IDT 89HPES device.
+
+Optional properties:
+  - read-only : Parameterless property disables writes to the EEPROM
+  - idt,eesize : Size of EEPROM device connected to IDT 89HPES i2c-master bus
+(default value is 4096 bytes if option isn't specified)
+  - idt,eeaddr : Custom address of EEPROM device
+(If not specified IDT 89HPESx device will try to communicate
+ with EEPROM sited by default address - 0x50)
+
+Example:
+   idt_pcie_sw@60 {
+   compatible = "idt,89hpes12nt3";
+   reg = <0x60>;
+   read-only;
+   idt,eesize = <65536>;
+   idt,eeaddr = <0x50>;
+   };
-- 
2.6.6

Re: [PATCH v2 1/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

On Tue, Nov 29, 2016 at 10:24:12PM +0100, Greg KH <gre...@linuxfoundation.org> 
wrote:
> On Wed, Nov 30, 2016 at 12:16:25AM +0300, Serge Semin wrote:
> > On Tue, Nov 29, 2016 at 08:37:50PM +0100, Greg KH 
> > <gre...@linuxfoundation.org> wrote:
> > > On Tue, Nov 29, 2016 at 01:38:20AM +0300, Serge Semin wrote:
> > > > +struct idt_89hpesx_dev {
> > > > +   u32 eesize;
> > > > +   bool eero;
> > > > +   u8 eeaddr;
> > > > +
> > > > +   u8 inieecmd;
> > > > +   u8 inicsrcmd;
> > > > +   u8 iniccode;
> > > > +
> > > > +   atomic_t csr;
> > > > +
> > > > +   int (*smb_write)(struct idt_89hpesx_dev *, const struct 
> > > > idt_smb_seq *);
> > > > +   int (*smb_read)(struct idt_89hpesx_dev *, struct idt_smb_seq *);
> > > > +   struct mutex smb_mtx;
> > > > +
> > > > +   struct i2c_client *client;
> > > > +
> > > > +   struct bin_attribute *ee_file;
> > > > +   struct dentry *csr_dir;
> > > > +   struct dentry *csr_file;
> > > > +};
> > > > +#define to_pdev_kobj(__kobj) \
> > > > +   dev_get_drvdata(container_of(__kobj, struct device, kobj))
> > > 
> > > Is it a struct device, or a kobject?  This is totally confusing to me.
> > > 
> > > And can't you just use kobj_to_dev()?
> > > 
> > 
> > I just didn't know about kobj_to_dev() inline function. Totally agree that
> > container_of() should be replaced with it.
> > What does look confusing to you? Do you mean the name "to_pdev_kobj" of the
> > macro?
> 
> Yes, the macro is odd.  As you are doing two different things here, just
> spell it out in the code and use kobj_to_dev() to make it easier to
> read please.
> 
> > > > +/*
> > > > + * eeprom_attribute - EEPROM sysfs-node attributes
> > > > + *
> > > > + * NOTE Size will be changed in compliance with OF node. EEPROM 
> > > > attribute will
> > > > + * be read-only as well if the corresponding flag is specified in OF 
> > > > node.
> > > > + */
> > > > +BIN_ATTR(eeprom, 0644, idt_sysfs_eeprom_read, idt_sysfs_eeprom_write,
> > > > +EEPROM_DEF_SIZE);
> > > 
> > > static?
> > > 
> > > And BIN_ATTR_RW()?
> > > 
> > > thanks,
> > > 
> > > greg k-h
> > 
> > Of course it should be static. Thanks for noticing that.
> > But I intentionally utilized BIN_ATTR() instead of BIN_ATTR_RW(), because
> > the last one implies to define the read/write methods with names
> > "_name##_read"/"_name##_write", which totally get out of naming within the
> > driver source code.
> 
> That's ok, use the names the macro wants you to, that's the best way,
> and it ensures that I don't have to audit your permissions are correct
> for the file.
> 
> > To tell the truth macro BIN_ATTR_RW() isn't that popular in the
> > kernel.
> 
> Yes, but it should be, I have patches floating around somewhere to fix
> almost all of these up.
> 
> > Neither is BIN_ATTR() macro, but it suites my driver better than the
> > another one.
> 
> a "raw" BIN_ATTR() shouldn't be used either, please use the _RW()
> variant.
> 
> thanks,
> 
> greg k-h

Agreed with all the notes. I will send patchset v3 within next hour.

Thanks,
-Sergey

Re: [PATCH v2 2/2] eeprom: Add IDT 89HPESx driver bindings file

[Serge Semin]

On Tue, Nov 29, 2016 at 08:34:36PM +0100, Greg KH <gre...@linuxfoundation.org> 
wrote:
> On Tue, Nov 29, 2016 at 01:38:21AM +0300, Serge Semin wrote:
> > See cover-letter for changelog
> 
> There is no cover letter in an individual patch when it gets committed
> to the tree...
> 
> So please fix, personally, I never read cover letters, each patch should
> be "obvious" on it's own :)
> 
> thanks,
> 
> greg k-h

Understood. I'll send the v3 of altered patchset over with individual messages
for each patch.

Thanks,
-Sergey

[PATCH v3 6/9] NTB: Add Messaging NTB API

[Serge Semin]

Some IDT NTB-capable PCIe-switches have message registers to communicate with
peer devices. This patch adds new NTB API callback methods, which can be used
to utilize these registers functionality:
 ntb_msg_count(); - get number of message registers
 ntb_msg_inbits(); - get bitfield of inbound message registers status
 ntb_msg_outbits(); - get bitfield of outbound message registers status
 ntb_msg_read_sts(); - read the inbound and outbound message registers status
 ntb_msg_clear_sts(); - clear status bits of message registers
 ntb_msg_set_mask(); - mask interrupts raised by status bits of message
registers.
 ntb_msg_clear_mask(); - clear interrupts mask bits of message registers
 ntb_msg_read(midx, *pidx); - read message register with specified index,
additionally getting peer port index which data received from
 ntb_msg_write(midx, pidx); - write data to the specified message register
sending it to the passed peer device connected over a pidx port
 ntb_msg_event(); - notify driver context of a new message event

Of course there is hardware which doesn't support Message registers, so
this API is made optional.

Acked-by: Allen Hubbe <allen.hu...@dell.com>
Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/ntb/ntb.c   |  13 
 include/linux/ntb.h | 205 
 2 files changed, 218 insertions(+)

diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index 2551bb2..03b80d8 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -193,6 +193,19 @@ void ntb_db_event(struct ntb_dev *ntb, int vector)
 }
 EXPORT_SYMBOL(ntb_db_event);
 
+void ntb_msg_event(struct ntb_dev *ntb)
+{
+   unsigned long irqflags;
+
+   spin_lock_irqsave(>ctx_lock, irqflags);
+   {
+   if (ntb->ctx_ops && ntb->ctx_ops->msg_event)
+   ntb->ctx_ops->msg_event(ntb->ctx);
+   }
+   spin_unlock_irqrestore(>ctx_lock, irqflags);
+}
+EXPORT_SYMBOL(ntb_msg_event);
+
 int ntb_default_port_number(struct ntb_dev *ntb)
 {
switch (ntb->topo) {
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index a54e2be..76c56d5 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -178,10 +178,12 @@ static inline int ntb_client_ops_is_valid(const struct 
ntb_client_ops *ops)
  * struct ntb_ctx_ops - ntb driver context operations
  * @link_event:See ntb_link_event().
  * @db_event:  See ntb_db_event().
+ * @msg_event: See ntb_msg_event().
  */
 struct ntb_ctx_ops {
void (*link_event)(void *ctx);
void (*db_event)(void *ctx, int db_vector);
+   void (*msg_event)(void *ctx);
 };
 
 static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
@@ -190,6 +192,7 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
return
/* ops->link_event  && */
/* ops->db_event&& */
+   /* ops->msg_event   && */
1;
 }
 
@@ -234,6 +237,15 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @peer_spad_addr:See ntb_peer_spad_addr().
  * @peer_spad_read:See ntb_peer_spad_read().
  * @peer_spad_write:   See ntb_peer_spad_write().
+ * @msg_count: See ntb_msg_count().
+ * @msg_inbits:See ntb_msg_inbits().
+ * @msg_outbits:   See ntb_msg_outbits().
+ * @msg_read_sts:  See ntb_msg_read_sts().
+ * @msg_clear_sts: See ntb_msg_clear_sts().
+ * @msg_set_mask:  See ntb_msg_set_mask().
+ * @msg_clear_mask:See ntb_msg_clear_mask().
+ * @msg_read:  See ntb_msg_read().
+ * @msg_write: See ntb_msg_write().
  */
 struct ntb_dev_ops {
int (*port_number)(struct ntb_dev *ntb);
@@ -296,6 +308,16 @@ struct ntb_dev_ops {
u32 (*peer_spad_read)(struct ntb_dev *ntb, int pidx, int sidx);
int (*peer_spad_write)(struct ntb_dev *ntb, int pidx, int sidx,
   u32 val);
+
+   int (*msg_count)(struct ntb_dev *ntb);
+   u64 (*msg_inbits)(struct ntb_dev *ntb);
+   u64 (*msg_outbits)(struct ntb_dev *ntb);
+   u64 (*msg_read_sts)(struct ntb_dev *ntb);
+   int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits);
+   int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_read)(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg);
+   int (*msg_write)(struct ntb_dev *ntb, int midx, int pidx, u32 msg);
 };
 
 static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
@@ -342,6 +364,15 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* !ops->peer_spad_addr == !ops->spad_count && */
/* !ops->peer_spad_read == !ops->spad_count && */

[PATCH v3 9/9] NTB: Add ntb.h comments

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 include/linux/ntb.h | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 6d46179..dab0a1b 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -326,12 +326,17 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
 {
/* commented callbacks are not required: */
return
+   /* Port operations are required */
!ops->peer_port_count == !ops->port_number  &&
!ops->peer_port_number == !ops->port_number &&
!ops->peer_port_idx == !ops->port_number&&
+
+   /* Link operations are requiered */
ops->link_is_up &&
ops->link_enable&&
ops->link_disable   &&
+
+   /* One or both MW interfaces should be developed */
ops->mw_count   &&
ops->mw_get_align   &&
(ops->mw_set_trans  ||
@@ -341,12 +346,11 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
ops->peer_mw_get_addr   &&
/* ops->peer_mw_clear_trans && */
 
+   /* Doorbell operations are mostly required */
/* ops->db_is_unsafe&& */
ops->db_valid_mask  &&
-
/* both set, or both unset */
-   (!ops->db_vector_count == !ops->db_vector_mask) &&
-
+   (!ops->db_vector_count == !ops->db_vector_mask) &&
ops->db_read&&
/* ops->db_set  && */
ops->db_clear   &&
@@ -360,6 +364,8 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* ops->peer_db_read_mask   && */
/* ops->peer_db_set_mask&& */
/* ops->peer_db_clear_mask  && */
+
+   /* Scrachpads interface is optional */
/* !ops->spad_is_unsafe == !ops->spad_count && */
!ops->spad_read == !ops->spad_count &&
!ops->spad_write == !ops->spad_count&&
@@ -367,6 +373,7 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* !ops->peer_spad_read == !ops->spad_count && */
!ops->peer_spad_write == !ops->spad_count   &&
 
+   /* Messaging interface is optional */
!ops->msg_inbits == !ops->msg_count &&
!ops->msg_outbits == !ops->msg_count&&
!ops->msg_read_sts == !ops->msg_count   &&
@@ -387,13 +394,12 @@ struct ntb_client {
struct device_driverdrv;
const struct ntb_client_ops ops;
 };
-
 #define drv_ntb_client(__drv) container_of((__drv), struct ntb_client, drv)
 
 /**
  * struct ntb_device - ntb device
  * @dev:   Linux device object.
- * @pdev:  Pci device entry of the ntb.
+ * @pdev:  PCI device entry of the ntb.
  * @topo:  Detected topology of the ntb.
  * @ops:   See _dev_ops.
  * @ctx:   See _ctx_ops.
@@ -414,7 +420,6 @@ struct ntb_dev {
/* block unregister until device is fully released */
struct completion   released;
 };
-
 #define dev_ntb(__dev) container_of((__dev), struct ntb_dev, dev)
 
 /**
@@ -511,7 +516,7 @@ void ntb_link_event(struct ntb_dev *ntb);
  * multiple interrupt vectors for doorbells, the vector number indicates which
  * vector received the interrupt.  The vector number is relative to the first
  * vector used for doorbells, starting at zero, and must be less than
- ** ntb_db_vector_count().  The driver may call ntb_db_read() to check which
+ * ntb_db_vector_count().  The driver may call ntb_db_read() to check which
  * doorbell bits need service, and ntb_db_vector_mask() to determine which of
  * those bits are associated with the vector number.
  */
-- 
2.6.6

[PATCH v3 7/9] NTB: Add new Memory Windows API documentation

[Serge Semin]

Since the new API slightly changes the way a typical NTB client driver
works, the documentation file needs to be appropriately updated.

Acked-by: Allen Hubbe <allen.hu...@dell.com>
Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 Documentation/ntb.txt | 99 ++-
 1 file changed, 91 insertions(+), 8 deletions(-)

diff --git a/Documentation/ntb.txt b/Documentation/ntb.txt
index 1d9bbab..a5af4f0 100644
--- a/Documentation/ntb.txt
+++ b/Documentation/ntb.txt
@@ -1,14 +1,16 @@
 # NTB Drivers
 
 NTB (Non-Transparent Bridge) is a type of PCI-Express bridge chip that connects
-the separate memory systems of two computers to the same PCI-Express fabric.
-Existing NTB hardware supports a common feature set, including scratchpad
-registers, doorbell registers, and memory translation windows.  Scratchpad
-registers are read-and-writable registers that are accessible from either side
-of the device, so that peers can exchange a small amount of information at a
-fixed address.  Doorbell registers provide a way for peers to send interrupt
-events.  Memory windows allow translated read and write access to the peer
-memory.
+the separate memory systems of two or more computers to the same PCI-Express
+fabric. Existing NTB hardware supports a common feature set: doorbell
+registers and memory translation windows, as well as non common features like
+scratchpad and message registers. Scratchpad registers are read-and-writable
+registers that are accessible from either side of the device, so that peers can
+exchange a small amount of information at a fixed address. Message registers 
can
+be utilized for the same purpose. Additionally they are provided with with
+special status bits to make sure the information isn't rewritten by another
+peer. Doorbell registers provide a way for peers to send interrupt events.
+Memory windows allow translated read and write access to the peer memory.
 
 ## NTB Core Driver (ntb)
 
@@ -26,6 +28,87 @@ as ntb hardware, or hardware drivers, are inserted and 
removed.  The
 registration uses the Linux Device framework, so it should feel familiar to
 anyone who has written a pci driver.
 
+### NTB Typical client driver implementation
+
+Primary purpose of NTB is to share some peace of memory between at least two
+systems. So the NTB device features like Scratchpad/Message registers are
+mainly used to perform the proper memory window initialization. Typically
+there are two types of memory window interfaces supported by the NTB API:
+inbound translation configured on the local ntb port and outbound translation
+configured by the peer, on the peer ntb port. The first type is
+depicted on the next figure
+
+Inbound translation:
+ Memory:  Local NTB Port:  Peer NTB Port:  Peer MMIO:
+  
+ | dma-mapped |-ntb_mw_set_trans(addr)  |
+ | memory |_v   |   __
+ | (addr) |<==| MW xlat addr |<| MW base addr |<== memory-mapped IO
+ ||   |--|  |  |--|
+
+So typical scenario of the first type memory window initialization looks:
+1) allocate a memory region, 2) put translated address to NTB config,
+3) somehow notify a peer device of performed initialization, 4) peer device
+maps corresponding outbound memory window so to have access to the shared
+memory region.
+
+The second type of interface, that implies the shared windows being
+initialized by a peer device, is depicted on the figure:
+
+Outbound translation:
+ Memory:Local NTB Port:Peer NTB Port:  Peer MMIO:
+    __
+ | dma-mapped ||   | MW base addr |<== memory-mapped IO
+ | memory ||   |--|
+ | (addr) |<===| MW xlat addr 
|<-ntb_peer_mw_set_trans(addr)
+ |||   |--|
+
+Typical scenario of the second type interface initialization would be:
+1) allocate a memory region, 2) somehow deliver a translated address to a peer
+device, 3) peer puts the translated address to NTB config, 4) peer device maps
+outbound memory window so to have access to the shared memory region.
+
+As one can see the described scenarios can be combined in one portable
+algorithm.
+ Local device:
+  1) Allocate memory for a shared window
+  2) Initialize memory window by translated address of the allocated region
+ (it may fail if local memory window initialization is unsupported)
+  3) Send the translated address and memory window index to a peer device
+ Peer device:
+  1) Initialize memory window with retrieved address of the allocated
+ by another device memory region (it may fail if peer memory window
+ initialization is unsupported)
+  2) Map outbound memory window
+
+In accordance with this scenario, the NTB Memory Window API can be used as
+follows:
+ Local device:
+  1) ntb_mw_count(pidx) - re

[PATCH v3 3/9] NTB: Alter link-state API to support multi-port devices

[Serge Semin]

Multi-port devices permit the NTB connections between multiple domains,
so a local device can have NTB link being up with one peer and being
down with another. NTB link-state API is appropriately altered to return
a bitfield of the link-states between the local device and possible peers.

Acked-by: Allen Hubbe <allen.hu...@dell.com>
Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/ntb/hw/amd/ntb_hw_amd.c |  2 +-
 drivers/ntb/hw/intel/ntb_hw_intel.c |  2 +-
 include/linux/ntb.h | 31 ---
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 6ccba0d..4d8d0bd 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -213,7 +213,7 @@ static int amd_link_is_up(struct amd_ntb_dev *ndev)
return 0;
 }
 
-static int amd_ntb_link_is_up(struct ntb_dev *ntb,
+static u64 amd_ntb_link_is_up(struct ntb_dev *ntb,
  enum ntb_speed *speed,
  enum ntb_width *width)
 {
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index d2ce280..725ffa4 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -1171,7 +1171,7 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, 
int idx,
return 0;
 }
 
-static int intel_ntb_link_is_up(struct ntb_dev *ntb,
+static u64 intel_ntb_link_is_up(struct ntb_dev *ntb,
enum ntb_speed *speed,
enum ntb_width *width)
 {
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index fe175c7..6eef109 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -235,7 +235,7 @@ struct ntb_dev_ops {
int (*peer_port_number)(struct ntb_dev *ntb, int pidx);
int (*peer_port_idx)(struct ntb_dev *ntb, int port);
 
-   int (*link_is_up)(struct ntb_dev *ntb,
+   u64 (*link_is_up)(struct ntb_dev *ntb,
  enum ntb_speed *speed, enum ntb_width *width);
int (*link_enable)(struct ntb_dev *ntb,
   enum ntb_speed max_speed, enum ntb_width max_width);
@@ -607,25 +607,26 @@ static inline int ntb_peer_port_idx(struct ntb_dev *ntb, 
int port)
  * state once after every link event.  It is safe to query the link state in
  * the context of the link event callback.
  *
- * Return: One if the link is up, zero if the link is down, otherwise a
- * negative value indicating the error number.
+ * Return: bitfield of indexed ports link state: bit is set/cleared if the
+ * link is up/down respectively.
  */
-static inline int ntb_link_is_up(struct ntb_dev *ntb,
+static inline u64 ntb_link_is_up(struct ntb_dev *ntb,
 enum ntb_speed *speed, enum ntb_width *width)
 {
return ntb->ops->link_is_up(ntb, speed, width);
 }
 
 /**
- * ntb_link_enable() - enable the link on the secondary side of the ntb
+ * ntb_link_enable() - enable the local port ntb connection
  * @ntb:   NTB device context.
  * @max_speed: The maximum link speed expressed as PCIe generation number.
  * @max_width: The maximum link width expressed as the number of PCIe lanes.
  *
- * Enable the link on the secondary side of the ntb.  This can only be done
- * from the primary side of the ntb in primary or b2b topology.  The ntb device
- * should train the link to its maximum speed and width, or the requested speed
- * and width, whichever is smaller, if supported.
+ * Enable the NTB/PCIe link on the local or remote (for bridge-to-bridge
+ * topology) side of the bridge. If it's supported the ntb device should train
+ * the link to its maximum speed and width, or the requested speed and width,
+ * whichever is smaller. Some hardware doesn't support PCIe link training, so
+ * the last two arguments will be ignored then.
  *
  * Return: Zero on success, otherwise an error number.
  */
@@ -637,14 +638,14 @@ static inline int ntb_link_enable(struct ntb_dev *ntb,
 }
 
 /**
- * ntb_link_disable() - disable the link on the secondary side of the ntb
+ * ntb_link_disable() - disable the local port ntb connection
  * @ntb:   NTB device context.
  *
- * Disable the link on the secondary side of the ntb.  This can only be
- * done from the primary side of the ntb in primary or b2b topology.  The ntb
- * device should disable the link.  Returning from this call must indicate that
- * a barrier has passed, though with no more writes may pass in either
- * direction across the link, except if this call returns an error number.
+ * Disable the link on the local or remote (for b2b topology) of the ntb.
+ * The ntb device should disable the link.  Returning from this call must
+ * indicate that a barrier has passed, though with no more writes may pass in
+ * either direction across the link, except if this call returns an error
+ * number.
  

[PATCH v3 5/9] NTB: Alter Scratchpads API to support multi-ports devices

[Serge Semin]

Even though there is no any real NTB hardware, which would have both more
than two ports and Scratchpad registers, it is logically correct to have
Scratchpad API accepting a peer port index as well. Intel/AMD drivers utilize
Primary and Secondary topology to split Scratchpad between connected root
devices. Since port-index API introduced, Intel/AMD NTB hardware drivers can
use device port to determine which Scratchpad registers actually belong to
local and peer devices. The same approach can be used if some potential
hardware in future will be multi-port and have some set of Scratchpads.
Here are the brief of changes in the API:
 ntb_spad_count() - return number of Scratchpads per each port
 ntb_peer_spad_addr(pidx, sidx) - address of Scratchpad register of the
peer device with pidx-index
 ntb_peer_spad_read(pidx, sidx) - read specified Scratchpad register of the
peer with pidx-index
 ntb_peer_spad_write(pidx, sidx) - write data to Scratchpad register of the
peer with pidx-index

Since there is hardware which doesn't support Scratchpad registers, the
corresponding API methods are now made optional.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/ntb/hw/amd/ntb_hw_amd.c | 14 +++
 drivers/ntb/hw/intel/ntb_hw_intel.c | 14 +++
 drivers/ntb/ntb_transport.c | 17 -
 drivers/ntb/test/ntb_perf.c |  6 +--
 drivers/ntb/test/ntb_pingpong.c |  8 +++-
 drivers/ntb/test/ntb_tool.c | 21 --
 include/linux/ntb.h | 76 +++--
 7 files changed, 98 insertions(+), 58 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 6a41c38..bc537aa 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -433,30 +433,30 @@ static int amd_ntb_spad_write(struct ntb_dev *ntb,
return 0;
 }
 
-static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
void __iomem *mmio = ndev->self_mmio;
u32 offset;
 
-   if (idx < 0 || idx >= ndev->spad_count)
+   if (sidx < 0 || sidx >= ndev->spad_count)
return -EINVAL;
 
-   offset = ndev->peer_spad + (idx << 2);
+   offset = ndev->peer_spad + (sidx << 2);
return readl(mmio + AMD_SPAD_OFFSET + offset);
 }
 
-static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
-  int idx, u32 val)
+static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
+  int sidx, u32 val)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
void __iomem *mmio = ndev->self_mmio;
u32 offset;
 
-   if (idx < 0 || idx >= ndev->spad_count)
+   if (sidx < 0 || sidx >= ndev->spad_count)
return -EINVAL;
 
-   offset = ndev->peer_spad + (idx << 2);
+   offset = ndev->peer_spad + (sidx << 2);
writel(val, mmio + AMD_SPAD_OFFSET + offset);
 
return 0;
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 4b84012..7bb14cb 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -1409,30 +1409,30 @@ static int intel_ntb_spad_write(struct ntb_dev *ntb,
   ndev->self_reg->spad);
 }
 
-static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
+static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
phys_addr_t *spad_addr)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
+   return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
  ndev->peer_reg->spad);
 }
 
-static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_read(ndev, idx,
+   return ndev_spad_read(ndev, sidx,
  ndev->peer_mmio +
  ndev->peer_reg->spad);
 }
 
-static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
-int idx, u32 val)
+static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
+int sidx, u32 val)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_write(ndev, idx, val,
+   return ndev_spad_write(ndev, sidx, val,
   ndev->peer_mmio +
   ndev->peer_reg->spad);
 }
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 4d5b1

[PATCH v3 8/9] NTB: Add PCIe Gen4 link speed

[Serge Semin]

Acked-by: Allen Hubbe <allen.hu...@dell.com>
Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 include/linux/ntb.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 76c56d5..6d46179 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -108,6 +108,7 @@ static inline char *ntb_topo_string(enum ntb_topo topo)
  * @NTB_SPEED_GEN1:Link is trained to gen1 speed.
  * @NTB_SPEED_GEN2:Link is trained to gen2 speed.
  * @NTB_SPEED_GEN3:Link is trained to gen3 speed.
+ * @NTB_SPEED_GEN4:Link is trained to gen4 speed.
  */
 enum ntb_speed {
NTB_SPEED_AUTO = -1,
@@ -115,6 +116,7 @@ enum ntb_speed {
NTB_SPEED_GEN1 = 1,
NTB_SPEED_GEN2 = 2,
NTB_SPEED_GEN3 = 3,
+   NTB_SPEED_GEN4 = 4
 };
 
 /**
-- 
2.6.6

[PATCH v3 0/9] NTB: Alter kernel API to support multi-port devices

[Serge Semin]

There are devices, like IDT PCIe-switches, which have more than just two ports.
Particularly one device can have up to eight ports with NTB-function activated.
In order to support such devices, NTB kernel API should be altered since
currently it's optimized to work with two-ports devices only.

Changelog v2:
 - Move comments from cover letter to individual patches
 - Combine patches to make code buildable
 - Alter patchset to support Intel SKX driver
 - Make sure all the API uses the same midx/widx/pidx/sidx arguments notation
 - Move new MW API usage description into Documention
 - Alter Spad/Msg API checking valid function to make spad and msg interfaces 
optional
 - Alter comments in ntb.h
 - Split: add NTB_SPEED_GEN4 and ntb.h comments into separate patches
 - Put copyrights into some of the existing patches
 - Get rid of TOPO updates

Changelog v3:
 - Get rid of code rearrangements within Intel/AMD drivers
 - Make two-ports NTB API being default
 - Check pidx argument for negative values
 - Translation address methods should return zero in case if not implemented
 - Simplify Scratchpad alterations of ntb_tool client driver
 - ntb.h comments are altered in compliance with port-related API changes
 - Spell check the text

Serge Semin (9):
  NTB: Make link-state API being declared first
  NTB: Add indexed ports NTB API
  NTB: Alter link-state API to support multi-port devices
  NTB: Alter MW API to support multi-ports devices
  NTB: Alter Scratchpads API to support multi-ports devices
  NTB: Add Messaging NTB API
  NTB: Add new Memory Windows API documentation
  NTB: Add PCIe Gen4 link speed
  NTB: Add ntb.h comments

 Documentation/ntb.txt   |  99 -
 drivers/ntb/hw/amd/ntb_hw_amd.c |  84 +++--
 drivers/ntb/hw/intel/ntb_hw_intel.c | 106 --
 drivers/ntb/ntb.c   |  69 
 drivers/ntb/ntb_transport.c |  44 ++-
 drivers/ntb/test/ntb_perf.c |  27 +-
 drivers/ntb/test/ntb_pingpong.c |  14 +-
 drivers/ntb/test/ntb_tool.c |  69 +++-
 include/linux/ntb.h | 724 ++--
 9 files changed, 1026 insertions(+), 210 deletions(-)

-- 
2.6.6

[PATCH v3 1/9] NTB: Make link-state API being declared first

[Serge Semin]

Since link operations are usually performed before memory window access
operations, it's logically better to declare link-related API before any
of MW/Doorbell/Scratchpad methods.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 include/linux/ntb.h | 137 ++--
 1 file changed, 69 insertions(+), 68 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 6f47562..5d1f260 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -179,13 +179,13 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
 
 /**
  * struct ntb_ctx_ops - ntb device operations
+ * @link_is_up:See ntb_link_is_up().
+ * @link_enable:   See ntb_link_enable().
+ * @link_disable:  See ntb_link_disable().
  * @mw_count:  See ntb_mw_count().
  * @mw_get_range:  See ntb_mw_get_range().
  * @mw_set_trans:  See ntb_mw_set_trans().
  * @mw_clear_trans:See ntb_mw_clear_trans().
- * @link_is_up:See ntb_link_is_up().
- * @link_enable:   See ntb_link_enable().
- * @link_disable:  See ntb_link_disable().
  * @db_is_unsafe:  See ntb_db_is_unsafe().
  * @db_valid_mask: See ntb_db_valid_mask().
  * @db_vector_count:   See ntb_db_vector_count().
@@ -212,6 +212,12 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @peer_spad_write:   See ntb_peer_spad_write().
  */
 struct ntb_dev_ops {
+   int (*link_is_up)(struct ntb_dev *ntb,
+ enum ntb_speed *speed, enum ntb_width *width);
+   int (*link_enable)(struct ntb_dev *ntb,
+  enum ntb_speed max_speed, enum ntb_width max_width);
+   int (*link_disable)(struct ntb_dev *ntb);
+
int (*mw_count)(struct ntb_dev *ntb);
int (*mw_get_range)(struct ntb_dev *ntb, int idx,
phys_addr_t *base, resource_size_t *size,
@@ -220,12 +226,6 @@ struct ntb_dev_ops {
dma_addr_t addr, resource_size_t size);
int (*mw_clear_trans)(struct ntb_dev *ntb, int idx);
 
-   int (*link_is_up)(struct ntb_dev *ntb,
- enum ntb_speed *speed, enum ntb_width *width);
-   int (*link_enable)(struct ntb_dev *ntb,
-  enum ntb_speed max_speed, enum ntb_width max_width);
-   int (*link_disable)(struct ntb_dev *ntb);
-
int (*db_is_unsafe)(struct ntb_dev *ntb);
u64 (*db_valid_mask)(struct ntb_dev *ntb);
int (*db_vector_count)(struct ntb_dev *ntb);
@@ -265,13 +265,14 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
 {
/* commented callbacks are not required: */
return
+   ops->link_is_up &&
+   ops->link_enable&&
+   ops->link_disable   &&
ops->mw_count   &&
ops->mw_get_range   &&
ops->mw_set_trans   &&
/* ops->mw_clear_trans  && */
-   ops->link_is_up &&
-   ops->link_enable&&
-   ops->link_disable   &&
+
/* ops->db_is_unsafe&& */
ops->db_valid_mask  &&
 
@@ -441,6 +442,62 @@ void ntb_link_event(struct ntb_dev *ntb);
 void ntb_db_event(struct ntb_dev *ntb, int vector);
 
 /**
+ * ntb_link_is_up() - get the current ntb link state
+ * @ntb:   NTB device context.
+ * @speed: OUT - The link speed expressed as PCIe generation number.
+ * @width: OUT - The link width expressed as the number of PCIe lanes.
+ *
+ * Get the current state of the ntb link.  It is recommended to query the link
+ * state once after every link event.  It is safe to query the link state in
+ * the context of the link event callback.
+ *
+ * Return: One if the link is up, zero if the link is down, otherwise a
+ * negative value indicating the error number.
+ */
+static inline int ntb_link_is_up(struct ntb_dev *ntb,
+enum ntb_speed *speed, enum ntb_width *width)
+{
+   return ntb->ops->link_is_up(ntb, speed, width);
+}
+
+/**
+ * ntb_link_enable() - enable the link on the secondary side of the ntb
+ * @ntb:   NTB device context.
+ * @max_speed: The maximum link speed expressed as PCIe generation number.
+ * @max_width: The maximum link width expressed as the number of PCIe lanes.
+ *
+ * Enable the link on the secondary side of the ntb.  This can only be done
+ * from the primary side of the ntb in primary or b2b topology.  The ntb device
+ * should train the link to its maximum speed and width, or the reques

[PATCH v3 2/9] NTB: Add indexed ports NTB API

[Serge Semin]

There is some NTB hardware, which can combine more than just two domains
over NTB. For instance, some IDT PCIe-switches can have NTB-functions
activated on more than two-ports. The different domains are distinguished
by ports they are connected to. So the new port-related methods are added to
the NTB API:
 ntb_port_number() - return local port
 ntb_peer_port_count() - return number of peers local port can connect to
 ntb_peer_port_number(pdix) - return port number by it index
 ntb_peer_port_idx(port) - return port index by it number

Current test-drivers aren't changed much. They still support two-ports devices
for the time being while multi-ports hardware drivers aren't added.

By default port-related API is declared for two-ports hardware.
So corresponding hardware drivers won't need to implement it.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/ntb/ntb.c   |  54 ++
 drivers/ntb/ntb_transport.c |   6 ++
 drivers/ntb/test/ntb_perf.c |   4 ++
 drivers/ntb/test/ntb_pingpong.c |   6 ++
 drivers/ntb/test/ntb_tool.c |   5 ++
 include/linux/ntb.h | 156 
 6 files changed, 231 insertions(+)

diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index 2e25307..1e92e52 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -191,6 +191,60 @@ void ntb_db_event(struct ntb_dev *ntb, int vector)
 }
 EXPORT_SYMBOL(ntb_db_event);
 
+int ntb_default_port_number(struct ntb_dev *ntb)
+{
+   switch (ntb->topo) {
+   case NTB_TOPO_PRI:
+   case NTB_TOPO_B2B_USD:
+   return NTB_PORT_PRI_USD;
+   case NTB_TOPO_SEC:
+   case NTB_TOPO_B2B_DSD:
+   return NTB_PORT_SEC_DSD;
+   default:
+   break;
+   }
+
+   return -EINVAL;
+}
+EXPORT_SYMBOL(ntb_default_port_number);
+
+int ntb_default_peer_port_count(struct ntb_dev *ntb)
+{
+   return NTB_DEF_PEER_CNT;
+}
+EXPORT_SYMBOL(ntb_default_peer_port_count);
+
+int ntb_default_peer_port_number(struct ntb_dev *ntb, int pidx)
+{
+   if (pidx != NTB_DEF_PEER_IDX)
+   return -EINVAL;
+
+   switch (ntb->topo) {
+   case NTB_TOPO_PRI:
+   case NTB_TOPO_B2B_USD:
+   return NTB_PORT_SEC_DSD;
+   case NTB_TOPO_SEC:
+   case NTB_TOPO_B2B_DSD:
+   return NTB_PORT_PRI_USD;
+   default:
+   break;
+   }
+
+   return -EINVAL;
+}
+EXPORT_SYMBOL(ntb_default_peer_port_number);
+
+int ntb_default_peer_port_idx(struct ntb_dev *ntb, int port)
+{
+   int peer_port = ntb_default_peer_port_number(ntb, NTB_DEF_PEER_IDX);
+
+   if (peer_port == -EINVAL || port != peer_port)
+   return -EINVAL;
+
+   return 0;
+}
+EXPORT_SYMBOL(ntb_default_peer_port_idx);
+
 static int ntb_probe(struct device *dev)
 {
struct ntb_dev *ntb;
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 4eb8adb..10518b7 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -94,6 +94,9 @@ MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large 
data copy");
 
 static struct dentry *nt_debugfs_dir;
 
+/* Only two-ports NTB devices are supported */
+#define PIDX   NTB_DEF_PEER_IDX
+
 struct ntb_queue_entry {
/* ntb_queue list reference */
struct list_head entry;
@@ -1083,6 +1086,9 @@ static int ntb_transport_probe(struct ntb_client *self, 
struct ntb_dev *ndev)
dev_dbg(>dev,
"scratchpad is unsafe, proceed anyway...\n");
 
+   if (ntb_peer_port_count(ndev) != NTB_DEF_PEER_CNT)
+   dev_warn(>dev, "Multi-port NTB devices unsupported\n");
+
node = dev_to_node(>dev);
 
nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node);
diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
index e75d4fd..c908b3a 100644
--- a/drivers/ntb/test/ntb_perf.c
+++ b/drivers/ntb/test/ntb_perf.c
@@ -76,6 +76,7 @@
 #define DMA_RETRIES20
 #define SZ_4G  (1ULL << 32)
 #define MAX_SEG_ORDER  20 /* no larger than 1M for kmalloc buffer */
+#define PIDX   NTB_DEF_PEER_IDX
 
 MODULE_LICENSE(DRIVER_LICENSE);
 MODULE_VERSION(DRIVER_VERSION);
@@ -764,6 +765,9 @@ static int perf_probe(struct ntb_client *client, struct 
ntb_dev *ntb)
return -EIO;
}
 
+   if (ntb_peer_port_count(ntb) != NTB_DEF_PEER_CNT)
+   dev_warn(>dev, "Multi-port NTB devices unsupported\n");
+
node = dev_to_node(>dev);
 
perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node);
diff --git a/drivers/ntb/test/ntb_pingpong.c b/drivers/ntb/test/ntb_pingpong.c
index 4358611..12f8b40 100644
--- a/drivers/ntb/test/ntb_pingpong.c
+++ b/drivers/ntb/test/ntb_pingpong.c
@@ -90,6 +90,9 @@ static unsigned long db_init = 0x7;
 module_param(db_init, ulong, 0644);
 MODULE_PARM_DESC(db_init, "Initial doorb

[PATCH v3 4/9] NTB: Alter MW API to support multi-ports devices

[Serge Semin]

Multi-port NTB devices permit to share a memory between all accessible peers.
Memory Windows API is altered to correspondingly initialize and map memory
windows for such devices:
 ntb_mw_count(pidx); - number of inbound memory windows, which can be allocated
for shared buffer with specified peer device.
 ntb_mw_get_align(pidx, widx); - get alignment and size restriction parameters
to properly allocate inbound memory region.
 ntb_peer_mw_count(); - get number of outbound memory windows.
 ntb_peer_mw_get_addr(widx); - get mapping address of an outbound memory window

If hardware supports inbound translation configured on the local ntb port:
 ntb_mw_set_trans(pidx, widx); - set translation address of allocated inbound
memory window so a peer device could access it.
 ntb_mw_clear_trans(pidx, widx); - clear the translation address of an inbound
memory window.

If hardware supports outbound translation configured on the peer ntb port:
 ntb_peer_mw_set_trans(pidx, widx); - set translation address of a memory
window retrieved from a peer device
 ntb_peer_mw_clear_trans(pidx, widx); - clear the translation address of an
outbound memory window

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/ntb/hw/amd/ntb_hw_amd.c |  68 +---
 drivers/ntb/hw/intel/ntb_hw_intel.c |  90 
 drivers/ntb/ntb.c   |   2 +
 drivers/ntb/ntb_transport.c |  21 +++-
 drivers/ntb/test/ntb_perf.c |  17 ++-
 drivers/ntb/test/ntb_tool.c |  43 +---
 include/linux/ntb.h | 208 
 7 files changed, 342 insertions(+), 107 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 4d8d0bd..6a41c38 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
  *   BSD LICENSE
  *
  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -79,40 +81,42 @@ static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
return 1 << idx;
 }
 
-static int amd_ntb_mw_count(struct ntb_dev *ntb)
+static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 {
+   if (pidx != NTB_DEF_PEER_IDX)
+   return -EINVAL;
+
return ntb_ndev(ntb)->mw_count;
 }
 
-static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
-   phys_addr_t *base,
-   resource_size_t *size,
-   resource_size_t *align,
-   resource_size_t *align_size)
+static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
+   resource_size_t *addr_align,
+   resource_size_t *size_align,
+   resource_size_t *size_max)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
int bar;
 
+   if (pidx != NTB_DEF_PEER_IDX)
+   return -EINVAL;
+
bar = ndev_mw_to_bar(ndev, idx);
if (bar < 0)
return bar;
 
-   if (base)
-   *base = pci_resource_start(ndev->ntb.pdev, bar);
-
-   if (size)
-   *size = pci_resource_len(ndev->ntb.pdev, bar);
+   if (addr_align)
+   *addr_align = SZ_4K;
 
-   if (align)
-   *align = SZ_4K;
+   if (size_align)
+   *size_align = 1;
 
-   if (align_size)
-   *align_size = 1;
+   if (size_max)
+   *size_max = pci_resource_len(ndev->ntb.pdev, bar);
 
return 0;
 }
 
-static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
+static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
dma_addr_t addr, resource_size_t size)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
@@ -122,6 +126,9 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int 
idx,
u64 base_addr, limit, reg_val;
int bar;
 
+   if (pidx != NTB_DEF_PEER_IDX)
+   return -EINVAL;
+
bar = ndev_mw_to_bar(ndev, idx);
if (bar < 0)
return bar;
@@ -285,6 +292,31 @@ static int amd_ntb_link_disable(struct ntb_dev *ntb)
return 0;
 }
 
+static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
+{
+   /* The same as for inbound MWs */
+   return ntb_ndev(ntb)->mw_count;
+}
+
+static int amd_ntb_peer_mw_get_

[PATCH v3 9/9] NTB: Add ntb.h comments

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 include/linux/ntb.h | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 6d46179..dab0a1b 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -326,12 +326,17 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
 {
/* commented callbacks are not required: */
return
+   /* Port operations are required for multiport devices */
!ops->peer_port_count == !ops->port_number  &&
!ops->peer_port_number == !ops->port_number &&
!ops->peer_port_idx == !ops->port_number&&
+
+   /* Link operations are required */
ops->link_is_up &&
ops->link_enable&&
ops->link_disable   &&
+
+   /* One or both MW interfaces should be developed */
ops->mw_count   &&
ops->mw_get_align   &&
(ops->mw_set_trans  ||
@@ -341,12 +346,11 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
ops->peer_mw_get_addr   &&
/* ops->peer_mw_clear_trans && */
 
+   /* Doorbell operations are mostly required */
/* ops->db_is_unsafe&& */
ops->db_valid_mask  &&
-
/* both set, or both unset */
-   (!ops->db_vector_count == !ops->db_vector_mask) &&
-
+   (!ops->db_vector_count == !ops->db_vector_mask) &&
ops->db_read&&
/* ops->db_set  && */
ops->db_clear   &&
@@ -360,6 +364,8 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* ops->peer_db_read_mask   && */
/* ops->peer_db_set_mask&& */
/* ops->peer_db_clear_mask  && */
+
+   /* Scrachpads interface is optional */
/* !ops->spad_is_unsafe == !ops->spad_count && */
!ops->spad_read == !ops->spad_count &&
!ops->spad_write == !ops->spad_count&&
@@ -367,6 +373,7 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* !ops->peer_spad_read == !ops->spad_count && */
!ops->peer_spad_write == !ops->spad_count   &&
 
+   /* Messaging interface is optional */
!ops->msg_inbits == !ops->msg_count &&
!ops->msg_outbits == !ops->msg_count&&
!ops->msg_read_sts == !ops->msg_count   &&
@@ -387,13 +394,12 @@ struct ntb_client {
struct device_driverdrv;
const struct ntb_client_ops ops;
 };
-
 #define drv_ntb_client(__drv) container_of((__drv), struct ntb_client, drv)
 
 /**
  * struct ntb_device - ntb device
  * @dev:   Linux device object.
- * @pdev:  Pci device entry of the ntb.
+ * @pdev:  PCI device entry of the ntb.
  * @topo:  Detected topology of the ntb.
  * @ops:   See _dev_ops.
  * @ctx:   See _ctx_ops.
@@ -414,7 +420,6 @@ struct ntb_dev {
/* block unregister until device is fully released */
struct completion   released;
 };
-
 #define dev_ntb(__dev) container_of((__dev), struct ntb_dev, dev)
 
 /**
@@ -511,7 +516,7 @@ void ntb_link_event(struct ntb_dev *ntb);
  * multiple interrupt vectors for doorbells, the vector number indicates which
  * vector received the interrupt.  The vector number is relative to the first
  * vector used for doorbells, starting at zero, and must be less than
- ** ntb_db_vector_count().  The driver may call ntb_db_read() to check which
+ * ntb_db_vector_count().  The driver may call ntb_db_read() to check which
  * doorbell bits need service, and ntb_db_vector_mask() to determine which of
  * those bits are associated with the vector number.
  */
-- 
2.6.6

Re: [PATCH v2 2/2] eeprom: Add IDT 89HPESx driver bindings file

[Serge Semin]

On Mon, Dec 12, 2016 at 05:04:31PM -0600, Rob Herring <r...@kernel.org> wrote:
> On Mon, Dec 5, 2016 at 1:04 PM, Serge Semin <fancer.lan...@gmail.com> wrote:
> > On Mon, Dec 05, 2016 at 11:27:07AM -0600, Rob Herring <r...@kernel.org> 
> > wrote:
> >> On Mon, Dec 5, 2016 at 9:25 AM, Serge Semin <fancer.lan...@gmail.com> 
> >> wrote:
> >> > On Mon, Dec 05, 2016 at 08:46:21AM -0600, Rob Herring <r...@kernel.org> 
> >> > wrote:
> >> >> On Tue, Nov 29, 2016 at 01:38:21AM +0300, Serge Semin wrote:
> >> >> > See cover-letter for changelog
> >> >> >
> >> >> > Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
> >> >> >
> >> >> > ---
> >> >> >  .../devicetree/bindings/misc/idt_89hpesx.txt   | 41 
> >> >> > ++
> >> >>
> >> >> There's not a better location for this? I can't tell because you don't
> >> >> describe what the device is.
> >> >>
> >> >
> >> > The device is PCIe-switch EEPROM driver with additional debug-interface 
> >> > to
> >> > access the switch CSRs. EEPROM is accesses via a separate i2c-slave
> >> > interface of the switch.
> >> >
> >> > There might be another place to put the binding file in. There is a 
> >> > special
> >> > location for EEPROM drivers bindings - 
> >> > Documentation/devicetree/bindings/eeprom/ .
> >> > But as far as I understood from the files put in there, it's intended for
> >> > pure EEPROM drivers only. On the other hand the directory I've chosen:
> >> > Documentation/devicetree/bindings/misc/
> >> > mostly intended for some unusual devices. My device isn't usual, since it
> >> > has CSRs debug-interface as well. Additionally I've found
> >> > eeprom-93xx46.txt binding file there, which describes EEPROM bindings.
> >> >
> >> > Anyway if you find the file should be placed in
> >> > Documentation/devicetree/bindings/eeprom/ instead, I'll move it, it's not
> >> > that a big problem.
> >> >
> >
> > What about this comment? Shall the file be left at the path I placed it?
> >
> >> >> >  1 file changed, 41 insertions(+)
> >> >> >  create mode 100644 
> >> >> > Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> >> >> >
> >> >> > diff --git a/Documentation/devicetree/bindings/misc/idt_89hpesx.txt 
> >> >> > b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> >> >> > index 000..469cc93
> >> >> > --- /dev/null
> >> >> > +++ b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> >> >> > @@ -0,0 +1,41 @@
> >> >> > +EEPROM / CSR SMBus-slave interface of IDT 89HPESx devices
> >> >> > +
> >> >> > +Required properties:
> >> >> > +  - compatible : should be ","
> >> >> > +Basically there is only one manufacturer: idt, but some
> >> >> > +compatible devices may be produced in future. Following 
> >> >> > devices
> >> >> > +are supported: 89hpes8nt2, 89hpes12nt3, 89hpes24nt6ag2,
> >> >> > +89hpes32nt8ag2, 89hpes32nt8bg2, 89hpes12nt12g2, 
> >> >> > 89hpes16nt16g2,
> >> >> > +89hpes24nt24g2, 89hpes32nt24ag2, 89hpes32nt24bg2;
> >> >> > +89hpes12n3, 89hpes12n3a, 89hpes24n3, 89hpes24n3a;
> >> >> > +89hpes32h8, 89hpes32h8g2, 89hpes48h12, 89hpes48h12g2,
> >> >> > +89hpes48h12ag2, 89hpes16h16, 89hpes22h16, 89hpes22h16g2,
> >> >> > +89hpes34h16, 89hpes34h16g2, 89hpes64h16, 89hpes64h16g2,
> >> >> > +89hpes64h16ag2;
> >> >> > +89hpes12t3g2, 89hpes24t3g2, 89hpes16t4, 89hpes4t4g2,
> >> >> > +89hpes10t4g2, 89hpes16t4g2, 89hpes16t4ag2, 89hpes5t5,
> >> >> > +89hpes6t5, 89hpes8t5, 89hpes8t5a, 89hpes24t6, 
> >> >> > 89hpes6t6g2,
> >> >> > +89hpes24t6g2, 89hpes16t7, 89hpes32t8, 89hpes32t8g2,
> >> >> > +89hpes48t12, 89hpes48t12g2.
> >> >> > +Current implementation of the driver doesn't have any 
> >> >> > device-
&g

[PATCH v4 1/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

  This driver provides an access to EEPROM of IDT PCIe-switches. IDT PCIe-
switches expose a simple SMBus interface to perform IO-operations from/to
EEPROM, which is located at private (so called Master) SMBus. The driver
creates a simple binary sysfs-file to have an access to the EEPROM using
the SMBus-slave interface in the i2c-device susfs-directory:
 /sys/bus/i2c/devices/-/eeprom
In case if read-only flag is specified at dts-node of the device, User-space
applications won't be able to write to the EEPROM sysfs-node.

  Additionally IDT 89HPESx SMBus interface has an ability to read/write
values of device CSRs. This driver exposes debugfs-file to perform simple
IO-operations using that ability for just basic debug purpose. Particularly
the next file is created in the specific debugfs-directory:
 /sys/kernel/debug/idt_csr/
Format of the debugfs-file value is:
 $ cat /sys/kernel/debug/idt_csr/-/;
 :
So reading the content of the file gives current CSR address and it value.
If User-space application wishes to change current CSR address, it can just
write a proper value to the sysfs-file:
 $ echo "" >
 /sys/kernel/debug/idt_csr/-/
If it wants to change the CSR value as well, the format of the write
operation is:
 $ echo ":" > \
 /sys/kernel/debug/idt_csr/-/;
CSR address and value can be any of hexadecimal, decimal or octal format.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/misc/eeprom/Kconfig   |   10 +
 drivers/misc/eeprom/Makefile  |1 +
 drivers/misc/eeprom/idt_89hpesx.c | 1634 +
 3 files changed, 1645 insertions(+)
 create mode 100644 drivers/misc/eeprom/idt_89hpesx.c

diff --git a/drivers/misc/eeprom/Kconfig b/drivers/misc/eeprom/Kconfig
index c4e41c2..de58762 100644
--- a/drivers/misc/eeprom/Kconfig
+++ b/drivers/misc/eeprom/Kconfig
@@ -100,4 +100,14 @@ config EEPROM_DIGSY_MTC_CFG
 
  If unsure, say N.
 
+config EEPROM_IDT_89HPESX
+   tristate "IDT 89HPESx PCIe-swtiches EEPROM / CSR support"
+   depends on I2C && SYSFS
+   help
+ Enable this driver to get read/write access to EEPROM / CSRs
+ over IDT PCIe-swtich i2c-slave interface.
+
+ This driver can also be built as a module. If so, the module
+ will be called idt_89hpesx.
+
 endmenu
diff --git a/drivers/misc/eeprom/Makefile b/drivers/misc/eeprom/Makefile
index fc1e81d..90a5262 100644
--- a/drivers/misc/eeprom/Makefile
+++ b/drivers/misc/eeprom/Makefile
@@ -5,3 +5,4 @@ obj-$(CONFIG_EEPROM_MAX6875)+= max6875.o
 obj-$(CONFIG_EEPROM_93CX6) += eeprom_93cx6.o
 obj-$(CONFIG_EEPROM_93XX46)+= eeprom_93xx46.o
 obj-$(CONFIG_EEPROM_DIGSY_MTC_CFG) += digsy_mtc_eeprom.o
+obj-$(CONFIG_EEPROM_IDT_89HPESX) += idt_89hpesx.o
diff --git a/drivers/misc/eeprom/idt_89hpesx.c 
b/drivers/misc/eeprom/idt_89hpesx.c
new file mode 100644
index 000..664e315
--- /dev/null
+++ b/drivers/misc/eeprom/idt_89hpesx.c
@@ -0,0 +1,1634 @@
+/*
+ *   This file is provided under a GPLv2 license.  When using or
+ *   redistributing this file, you may do so under that license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify it
+ *   under the terms and conditions of the GNU General Public License,
+ *   version 2, as published by the Free Software Foundation.
+ *
+ *   This program is distributed in the hope that it will be useful, but 
WITHOUT
+ *   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ *   more details.
+ *
+ *   You should have received a copy of the GNU General Public License along
+ *   with this program; if not, it can be found <http://www.gnu.org/licenses/>.
+ *
+ *   The full GNU General Public License is included in this distribution in
+ *   the file called "COPYING".
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * IDT PCIe-switch NTB Linux driver
+ *
+ * Contact Information:
+ * Serge Semin <fancer.lan...@gmail.com>, 

[PATCH v4 2/2] eeprom: Add IDT 89HPESx driver bindings file

[Serge Semin]

IDT 89HPESx PCIe-switches exposes SMBus interface to have an access to
the device CSRs and EEPROM. So to properly utilize the interface
functionality, developer should declare a valid dts-file node, which
would refer to the corresponding 89HPESx device.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 .../devicetree/bindings/misc/idt_89hpesx.txt   | 44 ++
 1 file changed, 44 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt

diff --git a/Documentation/devicetree/bindings/misc/idt_89hpesx.txt 
b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
new file mode 100644
index 000..b9093b7
--- /dev/null
+++ b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
@@ -0,0 +1,44 @@
+EEPROM / CSR SMBus-slave interface of IDT 89HPESx devices
+
+Required properties:
+  - compatible : should be ","
+Basically there is only one manufacturer: idt, but some
+compatible devices may be produced in future. Following devices
+are supported: 89hpes8nt2, 89hpes12nt3, 89hpes24nt6ag2,
+89hpes32nt8ag2, 89hpes32nt8bg2, 89hpes12nt12g2, 89hpes16nt16g2,
+89hpes24nt24g2, 89hpes32nt24ag2, 89hpes32nt24bg2;
+89hpes12n3, 89hpes12n3a, 89hpes24n3, 89hpes24n3a;
+89hpes32h8, 89hpes32h8g2, 89hpes48h12, 89hpes48h12g2,
+89hpes48h12ag2, 89hpes16h16, 89hpes22h16, 89hpes22h16g2,
+89hpes34h16, 89hpes34h16g2, 89hpes64h16, 89hpes64h16g2,
+89hpes64h16ag2;
+89hpes12t3g2, 89hpes24t3g2, 89hpes16t4, 89hpes4t4g2,
+89hpes10t4g2, 89hpes16t4g2, 89hpes16t4ag2, 89hpes5t5,
+89hpes6t5, 89hpes8t5, 89hpes8t5a, 89hpes24t6, 89hpes6t6g2,
+89hpes24t6g2, 89hpes16t7, 89hpes32t8, 89hpes32t8g2,
+89hpes48t12, 89hpes48t12g2.
+  - reg :   I2C address of the IDT 89HPESx device.
+
+Optionally there can be EEPROM-compatible subnode:
+  - compatible:  There are five EEPROM devices supported: 24c32, 24c64, 24c128,
+24c256 and 24c512 differed by size.
+  - reg: Custom address of EEPROM device (If not specified IDT 89HPESx
+(optional)  device will try to communicate with EEPROM sited by default
+address - 0x50)
+  - read-only : Parameterless property disables writes to the EEPROM
+(optional)
+
+Example:
+   idt@60 {
+   compatible = "idt,89hpes32nt8ag2";
+   reg = <0x74>;
+   #address-cells = <1>;
+   #size-cells = <0>;
+
+   eeprom@50 {
+   compatible = "onsemi,24c64";
+   reg = <0x50>;
+   read-only;
+   };
+   };
+
-- 
2.6.6

[PATCH v4 0/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

Changelog v3:
- Get rid of dev_*_idt() macros
- Replace to_pdev_kobj() macro with naked dev_get_drvdata() call
- Return naked 0 instead of SUCCESS macro
- IDT CSR debug file is moved to debugfs
- BIN_ATTR_RW is used to declare sysfs binary attribute
- Moved bindings file to a separate patch
- Need to create a specific bin_attribute structure for each device
- Perform a few read retries with delays if EEPROM is busy

Changelog v4:
- Make 89HPESx device bindings to have one EEPROM subnode
- Alter 89HPESx device bindings text

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

Serge Semin (2):
  eeprom: Add IDT 89HPESx EEPROM/CSR driver
  eeprom: Add IDT 89HPESx driver bindings file

 .../devicetree/bindings/misc/idt_89hpesx.txt   |   44 +
 drivers/misc/eeprom/Kconfig|   10 +
 drivers/misc/eeprom/Makefile   |1 +
 drivers/misc/eeprom/idt_89hpesx.c  | 1634 
 4 files changed, 1689 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt
 create mode 100644 drivers/misc/eeprom/idt_89hpesx.c

-- 
2.6.6

[PATCH v2 4/9] NTB: Alter MW API to support multi-ports devices

[Serge Semin]

Multi-port NTB devices permit to share a memory between all accessible peers.
Memory Windows API is altered to correspondingly initialize and map memory
windows for such devices:
 ntb_mw_count(pidx); - number of inbound memory windows, which can be allocated
for shared buffer with specified peer device.
 ntb_mw_get_align(pidx, widx); - get alignment and size restrition parameters
to properly allocate inbound memory region.
 ntb_peer_mw_count(); - get number of outbound memory windows.
 ntb_peer_mw_get_addr(widx); - get mapping address of an outbound memory window

If hardware supports inbound translation configured on the local ntb port:
 ntb_mw_set_trans(pidx, widx); - set translation address of allocated inbound
memory window so a peer device could access it.
 ntb_mw_clear_trans(pidx, widx); - clear the translation address of an inbound
memory window.

If hadrware supports outbound translation configured on the peer ntb port:
 ntb_peer_mw_set_trans(pidx, widx); - set translation address of a memory
window retrieved from a peer device
 ntb_peer_mw_clear_trans(pidx, widx); - clear the translation address of an
outbound memory window

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/amd/ntb_hw_amd.c |  68 +---
 drivers/ntb/hw/amd/ntb_hw_amd.h |   2 +
 drivers/ntb/hw/intel/ntb_hw_intel.c |  90 
 drivers/ntb/hw/intel/ntb_hw_intel.h |   2 +
 drivers/ntb/ntb.c   |   2 +
 drivers/ntb/ntb_transport.c |  21 +++-
 drivers/ntb/test/ntb_perf.c |  17 ++-
 drivers/ntb/test/ntb_tool.c |  43 +---
 include/linux/ntb.h | 208 
 9 files changed, 346 insertions(+), 107 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index b6a4291..74fe9b8 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
  *   BSD LICENSE
  *
  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -213,40 +215,42 @@ static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int 
idx)
return 1 << idx;
 }
 
-static int amd_ntb_mw_count(struct ntb_dev *ntb)
+static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 {
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
return ntb_ndev(ntb)->mw_count;
 }
 
-static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
-   phys_addr_t *base,
-   resource_size_t *size,
-   resource_size_t *align,
-   resource_size_t *align_size)
+static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
+   resource_size_t *addr_align,
+   resource_size_t *size_align,
+   resource_size_t *size_max)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
int bar;
 
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
bar = ndev_mw_to_bar(ndev, idx);
if (bar < 0)
return bar;
 
-   if (base)
-   *base = pci_resource_start(ndev->ntb.pdev, bar);
-
-   if (size)
-   *size = pci_resource_len(ndev->ntb.pdev, bar);
+   if (addr_align)
+   *addr_align = SZ_4K;
 
-   if (align)
-   *align = SZ_4K;
+   if (size_align)
+   *size_align = 1;
 
-   if (align_size)
-   *align_size = 1;
+   if (size_max)
+   *size_max = pci_resource_len(ndev->ntb.pdev, bar);
 
return 0;
 }
 
-static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
+static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
dma_addr_t addr, resource_size_t size)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
@@ -256,6 +260,9 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int 
idx,
u64 base_addr, limit, reg_val;
int bar;
 
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
bar = ndev_mw_to_bar(ndev, idx);
if (bar < 0)
return bar;
@@ -328,6 +335,31 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int 
idx,
return 0;
 }
 
+static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
+{
+   /* The same as for inb

[PATCH v2 3/9] NTB: Alter link-state API to support multi-port devices

[Serge Semin]

Multi-port devices permit the NTB connections between multiple domains,
so a local device can have NTB link being up with one peer and being
down with another. NTB link-state API is appropriately altered to return
a bitfield of the link-states between the local device and possible peers.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/amd/ntb_hw_amd.c |  2 +-
 drivers/ntb/hw/intel/ntb_hw_intel.c |  2 +-
 include/linux/ntb.h | 31 ---
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 0b767ef..b6a4291 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -133,7 +133,7 @@ static int amd_link_is_up(struct amd_ntb_dev *ndev)
return 0;
 }
 
-static int amd_ntb_link_is_up(struct ntb_dev *ntb,
+static u64 amd_ntb_link_is_up(struct ntb_dev *ntb,
  enum ntb_speed *speed,
  enum ntb_width *width)
 {
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 7e44dc3..f37b6fb 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -1078,7 +1078,7 @@ static int intel_ntb_peer_port_idx(struct ntb_dev *ntb, 
int port)
return 0;
 }
 
-static int intel_ntb_link_is_up(struct ntb_dev *ntb,
+static u64 intel_ntb_link_is_up(struct ntb_dev *ntb,
enum ntb_speed *speed,
enum ntb_width *width)
 {
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 3216689..47ec611 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -221,7 +221,7 @@ struct ntb_dev_ops {
int (*peer_port_number)(struct ntb_dev *ntb, int pidx);
int (*peer_port_idx)(struct ntb_dev *ntb, int port);
 
-   int (*link_is_up)(struct ntb_dev *ntb,
+   u64 (*link_is_up)(struct ntb_dev *ntb,
  enum ntb_speed *speed, enum ntb_width *width);
int (*link_enable)(struct ntb_dev *ntb,
   enum ntb_speed max_speed, enum ntb_width max_width);
@@ -522,25 +522,26 @@ static inline int ntb_peer_port_idx(struct ntb_dev *ntb, 
int port)
  * state once after every link event.  It is safe to query the link state in
  * the context of the link event callback.
  *
- * Return: One if the link is up, zero if the link is down, otherwise a
- * negative value indicating the error number.
+ * Return: bitfield of indexed ports link state: bit is set/cleared if the
+ * link is up/down respectively.
  */
-static inline int ntb_link_is_up(struct ntb_dev *ntb,
+static inline u64 ntb_link_is_up(struct ntb_dev *ntb,
 enum ntb_speed *speed, enum ntb_width *width)
 {
return ntb->ops->link_is_up(ntb, speed, width);
 }
 
 /**
- * ntb_link_enable() - enable the link on the secondary side of the ntb
+ * ntb_link_enable() - enable the local port ntb connection
  * @ntb:   NTB device context.
  * @max_speed: The maximum link speed expressed as PCIe generation number.
  * @max_width: The maximum link width expressed as the number of PCIe lanes.
  *
- * Enable the link on the secondary side of the ntb.  This can only be done
- * from the primary side of the ntb in primary or b2b topology.  The ntb device
- * should train the link to its maximum speed and width, or the requested speed
- * and width, whichever is smaller, if supported.
+ * Enable the NTB/PCIe link on the local or remote (for bridge-to-bridge
+ * topology) side of the bridge. If it's supported the ntb device should train
+ * the link to its maximum speed and width, or the requested speed and width,
+ * whichever is smaller. Some hardware doesn't support PCIe link training, so
+ * the last two arguments will be ignored then.
  *
  * Return: Zero on success, otherwise an error number.
  */
@@ -552,14 +553,14 @@ static inline int ntb_link_enable(struct ntb_dev *ntb,
 }
 
 /**
- * ntb_link_disable() - disable the link on the secondary side of the ntb
+ * ntb_link_disable() - disable the local port ntb connection
  * @ntb:   NTB device context.
  *
- * Disable the link on the secondary side of the ntb.  This can only be
- * done from the primary side of the ntb in primary or b2b topology.  The ntb
- * device should disable the link.  Returning from this call must indicate that
- * a barrier has passed, though with no more writes may pass in either
- * direction across the link, except if this call returns an error number.
+ * Disable the link on the local or remote (for b2b topology) of the ntb.
+ * The ntb device should disable the link.  Returning from this call must
+ * indicate that a barrier has passed, though with no more writes may pass in
+ * either direction across the link, except if this call returns an error
+ * number.
  *
  * Return: Zero on success, otherwise an 

[PATCH v2 1/9] NTB: Make link-state API being declared first

[Serge Semin]

Since link operations are usually performed before memory window access
operations, it's logically better to declared link-related API before any
other methods. Additionally it's good practice for readability to declare
NTB device callback methods of hadrware drivers with the same order as it's
done within ntb.h.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/amd/ntb_hw_amd.c | 188 ++--
 drivers/ntb/hw/intel/ntb_hw_intel.c | 168 
 include/linux/ntb.h | 137 +-
 3 files changed, 247 insertions(+), 246 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 6ccba0d..6704327 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -71,6 +71,97 @@ MODULE_AUTHOR("AMD Inc.");
 static const struct file_operations amd_ntb_debugfs_info;
 static struct dentry *debugfs_dir;
 
+static int amd_link_is_up(struct amd_ntb_dev *ndev)
+{
+   if (!ndev->peer_sta)
+   return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
+
+   /* If peer_sta is reset or D0 event, the ISR has
+* started a timer to check link status of hardware.
+* So here just clear status bit. And if peer_sta is
+* D3 or PME_TO, D0/reset event will be happened when
+* system wakeup/poweron, so do nothing here.
+*/
+   if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
+   ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
+   else if (ndev->peer_sta & AMD_PEER_D0_EVENT)
+   ndev->peer_sta = 0;
+
+   return 0;
+}
+
+static int amd_ntb_link_is_up(struct ntb_dev *ntb,
+ enum ntb_speed *speed,
+ enum ntb_width *width)
+{
+   struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+   int ret = 0;
+
+   if (amd_link_is_up(ndev)) {
+   if (speed)
+   *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
+   if (width)
+   *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
+
+   dev_dbg(ndev_dev(ndev), "link is up.\n");
+
+   ret = 1;
+   } else {
+   if (speed)
+   *speed = NTB_SPEED_NONE;
+   if (width)
+   *width = NTB_WIDTH_NONE;
+
+   dev_dbg(ndev_dev(ndev), "link is down.\n");
+   }
+
+   return ret;
+}
+
+static int amd_ntb_link_enable(struct ntb_dev *ntb,
+  enum ntb_speed max_speed,
+  enum ntb_width max_width)
+{
+   struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+   void __iomem *mmio = ndev->self_mmio;
+   u32 ntb_ctl;
+
+   /* Enable event interrupt */
+   ndev->int_mask &= ~AMD_EVENT_INTMASK;
+   writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
+
+   if (ndev->ntb.topo == NTB_TOPO_SEC)
+   return -EINVAL;
+   dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
+
+   ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
+   ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
+   writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
+
+   return 0;
+}
+
+static int amd_ntb_link_disable(struct ntb_dev *ntb)
+{
+   struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+   void __iomem *mmio = ndev->self_mmio;
+   u32 ntb_ctl;
+
+   /* Disable event interrupt */
+   ndev->int_mask |= AMD_EVENT_INTMASK;
+   writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
+
+   if (ndev->ntb.topo == NTB_TOPO_SEC)
+   return -EINVAL;
+   dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
+
+   ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
+   ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
+   writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
+
+   return 0;
+}
+
 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
 {
if (idx < 0 || idx > ndev->mw_count)
@@ -194,97 +285,6 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int 
idx,
return 0;
 }
 
-static int amd_link_is_up(struct amd_ntb_dev *ndev)
-{
-   if (!ndev->peer_sta)
-   return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
-
-   /* If peer_sta is reset or D0 event, the ISR has
-* started a timer to check link status of hardware.
-* So here just clear status bit. And if peer_sta is
-* D3 or PME_TO, D0/reset event will be happened when
-* system wakeup/poweron, so do nothing here.
-*/
-   if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
-   ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
-   else if (ndev->peer_sta & AMD_PEER_D0_EVENT)
-   ndev->peer_sta = 0;
-
-   return 0;
-}
-
-static int amd_ntb_link_is_up(struct ntb_dev *ntb,
- enum ntb_speed *spee

[PATCH v2 2/9] NTB: Add indexed ports NTB API

[Serge Semin]

There are some NTB hardware, which can combine more than just two domains
over NTB. For instance, some IDT PCIe-switches can have NTB-functions
activated on more than two-ports. The different domains are distinguished
by ports they are connected to. So the new port-related methods are added to
the NTB API:
 ntb_port_number() - return local port
 ntb_peer_port_count() - return number of peers local port can connect to
 ntb_peer_port_number(pdix) - return port number by it index
 ntb_peer_port_idx(port) - return port index by it number

Current test-drivers aren't changed much. They still support two-ports devices
for the time being while multi-ports hardware drivers aren't added.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/amd/ntb_hw_amd.c | 47 
 drivers/ntb/hw/amd/ntb_hw_amd.h |  9 +
 drivers/ntb/hw/intel/ntb_hw_intel.c | 52 ++-
 drivers/ntb/hw/intel/ntb_hw_intel.h |  9 +
 drivers/ntb/ntb_transport.c |  6 
 drivers/ntb/test/ntb_perf.c |  4 +++
 drivers/ntb/test/ntb_pingpong.c |  6 
 drivers/ntb/test/ntb_tool.c |  5 +++
 include/linux/ntb.h | 71 +
 9 files changed, 208 insertions(+), 1 deletion(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 6704327..0b767ef 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -71,6 +71,49 @@ MODULE_AUTHOR("AMD Inc.");
 static const struct file_operations amd_ntb_debugfs_info;
 static struct dentry *debugfs_dir;
 
+static int amd_ntb_port_number(struct ntb_dev *ntb)
+{
+   switch (ntb->topo) {
+   case NTB_TOPO_PRI:
+   case NTB_TOPO_B2B_USD:
+   return NTB_PORT_PRI_USD;
+   case NTB_TOPO_SEC:
+   case NTB_TOPO_B2B_DSD:
+   return NTB_PORT_SEC_DSD;
+   default:
+   break;
+   }
+
+   return -EINVAL;
+}
+
+static int amd_ntb_peer_port_count(struct ntb_dev *ntb)
+{
+   return NTB_PEER_CNT;
+}
+
+static int amd_ntb_peer_port_number(struct ntb_dev *ntb, int pidx)
+{
+   int local_port = amd_ntb_port_number(ntb);
+
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
+   return (local_port == NTB_PORT_PRI_USD ?
+   NTB_PORT_SEC_DSD : NTB_PORT_PRI_USD);
+}
+
+static int amd_ntb_peer_port_idx(struct ntb_dev *ntb, int port)
+{
+   int local_port = amd_ntb_port_number(ntb);
+
+   if ((local_port == NTB_PORT_PRI_USD && port != NTB_PORT_SEC_DSD) ||
+   (local_port == NTB_PORT_SEC_DSD && port != NTB_PORT_PRI_USD))
+   return -EINVAL;
+
+   return 0;
+}
+
 static int amd_link_is_up(struct amd_ntb_dev *ndev)
 {
if (!ndev->peer_sta)
@@ -431,6 +474,10 @@ static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
 }
 
 static const struct ntb_dev_ops amd_ntb_ops = {
+   .port_number= amd_ntb_port_number,
+   .peer_port_count= amd_ntb_peer_port_count,
+   .peer_port_number   = amd_ntb_peer_port_number,
+   .peer_port_idx  = amd_ntb_peer_port_idx,
.link_is_up = amd_ntb_link_is_up,
.link_enable= amd_ntb_link_enable,
.link_disable   = amd_ntb_link_disable,
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.h b/drivers/ntb/hw/amd/ntb_hw_amd.h
index 2eac3cd..1aeb08f 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.h
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.h
@@ -62,6 +62,10 @@
 #define NTB_LNK_STA_SPEED(x)   (((x) & NTB_LNK_STA_SPEED_MASK) >> 16)
 #define NTB_LNK_STA_WIDTH(x)   (((x) & NTB_LNK_STA_WIDTH_MASK) >> 20)
 
+/* port related constants */
+#define NTB_PEER_CNT   (1)
+#define NTB_PIDX_MAX   (0)
+
 #ifndef read64
 #ifdef readq
 #define read64 readq
@@ -91,6 +95,11 @@ static inline void _write64(u64 val, void __iomem *mmio)
 #endif
 #endif
 
+enum amd_ntb_port {
+   NTB_PORT_PRI_USD,
+   NTB_PORT_SEC_DSD
+};
+
 enum {
/* AMD NTB Capability */
AMD_MW_CNT  = 3,
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 68d9908..7e44dc3 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -1035,6 +1035,49 @@ static void ndev_deinit_debugfs(struct intel_ntb_dev 
*ndev)
debugfs_remove_recursive(ndev->debugfs_dir);
 }
 
+static int intel_ntb_port_number(struct ntb_dev *ntb)
+{
+   switch (ntb->topo) {
+   case NTB_TOPO_PRI:
+   case NTB_TOPO_B2B_USD:
+   return NTB_PORT_PRI_USD;
+   case NTB_TOPO_SEC:
+   case NTB_TOPO_B2B_DSD:
+   return NTB_PORT_SEC_DSD;
+   default:
+   break;
+   }
+
+   return -EINVAL;
+}
+
+static int intel_ntb_peer_port_count(struct ntb_dev *ntb)
+{
+   return NTB_PEER_CNT;
+}
+
+static int intel_ntb_peer_p

[PATCH v2 7/9] NTB: Add new Memory Windows API documentation

[Serge Semin]

Since the new API slightly changes the way a typical NTB client driver
works, the documentation file needs to be appropriately updated.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 Documentation/ntb.txt | 99 ++-
 1 file changed, 91 insertions(+), 8 deletions(-)

diff --git a/Documentation/ntb.txt b/Documentation/ntb.txt
index 1d9bbab..d01bb69 100644
--- a/Documentation/ntb.txt
+++ b/Documentation/ntb.txt
@@ -1,14 +1,16 @@
 # NTB Drivers
 
 NTB (Non-Transparent Bridge) is a type of PCI-Express bridge chip that connects
-the separate memory systems of two computers to the same PCI-Express fabric.
-Existing NTB hardware supports a common feature set, including scratchpad
-registers, doorbell registers, and memory translation windows.  Scratchpad
-registers are read-and-writable registers that are accessible from either side
-of the device, so that peers can exchange a small amount of information at a
-fixed address.  Doorbell registers provide a way for peers to send interrupt
-events.  Memory windows allow translated read and write access to the peer
-memory.
+the separate memory systems of two or more computers to the same PCI-Express
+fabric. Existing NTB hardware supports a common feature set: doorbell
+registers and memory translation windows, as well as non common features like
+scratchpad and message registers. Scratchpad registers are read-and-writable
+registers that are accessible from either side of the device, so that peers can
+exchange a small amount of information at a fixed address. Message registers 
can
+be utialized for the same purpose. Additionally they are provided with with
+special status bits to make sure the information isn't rewritten by another
+peer. Doorbell registers provide a way for peers to send interrupt events.
+Memory windows allow translated read and write access to the peer memory.
 
 ## NTB Core Driver (ntb)
 
@@ -26,6 +28,87 @@ as ntb hardware, or hardware drivers, are inserted and 
removed.  The
 registration uses the Linux Device framework, so it should feel familiar to
 anyone who has written a pci driver.
 
+### NTB Typical client driver implementation
+
+Primary purpose of NTB is to share some peace of memory between at least two
+systems. So the NTB device features like Scratchpad/Message regiesters are
+mainly used to perform the proper memory window initialization. Typically
+there are two types of memory window interfaces supported by the NTB API:
+inbound translation configured on the local ntb port and outbound translation
+configured by the peer, on the peer ntb port. The first type is
+depicted on the next figure
+
+Inbound translation:
+ Memory:  Local NTB Port:  Peer NTB Port:  Peer MMIO:
+  
+ | dma-mapped |-ntb_mw_set_trans(addr)  |
+ | memory |_v   |   __
+ | (addr) |<==| MW xlat addr |<| MW base addr |<== memory-mapped IO
+ ||   |--|  |  |--|
+
+So typical scenario of the first type memory window initialization looks:
+1) allocate a memory region, 2) put translated address to NTB config,
+3) somehow notify a peer device of performed initialization, 4) peer device
+maps corresponding outbound memory window so to have access to the shared
+memory region.
+
+The second type of interface, that implies the shared windows being
+initialized by a peer device, is depicted on the figure:
+
+Outbound translation:
+ Memory:Local NTB Port:Peer NTB Port:  Peer MMIO:
+    __
+ | dma-mapped ||   | MW base addr |<== memory-mapped IO
+ | memory ||   |--|
+ | (addr) |<===| MW xlat addr 
|<-ntb_peer_mw_set_trans(addr)
+ |||   |--|
+
+Typical scenario of the second type interface initialization would be:
+1) allocate a memory region, 2) somehow deliver a translated address to a peer
+device, 3) peer puts the translated address to NTB config, 4) peer device maps
+outbound memory window so to have access to the shared memory region.
+
+As one can see the described scenarios can be combined in one portable
+algorithm.
+ Local device:
+  1) Allocate memory for a shared window
+  2) Initialize memory window by translated address of the allocated region
+ (it may fail if local memory window initialzation is unsupported)
+  3) Send the translated address and memory window index to a peer device
+ Peer device:
+  1) Initialize memory window with retrieved address of the allocated
+ by another device memory region (it may fail if peer memory window
+ initialization is unsupported)
+  2) Map outbound memory window
+
+In accordance with this scenario, the NTB Memory Window API can be used as
+follows:
+ Local device:
+  1) ntb_mw_count(pidx) - retrieve number of memory ranges, which can
+ be allo

[PATCH v2 0/9] NTB: Alter kernel API to support multi-port devices

[Serge Semin]

There are devices, like IDT PCIe-switches, which have more than just two ports.
Particularly one device can have up to eight ports with NTB-function activated.
In order to support such devices, NTB kernel API should be altered since
currently it's optimized to work with two-ports devices only.

Changelog v2:
 - Move comments from cover letter to individual patches
 - Combine patches to make code buildable
 - Alter patchset to support Intel SKX driver
 - Make sure all the API uses the same midx/widx/pidx/sidx arguments notation
 - Move new MW API usage description into Documention
 - Alter Spad/Msg API checking valid function to make spad and msg interfaces 
optional
 - Alter comments in ntb.h
 - Split: add NTB_SPEED_GEN4 and ntb.h comments into separate patches
 - Put copyrights into some of the existing patches
 - Get rid of TOPO updates

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

Serge Semin (9):
  NTB: Make link-state API being declared first
  NTB: Add indexed ports NTB API
  NTB: Alter link-state API to support multi-port devices
  NTB: Alter MW API to support multi-ports devices
  NTB: Alter Scratchpads API to support multi-ports devices
  NTB: Add Messaging NTB API
  NTB: Add new Memory Windows API documentation
  NTB: Add PCIe Gen4 link speed
  NTB: Add ntb.h comments

 Documentation/ntb.txt   |  99 +-
 drivers/ntb/hw/amd/ntb_hw_amd.c | 291 ++--
 drivers/ntb/hw/amd/ntb_hw_amd.h |  11 +
 drivers/ntb/hw/intel/ntb_hw_intel.c | 296 +++--
 drivers/ntb/hw/intel/ntb_hw_intel.h |  11 +
 drivers/ntb/ntb.c   |  15 +
 drivers/ntb/ntb_transport.c |  44 ++-
 drivers/ntb/test/ntb_perf.c |  27 +-
 drivers/ntb/test/ntb_pingpong.c |  14 +-
 drivers/ntb/test/ntb_tool.c |  93 --
 include/linux/ntb.h | 639 ++--
 11 files changed, 1173 insertions(+), 367 deletions(-)

-- 
2.6.6

[PATCH v2 8/9] NTB: Add PCIe Gen4 link speed

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 include/linux/ntb.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 90746df..fe0437c 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -108,6 +108,7 @@ static inline char *ntb_topo_string(enum ntb_topo topo)
  * @NTB_SPEED_GEN1:Link is trained to gen1 speed.
  * @NTB_SPEED_GEN2:Link is trained to gen2 speed.
  * @NTB_SPEED_GEN3:Link is trained to gen3 speed.
+ * @NTB_SPEED_GEN4:Link is trained to gen4 speed.
  */
 enum ntb_speed {
NTB_SPEED_AUTO = -1,
@@ -115,6 +116,7 @@ enum ntb_speed {
NTB_SPEED_GEN1 = 1,
NTB_SPEED_GEN2 = 2,
NTB_SPEED_GEN3 = 3,
+   NTB_SPEED_GEN4 = 4
 };
 
 /**
-- 
2.6.6

[PATCH v2 5/9] NTB: Alter Scratchpads API to support multi-ports devices

[Serge Semin]

Even though there is no any real NTB hardware, which would have both more
than two ports and Scratchpad registers, it is logically correct to have
Scratchpad API accepting a peer port index as well. Intel/AMD drivers utilize
Primary and Secondary topology to split Scratchpad between connected root
devices. Since port-index API introduced, Intel/AMD NTB hadrware drivers can
use device port to determine which Scratchpad registers actually belong to
local and peer devices. The same approach can be used if some potential
hardware in future will be multi-port and have some set of Scratchpads.
Here are the brief of changes in the API:
 ntb_spad_count() - return number of Scratchpad per each port
 ntb_peer_spad_addr(pidx, sidx) - address of Scratchpad register of the
peer device with pidx-index
 ntb_peer_spad_read(pidx, sidx) - read specified Scratchpad register of the
peer with pidx-index
 ntb_peer_spad_write(pidx, sidx) - write data to Scratchpad register of the
peer with pidx-index

Since there is hardware which doesn't support Scratchpad registers, the
corresponding API methods are now made optional.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/amd/ntb_hw_amd.c | 14 +++
 drivers/ntb/hw/intel/ntb_hw_intel.c | 14 +++
 drivers/ntb/ntb_transport.c | 17 -
 drivers/ntb/test/ntb_perf.c |  6 +--
 drivers/ntb/test/ntb_pingpong.c |  8 +++-
 drivers/ntb/test/ntb_tool.c | 45 +-
 include/linux/ntb.h | 76 +++--
 7 files changed, 115 insertions(+), 65 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 74fe9b8..a2596ad 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -476,30 +476,30 @@ static int amd_ntb_spad_write(struct ntb_dev *ntb,
return 0;
 }
 
-static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
void __iomem *mmio = ndev->self_mmio;
u32 offset;
 
-   if (idx < 0 || idx >= ndev->spad_count)
+   if (sidx < 0 || sidx >= ndev->spad_count)
return -EINVAL;
 
-   offset = ndev->peer_spad + (idx << 2);
+   offset = ndev->peer_spad + (sidx << 2);
return readl(mmio + AMD_SPAD_OFFSET + offset);
 }
 
-static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
-  int idx, u32 val)
+static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
+  int sidx, u32 val)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
void __iomem *mmio = ndev->self_mmio;
u32 offset;
 
-   if (idx < 0 || idx >= ndev->spad_count)
+   if (sidx < 0 || sidx >= ndev->spad_count)
return -EINVAL;
 
-   offset = ndev->peer_spad + (idx << 2);
+   offset = ndev->peer_spad + (sidx << 2);
writel(val, mmio + AMD_SPAD_OFFSET + offset);
 
return 0;
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 5a57d9e..471b0ba 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -1452,30 +1452,30 @@ static int intel_ntb_spad_write(struct ntb_dev *ntb,
   ndev->self_reg->spad);
 }
 
-static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
+static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
phys_addr_t *spad_addr)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
+   return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
  ndev->peer_reg->spad);
 }
 
-static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_read(ndev, idx,
+   return ndev_spad_read(ndev, sidx,
  ndev->peer_mmio +
  ndev->peer_reg->spad);
 }
 
-static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
-int idx, u32 val)
+static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
+int sidx, u32 val)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_write(ndev, idx, val,
+   return ndev_spad_write(ndev, sidx, val,
   ndev->peer_mmio +
   ndev->peer_reg->spad);
 }
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index cb4f998

[PATCH v2 6/9] NTB: Add Messaging NTB API

[Serge Semin]

Some IDT NTB-capable PCIe-switches have message registers to communicate with
peer devices. This patch adds new NTB API callback methods, which can be used
to utilize these registers functionality:
 ntb_msg_count(); - get number of message registers
 ntb_msg_inbits(); - get bitfield of inbound message registers status
 ntb_msg_outbits(); - get bitfield of outbound message registers status
 ntb_msg_read_sts(); - read the inbound and outbound message registers status
 ntb_msg_clear_sts(); - clear status bits of message registers
 ntb_msg_set_mask(); - mask interrupts raised by status bits of message
registers.
 ntb_msg_clear_mask(); - clear interrupts mask bits of message registers
 ntb_msg_read(midx, *pidx); - read message register with specified index,
additionally getting peer port index which data received from
 ntb_msg_write(midx, pidx); - write data to the specified message register
sending it to the passed peer device connected over a pidx port
 ntb_msg_event(); - notify driver context of a new message event

Of course there is hadrware which doesn't support Message registers, so
this API is made optional.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/ntb.c   |  13 
 include/linux/ntb.h | 205 
 2 files changed, 218 insertions(+)

diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index f6153af..06574f8 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -193,6 +193,19 @@ void ntb_db_event(struct ntb_dev *ntb, int vector)
 }
 EXPORT_SYMBOL(ntb_db_event);
 
+void ntb_msg_event(struct ntb_dev *ntb)
+{
+   unsigned long irqflags;
+
+   spin_lock_irqsave(>ctx_lock, irqflags);
+   {
+   if (ntb->ctx_ops && ntb->ctx_ops->msg_event)
+   ntb->ctx_ops->msg_event(ntb->ctx);
+   }
+   spin_unlock_irqrestore(>ctx_lock, irqflags);
+}
+EXPORT_SYMBOL(ntb_msg_event);
+
 static int ntb_probe(struct device *dev)
 {
struct ntb_dev *ntb;
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index a6bf15d..90746df 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -164,10 +164,12 @@ static inline int ntb_client_ops_is_valid(const struct 
ntb_client_ops *ops)
  * struct ntb_ctx_ops - ntb driver context operations
  * @link_event:See ntb_link_event().
  * @db_event:  See ntb_db_event().
+ * @msg_event: See ntb_msg_event().
  */
 struct ntb_ctx_ops {
void (*link_event)(void *ctx);
void (*db_event)(void *ctx, int db_vector);
+   void (*msg_event)(void *ctx);
 };
 
 static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
@@ -176,6 +178,7 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
return
/* ops->link_event  && */
/* ops->db_event&& */
+   /* ops->msg_event   && */
1;
 }
 
@@ -220,6 +223,15 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @peer_spad_addr:See ntb_peer_spad_addr().
  * @peer_spad_read:See ntb_peer_spad_read().
  * @peer_spad_write:   See ntb_peer_spad_write().
+ * @msg_count: See ntb_msg_count().
+ * @msg_inbits:See ntb_msg_inbits().
+ * @msg_outbits:   See ntb_msg_outbits().
+ * @msg_read_sts:  See ntb_msg_read_sts().
+ * @msg_clear_sts: See ntb_msg_clear_sts().
+ * @msg_set_mask:  See ntb_msg_set_mask().
+ * @msg_clear_mask:See ntb_msg_clear_mask().
+ * @msg_read:  See ntb_msg_read().
+ * @msg_write: See ntb_msg_write().
  */
 struct ntb_dev_ops {
int (*port_number)(struct ntb_dev *ntb);
@@ -282,6 +294,16 @@ struct ntb_dev_ops {
u32 (*peer_spad_read)(struct ntb_dev *ntb, int pidx, int sidx);
int (*peer_spad_write)(struct ntb_dev *ntb, int pidx, int sidx,
   u32 val);
+
+   int (*msg_count)(struct ntb_dev *ntb);
+   u64 (*msg_inbits)(struct ntb_dev *ntb);
+   u64 (*msg_outbits)(struct ntb_dev *ntb);
+   u64 (*msg_read_sts)(struct ntb_dev *ntb);
+   int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits);
+   int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_read)(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg);
+   int (*msg_write)(struct ntb_dev *ntb, int midx, int pidx, u32 msg);
 };
 
 static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
@@ -329,6 +351,15 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* !ops->peer_spad_addr == !ops->spad_count && */
/* !ops->peer_spad_read == !ops->spad_count && */
!ops->peer_spad_write == !ops->spad_count &&
+
+   !ops->

[PATCH v2 9/9] NTB: Add ntb.h comments

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 include/linux/ntb.h | 17 +++--
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index fe0437c..c5a369c 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -312,13 +312,18 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
 {
/* commented callbacks are not required: */
return
+   /* Port operations are required */
ops->port_number&&
ops->peer_port_count&&
ops->peer_port_number   &&
ops->peer_port_idx  &&
+
+   /* Link operations are requiered */
ops->link_is_up &&
ops->link_enable&&
ops->link_disable   &&
+
+   /* One or both MW interfaces should be developed */
ops->mw_count   &&
ops->mw_get_align   &&
(ops->mw_set_trans  ||
@@ -328,12 +333,11 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
ops->peer_mw_get_addr   &&
/* ops->peer_mw_clear_trans && */
 
+   /* Doorbell operations are mostly required */
/* ops->db_is_unsafe&& */
ops->db_valid_mask  &&
-
/* both set, or both unset */
(!ops->db_vector_count == !ops->db_vector_mask) &&
-
ops->db_read&&
/* ops->db_set  && */
ops->db_clear   &&
@@ -347,6 +351,8 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* ops->peer_db_read_mask   && */
/* ops->peer_db_set_mask&& */
/* ops->peer_db_clear_mask  && */
+
+   /* Scrachpad interface is optional */
/* !ops->spad_is_unsafe == !ops->spad_count && */
!ops->spad_read == !ops->spad_count &&
!ops->spad_write == !ops->spad_count&&
@@ -354,6 +360,7 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* !ops->peer_spad_read == !ops->spad_count && */
!ops->peer_spad_write == !ops->spad_count &&
 
+   /* Message registers interface is optional */
!ops->msg_inbits == !ops->msg_count &&
!ops->msg_outbits == !ops->msg_count&&
!ops->msg_read_sts == !ops->msg_count   &&
@@ -374,13 +381,12 @@ struct ntb_client {
struct device_driverdrv;
const struct ntb_client_ops ops;
 };
-
 #define drv_ntb_client(__drv) container_of((__drv), struct ntb_client, drv)
 
 /**
  * struct ntb_device - ntb device
  * @dev:   Linux device object.
- * @pdev:  Pci device entry of the ntb.
+ * @pdev:  PCI device entry of the ntb.
  * @topo:  Detected topology of the ntb.
  * @ops:   See _dev_ops.
  * @ctx:   See _ctx_ops.
@@ -401,7 +407,6 @@ struct ntb_dev {
/* block unregister until device is fully released */
struct completion   released;
 };
-
 #define dev_ntb(__dev) container_of((__dev), struct ntb_dev, dev)
 
 /**
@@ -498,7 +503,7 @@ void ntb_link_event(struct ntb_dev *ntb);
  * multiple interrupt vectors for doorbells, the vector number indicates which
  * vector received the interrupt.  The vector number is relative to the first
  * vector used for doorbells, starting at zero, and must be less than
- ** ntb_db_vector_count().  The driver may call ntb_db_read() to check which
+ * ntb_db_vector_count().  The driver may call ntb_db_read() to check which
  * doorbell bits need service, and ntb_db_vector_mask() to determine which of
  * those bits are associated with the vector number.
  */
-- 
2.6.6

Re: [PATCH v2 2/2] eeprom: Add IDT 89HPESx driver bindings file

[Serge Semin]

Rob,
Could you please respond on these comments? I've got some free time, so I wanna
rewrite the code until I've not got busy again.

Regards,
-Sergey

On Mon, Dec 05, 2016 at 10:04:56PM +0300, Serge Semin <fancer.lan...@gmail.com> 
wrote:
> On Mon, Dec 05, 2016 at 11:27:07AM -0600, Rob Herring <r...@kernel.org> wrote:
> > On Mon, Dec 5, 2016 at 9:25 AM, Serge Semin <fancer.lan...@gmail.com> wrote:
> > > On Mon, Dec 05, 2016 at 08:46:21AM -0600, Rob Herring <r...@kernel.org> 
> > > wrote:
> > >> On Tue, Nov 29, 2016 at 01:38:21AM +0300, Serge Semin wrote:
> > >> > See cover-letter for changelog
> > >> >
> > >> > Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
> > >> >
> > >> > ---
> > >> >  .../devicetree/bindings/misc/idt_89hpesx.txt   | 41 
> > >> > ++
> > >>
> > >> There's not a better location for this? I can't tell because you don't
> > >> describe what the device is.
> > >>
> > >
> > > The device is PCIe-switch EEPROM driver with additional debug-interface to
> > > access the switch CSRs. EEPROM is accesses via a separate i2c-slave
> > > interface of the switch.
> > >
> > > There might be another place to put the binding file in. There is a 
> > > special
> > > location for EEPROM drivers bindings - 
> > > Documentation/devicetree/bindings/eeprom/ .
> > > But as far as I understood from the files put in there, it's intended for
> > > pure EEPROM drivers only. On the other hand the directory I've chosen:
> > > Documentation/devicetree/bindings/misc/
> > > mostly intended for some unusual devices. My device isn't usual, since it
> > > has CSRs debug-interface as well. Additionally I've found
> > > eeprom-93xx46.txt binding file there, which describes EEPROM bindings.
> > >
> > > Anyway if you find the file should be placed in
> > > Documentation/devicetree/bindings/eeprom/ instead, I'll move it, it's not
> > > that a big problem.
> > >
> 
> What about this comment? Shall the file be left at the path I placed it?
> 
> > >> >  1 file changed, 41 insertions(+)
> > >> >  create mode 100644 
> > >> > Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> > >> >
> > >> > diff --git a/Documentation/devicetree/bindings/misc/idt_89hpesx.txt 
> > >> > b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> > >> > index 000..469cc93
> > >> > --- /dev/null
> > >> > +++ b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> > >> > @@ -0,0 +1,41 @@
> > >> > +EEPROM / CSR SMBus-slave interface of IDT 89HPESx devices
> > >> > +
> > >> > +Required properties:
> > >> > +  - compatible : should be ","
> > >> > +Basically there is only one manufacturer: idt, but some
> > >> > +compatible devices may be produced in future. Following 
> > >> > devices
> > >> > +are supported: 89hpes8nt2, 89hpes12nt3, 89hpes24nt6ag2,
> > >> > +89hpes32nt8ag2, 89hpes32nt8bg2, 89hpes12nt12g2, 
> > >> > 89hpes16nt16g2,
> > >> > +89hpes24nt24g2, 89hpes32nt24ag2, 89hpes32nt24bg2;
> > >> > +89hpes12n3, 89hpes12n3a, 89hpes24n3, 89hpes24n3a;
> > >> > +89hpes32h8, 89hpes32h8g2, 89hpes48h12, 89hpes48h12g2,
> > >> > +89hpes48h12ag2, 89hpes16h16, 89hpes22h16, 89hpes22h16g2,
> > >> > +89hpes34h16, 89hpes34h16g2, 89hpes64h16, 89hpes64h16g2,
> > >> > +89hpes64h16ag2;
> > >> > +89hpes12t3g2, 89hpes24t3g2, 89hpes16t4, 89hpes4t4g2,
> > >> > +89hpes10t4g2, 89hpes16t4g2, 89hpes16t4ag2, 89hpes5t5,
> > >> > +89hpes6t5, 89hpes8t5, 89hpes8t5a, 89hpes24t6, 89hpes6t6g2,
> > >> > +89hpes24t6g2, 89hpes16t7, 89hpes32t8, 89hpes32t8g2,
> > >> > +89hpes48t12, 89hpes48t12g2.
> > >> > +Current implementation of the driver doesn't have any 
> > >> > device-
> > >>
> > >> Driver capabilties are irrelevant to bindings.
> > >>
> > >
> > > Why? I've told in the comment, that the devices actually differ by the 
> > > CSRs
> > > map. Even though it's not reflected in 

[PATCH v3 4/9] NTB: Alter MW API to support multi-ports devices

[Serge Semin]

Multi-port NTB devices permit to share a memory between all accessible peers.
Memory Windows API is altered to correspondingly initialize and map memory
windows for such devices:
 ntb_mw_count(pidx); - number of inbound memory windows, which can be allocated
for shared buffer with specified peer device.
 ntb_mw_get_align(pidx, widx); - get alignment and size restriction parameters
to properly allocate inbound memory region.
 ntb_peer_mw_count(); - get number of outbound memory windows.
 ntb_peer_mw_get_addr(widx); - get mapping address of an outbound memory window

If hardware supports inbound translation configured on the local ntb port:
 ntb_mw_set_trans(pidx, widx); - set translation address of allocated inbound
memory window so a peer device could access it.
 ntb_mw_clear_trans(pidx, widx); - clear the translation address of an inbound
memory window.

If hardware supports outbound translation configured on the peer ntb port:
 ntb_peer_mw_set_trans(pidx, widx); - set translation address of a memory
window retrieved from a peer device
 ntb_peer_mw_clear_trans(pidx, widx); - clear the translation address of an
outbound memory window

Acked-by: Allen Hubbe <allen.hu...@dell.com>
Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/ntb/hw/amd/ntb_hw_amd.c |  68 +---
 drivers/ntb/hw/intel/ntb_hw_intel.c |  90 
 drivers/ntb/ntb.c   |   2 +
 drivers/ntb/ntb_transport.c |  21 +++-
 drivers/ntb/test/ntb_perf.c |  17 ++-
 drivers/ntb/test/ntb_tool.c |  43 +---
 include/linux/ntb.h | 208 
 7 files changed, 342 insertions(+), 107 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 4d8d0bd..6a41c38 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
  *   BSD LICENSE
  *
  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -79,40 +81,42 @@ static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
return 1 << idx;
 }
 
-static int amd_ntb_mw_count(struct ntb_dev *ntb)
+static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 {
+   if (pidx != NTB_DEF_PEER_IDX)
+   return -EINVAL;
+
return ntb_ndev(ntb)->mw_count;
 }
 
-static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
-   phys_addr_t *base,
-   resource_size_t *size,
-   resource_size_t *align,
-   resource_size_t *align_size)
+static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
+   resource_size_t *addr_align,
+   resource_size_t *size_align,
+   resource_size_t *size_max)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
int bar;
 
+   if (pidx != NTB_DEF_PEER_IDX)
+   return -EINVAL;
+
bar = ndev_mw_to_bar(ndev, idx);
if (bar < 0)
return bar;
 
-   if (base)
-   *base = pci_resource_start(ndev->ntb.pdev, bar);
-
-   if (size)
-   *size = pci_resource_len(ndev->ntb.pdev, bar);
+   if (addr_align)
+   *addr_align = SZ_4K;
 
-   if (align)
-   *align = SZ_4K;
+   if (size_align)
+   *size_align = 1;
 
-   if (align_size)
-   *align_size = 1;
+   if (size_max)
+   *size_max = pci_resource_len(ndev->ntb.pdev, bar);
 
return 0;
 }
 
-static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
+static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
dma_addr_t addr, resource_size_t size)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
@@ -122,6 +126,9 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int 
idx,
u64 base_addr, limit, reg_val;
int bar;
 
+   if (pidx != NTB_DEF_PEER_IDX)
+   return -EINVAL;
+
bar = ndev_mw_to_bar(ndev, idx);
if (bar < 0)
return bar;
@@ -285,6 +292,31 @@ static int amd_ntb_link_disable(struct ntb_dev *ntb)
return 0;
 }
 
+static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
+{
+   /* The same as for inbound MWs */
+   return ntb_n

[PATCH v3 5/9] NTB: Alter Scratchpads API to support multi-ports devices

[Serge Semin]

Even though there is no any real NTB hardware, which would have both more
than two ports and Scratchpad registers, it is logically correct to have
Scratchpad API accepting a peer port index as well. Intel/AMD drivers utilize
Primary and Secondary topology to split Scratchpad between connected root
devices. Since port-index API introduced, Intel/AMD NTB hardware drivers can
use device port to determine which Scratchpad registers actually belong to
local and peer devices. The same approach can be used if some potential
hardware in future will be multi-port and have some set of Scratchpads.
Here are the brief of changes in the API:
 ntb_spad_count() - return number of Scratchpads per each port
 ntb_peer_spad_addr(pidx, sidx) - address of Scratchpad register of the
peer device with pidx-index
 ntb_peer_spad_read(pidx, sidx) - read specified Scratchpad register of the
peer with pidx-index
 ntb_peer_spad_write(pidx, sidx) - write data to Scratchpad register of the
peer with pidx-index

Since there is hardware which doesn't support Scratchpad registers, the
corresponding API methods are now made optional.

Acked-by: Allen Hubbe <allen.hu...@dell.com>
Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/ntb/hw/amd/ntb_hw_amd.c | 14 +++
 drivers/ntb/hw/intel/ntb_hw_intel.c | 14 +++
 drivers/ntb/ntb_transport.c | 17 -
 drivers/ntb/test/ntb_perf.c |  6 +--
 drivers/ntb/test/ntb_pingpong.c |  8 +++-
 drivers/ntb/test/ntb_tool.c | 21 --
 include/linux/ntb.h | 76 +++--
 7 files changed, 98 insertions(+), 58 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 6a41c38..bc537aa 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -433,30 +433,30 @@ static int amd_ntb_spad_write(struct ntb_dev *ntb,
return 0;
 }
 
-static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
void __iomem *mmio = ndev->self_mmio;
u32 offset;
 
-   if (idx < 0 || idx >= ndev->spad_count)
+   if (sidx < 0 || sidx >= ndev->spad_count)
return -EINVAL;
 
-   offset = ndev->peer_spad + (idx << 2);
+   offset = ndev->peer_spad + (sidx << 2);
return readl(mmio + AMD_SPAD_OFFSET + offset);
 }
 
-static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
-  int idx, u32 val)
+static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
+  int sidx, u32 val)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
void __iomem *mmio = ndev->self_mmio;
u32 offset;
 
-   if (idx < 0 || idx >= ndev->spad_count)
+   if (sidx < 0 || sidx >= ndev->spad_count)
return -EINVAL;
 
-   offset = ndev->peer_spad + (idx << 2);
+   offset = ndev->peer_spad + (sidx << 2);
writel(val, mmio + AMD_SPAD_OFFSET + offset);
 
return 0;
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 4b84012..7bb14cb 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -1409,30 +1409,30 @@ static int intel_ntb_spad_write(struct ntb_dev *ntb,
   ndev->self_reg->spad);
 }
 
-static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
+static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
phys_addr_t *spad_addr)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
+   return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
  ndev->peer_reg->spad);
 }
 
-static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_read(ndev, idx,
+   return ndev_spad_read(ndev, sidx,
  ndev->peer_mmio +
  ndev->peer_reg->spad);
 }
 
-static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
-int idx, u32 val)
+static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
+int sidx, u32 val)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_write(ndev, idx, val,
+   return ndev_spad_write(ndev, sidx, val,
   ndev->peer_mmio +
   ndev->peer_reg->spad);
 }
diff --git a/drivers/ntb/ntb_transport.c b/drivers/nt

Re: [PATCH v4 1/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

On Fri, Jan 13, 2017 at 08:22:35AM +0100, Greg KH <gre...@linuxfoundation.org> 
wrote:
> On Fri, Jan 13, 2017 at 01:54:17AM +0300, Serge Semin wrote:
> > On Wed, Jan 11, 2017 at 09:21:19AM +0100, Greg KH 
> > <gre...@linuxfoundation.org> wrote:
> > > > +   /* Return failure if root directory doesn't exist */
> > > > +   if (!csr_dbgdir) {
> > > > +   dev_dbg(dev, "No Debugfs root directory");
> > > > +   return -EINVAL;
> > > > +   }
> > > 
> > > If debugfs is not enabled, don't error out, just keep going, it should
> > > never stop kernel code from running properly.
> > > 
> > > Also, this test isn't really doing what you think it is doing...
> > > 
> > 
> > I see, it must be replaced with IS_ERR_OR_NULL() test.
> 
> No!  That's a pain, when the debugfs interface was created its goal was
> to make it _easy_ to use, not hard.  IS_ERR_OR_NULL() is hard, and
> messy, don't do that.
> 
> > But I don't think,
> > it would be good to get rid of dev_dbg() completely here. In case if
> > debugging is enabled, user would understand why csr-node isn't created 
> > within
> > DebugFS directory. I don't see the reasoning why one shouldn't know a source
> > of possible problems.
> > (See the next comment as continue of the discussion)
> 
> Why would a user care about debugfs?
> 
> > > > +   /* Create Debugfs directory for CSR file */
> > > > +   snprintf(fname, CSRNAME_LEN, "%d-%04hx", cli->adapter->nr, 
> > > > cli->addr);
> > > > +   pdev->csr_dir = debugfs_create_dir(fname, csr_dbgdir);
> > > > +   if (IS_ERR_OR_NULL(pdev->csr_dir)) {
> > > > +   dev_err(dev, "Failed to create CSR node directory");
> > > > +   return -EINVAL;
> > > 
> > > Again, don't do this, you really don't care if debugfs worked or not.
> > > 
> > 
> > Actually the driver doesn't stop the kernel code from running, if it finds 
> > out
> > any problem with DebugFS CSR-node creation. The function just logs the error
> > and return error status. Take a look the place the method is called:
> > 1489/* Create debugfs files */
> > 1490(void)idt_create_dbgfs_files(pdev);
> > The initialization code doesn't check the return value at all, so the driver
> > will proceed with further code.
> > Why did I make the function with return value? Because it's a good style to
> > always return a status of function code execution if it may fail, but only
> > caller will decide whether to check the return value or not.
> 
> There is only one type of error that a debugfs call can return, and that
> is if debugfs is not enabled in the build.  That's it, you don't need to
> care about any of that.
> 
> > Regarding the error printing. In case if the code gets to this check, one 
> > can
> > be sure the DebugFS works properly, so in case if the driver failed to 
> > create
> > the corresponding sub-directory or node, it is really error to have any 
> > failure
> > at this point, and a user should be notified. But still the driver won't 
> > stop
> > functioning, since the caller doesn't check the return value.
> > 
> > Hopefully you'll understand my point.
> 
> Please understand mine, debugfs is supposed to be easy to use, you are
> not testing things properly here, and when you are, it doesn't matter.
> Just call the functions, save the return results if you need to (for
> dentries and the like), and move on.  No error handling needed AT ALL!
> 
> Yes, it feels "odd" for kernel code, but remember, this is only for
> debugging.  Your code should not have any different codepaths for if the
> debugging logic worked or not.  It doesn't care at all.  So please, make
> it simple.
> 
> > > > +   dev_dbg(dev, "Debugfs-files created");
> > > 
> > > You do know about ftrace, right?  Please remove all of these
> > > "trace-like" debugging lines, they aren't needed for anyone.
> > > 
> > 
> > Ok, I'll remove all these prints, even though I do find these prints being
> > handy to have initialization process printed on debugging stage.
> 
> Then use ftrace, that is what it is there for, don't roll your own
> driver-specific-functionality please.
> 
> thanks,
> 
> greg k-h

Ok, I see your point and do as you say.

Thanks,
Serge

[PATCH v5 0/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

Changelog v3:
- Get rid of dev_*_idt() macros
- Replace to_pdev_kobj() macro with naked dev_get_drvdata() call
- Return naked 0 instead of SUCCESS macro
- IDT CSR debug file is moved to debugfs
- BIN_ATTR_RW is used to declare sysfs binary attribute
- Moved bindings file to a separate patch
- Need to create a specific bin_attribute structure for each device
- Perform a few read retries with delays if EEPROM is busy

Changelog v4:
- Make 89HPESx device bindings to have one EEPROM subnode
- Alter 89HPESx device bindings text

Changelog v5:
- Replace atomic_t CSR field with of u16 type
- Get rid of dev_dbg() print outs of initialization methods
- Simplify DebugFS initialization method

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

Serge Semin (2):
  eeprom: Add IDT 89HPESx EEPROM/CSR driver
  eeprom: Add IDT 89HPESx driver bindings file

 .../devicetree/bindings/misc/idt_89hpesx.txt   |   44 +
 drivers/misc/eeprom/Kconfig|   10 +
 drivers/misc/eeprom/Makefile   |1 +
 drivers/misc/eeprom/idt_89hpesx.c  | 1587 
 4 files changed, 1642 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt
 create mode 100644 drivers/misc/eeprom/idt_89hpesx.c

-- 
2.6.6

[PATCH v5 1/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

  This driver provides an access to EEPROM of IDT PCIe-switches. IDT PCIe-
switches expose a simple SMBus interface to perform IO-operations from/to
EEPROM, which is located at private (so called Master) SMBus. The driver
creates a simple binary sysfs-file to have an access to the EEPROM using
the SMBus-slave interface in the i2c-device susfs-directory:
 /sys/bus/i2c/devices/-/eeprom
In case if read-only flag is specified at dts-node of the device, User-space
applications won't be able to write to the EEPROM sysfs-node.

  Additionally IDT 89HPESx SMBus interface has an ability to read/write
values of device CSRs. This driver exposes debugfs-file to perform simple
IO-operations using that ability for just basic debug purpose. Particularly
the next file is created in the specific debugfs-directory:
 /sys/kernel/debug/idt_csr/
Format of the debugfs-file value is:
 $ cat /sys/kernel/debug/idt_csr/-/;
 :
So reading the content of the file gives current CSR address and it value.
If User-space application wishes to change current CSR address, it can just
write a proper value to the sysfs-file:
 $ echo "" >
 /sys/kernel/debug/idt_csr/-/
If it wants to change the CSR value as well, the format of the write
operation is:
 $ echo ":" > \
 /sys/kernel/debug/idt_csr/-/;
CSR address and value can be any of hexadecimal, decimal or octal format.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 drivers/misc/eeprom/Kconfig   |   10 +
 drivers/misc/eeprom/Makefile  |1 +
 drivers/misc/eeprom/idt_89hpesx.c | 1587 +
 3 files changed, 1598 insertions(+)
 create mode 100644 drivers/misc/eeprom/idt_89hpesx.c

diff --git a/drivers/misc/eeprom/Kconfig b/drivers/misc/eeprom/Kconfig
index c4e41c2..de58762 100644
--- a/drivers/misc/eeprom/Kconfig
+++ b/drivers/misc/eeprom/Kconfig
@@ -100,4 +100,14 @@ config EEPROM_DIGSY_MTC_CFG
 
  If unsure, say N.
 
+config EEPROM_IDT_89HPESX
+   tristate "IDT 89HPESx PCIe-swtiches EEPROM / CSR support"
+   depends on I2C && SYSFS
+   help
+ Enable this driver to get read/write access to EEPROM / CSRs
+ over IDT PCIe-swtich i2c-slave interface.
+
+ This driver can also be built as a module. If so, the module
+ will be called idt_89hpesx.
+
 endmenu
diff --git a/drivers/misc/eeprom/Makefile b/drivers/misc/eeprom/Makefile
index fc1e81d..90a5262 100644
--- a/drivers/misc/eeprom/Makefile
+++ b/drivers/misc/eeprom/Makefile
@@ -5,3 +5,4 @@ obj-$(CONFIG_EEPROM_MAX6875)+= max6875.o
 obj-$(CONFIG_EEPROM_93CX6) += eeprom_93cx6.o
 obj-$(CONFIG_EEPROM_93XX46)+= eeprom_93xx46.o
 obj-$(CONFIG_EEPROM_DIGSY_MTC_CFG) += digsy_mtc_eeprom.o
+obj-$(CONFIG_EEPROM_IDT_89HPESX) += idt_89hpesx.o
diff --git a/drivers/misc/eeprom/idt_89hpesx.c 
b/drivers/misc/eeprom/idt_89hpesx.c
new file mode 100644
index 000..c53f0e5
--- /dev/null
+++ b/drivers/misc/eeprom/idt_89hpesx.c
@@ -0,0 +1,1587 @@
+/*
+ *   This file is provided under a GPLv2 license.  When using or
+ *   redistributing this file, you may do so under that license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify it
+ *   under the terms and conditions of the GNU General Public License,
+ *   version 2, as published by the Free Software Foundation.
+ *
+ *   This program is distributed in the hope that it will be useful, but 
WITHOUT
+ *   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ *   more details.
+ *
+ *   You should have received a copy of the GNU General Public License along
+ *   with this program; if not, it can be found <http://www.gnu.org/licenses/>.
+ *
+ *   The full GNU General Public License is included in this distribution in
+ *   the file called "COPYING".
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * IDT PCIe-switch NTB Linux driver
+ *
+ * Contact Information:
+ * Serge Semin <fancer.lan...@gmail.com>, 

[PATCH v5 2/2] eeprom: Add IDT 89HPESx driver bindings file

[Serge Semin]

IDT 89HPESx PCIe-switches exposes SMBus interface to have an access to
the device CSRs and EEPROM. So to properly utilize the interface
functionality, developer should declare a valid dts-file node, which
would refer to the corresponding 89HPESx device.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 .../devicetree/bindings/misc/idt_89hpesx.txt   | 44 ++
 1 file changed, 44 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt

diff --git a/Documentation/devicetree/bindings/misc/idt_89hpesx.txt 
b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
new file mode 100644
index 000..b9093b7
--- /dev/null
+++ b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
@@ -0,0 +1,44 @@
+EEPROM / CSR SMBus-slave interface of IDT 89HPESx devices
+
+Required properties:
+  - compatible : should be ","
+Basically there is only one manufacturer: idt, but some
+compatible devices may be produced in future. Following devices
+are supported: 89hpes8nt2, 89hpes12nt3, 89hpes24nt6ag2,
+89hpes32nt8ag2, 89hpes32nt8bg2, 89hpes12nt12g2, 89hpes16nt16g2,
+89hpes24nt24g2, 89hpes32nt24ag2, 89hpes32nt24bg2;
+89hpes12n3, 89hpes12n3a, 89hpes24n3, 89hpes24n3a;
+89hpes32h8, 89hpes32h8g2, 89hpes48h12, 89hpes48h12g2,
+89hpes48h12ag2, 89hpes16h16, 89hpes22h16, 89hpes22h16g2,
+89hpes34h16, 89hpes34h16g2, 89hpes64h16, 89hpes64h16g2,
+89hpes64h16ag2;
+89hpes12t3g2, 89hpes24t3g2, 89hpes16t4, 89hpes4t4g2,
+89hpes10t4g2, 89hpes16t4g2, 89hpes16t4ag2, 89hpes5t5,
+89hpes6t5, 89hpes8t5, 89hpes8t5a, 89hpes24t6, 89hpes6t6g2,
+89hpes24t6g2, 89hpes16t7, 89hpes32t8, 89hpes32t8g2,
+89hpes48t12, 89hpes48t12g2.
+  - reg :   I2C address of the IDT 89HPESx device.
+
+Optionally there can be EEPROM-compatible subnode:
+  - compatible:  There are five EEPROM devices supported: 24c32, 24c64, 24c128,
+24c256 and 24c512 differed by size.
+  - reg: Custom address of EEPROM device (If not specified IDT 89HPESx
+(optional)  device will try to communicate with EEPROM sited by default
+address - 0x50)
+  - read-only : Parameterless property disables writes to the EEPROM
+(optional)
+
+Example:
+   idt@60 {
+   compatible = "idt,89hpes32nt8ag2";
+   reg = <0x74>;
+   #address-cells = <1>;
+   #size-cells = <0>;
+
+   eeprom@50 {
+   compatible = "onsemi,24c64";
+   reg = <0x50>;
+   read-only;
+   };
+   };
+
-- 
2.6.6

Re: [PATCH v4 1/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

On Wed, Jan 11, 2017 at 09:21:19AM +0100, Greg KH <gre...@linuxfoundation.org> 
wrote:
> On Tue, Dec 13, 2016 at 05:22:50PM +0300, Serge Semin wrote:
> > +struct idt_89hpesx_dev {
> > +   u32 eesize;
> > +   bool eero;
> > +   u8 eeaddr;
> > +
> > +   u8 inieecmd;
> > +   u8 inicsrcmd;
> > +   u8 iniccode;
> > +
> > +   atomic_t csr;
> 
> Why is this an atomic_t and not just a "normal" u32 or u64?  I don't see
> the need for an atomic variable at all here, do you?
> 

Of course, I did. Since it was shared resource before it was necessary to have
it atomically accessed. But since we moved "csr" sysfs node to DebugFS, it 
might not
be necessary of atomic_t type.
Ok, I'll make it unsigned int.

> 
> > +
> > +   int (*smb_write)(struct idt_89hpesx_dev *, const struct idt_smb_seq *);
> > +   int (*smb_read)(struct idt_89hpesx_dev *, struct idt_smb_seq *);
> > +   struct mutex smb_mtx;
> > +
> > +   struct i2c_client *client;
> > +
> > +   struct bin_attribute *ee_file;
> > +   struct dentry *csr_dir;
> > +   struct dentry *csr_file;
> > +};
> 
> 
> 
> > +
> > +static int idt_create_dbgfs_files(struct idt_89hpesx_dev *pdev)
> > +{
> > +   struct device *dev = >client->dev;
> > +   struct i2c_client *cli = pdev->client;
> > +   char fname[CSRNAME_LEN];
> > +
> > +   /* Initialize basic value of CSR debugfs dentries */
> > +   pdev->csr_dir = NULL;
> > +   pdev->csr_file = NULL;
> > +
> > +   /* Return failure if root directory doesn't exist */
> > +   if (!csr_dbgdir) {
> > +   dev_dbg(dev, "No Debugfs root directory");
> > +   return -EINVAL;
> > +   }
> 
> If debugfs is not enabled, don't error out, just keep going, it should
> never stop kernel code from running properly.
> 
> Also, this test isn't really doing what you think it is doing...
> 

I see, it must be replaced with IS_ERR_OR_NULL() test. But I don't think,
it would be good to get rid of dev_dbg() completely here. In case if
debugging is enabled, user would understand why csr-node isn't created within
DebugFS directory. I don't see the reasoning why one shouldn't know a source
of possible problems.
(See the next comment as continue of the discussion)

> > +   /* Create Debugfs directory for CSR file */
> > +   snprintf(fname, CSRNAME_LEN, "%d-%04hx", cli->adapter->nr, cli->addr);
> > +   pdev->csr_dir = debugfs_create_dir(fname, csr_dbgdir);
> > +   if (IS_ERR_OR_NULL(pdev->csr_dir)) {
> > +   dev_err(dev, "Failed to create CSR node directory");
> > +   return -EINVAL;
> 
> Again, don't do this, you really don't care if debugfs worked or not.
> 

Actually the driver doesn't stop the kernel code from running, if it finds out
any problem with DebugFS CSR-node creation. The function just logs the error
and return error status. Take a look the place the method is called:
1489/* Create debugfs files */
1490(void)idt_create_dbgfs_files(pdev);
The initialization code doesn't check the return value at all, so the driver
will proceed with further code.
Why did I make the function with return value? Because it's a good style to
always return a status of function code execution if it may fail, but only
caller will decide whether to check the return value or not.

Regarding the error printing. In case if the code gets to this check, one can
be sure the DebugFS works properly, so in case if the driver failed to create
the corresponding sub-directory or node, it is really error to have any failure
at this point, and a user should be notified. But still the driver won't stop
functioning, since the caller doesn't check the return value.

Hopefully you'll understand my point.

> > +   }
> > +
> > +   /* Create Debugfs file for CSR read/write operations */
> > +   pdev->csr_file = debugfs_create_file(cli->name, 0600,
> > +   pdev->csr_dir, pdev, _dbgfs_ops);
> > +   if (IS_ERR_OR_NULL(pdev->csr_file)) {
> > +   dev_err(dev, "Failed to create CSR dbgfs-node");
> > +   debugfs_remove_recursive(pdev->csr_dir);
> > +   return -EINVAL;
> 
> Same here, just create the file and move on.
> 
> > +   }
> > +
> > +   dev_dbg(dev, "Debugfs-files created");
> 
> You do know about ftrace, right?  Please remove all of these
> "trace-like" debugging lines, they aren't needed for anyone.
> 

Ok, I'll remove all these prints, even though I do find these prints being
handy to have initialization process printed on debugging stage.

> thanks,
> 
> greg k-h

thanks,
Sergey

Re: [PATCH v2 1/2] eeprom: Add IDT 89HPESx EEPROM/CSR driver

[Serge Semin]

On Tue, Nov 29, 2016 at 08:37:50PM +0100, Greg KH <gre...@linuxfoundation.org> 
wrote:
> On Tue, Nov 29, 2016 at 01:38:20AM +0300, Serge Semin wrote:
> > +struct idt_89hpesx_dev {
> > +   u32 eesize;
> > +   bool eero;
> > +   u8 eeaddr;
> > +
> > +   u8 inieecmd;
> > +   u8 inicsrcmd;
> > +   u8 iniccode;
> > +
> > +   atomic_t csr;
> > +
> > +   int (*smb_write)(struct idt_89hpesx_dev *, const struct idt_smb_seq *);
> > +   int (*smb_read)(struct idt_89hpesx_dev *, struct idt_smb_seq *);
> > +   struct mutex smb_mtx;
> > +
> > +   struct i2c_client *client;
> > +
> > +   struct bin_attribute *ee_file;
> > +   struct dentry *csr_dir;
> > +   struct dentry *csr_file;
> > +};
> > +#define to_pdev_kobj(__kobj) \
> > +   dev_get_drvdata(container_of(__kobj, struct device, kobj))
> 
> Is it a struct device, or a kobject?  This is totally confusing to me.
> 
> And can't you just use kobj_to_dev()?
> 

I just didn't know about kobj_to_dev() inline function. Totally agree that
container_of() should be replaced with it.
What does look confusing to you? Do you mean the name "to_pdev_kobj" of the
macro?

> > +/*
> > + * eeprom_attribute - EEPROM sysfs-node attributes
> > + *
> > + * NOTE Size will be changed in compliance with OF node. EEPROM attribute 
> > will
> > + * be read-only as well if the corresponding flag is specified in OF node.
> > + */
> > +BIN_ATTR(eeprom, 0644, idt_sysfs_eeprom_read, idt_sysfs_eeprom_write,
> > +EEPROM_DEF_SIZE);
> 
> static?
> 
> And BIN_ATTR_RW()?
> 
> thanks,
> 
> greg k-h

Of course it should be static. Thanks for noticing that.
But I intentionally utilized BIN_ATTR() instead of BIN_ATTR_RW(), because
the last one implies to define the read/write methods with names
"_name##_read"/"_name##_write", which totally get out of naming within the
driver source code. To tell the truth macro BIN_ATTR_RW() isn't that
popular in the kernel. Neither is BIN_ATTR() macro, but it suites my driver
better than the another one.

Thanks,
-Sergey

[PATCH 01/22] NTB: Move link state API being first in sources

[Serge Semin]

Since link operations are usually performed before memory window access
operations, it's logically better to declared link-related API before any
other methods. Additionally it's good practice for readability to declare
NTB device callback methods of hadrware drivers with the same order as it's
done within ntb.h.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 include/linux/ntb.h | 137 ++--
 1 file changed, 69 insertions(+), 68 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 6f47562..5d1f260 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -179,13 +179,13 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
 
 /**
  * struct ntb_ctx_ops - ntb device operations
+ * @link_is_up:See ntb_link_is_up().
+ * @link_enable:   See ntb_link_enable().
+ * @link_disable:  See ntb_link_disable().
  * @mw_count:  See ntb_mw_count().
  * @mw_get_range:  See ntb_mw_get_range().
  * @mw_set_trans:  See ntb_mw_set_trans().
  * @mw_clear_trans:See ntb_mw_clear_trans().
- * @link_is_up:See ntb_link_is_up().
- * @link_enable:   See ntb_link_enable().
- * @link_disable:  See ntb_link_disable().
  * @db_is_unsafe:  See ntb_db_is_unsafe().
  * @db_valid_mask: See ntb_db_valid_mask().
  * @db_vector_count:   See ntb_db_vector_count().
@@ -212,6 +212,12 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @peer_spad_write:   See ntb_peer_spad_write().
  */
 struct ntb_dev_ops {
+   int (*link_is_up)(struct ntb_dev *ntb,
+ enum ntb_speed *speed, enum ntb_width *width);
+   int (*link_enable)(struct ntb_dev *ntb,
+  enum ntb_speed max_speed, enum ntb_width max_width);
+   int (*link_disable)(struct ntb_dev *ntb);
+
int (*mw_count)(struct ntb_dev *ntb);
int (*mw_get_range)(struct ntb_dev *ntb, int idx,
phys_addr_t *base, resource_size_t *size,
@@ -220,12 +226,6 @@ struct ntb_dev_ops {
dma_addr_t addr, resource_size_t size);
int (*mw_clear_trans)(struct ntb_dev *ntb, int idx);
 
-   int (*link_is_up)(struct ntb_dev *ntb,
- enum ntb_speed *speed, enum ntb_width *width);
-   int (*link_enable)(struct ntb_dev *ntb,
-  enum ntb_speed max_speed, enum ntb_width max_width);
-   int (*link_disable)(struct ntb_dev *ntb);
-
int (*db_is_unsafe)(struct ntb_dev *ntb);
u64 (*db_valid_mask)(struct ntb_dev *ntb);
int (*db_vector_count)(struct ntb_dev *ntb);
@@ -265,13 +265,14 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
 {
/* commented callbacks are not required: */
return
+   ops->link_is_up &&
+   ops->link_enable&&
+   ops->link_disable   &&
ops->mw_count   &&
ops->mw_get_range   &&
ops->mw_set_trans   &&
/* ops->mw_clear_trans  && */
-   ops->link_is_up &&
-   ops->link_enable&&
-   ops->link_disable   &&
+
/* ops->db_is_unsafe&& */
ops->db_valid_mask  &&
 
@@ -441,6 +442,62 @@ void ntb_link_event(struct ntb_dev *ntb);
 void ntb_db_event(struct ntb_dev *ntb, int vector);
 
 /**
+ * ntb_link_is_up() - get the current ntb link state
+ * @ntb:   NTB device context.
+ * @speed: OUT - The link speed expressed as PCIe generation number.
+ * @width: OUT - The link width expressed as the number of PCIe lanes.
+ *
+ * Get the current state of the ntb link.  It is recommended to query the link
+ * state once after every link event.  It is safe to query the link state in
+ * the context of the link event callback.
+ *
+ * Return: One if the link is up, zero if the link is down, otherwise a
+ * negative value indicating the error number.
+ */
+static inline int ntb_link_is_up(struct ntb_dev *ntb,
+enum ntb_speed *speed, enum ntb_width *width)
+{
+   return ntb->ops->link_is_up(ntb, speed, width);
+}
+
+/**
+ * ntb_link_enable() - enable the link on the secondary side of the ntb
+ * @ntb:   NTB device context.
+ * @max_speed: The maximum link speed expressed as PCIe generation number.
+ * @max_width: The maximum link width expressed as the number of PCIe lanes.
+ *
+ * Enable the link on the secondary side of the ntb.  This can only be done
+ * from the prim

[PATCH 11/22] NTB Intel: Alter MW interface to fit new NTB API

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/intel/ntb_hw_intel.c | 79 -
 1 file changed, 61 insertions(+), 18 deletions(-)

diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 724ccfe..c41da32 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -941,20 +941,26 @@ static int intel_ntb_link_disable(struct ntb_dev *ntb)
return 0;
 }
 
-static int intel_ntb_mw_count(struct ntb_dev *ntb)
+static int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 {
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
return ntb_ndev(ntb)->mw_count;
 }
 
-static int intel_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
- phys_addr_t *base,
- resource_size_t *size,
- resource_size_t *align,
- resource_size_t *align_size)
+static int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
+ resource_size_t *addr_align,
+ resource_size_t *size_align,
+ resource_size_t *size_max)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+   resource_size_t bar_size, mw_size;
int bar;
 
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
if (idx >= ndev->b2b_idx && !ndev->b2b_off)
idx += 1;
 
@@ -962,24 +968,26 @@ static int intel_ntb_mw_get_range(struct ntb_dev *ntb, 
int idx,
if (bar < 0)
return bar;
 
-   if (base)
-   *base = pci_resource_start(ndev->ntb.pdev, bar) +
-   (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
+   bar_size = pci_resource_len(ndev->ntb.pdev, bar);
 
-   if (size)
-   *size = pci_resource_len(ndev->ntb.pdev, bar) -
-   (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
+   if (idx == ndev->b2b_idx)
+   mw_size = bar_size - ndev->b2b_off;
+   else
+   mw_size = bar_size;
 
-   if (align)
-   *align = pci_resource_len(ndev->ntb.pdev, bar);
+   if (addr_align)
+   *addr_align = pci_resource_len(ndev->ntb.pdev, bar);
 
-   if (align_size)
-   *align_size = 1;
+   if (size_align)
+   *size_align = 1;
+
+   if (size_max)
+   *size_max = mw_size;
 
return 0;
 }
 
-static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
+static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
  dma_addr_t addr, resource_size_t size)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
@@ -989,6 +997,9 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int 
idx,
u64 base, limit, reg_val;
int bar;
 
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
if (idx >= ndev->b2b_idx && !ndev->b2b_off)
idx += 1;
 
@@ -1077,6 +1088,36 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, 
int idx,
return 0;
 }
 
+static int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
+{
+   /* Numbers of inbound and outbound memory windows match */
+   return ntb_ndev(ntb)->mw_count;
+}
+
+static int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
+phys_addr_t *base, resource_size_t *size)
+{
+   struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+   int bar;
+
+   if (idx >= ndev->b2b_idx && !ndev->b2b_off)
+   idx += 1;
+
+   bar = ndev_mw_to_bar(ndev, idx);
+   if (bar < 0)
+   return bar;
+
+   if (base)
+   *base = pci_resource_start(ndev->ntb.pdev, bar) +
+   (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
+
+   if (size)
+   *size = pci_resource_len(ndev->ntb.pdev, bar) -
+   (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
+
+   return 0;
+}
+
 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
 {
return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
@@ -2296,8 +2337,10 @@ static const struct ntb_dev_ops intel_ntb_ops = {
.link_enable= intel_ntb_link_enable,
.link_disable   = intel_ntb_link_disable,
.mw_count   = intel_ntb_mw_count,
-   .mw_get_range   = intel_ntb_mw_get_range,
+   .mw_get_align   = intel_ntb_mw_get_align,
.mw_set_trans   = intel_ntb_mw_set_trans,
+   .peer_mw_count  = intel_ntb_peer_mw_count,
+   .peer_mw_get_addr   = intel_ntb_peer_mw_get_addr,
.db_is_unsafe   = intel_ntb_db_is_unsafe,
.db_valid_mask  = intel_ntb_db_valid_mask,
.db_vector_count= intel_ntb_db_vector_count,
-- 
2.6.6

[PATCH 08/22] NTB: Add T-Platforms copyrights to NTB API

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/ntb.c   | 2 ++
 include/linux/ntb.h | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index 4b2cc60..06574f8 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -18,6 +19,7 @@
  *   BSD LICENSE
  *
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 8b19327..9edd9dc 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -18,6 +19,7 @@
  *   BSD LICENSE
  *
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
-- 
2.6.6

[PATCH 09/22] NTB Intel: Move link-related methods being first in the driver

[Serge Semin]

See patch 01 for reasoning of this movement.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/intel/ntb_hw_intel.c | 162 ++--
 1 file changed, 81 insertions(+), 81 deletions(-)

diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 7310a26..d3da0ce 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -836,6 +836,84 @@ static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
debugfs_remove_recursive(ndev->debugfs_dir);
 }
 
+static int intel_ntb_link_is_up(struct ntb_dev *ntb,
+   enum ntb_speed *speed,
+   enum ntb_width *width)
+{
+   struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+   if (ndev->reg->link_is_up(ndev)) {
+   if (speed)
+   *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
+   if (width)
+   *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
+   return 1;
+   } else {
+   /* TODO MAYBE: is it possible to observe the link speed and
+* width while link is training? */
+   if (speed)
+   *speed = NTB_SPEED_NONE;
+   if (width)
+   *width = NTB_WIDTH_NONE;
+   return 0;
+   }
+}
+
+static int intel_ntb_link_enable(struct ntb_dev *ntb,
+enum ntb_speed max_speed,
+enum ntb_width max_width)
+{
+   struct intel_ntb_dev *ndev;
+   u32 ntb_ctl;
+
+   ndev = container_of(ntb, struct intel_ntb_dev, ntb);
+
+   if (ndev->ntb.topo == NTB_TOPO_SEC)
+   return -EINVAL;
+
+   dev_dbg(ndev_dev(ndev),
+   "Enabling link with max_speed %d max_width %d\n",
+   max_speed, max_width);
+   if (max_speed != NTB_SPEED_AUTO)
+   dev_dbg(ndev_dev(ndev), "ignoring max_speed %d\n", max_speed);
+   if (max_width != NTB_WIDTH_AUTO)
+   dev_dbg(ndev_dev(ndev), "ignoring max_width %d\n", max_width);
+
+   ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
+   ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
+   ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
+   ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
+   if (ndev->bar4_split)
+   ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
+   iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
+
+   return 0;
+}
+
+static int intel_ntb_link_disable(struct ntb_dev *ntb)
+{
+   struct intel_ntb_dev *ndev;
+   u32 ntb_cntl;
+
+   ndev = container_of(ntb, struct intel_ntb_dev, ntb);
+
+   if (ndev->ntb.topo == NTB_TOPO_SEC)
+   return -EINVAL;
+
+   dev_dbg(ndev_dev(ndev), "Disabling link\n");
+
+   /* Bring NTB link down */
+   ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
+   ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
+   ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
+   if (ndev->bar4_split)
+   ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
+   ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
+   iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
+
+   return 0;
+}
+
 static int intel_ntb_mw_count(struct ntb_dev *ntb)
 {
return ntb_ndev(ntb)->mw_count;
@@ -972,84 +1050,6 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, 
int idx,
return 0;
 }
 
-static int intel_ntb_link_is_up(struct ntb_dev *ntb,
-   enum ntb_speed *speed,
-   enum ntb_width *width)
-{
-   struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-   if (ndev->reg->link_is_up(ndev)) {
-   if (speed)
-   *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
-   if (width)
-   *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
-   return 1;
-   } else {
-   /* TODO MAYBE: is it possible to observe the link speed and
-* width while link is training? */
-   if (speed)
-   *speed = NTB_SPEED_NONE;
-   if (width)
-   *width = NTB_WIDTH_NONE;
-   return 0;
-   }
-}
-
-static int intel_ntb_link_enable(struct ntb_dev *ntb,
-enum ntb_speed max_speed,
-enum ntb_width max_width)
-{
-   struct intel_ntb_dev *ndev;
-   u32 ntb_ctl;
-
-   ndev = container_of(ntb, struct intel_ntb_dev, ntb);
-
-   if (ndev->ntb.topo == NTB_TOPO_SEC)
-   return -EINVAL;
-
-   dev_dbg(ndev_dev(ndev),
-

[PATCH 12/22] NTB Intel: Alter Scratchpads interface to fit new NTB API

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/intel/ntb_hw_intel.c | 14 +++---
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index c41da32..e308fbe 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -1235,30 +1235,30 @@ static int intel_ntb_spad_write(struct ntb_dev *ntb,
   ndev->self_reg->spad);
 }
 
-static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
+static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
phys_addr_t *spad_addr)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
+   return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
  ndev->peer_reg->spad);
 }
 
-static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_read(ndev, idx,
+   return ndev_spad_read(ndev, sidx,
  ndev->peer_mmio +
  ndev->peer_reg->spad);
 }
 
-static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
-int idx, u32 val)
+static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
+int sidx, u32 val)
 {
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
-   return ndev_spad_write(ndev, idx, val,
+   return ndev_spad_write(ndev, sidx, val,
   ndev->peer_mmio +
   ndev->peer_reg->spad);
 }
-- 
2.6.6

[PATCH 06/22] NTB: Slightly alter link state NTB API

[Serge Semin]

Some minor changes of link state NTB API. Particularly link_is_up()
method from now shall return a bitfield of link states for all accessible
port indexes.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 include/linux/ntb.h | 31 ---
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index fc9d034..a59a155 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -221,7 +221,7 @@ struct ntb_dev_ops {
int (*peer_port_number)(struct ntb_dev *ntb, int pidx);
int (*peer_port_idx)(struct ntb_dev *ntb, int port);
 
-   int (*link_is_up)(struct ntb_dev *ntb,
+   u64 (*link_is_up)(struct ntb_dev *ntb,
  enum ntb_speed *speed, enum ntb_width *width);
int (*link_enable)(struct ntb_dev *ntb,
   enum ntb_speed max_speed, enum ntb_width max_width);
@@ -567,25 +567,26 @@ static inline int ntb_peer_port_idx(struct ntb_dev *ntb, 
int port)
  * state once after every link event.  It is safe to query the link state in
  * the context of the link event callback.
  *
- * Return: One if the link is up, zero if the link is down, otherwise a
- * negative value indicating the error number.
+ * Return: bitfield of indexed ports link state: bit is set/cleared if the
+ * link is up/down respectively, otherwise a negative value indicating
+ * an error number.
  */
-static inline int ntb_link_is_up(struct ntb_dev *ntb,
+static inline u64 ntb_link_is_up(struct ntb_dev *ntb,
 enum ntb_speed *speed, enum ntb_width *width)
 {
return ntb->ops->link_is_up(ntb, speed, width);
 }
 
 /**
- * ntb_link_enable() - enable the link on the secondary side of the ntb
+ * ntb_link_enable() - enable the link of the ntb
  * @ntb:   NTB device context.
  * @max_speed: The maximum link speed expressed as PCIe generation number.
  * @max_width: The maximum link width expressed as the number of PCIe lanes.
  *
- * Enable the link on the secondary side of the ntb.  This can only be done
- * from the primary side of the ntb in primary or b2b topology.  The ntb device
- * should train the link to its maximum speed and width, or the requested speed
- * and width, whichever is smaller, if supported.
+ * Enable the NTB/PCIe link on the local or remote (for bridge-to-bridge
+ * topology) side of the bridge. The ntb device should train the link to its
+ * maximum speed and width, or the requested speed and width, whichever is
+ * smaller, if supported.
  *
  * Return: Zero on success, otherwise an error number.
  */
@@ -597,14 +598,14 @@ static inline int ntb_link_enable(struct ntb_dev *ntb,
 }
 
 /**
- * ntb_link_disable() - disable the link on the secondary side of the ntb
+ * ntb_link_disable() - disable the link of the ntb
  * @ntb:   NTB device context.
  *
- * Disable the link on the secondary side of the ntb.  This can only be
- * done from the primary side of the ntb in primary or b2b topology.  The ntb
- * device should disable the link.  Returning from this call must indicate that
- * a barrier has passed, though with no more writes may pass in either
- * direction across the link, except if this call returns an error number.
+ * Disable the link on the local or remote (for b2b topology) of the ntb.
+ * The ntb device should disable the link.  Returning from this call must
+ * indicate that a barrier has passed, though with no more writes may pass in
+ * either direction across the link, except if this call returns an error
+ * number.
  *
  * Return: Zero on success, otherwise an error number.
  */
-- 
2.6.6

[PATCH 10/22] NTB Intel: Add port-related NTB API callback methods

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/intel/ntb_hw_intel.c | 195 +---
 drivers/ntb/hw/intel/ntb_hw_intel.h |  10 ++
 2 files changed, 124 insertions(+), 81 deletions(-)

diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index d3da0ce..724ccfe 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -213,7 +213,7 @@ static inline void ndev_reset_unsafe_flags(struct 
intel_ntb_dev *ndev)
 
/* Only B2B has a workaround to avoid SDOORBELL */
if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
-   if (!ntb_topo_is_b2b(ndev->ntb.topo))
+   if (ndev->ntb.topo != NTB_TOPO_B2B)
ndev->unsafe_flags |= NTB_UNSAFE_DB;
 
/* No low level workaround to avoid SB01BASE */
@@ -574,8 +574,8 @@ static ssize_t ndev_debugfs_read(struct file *filp, char 
__user *ubuf,
 "NTB Device Information:\n");
 
off += scnprintf(buf + off, buf_size - off,
-"Connection Topology -\t%s\n",
-ntb_topo_string(ndev->ntb.topo));
+"Connection Topology -\t%s:%d\n",
+ntb_topo_string(ndev->ntb.topo), ndev->ntb.port);
 
if (ndev->b2b_idx != UINT_MAX) {
off += scnprintf(buf + off, buf_size - off,
@@ -707,7 +707,7 @@ static ssize_t ndev_debugfs_read(struct file *filp, char 
__user *ubuf,
}
 
if (pdev_is_xeon(pdev)) {
-   if (ntb_topo_is_b2b(ndev->ntb.topo)) {
+   if (ndev->ntb.topo == NTB_TOPO_B2B) {
off += scnprintf(buf + off, buf_size - off,
 "\nNTB Outgoing B2B XLAT:\n");
 
@@ -836,7 +836,34 @@ static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
debugfs_remove_recursive(ndev->debugfs_dir);
 }
 
-static int intel_ntb_link_is_up(struct ntb_dev *ntb,
+static int intel_ntb_port_number(struct ntb_dev *ntb)
+{
+   return ntb->port;
+}
+
+static int intel_ntb_peer_port_count(struct ntb_dev *ntb)
+{
+   return NTB_PEER_CNT;
+}
+
+static int intel_ntb_peer_port_number(struct ntb_dev *ntb, int pidx)
+{
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
+   return (ntb->port == NTB_PORT_PRI ? NTB_PORT_SEC : NTB_PORT_PRI);
+}
+
+static int intel_ntb_peer_port_idx(struct ntb_dev *ntb, int port)
+{
+   if ((ntb->port == NTB_PORT_PRI && port != NTB_PORT_SEC) ||
+   (ntb->port == NTB_PORT_SEC && port != NTB_PORT_PRI))
+   return -EINVAL;
+
+   return 0;
+}
+
+static u64 intel_ntb_link_is_up(struct ntb_dev *ntb,
enum ntb_speed *speed,
enum ntb_width *width)
 {
@@ -868,7 +895,7 @@ static int intel_ntb_link_enable(struct ntb_dev *ntb,
 
ndev = container_of(ntb, struct intel_ntb_dev, ntb);
 
-   if (ndev->ntb.topo == NTB_TOPO_SEC)
+   if (ndev->ntb.topo == NTB_TOPO_P2P && ndev->ntb.port == NTB_PORT_SEC)
return -EINVAL;
 
dev_dbg(ndev_dev(ndev),
@@ -897,7 +924,7 @@ static int intel_ntb_link_disable(struct ntb_dev *ntb)
 
ndev = container_of(ntb, struct intel_ntb_dev, ntb);
 
-   if (ndev->ntb.topo == NTB_TOPO_SEC)
+   if (ndev->ntb.topo == NTB_TOPO_P2P && ndev->ntb.port == NTB_PORT_SEC)
return -EINVAL;
 
dev_dbg(ndev_dev(ndev), "Disabling link\n");
@@ -1241,27 +1268,32 @@ static int atom_link_is_err(struct intel_ntb_dev *ndev)
return 0;
 }
 
-static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
+static inline int atom_ppd_init_topo(struct intel_ntb_dev *ndev, u32 ppd)
 {
switch (ppd & ATOM_PPD_TOPO_MASK) {
case ATOM_PPD_TOPO_B2B_USD:
dev_dbg(ndev_dev(ndev), "PPD %d B2B USD\n", ppd);
-   return NTB_TOPO_B2B_USD;
-
+   ndev->ntb.topo = NTB_TOPO_B2B;
+   ndev->ntb.port = NTB_PORT_PRI;
+   return 0;
case ATOM_PPD_TOPO_B2B_DSD:
dev_dbg(ndev_dev(ndev), "PPD %d B2B DSD\n", ppd);
-   return NTB_TOPO_B2B_DSD;
+   ndev->ntb.topo = NTB_TOPO_B2B;
+   ndev->ntb.port = NTB_PORT_SEC;
+   return 0;
 
case ATOM_PPD_TOPO_PRI_USD:
case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
case ATOM_PPD_TOPO_SEC_USD:
case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
-   dev_dbg(ndev_dev(ndev), "PPD %d non B2B disabled\n", ppd);
-   return NTB_TOPO_NONE;
+   dev_err(ndev_dev(ndev), "PPD %d non B2B disabled\n", ppd);
+   ndev->ntb.topo = NTB_TOP

[PATCH 07/22] NTB: Fix a few ntb.h issues

[Serge Semin]

Fix some minor issues found in ntb.h file.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 include/linux/ntb.h | 18 --
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index a59a155..8b19327 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -90,6 +90,7 @@ static inline char *ntb_topo_string(enum ntb_topo topo)
  * @NTB_SPEED_GEN1:Link is trained to gen1 speed.
  * @NTB_SPEED_GEN2:Link is trained to gen2 speed.
  * @NTB_SPEED_GEN3:Link is trained to gen3 speed.
+ * @NTB_SPEED_GEN4:Link is trained to gen4 speed.
  */
 enum ntb_speed {
NTB_SPEED_AUTO = -1,
@@ -97,6 +98,7 @@ enum ntb_speed {
NTB_SPEED_GEN1 = 1,
NTB_SPEED_GEN2 = 2,
NTB_SPEED_GEN3 = 3,
+   NTB_SPEED_GEN4 = 4
 };
 
 /**
@@ -292,13 +294,18 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
 {
/* commented callbacks are not required: */
return
+   /* Port operations are required */
ops->port_number&&
ops->peer_port_count&&
ops->peer_port_number   &&
ops->peer_port_idx  &&
+
+   /* Link operations are requiered */
ops->link_is_up &&
ops->link_enable&&
ops->link_disable   &&
+
+   /* One or both MW interfaces should be developed */
ops->mw_count   &&
ops->mw_get_align   &&
(ops->mw_set_trans  ||
@@ -308,12 +315,11 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
ops->peer_mw_get_addr   &&
/* ops->peer_mw_clear_trans && */
 
+   /* Doorbell operations are mostly required */
/* ops->db_is_unsafe&& */
ops->db_valid_mask  &&
-
/* both set, or both unset */
(!ops->db_vector_count == !ops->db_vector_mask) &&
-
ops->db_read&&
/* ops->db_set  && */
ops->db_clear   &&
@@ -327,6 +333,8 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* ops->peer_db_read_mask   && */
/* ops->peer_db_set_mask&& */
/* ops->peer_db_clear_mask  && */
+
+   /* Scrachpad or messaging interfaces should be developed */
((/* ops->spad_is_unsafe&& */
  ops->spad_count   &&
  ops->spad_read&&
@@ -355,13 +363,12 @@ struct ntb_client {
struct device_driverdrv;
const struct ntb_client_ops ops;
 };
-
 #define drv_ntb_client(__drv) container_of((__drv), struct ntb_client, drv)
 
 /**
  * struct ntb_device - ntb device
  * @dev:   Linux device object.
- * @pdev:  Pci device entry of the ntb.
+ * @pdev:  PCI device entry of the ntb.
  * @topo:  Detected topology of the ntb.
  * @port:  Local port of the ntb.
  * @ops:   See _dev_ops.
@@ -384,7 +391,6 @@ struct ntb_dev {
/* block unregister until device is fully released */
struct completion   released;
 };
-
 #define dev_ntb(__dev) container_of((__dev), struct ntb_dev, dev)
 
 /**
@@ -481,7 +487,7 @@ void ntb_link_event(struct ntb_dev *ntb);
  * multiple interrupt vectors for doorbells, the vector number indicates which
  * vector received the interrupt.  The vector number is relative to the first
  * vector used for doorbells, starting at zero, and must be less than
- ** ntb_db_vector_count().  The driver may call ntb_db_read() to check which
+ * ntb_db_vector_count().  The driver may call ntb_db_read() to check which
  * doorbell bits need service, and ntb_db_vector_mask() to determine which of
  * those bits are associated with the vector number.
  */
-- 
2.6.6

[PATCH 04/22] NTB: Add messaging NTB API

[Serge Semin]

IDT PCIe-switches have message registers to communicate with peer devices.
This patch adds new NTB API callback methods, which can be used to utilize
these registers functionality.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/ntb.c   |  13 +++
 include/linux/ntb.h | 236 ++--
 2 files changed, 241 insertions(+), 8 deletions(-)

diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index 2e25307..4b2cc60 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -191,6 +191,19 @@ void ntb_db_event(struct ntb_dev *ntb, int vector)
 }
 EXPORT_SYMBOL(ntb_db_event);
 
+void ntb_msg_event(struct ntb_dev *ntb)
+{
+   unsigned long irqflags;
+
+   spin_lock_irqsave(>ctx_lock, irqflags);
+   {
+   if (ntb->ctx_ops && ntb->ctx_ops->msg_event)
+   ntb->ctx_ops->msg_event(ntb->ctx);
+   }
+   spin_unlock_irqrestore(>ctx_lock, irqflags);
+}
+EXPORT_SYMBOL(ntb_msg_event);
+
 static int ntb_probe(struct device *dev)
 {
struct ntb_dev *ntb;
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 4a150b5..59de1f6 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -146,10 +146,12 @@ static inline int ntb_client_ops_is_valid(const struct 
ntb_client_ops *ops)
  * struct ntb_ctx_ops - ntb driver context operations
  * @link_event:See ntb_link_event().
  * @db_event:  See ntb_db_event().
+ * @msg_event: See ntb_msg_event().
  */
 struct ntb_ctx_ops {
void (*link_event)(void *ctx);
void (*db_event)(void *ctx, int db_vector);
+   void (*msg_event)(void *ctx);
 };
 
 static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
@@ -158,6 +160,7 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
return
/* ops->link_event  && */
/* ops->db_event&& */
+   /* ops->msg_event   && */
1;
 }
 
@@ -202,6 +205,15 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @peer_spad_addr:See ntb_peer_spad_addr().
  * @peer_spad_read:See ntb_peer_spad_read().
  * @peer_spad_write:   See ntb_peer_spad_write().
+ * @msg_count: See ntb_msg_count().
+ * @msg_inbits:See ntb_msg_inbits().
+ * @msg_outbits:   See ntb_msg_outbits().
+ * @msg_read_sts:  See ntb_msg_read_sts().
+ * @msg_clear_sts: See ntb_msg_clear_sts().
+ * @msg_set_mask:  See ntb_msg_set_mask().
+ * @msg_clear_mask:See ntb_msg_clear_mask().
+ * @msg_read:  See ntb_msg_read().
+ * @msg_write: See ntb_msg_write().
  */
 struct ntb_dev_ops {
int (*port_number)(struct ntb_dev *ntb);
@@ -263,6 +275,16 @@ struct ntb_dev_ops {
  phys_addr_t *spad_addr);
u32 (*peer_spad_read)(struct ntb_dev *ntb, int idx);
int (*peer_spad_write)(struct ntb_dev *ntb, int idx, u32 val);
+
+   int (*msg_count)(struct ntb_dev *ntb);
+   u64 (*msg_inbits)(struct ntb_dev *ntb);
+   u64 (*msg_outbits)(struct ntb_dev *ntb);
+   u64 (*msg_read_sts)(struct ntb_dev *ntb);
+   int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits);
+   int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits);
+   int (*msg_read)(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg);
+   int (*msg_write)(struct ntb_dev *ntb, int midx, int pidx, u32 msg);
 };
 
 static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
@@ -304,13 +326,22 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
/* ops->peer_db_read_mask   && */
/* ops->peer_db_set_mask&& */
/* ops->peer_db_clear_mask  && */
-   /* ops->spad_is_unsafe  && */
-   ops->spad_count &&
-   ops->spad_read  &&
-   ops->spad_write &&
-   /* ops->peer_spad_addr  && */
-   /* ops->peer_spad_read  && */
-   ops->peer_spad_write&&
+   ((/* ops->spad_is_unsafe&& */
+ ops->spad_count   &&
+ ops->spad_read&&
+ ops->spad_write   &&
+ /* ops->peer_spad_addr&& */
+ /* ops->peer_spad_read&& */
+ ops-&g

[PATCH 05/22] NTB: Alter Scratchpads NTB API to support multi-ports interface

[Serge Semin]

Even though there is no any real NTB hardware, which would have both more
than two ports and Scratchpad registers, it is logically correct to have
Scratchpad API accepting a peer port index as well. Intel/AMD drivers used
to utilize Primary and Secondary topology to split Scratchpad between
connected root devices. Since port-index API replaced Primary and Secondary
topology, Intel/AMD NTB hadrware drivers can use device port to determine
which Scratchpad registers actually belong to local and peer devices.
The same approach can be used if some potential hardware in future will be
multi-port and have some set of Scratchpads.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 include/linux/ntb.h | 46 ++
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 59de1f6..fc9d034 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -268,13 +268,14 @@ struct ntb_dev_ops {
int (*spad_is_unsafe)(struct ntb_dev *ntb);
int (*spad_count)(struct ntb_dev *ntb);
 
-   u32 (*spad_read)(struct ntb_dev *ntb, int idx);
-   int (*spad_write)(struct ntb_dev *ntb, int idx, u32 val);
+   u32 (*spad_read)(struct ntb_dev *ntb, int sidx);
+   int (*spad_write)(struct ntb_dev *ntb, int sidx, u32 val);
 
-   int (*peer_spad_addr)(struct ntb_dev *ntb, int idx,
+   int (*peer_spad_addr)(struct ntb_dev *ntb, int pidx, int sidx,
  phys_addr_t *spad_addr);
-   u32 (*peer_spad_read)(struct ntb_dev *ntb, int idx);
-   int (*peer_spad_write)(struct ntb_dev *ntb, int idx, u32 val);
+   u32 (*peer_spad_read)(struct ntb_dev *ntb, int pidx, int sidx);
+   int (*peer_spad_write)(struct ntb_dev *ntb, int pidx, int sidx,
+  u32 val);
 
int (*msg_count)(struct ntb_dev *ntb);
u64 (*msg_inbits)(struct ntb_dev *ntb);
@@ -1201,6 +1202,7 @@ static inline int ntb_spad_is_unsafe(struct ntb_dev *ntb)
  * @ntb:   NTB device context.
  *
  * Hardware and topology may support a different number of scratchpads.
+ * Although it must be the same for all ports per NTB device.
  *
  * Return: the number of scratchpads.
  */
@@ -1215,42 +1217,43 @@ static inline int ntb_spad_count(struct ntb_dev *ntb)
 /**
  * ntb_spad_read() - read the local scratchpad register
  * @ntb:   NTB device context.
- * @idx:   Scratchpad index.
+ * @sidx:  Scratchpad index.
  *
  * Read the local scratchpad register, and return the value.
  *
  * Return: The value of the local scratchpad register.
  */
-static inline u32 ntb_spad_read(struct ntb_dev *ntb, int idx)
+static inline u32 ntb_spad_read(struct ntb_dev *ntb, int sidx)
 {
if (!ntb->ops->spad_read)
return ~(u32)0;
 
-   return ntb->ops->spad_read(ntb, idx);
+   return ntb->ops->spad_read(ntb, sidx);
 }
 
 /**
  * ntb_spad_write() - write the local scratchpad register
  * @ntb:   NTB device context.
- * @idx:   Scratchpad index.
+ * @sidx:  Scratchpad index.
  * @val:   Scratchpad value.
  *
  * Write the value to the local scratchpad register.
  *
  * Return: Zero on success, otherwise an error number.
  */
-static inline int ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
+static inline int ntb_spad_write(struct ntb_dev *ntb, int sidx, u32 val)
 {
if (!ntb->ops->spad_write)
return -EINVAL;
 
-   return ntb->ops->spad_write(ntb, idx, val);
+   return ntb->ops->spad_write(ntb, sidx, val);
 }
 
 /**
  * ntb_peer_spad_addr() - address of the peer scratchpad register
  * @ntb:   NTB device context.
- * @idx:   Scratchpad index.
+ * @pidx:  Port index of peer device.
+ * @sidx:  Scratchpad index.
  * @spad_addr: OUT - The address of the peer scratchpad register.
  *
  * Return the address of the peer doorbell register.  This may be used, for
@@ -1258,48 +1261,51 @@ static inline int ntb_spad_write(struct ntb_dev *ntb, 
int idx, u32 val)
  *
  * Return: Zero on success, otherwise an error number.
  */
-static inline int ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
+static inline int ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
 phys_addr_t *spad_addr)
 {
if (!ntb->ops->peer_spad_addr)
return -EINVAL;
 
-   return ntb->ops->peer_spad_addr(ntb, idx, spad_addr);
+   return ntb->ops->peer_spad_addr(ntb, pidx, sidx, spad_addr);
 }
 
 /**
  * ntb_peer_spad_read() - read the peer scratchpad register
  * @ntb:   NTB device context.
- * @idx:   Scratchpad index.
+ * @pidx:  Port index of peer device.
+ * @sidx:  Scratchpad index.
  *
  * Read the peer scratchpad register, and return the value.
  *
  * Return: The value of the local scratchpad register.
  */
-static inline u32 ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+s

[PATCH 02/22] NTB: Add peer indexed ports NTB API

[Serge Semin]

Add new port-index NTB API. Additionally lets get rid of Primary and
Secondary topologies, since port-number can be effectively used instead.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 include/linux/ntb.h | 101 
 1 file changed, 79 insertions(+), 22 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 5d1f260..0941a43 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -64,37 +64,21 @@ struct pci_dev;
 /**
  * enum ntb_topo - NTB connection topology
  * @NTB_TOPO_NONE: Topology is unknown or invalid.
- * @NTB_TOPO_PRI:  On primary side of local ntb.
- * @NTB_TOPO_SEC:  On secondary side of remote ntb.
- * @NTB_TOPO_B2B_USD:  On primary side of local ntb upstream of remote ntb.
- * @NTB_TOPO_B2B_DSD:  On primary side of local ntb downstream of remote ntb.
+ * @NTB_TOPO_P2P:  Simple port-to-port NTB topology
+ * @NTB_TOPO_B2B:  Bridge-to-bridge NTB topology
  */
 enum ntb_topo {
NTB_TOPO_NONE = -1,
-   NTB_TOPO_PRI,
-   NTB_TOPO_SEC,
-   NTB_TOPO_B2B_USD,
-   NTB_TOPO_B2B_DSD,
+   NTB_TOPO_P2P,
+   NTB_TOPO_B2B
 };
 
-static inline int ntb_topo_is_b2b(enum ntb_topo topo)
-{
-   switch ((int)topo) {
-   case NTB_TOPO_B2B_USD:
-   case NTB_TOPO_B2B_DSD:
-   return 1;
-   }
-   return 0;
-}
-
 static inline char *ntb_topo_string(enum ntb_topo topo)
 {
switch (topo) {
case NTB_TOPO_NONE: return "NTB_TOPO_NONE";
-   case NTB_TOPO_PRI:  return "NTB_TOPO_PRI";
-   case NTB_TOPO_SEC:  return "NTB_TOPO_SEC";
-   case NTB_TOPO_B2B_USD:  return "NTB_TOPO_B2B_USD";
-   case NTB_TOPO_B2B_DSD:  return "NTB_TOPO_B2B_DSD";
+   case NTB_TOPO_P2P:  return "NTB_TOPO_P2P";
+   case NTB_TOPO_B2B:  return "NTB_TOPO_B2B";
}
return "NTB_TOPO_INVALID";
 }
@@ -179,6 +163,10 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
 
 /**
  * struct ntb_ctx_ops - ntb device operations
+ * @port_number:   See ntb_port_number().
+ * @peer_port_count:   See ntb_peer_port_count().
+ * @peer_port_number:  See ntb_peer_port_number().
+ * @peer_port_idx: See ntb_peer_port_idx().
  * @link_is_up:See ntb_link_is_up().
  * @link_enable:   See ntb_link_enable().
  * @link_disable:  See ntb_link_disable().
@@ -212,6 +200,11 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @peer_spad_write:   See ntb_peer_spad_write().
  */
 struct ntb_dev_ops {
+   int (*port_number)(struct ntb_dev *ntb);
+   int (*peer_port_count)(struct ntb_dev *ntb);
+   int (*peer_port_number)(struct ntb_dev *ntb, int pidx);
+   int (*peer_port_idx)(struct ntb_dev *ntb, int port);
+
int (*link_is_up)(struct ntb_dev *ntb,
  enum ntb_speed *speed, enum ntb_width *width);
int (*link_enable)(struct ntb_dev *ntb,
@@ -265,6 +258,10 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
 {
/* commented callbacks are not required: */
return
+   ops->port_number&&
+   ops->peer_port_count&&
+   ops->peer_port_number   &&
+   ops->peer_port_idx  &&
ops->link_is_up &&
ops->link_enable&&
ops->link_disable   &&
@@ -319,6 +316,7 @@ struct ntb_client {
  * @dev:   Linux device object.
  * @pdev:  Pci device entry of the ntb.
  * @topo:  Detected topology of the ntb.
+ * @port:  Local port of the ntb.
  * @ops:   See _dev_ops.
  * @ctx:   See _ctx_ops.
  * @ctx_ops:   See _ctx_ops.
@@ -327,6 +325,7 @@ struct ntb_dev {
struct device   dev;
struct pci_dev  *pdev;
enum ntb_topo   topo;
+   int port;
const struct ntb_dev_ops*ops;
void*ctx;
const struct ntb_ctx_ops*ctx_ops;
@@ -442,6 +441,64 @@ void ntb_link_event(struct ntb_dev *ntb);
 void ntb_db_event(struct ntb_dev *ntb, int vector);
 
 /**
+ * ntb_port_number() - get the local port number
+ * @ntb:   NTB device context.
+ *
+ * Hardware must support at least simple two-ports topology
+ *
+ * Return: the local port number
+ */
+static inline int ntb_port_number(struct ntb_dev *ntb)
+{
+   return ntb->ops->port_number(ntb);
+}
+
+/**
+ * ntb_peer_port_count() - get the number of peer device ports
+ * @ntb:   NTB device context.
+ *
+ * Hardware ma

[PATCH 03/22] NTB: Alter NTB API to support both inbound and outbound MW based interfaces

[Serge Semin]

Alter NTB API to support inbound and outbound MW based interfaces.
Additionally I made it supporting multi-port devices as well. Useful
infographics is added right before MW API is declared. It shall help to
better understand how the new API really works and how it can be utilized
within client drivers.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 include/linux/ntb.h | 290 
 1 file changed, 245 insertions(+), 45 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 0941a43..4a150b5 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -171,9 +171,13 @@ static inline int ntb_ctx_ops_is_valid(const struct 
ntb_ctx_ops *ops)
  * @link_enable:   See ntb_link_enable().
  * @link_disable:  See ntb_link_disable().
  * @mw_count:  See ntb_mw_count().
- * @mw_get_range:  See ntb_mw_get_range().
+ * @mw_get_align:  See ntb_mw_get_align().
  * @mw_set_trans:  See ntb_mw_set_trans().
  * @mw_clear_trans:See ntb_mw_clear_trans().
+ * @peer_mw_count: See ntb_peer_mw_count().
+ * @peer_mw_get_addr:  See ntb_peer_mw_get_addr().
+ * @peer_mw_set_trans: See ntb_peer_mw_set_trans().
+ * @peer_mw_clear_trans:See ntb_peer_mw_clear_trans().
  * @db_is_unsafe:  See ntb_db_is_unsafe().
  * @db_valid_mask: See ntb_db_valid_mask().
  * @db_vector_count:   See ntb_db_vector_count().
@@ -211,13 +215,20 @@ struct ntb_dev_ops {
   enum ntb_speed max_speed, enum ntb_width max_width);
int (*link_disable)(struct ntb_dev *ntb);
 
-   int (*mw_count)(struct ntb_dev *ntb);
-   int (*mw_get_range)(struct ntb_dev *ntb, int idx,
-   phys_addr_t *base, resource_size_t *size,
-   resource_size_t *align, resource_size_t *align_size);
-   int (*mw_set_trans)(struct ntb_dev *ntb, int idx,
+   int (*mw_count)(struct ntb_dev *ntb, int pidx);
+   int (*mw_get_align)(struct ntb_dev *ntb, int pidx, int widx,
+   resource_size_t *addr_align,
+   resource_size_t *size_align,
+   resource_size_t *size_max);
+   int (*mw_set_trans)(struct ntb_dev *ntb, int pidx, int widx,
dma_addr_t addr, resource_size_t size);
-   int (*mw_clear_trans)(struct ntb_dev *ntb, int idx);
+   int (*mw_clear_trans)(struct ntb_dev *ntb, int pidx, int widx);
+   int (*peer_mw_count)(struct ntb_dev *ntb);
+   int (*peer_mw_get_addr)(struct ntb_dev *ntb, int widx,
+   phys_addr_t *base, resource_size_t *size);
+   int (*peer_mw_set_trans)(struct ntb_dev *ntb, int pidx, int widx,
+u64 addr, resource_size_t size);
+   int (*peer_mw_clear_trans)(struct ntb_dev *ntb, int pidx, int widx);
 
int (*db_is_unsafe)(struct ntb_dev *ntb);
u64 (*db_valid_mask)(struct ntb_dev *ntb);
@@ -266,9 +277,13 @@ static inline int ntb_dev_ops_is_valid(const struct 
ntb_dev_ops *ops)
ops->link_enable&&
ops->link_disable   &&
ops->mw_count   &&
-   ops->mw_get_range   &&
-   ops->mw_set_trans   &&
+   ops->mw_get_align   &&
+   (ops->mw_set_trans  ||
+ops->peer_mw_set_trans)&&
/* ops->mw_clear_trans  && */
+   ops->peer_mw_count  &&
+   ops->peer_mw_get_addr   &&
+   /* ops->peer_mw_clear_trans && */
 
/* ops->db_is_unsafe&& */
ops->db_valid_mask  &&
@@ -555,79 +570,264 @@ static inline int ntb_link_disable(struct ntb_dev *ntb)
 }
 
 /**
- * ntb_mw_count() - get the number of memory windows
+ *   NTB Memory Windows description
+ *
+ * There are two types of memory window interfaces supported by the NTB API:
+ * local and peer side initialization of memory sharing. The first type is
+ * depicted on the next figure:
+ *
+ *  Local device: | Peer device:
+ *   NTB config   |
+ * Physical memory (RAM) __   | Memory mapped IO
+ *      +-->|   addr   |  |  _
+ *  ||  |   |--|  | | |
+ *  ||addr--+ | |-|
+ *  | Inbound MW |PCI Express + NTB | Outbound MW |
+ *  ||<=| |
+ 

[PATCH 00/22] NTB: Alter kernel API to support multi-port devices

[Serge Semin]

There are devices, like IDT PCIe-switches, which have more than just two ports.
Particularly one device can have up to eight ports with NTB-function activated.
In order to support such devices, NTB kernel API should be altered since
currently it's optimized to work with two-ports devices only.

Here are the changes I made to conform the design we discussed a few months ago:

1) Port-index-related methods are added to KAPI
 ntb_port_number();
 ntb_peer_port_count();
 ntb_peer_port_number(pdix);
 ntb_peer_port_idx(port);

2) Link state method returns bitfield of link states for each reachable port
 u64 ntb_link_is_up();
 
3) Link enable/disable methods work with local link side of NTB
 ntb_link_enable()/ntb_link_disable();
 
4) NTB memory window related interface does the following things
 ntb_mw_count(pidx); - number of inbound memory windows, which can be allocated
for shared buffer with specified peer device.
 ntb_mw_get_align(pidx, widx); - get alignment and size restrition parameters
to properly allocate inbound memory region.
 ntb_peer_mw_count(); - get number of outbound memory windows.
 ntb_peer_mw_get_addr(widx); - get mapping address of an outbound memory window 

Inbound MW based hardware:
 ntb_mw_set_trans(pidx, widx); - set translation address of allocated inbound
memory window so a peer device could access it.
 ntb_mw_clear_trans(pidx, widx); - clear the translation address of an inbound
memory window.

Outbound MW based hardware:
 ntb_peer_mw_set_trans(pidx, widx); - set translation address of a memory
window retrieved from a peer device
 ntb_peer_mw_clear_trans(pidx, widx); - clear the translation address of an
outbound memory window

5) Scratchpad interface needs to support multi-port devices as well
 ntb_spad_count() - return number of Scratchpad per each port
 ntb_peer_spad_addr(pidx, sidx) - address of Scratchpad register of the
peer device with pidx-index
 ntb_peer_spad_read(pidx, sidx) - read specified Scratchpad register of the
peer with pidx-index
 ntb_peer_spad_write(pidx, sidx) - write data to Scratchpad register of the
peer with pidx-index

6) Introduce new messaging interface of NTB KAPI
 ntb_msg_count(); - get number of message registers
 ntb_msg_inbits(); - get bitfield of inbound message registers status
 ntb_msg_outbits(); - get bitfield of outbound message registers status
 ntb_msg_read_sts(); - read the inbound and outbound message registers status
 ntb_msg_clear_sts(); - clear status bits of message registers
 ntb_msg_set_mask(); - mask interrupts raised by status bits of message
registers.
 ntb_msg_clear_mask(); - clear interrupts mask bits of message registers
 ntb_msg_recv(midx, *pidx); - read message register with specified index,
additionally getting peer port index which data received from
 ntb_msg_send(midx, pidx); - write data to the specified message register
sending it to the passed peer device connected over a pidx port
 ntb_msg_event(); - notify driver context of a new message event
 
7) Topology reduced to be either P2P (port-to-port) or B2B (bridge-to-bridge).
Since there is port number introduced to be part of ntb_dev structure, real
port number can be used to determine Primary and Secondary sides. Intel and AMD
driver are altered to support this novation.

8) Standard test drivers: PingPong, Debugging tool and Raw Perf as well as NTB
Transport drivers don't support multi-port devices at the moment.
Since we haven't got any real multi-port hadrware driver, it's dangerous to
make any serious alterations in the debugging tools. So I have made those
drivers to work the way they wokred before, but using the new NTB API.
The situation will change when I finish porting my current IDT NTB driver to
support new API. Then I'll be able to refactor the tools and test them using
real multi-port hardware.

The changes are made on top of the NTB-fork of the kernel:
https://github.com/jonmason/ntb
of "ntb"-branch with last commit:
9c763584b7c8911106bb77af7e648bef09af9d80 Linux 4.9-rc6

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

Serge Semin (22):
  NTB: Move link state API being first in sources
  NTB: Add peer indexed ports NTB API
  NTB: Alter NTB API to support both inbound and outbound MW based
interfaces
  NTB: Add messaging NTB API
  NTB: Alter Scratchpads NTB API to support multi-ports interface
  NTB: Slightly alter link state NTB API
  NTB: Fix a few ntb.h issues
  NTB: Add T-Platforms copyrights to NTB API
  NTB Intel: Move link-related methods being first in the driver
  NTB Intel: Add port-related NTB API callback methods
  NTB Intel: Alter MW interface to fit new NTB API
  NTB Intel: Alter Scratchpads interface to fit new NTB API
  NTB Intel: Add T-Platforms copyrights to Intel NTB driver
  NTB AMD: Move link-related methods being first in the driver
  NTB AMD: Add port-related NTB API callback methods
  NTB AMD: Alter MW interface to fit new NTB API
  NTB AMD: Alter Scratchpads interface to fit new NTB API
  NTB AMD: Add T-Plat

[PATCH 18/22] NTB AMD: Add T-Platforms copyrights to AMD NTB driver

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/amd/ntb_hw_amd.c | 2 ++
 drivers/ntb/hw/amd/ntb_hw_amd.h | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 3479c2b..2e1aefd 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
  *   BSD LICENSE
  *
  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.h b/drivers/ntb/hw/amd/ntb_hw_amd.h
index 993e053..6106f50 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.h
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.h
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
  *   BSD LICENSE
  *
  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
-- 
2.6.6

[PATCH 14/22] NTB AMD: Move link-related methods being first in the driver

[Serge Semin]

See patch 01 for reasoning of this movement.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/amd/ntb_hw_amd.c | 188 
 1 file changed, 94 insertions(+), 94 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 6ccba0d..6704327 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -71,6 +71,97 @@ MODULE_AUTHOR("AMD Inc.");
 static const struct file_operations amd_ntb_debugfs_info;
 static struct dentry *debugfs_dir;
 
+static int amd_link_is_up(struct amd_ntb_dev *ndev)
+{
+   if (!ndev->peer_sta)
+   return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
+
+   /* If peer_sta is reset or D0 event, the ISR has
+* started a timer to check link status of hardware.
+* So here just clear status bit. And if peer_sta is
+* D3 or PME_TO, D0/reset event will be happened when
+* system wakeup/poweron, so do nothing here.
+*/
+   if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
+   ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
+   else if (ndev->peer_sta & AMD_PEER_D0_EVENT)
+   ndev->peer_sta = 0;
+
+   return 0;
+}
+
+static int amd_ntb_link_is_up(struct ntb_dev *ntb,
+ enum ntb_speed *speed,
+ enum ntb_width *width)
+{
+   struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+   int ret = 0;
+
+   if (amd_link_is_up(ndev)) {
+   if (speed)
+   *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
+   if (width)
+   *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
+
+   dev_dbg(ndev_dev(ndev), "link is up.\n");
+
+   ret = 1;
+   } else {
+   if (speed)
+   *speed = NTB_SPEED_NONE;
+   if (width)
+   *width = NTB_WIDTH_NONE;
+
+   dev_dbg(ndev_dev(ndev), "link is down.\n");
+   }
+
+   return ret;
+}
+
+static int amd_ntb_link_enable(struct ntb_dev *ntb,
+  enum ntb_speed max_speed,
+  enum ntb_width max_width)
+{
+   struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+   void __iomem *mmio = ndev->self_mmio;
+   u32 ntb_ctl;
+
+   /* Enable event interrupt */
+   ndev->int_mask &= ~AMD_EVENT_INTMASK;
+   writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
+
+   if (ndev->ntb.topo == NTB_TOPO_SEC)
+   return -EINVAL;
+   dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
+
+   ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
+   ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
+   writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
+
+   return 0;
+}
+
+static int amd_ntb_link_disable(struct ntb_dev *ntb)
+{
+   struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+   void __iomem *mmio = ndev->self_mmio;
+   u32 ntb_ctl;
+
+   /* Disable event interrupt */
+   ndev->int_mask |= AMD_EVENT_INTMASK;
+   writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
+
+   if (ndev->ntb.topo == NTB_TOPO_SEC)
+   return -EINVAL;
+   dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
+
+   ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
+   ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
+   writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
+
+   return 0;
+}
+
 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
 {
if (idx < 0 || idx > ndev->mw_count)
@@ -194,97 +285,6 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int 
idx,
return 0;
 }
 
-static int amd_link_is_up(struct amd_ntb_dev *ndev)
-{
-   if (!ndev->peer_sta)
-   return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
-
-   /* If peer_sta is reset or D0 event, the ISR has
-* started a timer to check link status of hardware.
-* So here just clear status bit. And if peer_sta is
-* D3 or PME_TO, D0/reset event will be happened when
-* system wakeup/poweron, so do nothing here.
-*/
-   if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
-   ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
-   else if (ndev->peer_sta & AMD_PEER_D0_EVENT)
-   ndev->peer_sta = 0;
-
-   return 0;
-}
-
-static int amd_ntb_link_is_up(struct ntb_dev *ntb,
- enum ntb_speed *speed,
- enum ntb_width *width)
-{
-   struct amd_ntb_dev *ndev = ntb_ndev(ntb);
-   int ret = 0;
-
-   if (amd_link_is_up(ndev)) {
-   if (speed)
-   *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
-   if (width)
-   *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
-
-   dev_dbg(ndev_dev(ndev), "link is up.\n")

[PATCH 20/22] NTB Tool: Alter driver to work with two-ports NTB API

[Serge Semin]

The same as for PingPong driver, this driver can't be used with hardware
whithout Scratchpads. Additionally it supports two-ports and inbound MW
based devices only at the moment.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/test/ntb_tool.c | 87 +++--
 1 file changed, 61 insertions(+), 26 deletions(-)

diff --git a/drivers/ntb/test/ntb_tool.c b/drivers/ntb/test/ntb_tool.c
index 61bf2ef..f9329c0 100644
--- a/drivers/ntb/test/ntb_tool.c
+++ b/drivers/ntb/test/ntb_tool.c
@@ -119,7 +119,10 @@ MODULE_VERSION(DRIVER_VERSION);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
 
-#define MAX_MWS 16
+/* It is rare to have hadrware with greater than six MWs */
+#define MAX_MWS 6
+/* Only two-ports devices are supported */
+#define PIDX 0
 
 static struct dentry *tool_dbgfs;
 
@@ -261,14 +264,17 @@ static ssize_t tool_dbfn_write(struct tool_ctx *tc,
 
 static ssize_t tool_spadfn_read(struct tool_ctx *tc, char __user *ubuf,
size_t size, loff_t *offp,
-   u32 (*spad_read_fn)(struct ntb_dev *, int))
+   u32 (*spad_read_fn)(struct ntb_dev *, int),
+   u32 (*spad_peer_read_fn)(struct ntb_dev *, int,
+int))
 {
size_t buf_size;
char *buf;
ssize_t pos, rc;
int i, spad_count;
+   u32 data;
 
-   if (!spad_read_fn)
+   if (!spad_read_fn && !spad_peer_read_fn)
return -EINVAL;
 
spad_count = ntb_spad_count(tc->ntb);
@@ -287,8 +293,12 @@ static ssize_t tool_spadfn_read(struct tool_ctx *tc, char 
__user *ubuf,
pos = 0;
 
for (i = 0; i < spad_count; ++i) {
+   if (spad_read_fn)
+   data = spad_read_fn(tc->ntb, i);
+   else
+   data = spad_peer_read_fn(tc->ntb, PIDX, i);
pos += scnprintf(buf + pos, buf_size - pos, "%d\t%#x\n",
-i, spad_read_fn(tc->ntb, i));
+i, data);
}
 
rc = simple_read_from_buffer(ubuf, size, offp, buf, pos);
@@ -302,7 +312,9 @@ static ssize_t tool_spadfn_write(struct tool_ctx *tc,
 const char __user *ubuf,
 size_t size, loff_t *offp,
 int (*spad_write_fn)(struct ntb_dev *,
- int, u32))
+ int, u32),
+int (*spad_peer_write_fn)(struct ntb_dev *,
+  int, int, u32))
 {
int spad_idx;
u32 spad_val;
@@ -310,7 +322,7 @@ static ssize_t tool_spadfn_write(struct tool_ctx *tc,
int pos, n;
ssize_t rc;
 
-   if (!spad_write_fn) {
+   if (!spad_write_fn || !spad_peer_write_fn) {
dev_dbg(>ntb->dev, "no spad write fn\n");
return -EINVAL;
}
@@ -330,7 +342,11 @@ static ssize_t tool_spadfn_write(struct tool_ctx *tc,
n = sscanf(buf_ptr, "%d %i%n", _idx, _val, );
while (n == 2) {
buf_ptr += pos;
-   rc = spad_write_fn(tc->ntb, spad_idx, spad_val);
+   if (spad_write_fn)
+   rc = spad_write_fn(tc->ntb, spad_idx, spad_val);
+   else
+   rc = spad_peer_write_fn(tc->ntb, PIDX, spad_idx,
+   spad_val);
if (rc)
break;
 
@@ -443,7 +459,7 @@ static ssize_t tool_spad_read(struct file *filep, char 
__user *ubuf,
struct tool_ctx *tc = filep->private_data;
 
return tool_spadfn_read(tc, ubuf, size, offp,
-   tc->ntb->ops->spad_read);
+   tc->ntb->ops->spad_read, NULL);
 }
 
 static ssize_t tool_spad_write(struct file *filep, const char __user *ubuf,
@@ -452,7 +468,7 @@ static ssize_t tool_spad_write(struct file *filep, const 
char __user *ubuf,
struct tool_ctx *tc = filep->private_data;
 
return tool_spadfn_write(tc, ubuf, size, offp,
-tc->ntb->ops->spad_write);
+tc->ntb->ops->spad_write, NULL);
 }
 
 static TOOL_FOPS_RDWR(tool_spad_fops,
@@ -464,7 +480,7 @@ static ssize_t tool_peer_spad_read(struct file *filep, char 
__user *ubuf,
 {
struct tool_ctx *tc = filep->private_data;
 
-   return tool_spadfn_read(tc, ubuf, size, offp,
+   return tool_spadfn_read(tc, ubuf, size, offp, NULL,
tc->ntb->ops->peer_spad_read);
 }
 
@@ -473,7 +489,7 @@ static ssize_t tool_peer_spad_write(struct

[PATCH 17/22] NTB AMD: Alter Scratchpads interface to fit new NTB API

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/amd/ntb_hw_amd.c | 14 +++---
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 9b3f78c..3479c2b 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -458,30 +458,30 @@ static int amd_ntb_spad_write(struct ntb_dev *ntb,
return 0;
 }
 
-static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
void __iomem *mmio = ndev->self_mmio;
u32 offset;
 
-   if (idx < 0 || idx >= ndev->spad_count)
+   if (sidx < 0 || sidx >= ndev->spad_count)
return -EINVAL;
 
-   offset = ndev->peer_spad + (idx << 2);
+   offset = ndev->peer_spad + (sidx << 2);
return readl(mmio + AMD_SPAD_OFFSET + offset);
 }
 
-static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
-  int idx, u32 val)
+static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
+  int sidx, u32 val)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
void __iomem *mmio = ndev->self_mmio;
u32 offset;
 
-   if (idx < 0 || idx >= ndev->spad_count)
+   if (sidx < 0 || sidx >= ndev->spad_count)
return -EINVAL;
 
-   offset = ndev->peer_spad + (idx << 2);
+   offset = ndev->peer_spad + (sidx << 2);
writel(val, mmio + AMD_SPAD_OFFSET + offset);
 
return 0;
-- 
2.6.6

[PATCH 15/22] NTB AMD: Add port-related NTB API callback methods

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/amd/ntb_hw_amd.c | 72 ++---
 drivers/ntb/hw/amd/ntb_hw_amd.h | 10 ++
 2 files changed, 63 insertions(+), 19 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 6704327..b7c9f67 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -71,6 +71,33 @@ MODULE_AUTHOR("AMD Inc.");
 static const struct file_operations amd_ntb_debugfs_info;
 static struct dentry *debugfs_dir;
 
+static int amd_ntb_port_number(struct ntb_dev *ntb)
+{
+   return ntb->port;
+}
+
+static int amd_ntb_peer_port_count(struct ntb_dev *ntb)
+{
+   return NTB_PEER_CNT;
+}
+
+static int amd_ntb_peer_port_number(struct ntb_dev *ntb, int pidx)
+{
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
+   return (ntb->port == NTB_PORT_PRI ? NTB_PORT_SEC : NTB_PORT_PRI);
+}
+
+static int amd_ntb_peer_port_idx(struct ntb_dev *ntb, int port)
+{
+   if ((ntb->port == NTB_PORT_PRI && port != NTB_PORT_SEC) ||
+   (ntb->port == NTB_PORT_SEC && port != NTB_PORT_PRI))
+   return -EINVAL;
+
+   return 0;
+}
+
 static int amd_link_is_up(struct amd_ntb_dev *ndev)
 {
if (!ndev->peer_sta)
@@ -90,7 +117,7 @@ static int amd_link_is_up(struct amd_ntb_dev *ndev)
return 0;
 }
 
-static int amd_ntb_link_is_up(struct ntb_dev *ntb,
+static u64 amd_ntb_link_is_up(struct ntb_dev *ntb,
  enum ntb_speed *speed,
  enum ntb_width *width)
 {
@@ -130,7 +157,7 @@ static int amd_ntb_link_enable(struct ntb_dev *ntb,
ndev->int_mask &= ~AMD_EVENT_INTMASK;
writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
 
-   if (ndev->ntb.topo == NTB_TOPO_SEC)
+   if (ndev->ntb.port == NTB_PORT_SEC)
return -EINVAL;
dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
 
@@ -151,7 +178,7 @@ static int amd_ntb_link_disable(struct ntb_dev *ntb)
ndev->int_mask |= AMD_EVENT_INTMASK;
writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
 
-   if (ndev->ntb.topo == NTB_TOPO_SEC)
+   if (ndev->ntb.port == NTB_PORT_SEC)
return -EINVAL;
dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
 
@@ -431,6 +458,10 @@ static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
 }
 
 static const struct ntb_dev_ops amd_ntb_ops = {
+   .port_number= amd_ntb_port_number,
+   .peer_port_count= amd_ntb_peer_port_count,
+   .peer_port_number   = amd_ntb_peer_port_number,
+   .peer_port_idx  = amd_ntb_peer_port_idx,
.link_is_up = amd_ntb_link_is_up,
.link_enable= amd_ntb_link_enable,
.link_disable   = amd_ntb_link_disable,
@@ -697,8 +728,8 @@ static ssize_t ndev_debugfs_read(struct file *filp, char 
__user *ubuf,
 "NTB Device Information:\n");
 
off += scnprintf(buf + off, buf_size - off,
-"Connection Topology -\t%s\n",
-ntb_topo_string(ndev->ntb.topo));
+"Connection Topology -\t%s:%d\n",
+ntb_topo_string(ndev->ntb.topo), ndev->ntb.port);
 
off += scnprintf(buf + off, buf_size - off,
 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
@@ -797,6 +828,7 @@ static inline void ndev_init_struct(struct amd_ntb_dev 
*ndev,
 {
ndev->ntb.pdev = pdev;
ndev->ntb.topo = NTB_TOPO_NONE;
+   ndev->ntb.port = NTB_PORT_NONE;
ndev->ntb.ops = _ntb_ops;
ndev->int_mask = AMD_EVENT_INTMASK;
spin_lock_init(>db_mask_lock);
@@ -876,23 +908,23 @@ static int amd_init_ntb(struct amd_ntb_dev *ndev)
ndev->spad_count = AMD_SPADS_CNT;
ndev->db_count = AMD_DB_CNT;
 
-   switch (ndev->ntb.topo) {
-   case NTB_TOPO_PRI:
-   case NTB_TOPO_SEC:
+   if (ndev->ntb.topo == NTB_TOPO_P2P) {
ndev->spad_count >>= 1;
-   if (ndev->ntb.topo == NTB_TOPO_PRI) {
+   if (ndev->ntb.port == NTB_PORT_PRI) {
ndev->self_spad = 0;
ndev->peer_spad = 0x20;
-   } else {
+   } else if (ndev->ntb.port == NTB_PORT_SEC) {
ndev->self_spad = 0x20;
ndev->peer_spad = 0;
+   } else {
+   dev_err(ndev_dev(ndev), "Invalid topology port %d.\n",
+   ndev->ntb.port);
+   return -EINVAL;
}
 
INIT_DELAYED_WORK(>hb_timer, amd_link_hb);
schedule_delayed_work(>hb_timer, AMD_LIN

[PATCH 13/22] NTB Intel: Add T-Platforms copyrights to Intel NTB driver

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/intel/ntb_hw_intel.c | 2 ++
 drivers/ntb/hw/intel/ntb_hw_intel.h | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c 
b/drivers/ntb/hw/intel/ntb_hw_intel.c
index e308fbe..7785e9d 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -6,6 +6,7 @@
  *
  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -15,6 +16,7 @@
  *
  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h 
b/drivers/ntb/hw/intel/ntb_hw_intel.h
index 7f1da03..099cbdd 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.h
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.h
@@ -6,6 +6,7 @@
  *
  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -15,6 +16,7 @@
  *
  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
-- 
2.6.6

[PATCH 21/22] NTB Perf: Alter driver to work with two-ports NTB API

[Serge Semin]

The same as for PingPong driver, this driver can't be used with hardware
whithout Scratchpads. Additionally it supports two-ports and inbound MW
based devices only at the moment.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/test/ntb_perf.c | 27 ---
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
index e75d4fd..99f1522 100644
--- a/drivers/ntb/test/ntb_perf.c
+++ b/drivers/ntb/test/ntb_perf.c
@@ -76,6 +76,7 @@
 #define DMA_RETRIES20
 #define SZ_4G  (1ULL << 32)
 #define MAX_SEG_ORDER  20 /* no larger than 1M for kmalloc buffer */
+#define PIDX   0
 
 MODULE_LICENSE(DRIVER_LICENSE);
 MODULE_VERSION(DRIVER_VERSION);
@@ -452,7 +453,7 @@ static void perf_free_mw(struct perf_ctx *perf)
if (!mw->virt_addr)
return;
 
-   ntb_mw_clear_trans(perf->ntb, 0);
+   ntb_mw_clear_trans(perf->ntb, PIDX, 0);
dma_free_coherent(>dev, mw->buf_size,
  mw->virt_addr, mw->dma_addr);
mw->xlat_size = 0;
@@ -488,7 +489,7 @@ static int perf_set_mw(struct perf_ctx *perf, 
resource_size_t size)
mw->buf_size = 0;
}
 
-   rc = ntb_mw_set_trans(perf->ntb, 0, mw->dma_addr, mw->xlat_size);
+   rc = ntb_mw_set_trans(perf->ntb, PIDX, 0, mw->dma_addr, mw->xlat_size);
if (rc) {
dev_err(>ntb->dev, "Unable to set mw0 translation\n");
perf_free_mw(perf);
@@ -515,9 +516,9 @@ static void perf_link_work(struct work_struct *work)
if (max_mw_size && size > max_mw_size)
size = max_mw_size;
 
-   ntb_peer_spad_write(ndev, MW_SZ_HIGH, upper_32_bits(size));
-   ntb_peer_spad_write(ndev, MW_SZ_LOW, lower_32_bits(size));
-   ntb_peer_spad_write(ndev, VERSION, PERF_VERSION);
+   ntb_peer_spad_write(ndev, PIDX, MW_SZ_HIGH, upper_32_bits(size));
+   ntb_peer_spad_write(ndev, PIDX, MW_SZ_LOW, lower_32_bits(size));
+   ntb_peer_spad_write(ndev, PIDX, VERSION, PERF_VERSION);
 
/* now read what peer wrote */
val = ntb_spad_read(ndev, VERSION);
@@ -559,8 +560,12 @@ static int perf_setup_mw(struct ntb_dev *ntb, struct 
perf_ctx *perf)
 
mw = >mw;
 
-   rc = ntb_mw_get_range(ntb, 0, >phys_addr, >phys_size,
- >xlat_align, >xlat_align_size);
+   rc = ntb_mw_get_align(ntb, PIDX, 0, >xlat_align,
+ >xlat_align_size, NULL);
+   if (rc)
+   return rc;
+
+   rc = ntb_peer_mw_get_addr(ntb, 0, >phys_addr, >phys_size);
if (rc)
return rc;
 
@@ -764,6 +769,14 @@ static int perf_probe(struct ntb_client *client, struct 
ntb_dev *ntb)
return -EIO;
}
 
+   if (!ntb->ops->mw_set_trans) {
+   dev_err(>dev, "Need inbound MW based NTB API\n");
+   return -EINVAL;
+   }
+
+   if (ntb_peer_port_count(ntb) != 1)
+   dev_warn(>dev, "Multi-port NTB devices unsupported\n");
+
node = dev_to_node(>dev);
 
perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node);
-- 
2.6.6

[PATCH 22/22] NTB Transport: Alter driver to work with two-ports NTB API

[Serge Semin]

The same as for PingPong driver, this driver can't be used with hardware
whithout Scratchpads. Additionally it supports two-ports and inbound MW
based devices only at the moment.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/ntb_transport.c | 43 ++-
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 4eb8adb..2390c65 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -94,6 +94,9 @@ MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large 
data copy");
 
 static struct dentry *nt_debugfs_dir;
 
+/* Only two-ports NTB devices are supported */
+#define PIDX   0
+
 struct ntb_queue_entry {
/* ntb_queue list reference */
struct list_head entry;
@@ -682,7 +685,7 @@ static void ntb_free_mw(struct ntb_transport_ctx *nt, int 
num_mw)
if (!mw->virt_addr)
return;
 
-   ntb_mw_clear_trans(nt->ndev, num_mw);
+   ntb_mw_clear_trans(nt->ndev, PIDX, num_mw);
dma_free_coherent(>dev, mw->buff_size,
  mw->virt_addr, mw->dma_addr);
mw->xlat_size = 0;
@@ -739,7 +742,8 @@ static int ntb_set_mw(struct ntb_transport_ctx *nt, int 
num_mw,
}
 
/* Notify HW the memory location of the receive buffer */
-   rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);
+   rc = ntb_mw_set_trans(nt->ndev, PIDX, num_mw, mw->dma_addr,
+ mw->xlat_size);
if (rc) {
dev_err(>dev, "Unable to set mw%d translation", num_mw);
ntb_free_mw(nt, num_mw);
@@ -871,17 +875,17 @@ static void ntb_transport_link_work(struct work_struct 
*work)
size = max_mw_size;
 
spad = MW0_SZ_HIGH + (i * 2);
-   ntb_peer_spad_write(ndev, spad, upper_32_bits(size));
+   ntb_peer_spad_write(ndev, PIDX, spad, upper_32_bits(size));
 
spad = MW0_SZ_LOW + (i * 2);
-   ntb_peer_spad_write(ndev, spad, lower_32_bits(size));
+   ntb_peer_spad_write(ndev, PIDX, spad, lower_32_bits(size));
}
 
-   ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);
+   ntb_peer_spad_write(ndev, PIDX, NUM_MWS, nt->mw_count);
 
-   ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);
+   ntb_peer_spad_write(ndev, PIDX, NUM_QPS, nt->qp_count);
 
-   ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION);
+   ntb_peer_spad_write(ndev, PIDX, VERSION, NTB_TRANSPORT_VERSION);
 
/* Query the remote side for its info */
val = ntb_spad_read(ndev, VERSION);
@@ -957,10 +961,10 @@ static void ntb_qp_link_work(struct work_struct *work)
 
val = ntb_spad_read(nt->ndev, QP_LINKS);
 
-   ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num));
+   ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num));
 
/* query remote spad for qp ready bits */
-   ntb_peer_spad_read(nt->ndev, QP_LINKS);
+   ntb_peer_spad_read(nt->ndev, PIDX, QP_LINKS);
dev_dbg_ratelimited(>dev, "Remote QP link status = %x\n", val);
 
/* See if the remote side is up */
@@ -1069,13 +1073,21 @@ static int ntb_transport_probe(struct ntb_client *self, 
struct ntb_dev *ndev)
int node;
int rc, i;
 
-   mw_count = ntb_mw_count(ndev);
+   mw_count = ntb_mw_count(ndev, PIDX);
if (ntb_spad_count(ndev) < (NUM_MWS + 1 + mw_count * 2)) {
dev_err(>dev, "Not enough scratch pad registers for %s",
NTB_TRANSPORT_NAME);
return -EIO;
}
 
+   if (!ndev->ops->mw_set_trans) {
+   dev_err(>dev, "Inbound MW based NTB API is required\n");
+   return -EINVAL;
+   }
+
+   if (ntb_peer_port_count(ndev) != 1)
+   dev_warn(>dev, "Multi-port NTB devices unsupported\n");
+
if (ntb_db_is_unsafe(ndev))
dev_dbg(>dev,
"doorbell is unsafe, proceed anyway...\n");
@@ -1103,8 +1115,13 @@ static int ntb_transport_probe(struct ntb_client *self, 
struct ntb_dev *ndev)
for (i = 0; i < mw_count; i++) {
mw = >mw_vec[i];
 
-   rc = ntb_mw_get_range(ndev, i, >phys_addr, >phys_size,
- >xlat_align, >xlat_align_size);
+   rc = ntb_mw_get_align(ndev, PIDX, i, >xlat_align,
+ >xlat_align_size, NULL);
+   if (rc)
+   goto err1;
+
+   rc = ntb_peer_mw_get_addr(ndev, i, >phys_addr,
+ >phys_size);
if (rc)

[PATCH 16/22] NTB AMD: Alter MW interface to fit new NTB API

[Serge Semin]

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/hw/amd/ntb_hw_amd.c | 66 ++---
 1 file changed, 49 insertions(+), 17 deletions(-)

diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index b7c9f67..9b3f78c 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -197,40 +197,42 @@ static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int 
idx)
return 1 << idx;
 }
 
-static int amd_ntb_mw_count(struct ntb_dev *ntb)
+static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 {
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
return ntb_ndev(ntb)->mw_count;
 }
 
-static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
-   phys_addr_t *base,
-   resource_size_t *size,
-   resource_size_t *align,
-   resource_size_t *align_size)
+static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
+   resource_size_t *addr_align,
+   resource_size_t *size_align,
+   resource_size_t *size_max)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
int bar;
 
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
bar = ndev_mw_to_bar(ndev, idx);
if (bar < 0)
return bar;
 
-   if (base)
-   *base = pci_resource_start(ndev->ntb.pdev, bar);
-
-   if (size)
-   *size = pci_resource_len(ndev->ntb.pdev, bar);
+   if (addr_align)
+   *addr_align = SZ_4K;
 
-   if (align)
-   *align = SZ_4K;
+   if (size_align)
+   *size_align = 1;
 
-   if (align_size)
-   *align_size = 1;
+   if (size_max)
+   *size_max = pci_resource_len(ndev->ntb.pdev, bar);
 
return 0;
 }
 
-static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
+static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
dma_addr_t addr, resource_size_t size)
 {
struct amd_ntb_dev *ndev = ntb_ndev(ntb);
@@ -240,6 +242,9 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int 
idx,
u64 base_addr, limit, reg_val;
int bar;
 
+   if (pidx > NTB_PIDX_MAX)
+   return -EINVAL;
+
bar = ndev_mw_to_bar(ndev, idx);
if (bar < 0)
return bar;
@@ -312,6 +317,31 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int 
idx,
return 0;
 }
 
+static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
+{
+   /* The same as for inbound MWs */
+   return ntb_ndev(ntb)->mw_count;
+}
+
+static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
+   phys_addr_t *base, resource_size_t *size)
+{
+   struct amd_ntb_dev *ndev = ntb_ndev(ntb);
+   int bar;
+
+   bar = ndev_mw_to_bar(ndev, idx);
+   if (bar < 0)
+   return bar;
+
+   if (base)
+   *base = pci_resource_start(ndev->ntb.pdev, bar);
+
+   if (size)
+   *size = pci_resource_len(ndev->ntb.pdev, bar);
+
+   return 0;
+}
+
 static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
 {
return ntb_ndev(ntb)->db_valid_mask;
@@ -466,8 +496,10 @@ static const struct ntb_dev_ops amd_ntb_ops = {
.link_enable= amd_ntb_link_enable,
.link_disable   = amd_ntb_link_disable,
.mw_count   = amd_ntb_mw_count,
-   .mw_get_range   = amd_ntb_mw_get_range,
+   .mw_get_align   = amd_ntb_mw_get_align,
.mw_set_trans   = amd_ntb_mw_set_trans,
+   .peer_mw_count  = amd_ntb_peer_mw_count,
+   .peer_mw_get_addr   = amd_ntb_peer_mw_get_addr,
.db_valid_mask  = amd_ntb_db_valid_mask,
.db_vector_count= amd_ntb_db_vector_count,
.db_vector_mask = amd_ntb_db_vector_mask,
-- 
2.6.6

[PATCH 19/22] NTB PingPong: Alter driver to work with two-ports NTB API

[Serge Semin]

PingPong driver doesn't do much, but it uses Scratchpads, which can be
unsupported by most of IDT hardware. So I restricted the usage of the
driver with hardware, which supports Scratchpads up until the IDT driver
is ported to the NTB API. When it's done I'll alter the pingpong driver to
support message registers as well.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>

---
 drivers/ntb/test/ntb_pingpong.c | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/drivers/ntb/test/ntb_pingpong.c b/drivers/ntb/test/ntb_pingpong.c
index 4358611..90d3281 100644
--- a/drivers/ntb/test/ntb_pingpong.c
+++ b/drivers/ntb/test/ntb_pingpong.c
@@ -90,6 +90,9 @@ static unsigned long db_init = 0x7;
 module_param(db_init, ulong, 0644);
 MODULE_PARM_DESC(db_init, "Initial doorbell bits to ring on the peer");
 
+/* Only two-ports NTB devices are supported */
+#define PIDX   0
+
 struct pp_ctx {
struct ntb_dev  *ntb;
u64 db_bits;
@@ -135,7 +138,7 @@ static void pp_ping(unsigned long ctx)
"Ping bits %#llx read %#x write %#x\n",
db_bits, spad_rd, spad_wr);
 
-   ntb_peer_spad_write(pp->ntb, 0, spad_wr);
+   ntb_peer_spad_write(pp->ntb, PIDX, 0, spad_wr);
ntb_peer_db_set(pp->ntb, db_bits);
ntb_db_clear_mask(pp->ntb, db_mask);
 
@@ -222,6 +225,12 @@ static int pp_probe(struct ntb_client *client,
}
}
 
+   if (ntb_spad_count(ntb) < 1) {
+   dev_dbg(>dev, "no enough scratchpads\n");
+   rc = -EINVAL;
+   goto err_pp;
+   }
+
if (ntb_spad_is_unsafe(ntb)) {
dev_dbg(>dev, "scratchpad is unsafe\n");
if (!unsafe) {
-- 
2.6.6

Re: [PATCH v2 2/2] eeprom: Add IDT 89HPESx driver bindings file

[Serge Semin]

On Mon, Dec 05, 2016 at 11:27:07AM -0600, Rob Herring <r...@kernel.org> wrote:
> On Mon, Dec 5, 2016 at 9:25 AM, Serge Semin <fancer.lan...@gmail.com> wrote:
> > On Mon, Dec 05, 2016 at 08:46:21AM -0600, Rob Herring <r...@kernel.org> 
> > wrote:
> >> On Tue, Nov 29, 2016 at 01:38:21AM +0300, Serge Semin wrote:
> >> > See cover-letter for changelog
> >> >
> >> > Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
> >> >
> >> > ---
> >> >  .../devicetree/bindings/misc/idt_89hpesx.txt   | 41 
> >> > ++
> >>
> >> There's not a better location for this? I can't tell because you don't
> >> describe what the device is.
> >>
> >
> > The device is PCIe-switch EEPROM driver with additional debug-interface to
> > access the switch CSRs. EEPROM is accesses via a separate i2c-slave
> > interface of the switch.
> >
> > There might be another place to put the binding file in. There is a special
> > location for EEPROM drivers bindings - 
> > Documentation/devicetree/bindings/eeprom/ .
> > But as far as I understood from the files put in there, it's intended for
> > pure EEPROM drivers only. On the other hand the directory I've chosen:
> > Documentation/devicetree/bindings/misc/
> > mostly intended for some unusual devices. My device isn't usual, since it
> > has CSRs debug-interface as well. Additionally I've found
> > eeprom-93xx46.txt binding file there, which describes EEPROM bindings.
> >
> > Anyway if you find the file should be placed in
> > Documentation/devicetree/bindings/eeprom/ instead, I'll move it, it's not
> > that a big problem.
> >

What about this comment? Shall the file be left at the path I placed it?

> >> >  1 file changed, 41 insertions(+)
> >> >  create mode 100644 
> >> > Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> >> >
> >> > diff --git a/Documentation/devicetree/bindings/misc/idt_89hpesx.txt 
> >> > b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> >> > index 000..469cc93
> >> > --- /dev/null
> >> > +++ b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> >> > @@ -0,0 +1,41 @@
> >> > +EEPROM / CSR SMBus-slave interface of IDT 89HPESx devices
> >> > +
> >> > +Required properties:
> >> > +  - compatible : should be ","
> >> > +Basically there is only one manufacturer: idt, but some
> >> > +compatible devices may be produced in future. Following 
> >> > devices
> >> > +are supported: 89hpes8nt2, 89hpes12nt3, 89hpes24nt6ag2,
> >> > +89hpes32nt8ag2, 89hpes32nt8bg2, 89hpes12nt12g2, 
> >> > 89hpes16nt16g2,
> >> > +89hpes24nt24g2, 89hpes32nt24ag2, 89hpes32nt24bg2;
> >> > +89hpes12n3, 89hpes12n3a, 89hpes24n3, 89hpes24n3a;
> >> > +89hpes32h8, 89hpes32h8g2, 89hpes48h12, 89hpes48h12g2,
> >> > +89hpes48h12ag2, 89hpes16h16, 89hpes22h16, 89hpes22h16g2,
> >> > +89hpes34h16, 89hpes34h16g2, 89hpes64h16, 89hpes64h16g2,
> >> > +89hpes64h16ag2;
> >> > +89hpes12t3g2, 89hpes24t3g2, 89hpes16t4, 89hpes4t4g2,
> >> > +89hpes10t4g2, 89hpes16t4g2, 89hpes16t4ag2, 89hpes5t5,
> >> > +89hpes6t5, 89hpes8t5, 89hpes8t5a, 89hpes24t6, 89hpes6t6g2,
> >> > +89hpes24t6g2, 89hpes16t7, 89hpes32t8, 89hpes32t8g2,
> >> > +89hpes48t12, 89hpes48t12g2.
> >> > +Current implementation of the driver doesn't have any 
> >> > device-
> >>
> >> Driver capabilties are irrelevant to bindings.
> >>
> >
> > Why? I've told in the comment, that the devices actually differ by the CSRs
> > map. Even though it's not reflected in the code at the moment, the CSRs
> > read/write restrictions can be added by some concerned programmer in
> > future. But If I left something like "compatible : idt,89hpesx" device
> > only, it will be problematic to add that functionality.
> 
> Bindings describe the h/w, not what the Linux, FreeBSD, etc. driver
> does. You don't want to be changing the binding doc when the driver
> changes.
> 
> > Howbeit If you think it's not necessary and "compatible = idt,89hpesx" is
> > ok, it's perfectly fine for me to make it this way. The property will be
> > even simpler, than current approach.
> 
> 

Re: [PATCH v2 2/2] eeprom: Add IDT 89HPESx driver bindings file

[Serge Semin]

On Mon, Dec 05, 2016 at 08:46:21AM -0600, Rob Herring <r...@kernel.org> wrote:
> On Tue, Nov 29, 2016 at 01:38:21AM +0300, Serge Semin wrote:
> > See cover-letter for changelog
> > 
> > Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
> > 
> > ---
> >  .../devicetree/bindings/misc/idt_89hpesx.txt   | 41 
> > ++
> 
> There's not a better location for this? I can't tell because you don't 
> describe what the device is.
> 

The device is PCIe-switch EEPROM driver with additional debug-interface to
access the switch CSRs. EEPROM is accesses via a separate i2c-slave
interface of the switch.

There might be another place to put the binding file in. There is a special
location for EEPROM drivers bindings - 
Documentation/devicetree/bindings/eeprom/ .
But as far as I understood from the files put in there, it's intended for
pure EEPROM drivers only. On the other hand the directory I've chosen:
Documentation/devicetree/bindings/misc/
mostly intended for some unusual devices. My device isn't usual, since it
has CSRs debug-interface as well. Additionally I've found
eeprom-93xx46.txt binding file there, which describes EEPROM bindings.

Anyway if you find the file should be placed in 
Documentation/devicetree/bindings/eeprom/ instead, I'll move it, it's not
that a big problem.

> >  1 file changed, 41 insertions(+)
> >  create mode 100644 Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> > 
> > diff --git a/Documentation/devicetree/bindings/misc/idt_89hpesx.txt 
> > b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> > index 000..469cc93
> > --- /dev/null
> > +++ b/Documentation/devicetree/bindings/misc/idt_89hpesx.txt
> > @@ -0,0 +1,41 @@
> > +EEPROM / CSR SMBus-slave interface of IDT 89HPESx devices
> > +
> > +Required properties:
> > +  - compatible : should be ","
> > +Basically there is only one manufacturer: idt, but some
> > +compatible devices may be produced in future. Following devices
> > +are supported: 89hpes8nt2, 89hpes12nt3, 89hpes24nt6ag2,
> > +89hpes32nt8ag2, 89hpes32nt8bg2, 89hpes12nt12g2, 89hpes16nt16g2,
> > +89hpes24nt24g2, 89hpes32nt24ag2, 89hpes32nt24bg2;
> > +89hpes12n3, 89hpes12n3a, 89hpes24n3, 89hpes24n3a;
> > +89hpes32h8, 89hpes32h8g2, 89hpes48h12, 89hpes48h12g2,
> > +89hpes48h12ag2, 89hpes16h16, 89hpes22h16, 89hpes22h16g2,
> > +89hpes34h16, 89hpes34h16g2, 89hpes64h16, 89hpes64h16g2,
> > +89hpes64h16ag2;
> > +89hpes12t3g2, 89hpes24t3g2, 89hpes16t4, 89hpes4t4g2,
> > +89hpes10t4g2, 89hpes16t4g2, 89hpes16t4ag2, 89hpes5t5,
> > +89hpes6t5, 89hpes8t5, 89hpes8t5a, 89hpes24t6, 89hpes6t6g2,
> > +89hpes24t6g2, 89hpes16t7, 89hpes32t8, 89hpes32t8g2,
> > +89hpes48t12, 89hpes48t12g2.
> > +Current implementation of the driver doesn't have any device-
> 
> Driver capabilties are irrelevant to bindings.
> 

Why? I've told in the comment, that the devices actually differ by the CSRs
map. Even though it's not reflected in the code at the moment, the CSRs
read/write restrictions can be added by some concerned programmer in
future. But If I left something like "compatible : idt,89hpesx" device
only, it will be problematic to add that functionality.

Howbeit If you think it's not necessary and "compatible = idt,89hpesx" is
ok, it's perfectly fine for me to make it this way. The property will be
even simpler, than current approach.

> > +specific functionalities. But since each of them differs
> > +by registers mapping, CSRs read/write restrictions can be
> > +added in future.
> > +  - reg :   I2C address of the IDT 89HPES device.
> > +
> > +Optional properties:
> > +  - read-only : Parameterless property disables writes to the EEPROM
> > +  - idt,eesize : Size of EEPROM device connected to IDT 89HPES i2c-master 
> > bus
> > +(default value is 4096 bytes if option isn't specified)
> > +  - idt,eeaddr : Custom address of EEPROM device
> > +(If not specified IDT 89HPESx device will try to communicate
> > + with EEPROM sited by default address - 0x50)
> 
> Don't we already have standard EEPROM properties that could be used 
> here?
> 

If we do, just tell me which one. There are standard options:
"compatible, reg, pagesize, read-only". There isn't any connected with
EEPROM actual size.
Why so? Because standard EEPROM-drivers determine the device size from the
compatible-string name. Such approach won't work in this case, becau

[PATCH 05/21] MIPS memblock: Alter initrd memory reservation method

[Serge Semin]

Since memblock is used, initrd memory region can be easily
verified and reserved if looks ok. Verification method will be
useful for other reservation methods.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 arch/mips/kernel/setup.c | 157 -
 1 file changed, 87 insertions(+), 70 deletions(-)

diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 789aafe..d2f38ac 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -82,6 +82,8 @@ static struct resource data_resource = { .name = "Kernel 
data", };
 
 static void *detect_magic __initdata = detect_memory_region;
 
+static phys_addr_t __initdata mips_lowmem_limit;
+
 /*
  * General method to add RAM regions to the system
  *
@@ -266,6 +268,38 @@ static int __init early_parse_mem(char *p)
 early_param("mem", early_parse_mem);
 
 /*
+ * Helper method checking whether passed lowmem region is valid
+ */
+static bool __init is_lowmem_and_valid(const char *name, phys_addr_t base,
+  phys_addr_t size)
+{
+   phys_addr_t end = base + size;
+
+   /* Check whether region belongs to actual memory */
+   if (!memblock_is_region_memory(base, size)) {
+   pr_err("%s %08zx @ %pa is not a memory region", name,
+   (size_t)size, );
+   return false;
+   }
+
+   /* Check whether region belongs to low memory */
+   if (end > mips_lowmem_limit) {
+   pr_err("%s %08zx @ %pa is out of low memory", name,
+   (size_t)size, );
+  return false;
+   }
+
+   /* Check whether region is free */
+   if (memblock_is_region_reserved(base, size)) {
+   pr_err("%s %08zx @ %pa overlaps in-use memory", name,
+   (size_t)size, );
+   return false;
+   }
+
+   return true;
+}
+
+/*
  * Manage initrd
  */
 #ifdef CONFIG_BLK_DEV_INITRD
@@ -292,47 +326,6 @@ static int __init rd_size_early(char *p)
 }
 early_param("rd_size", rd_size_early);
 
-/* it returns the next free pfn after initrd */
-static unsigned long __init init_initrd(void)
-{
-   unsigned long end;
-
-   /*
-* Board specific code or command line parser should have
-* already set up initrd_start and initrd_end. In these cases
-* perfom sanity checks and use them if all looks good.
-*/
-   if (!initrd_start || initrd_end <= initrd_start)
-   goto disable;
-
-   if (initrd_start & ~PAGE_MASK) {
-   pr_err("initrd start must be page aligned\n");
-   goto disable;
-   }
-   if (initrd_start < PAGE_OFFSET) {
-   pr_err("initrd start < PAGE_OFFSET\n");
-   goto disable;
-   }
-
-   /*
-* Sanitize initrd addresses. For example firmware
-* can't guess if they need to pass them through
-* 64-bits values if the kernel has been built in pure
-* 32-bit. We need also to switch from KSEG0 to XKPHYS
-* addresses now, so the code can now safely use __pa().
-*/
-   end = __pa(initrd_end);
-   initrd_end = (unsigned long)__va(end);
-   initrd_start = (unsigned long)__va(__pa(initrd_start));
-
-   ROOT_DEV = Root_RAM0;
-   return PFN_UP(end);
-disable:
-   initrd_start = 0;
-   initrd_end = 0;
-   return 0;
-}
-
 /* In some conditions (e.g. big endian bootloader with a little endian
kernel), the initrd might appear byte swapped.  Try to detect this and
byte swap it if needed.  */
@@ -362,26 +355,64 @@ static void __init maybe_bswap_initrd(void)
 #endif
 }
 
-static void __init finalize_initrd(void)
+/*
+ * Check and reserve memory occupied by initrd
+ */
+static void __init mips_reserve_initrd_mem(void)
 {
-   unsigned long size = initrd_end - initrd_start;
+   phys_addr_t phys_initrd_start, phys_initrd_end, phys_initrd_size;
 
-   if (size == 0) {
-   printk(KERN_INFO "Initrd not found or empty");
+   /*
+* Board specific code or command line parser should have already set
+* up initrd_start and initrd_end. In these cases perform sanity checks
+* and use them if all looks good.
+*/
+   if (!initrd_start || initrd_end <= initrd_start) {
+   pr_info("No initrd found");
goto disable;
}
-   if (__pa(initrd_end) > PFN_PHYS(max_low_pfn)) {
-   printk(KERN_ERR "Initrd extends beyond end of memory");
+   if (initrd_start & ~PAGE_MASK) {
+   pr_err("Initrd start must be page aligned");
goto disable;
}
+   if (initrd_start < PAGE_OFFSET) {
+   pr_err("Initrd start < PAGE_OFFSET");
+   goto disable;
+   }
+
+   /*
+*

[PATCH 19/21] MIPS memblock: Add print out method of kernel virtual memory layout

[Serge Semin]

It's useful to have some printed map of the kernel virtual memory,
at least for debugging purpose.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 arch/mips/mm/init.c | 47 +++
 1 file changed, 47 insertions(+)

diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 13a032f..35e7ba8 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -32,6 +32,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -106,6 +107,49 @@ static void __init zone_sizes_init(void)
 }
 
 /*
+ * Print out kernel memory layout
+ */
+#define MLK(b, t) b, t, ((t) - (b)) >> 10
+#define MLM(b, t) b, t, ((t) - (b)) >> 20
+#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
+static void __init mem_print_kmap_info(void)
+{
+   pr_notice("Virtual kernel memory layout:\n"
+ "lowmem  : 0x%08lx - 0x%08lx   (%4ld MB)\n"
+ "vmalloc : 0x%08lx - 0x%08lx   (%4ld MB)\n"
+#ifdef CONFIG_HIGHMEM
+ "pkmap   : 0x%08lx - 0x%08lx   (%4ld MB)\n"
+#endif
+ "fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
+ "  .text : 0x%p" " - 0x%p" "   (%4td kB)\n"
+ "  .data : 0x%p" " - 0x%p" "   (%4td kB)\n"
+ "  .init : 0x%p" " - 0x%p" "   (%4td kB)\n",
+   MLM(PAGE_OFFSET, (unsigned long)high_memory),
+   MLM(VMALLOC_START, VMALLOC_END),
+#ifdef CONFIG_HIGHMEM
+   MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP)*(PAGE_SIZE)),
+#endif
+   MLK(FIXADDR_START, FIXADDR_TOP),
+   MLK_ROUNDUP(_text, _etext),
+   MLK_ROUNDUP(_sdata, _edata),
+   MLK_ROUNDUP(__init_begin, __init_end));
+
+   /* Check some fundamental inconsistencies. May add something else? */
+#ifdef CONFIG_HIGHMEM
+   BUILD_BUG_ON(VMALLOC_END < PAGE_OFFSET);
+   BUG_ON(VMALLOC_END < (unsigned long)high_memory);
+#endif
+   BUILD_BUG_ON((PKMAP_BASE) + (LAST_PKMAP)*(PAGE_SIZE) < PAGE_OFFSET);
+   BUG_ON((PKMAP_BASE) + (LAST_PKMAP)*(PAGE_SIZE) <
+   (unsigned long)high_memory);
+   BUILD_BUG_ON(FIXADDR_TOP < PAGE_OFFSET);
+   BUG_ON(FIXADDR_TOP < (unsigned long)high_memory);
+}
+#undef MLK
+#undef MLM
+#undef MLK_ROUNDUP
+
+/*
  * Not static inline because used by IP27 special magic initialization code
  */
 void setup_zero_pages(void)
@@ -492,6 +536,9 @@ void __init mem_init(void)
/* Free highmemory registered in memblocks */
mem_init_free_highmem();
 
+   /* Print out kernel memory layout */
+   mem_print_kmap_info();
+
/* Print out memory areas statistics */
mem_init_print_info(NULL);
 
-- 
2.6.6

[PATCH 06/21] MIPS memblock: Alter kexec-crashkernel parameters parser

[Serge Semin]

Memblock API can be successfully used to verify whether crashkernel
memory region belongs to low memory, then it can be reserved within
memblock allocator.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 arch/mips/kernel/setup.c | 105 ++---
 1 file changed, 52 insertions(+), 53 deletions(-)

diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index d2f38ac..cc6d06b 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -426,6 +426,55 @@ static void __init mips_reserve_initrd_mem(void) { }
 
 #endif
 
+#ifdef CONFIG_KEXEC
+/*
+ * Parse passed crashkernel parameter and reserve corresponding memory
+ */
+static void __init mips_parse_crashkernel(void)
+{
+   unsigned long long total_mem;
+   unsigned long long crash_size, crash_base;
+   int ret;
+
+   /* Parse crachkernel parameter */
+   total_mem = memblock_phys_mem_size();
+   ret = parse_crashkernel(boot_command_line, total_mem,
+   _size, _base);
+   if (ret != 0 || crash_size <= 0)
+   return;
+
+   crashk_res.start = crash_base;
+   crashk_res.end   = crash_base + crash_size - 1;
+
+   /* Check whether the region belogs to lowmem and valid */
+   if (!is_lowmem_and_valid("Crashkernel", crash_base, crash_size))
+   return;
+
+   /* Reserve crashkernel resource */
+   memblock_reserve(crash_base, crash_size);
+}
+
+/*
+ * Reserve crashkernel memory within passed RAM resource
+ */
+static void __init request_crashkernel(struct resource *res)
+{
+   int ret;
+
+   ret = request_resource(res, _res);
+   if (!ret)
+   pr_info("Reserving %ldMB of memory at %ldMB for crashkernel\n",
+   (unsigned long)((crashk_res.end -
+crashk_res.start + 1) >> 20),
+   (unsigned long)(crashk_res.start  >> 20));
+}
+#else /* !CONFIG_KEXEC */
+
+static void __init mips_parse_crashkernel(void) { }
+static void __init request_crashkernel(struct resource *res) { }
+
+#endif /* !CONFIG_KEXEC */
+
 /*
  * Initialize the bootmem allocator. It also setup initrd related data
  * if needed.
@@ -450,6 +499,9 @@ static void __init bootmem_init(void)
/* Check and reserve memory occupied by initrd */
mips_reserve_initrd_mem();
 
+   /* Parse crashkernel parameter */
+   mips_parse_crashkernel();
+
reserved_end = (unsigned long) PFN_UP(__pa_symbol(&_end));
 
/*
@@ -717,52 +769,6 @@ static void __init arch_mem_addpart(phys_addr_t mem, 
phys_addr_t end, int type)
add_memory_region(mem, size, type);
 }
 
-#ifdef CONFIG_KEXEC
-static inline unsigned long long get_total_mem(void)
-{
-   unsigned long long total;
-
-   total = max_pfn - min_low_pfn;
-   return total << PAGE_SHIFT;
-}
-
-static void __init mips_parse_crashkernel(void)
-{
-   unsigned long long total_mem;
-   unsigned long long crash_size, crash_base;
-   int ret;
-
-   total_mem = get_total_mem();
-   ret = parse_crashkernel(boot_command_line, total_mem,
-   _size, _base);
-   if (ret != 0 || crash_size <= 0)
-   return;
-
-   crashk_res.start = crash_base;
-   crashk_res.end   = crash_base + crash_size - 1;
-}
-
-static void __init request_crashkernel(struct resource *res)
-{
-   int ret;
-
-   ret = request_resource(res, _res);
-   if (!ret)
-   pr_info("Reserving %ldMB of memory at %ldMB for crashkernel\n",
-   (unsigned long)((crashk_res.end -
-crashk_res.start + 1) >> 20),
-   (unsigned long)(crashk_res.start  >> 20));
-}
-#else /* !defined(CONFIG_KEXEC)*/
-static void __init mips_parse_crashkernel(void)
-{
-}
-
-static void __init request_crashkernel(struct resource *res)
-{
-}
-#endif /* !defined(CONFIG_KEXEC)  */
-
 #define USE_PROM_CMDLINE   IS_ENABLED(CONFIG_MIPS_CMDLINE_FROM_BOOTLOADER)
 #define USE_DTB_CMDLINEIS_ENABLED(CONFIG_MIPS_CMDLINE_FROM_DTB)
 #define EXTEND_WITH_PROM   IS_ENABLED(CONFIG_MIPS_CMDLINE_DTB_EXTEND)
@@ -836,13 +842,6 @@ static void __init arch_mem_init(char **cmdline_p)
}
 #endif
 
-   mips_parse_crashkernel();
-#ifdef CONFIG_KEXEC
-   if (crashk_res.start != crashk_res.end)
-   reserve_bootmem(crashk_res.start,
-   crashk_res.end - crashk_res.start + 1,
-   BOOTMEM_DEFAULT);
-#endif
device_tree_init();
sparse_init();
plat_swiotlb_setup();
-- 
2.6.6

[PATCH 07/21] MIPS memblock: Alter elfcorehdr parameters parser

[Serge Semin]

Memblock API can be successfully used to verify whether elfcorehdr
memory region belongs to lowmemory, then it can be reserved within
memblock allocator. There is also available default method for
early parameters parser in kernel/crash_dump.c: setup_elfcorehdr(),
so it's wise to use one instead of creating our own doing actually
the same thing.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 arch/mips/kernel/setup.c | 91 +-
 1 file changed, 52 insertions(+), 39 deletions(-)

diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index cc6d06b..52205fb 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -426,6 +426,55 @@ static void __init mips_reserve_initrd_mem(void) { }
 
 #endif
 
+/*
+ * Reserve memory occupied by elfcorehdr
+ */
+static void __init mips_reserve_elfcorehdr(void)
+{
+#ifdef CONFIG_PROC_VMCORE
+   /*
+* Don't reserve anything if kernel isn't booting after a panic and
+* vmcore is usable (see linux/crash_dump.h for details)
+*/
+   if (!is_vmcore_usable())
+   return;
+
+   /* Check whether the passed address belongs to low memory */
+   if (elfcorehdr_addr + elfcorehdr_size >= mips_lowmem_limit) {
+   pr_err("Elfcorehdr %08zx @ %pa doesn't belong to low memory",
+   (size_t)elfcorehdr_size, _addr);
+   return;
+   }
+
+   /*
+* If elfcorehdr_size hasn't been specified, then try to reserve upto
+* low memory limit
+*/
+   if (!elfcorehdr_size)
+   elfcorehdr_size = mips_lowmem_limit - elfcorehdr_addr;
+
+   /* Check the region belongs to actual memory (size can be zero) */
+   if (!memblock_is_region_memory(elfcorehdr_addr, elfcorehdr_size)) {
+   pr_err("Elfcorehdr %08zx @ %pa is not a memory region",
+   (size_t)elfcorehdr_size, _addr);
+   return;
+   }
+
+   /* Check whether elfcorehdr region is free */
+   if (memblock_is_region_reserved(elfcorehdr_addr, elfcorehdr_size)) {
+   pr_err("Elfcorehdr %08zx @ %pa overlaps in-use memory",
+   (size_t)elfcorehdr_size, _addr);
+   return;
+   }
+
+   /* Reserve elfcorehdr within memblock */
+   memblock_reserve(elfcorehdr_addr, PAGE_ALIGN(elfcorehdr_size));
+
+   pr_info("Reserved memory for kdump at %08zx @ %pa\n",
+   (size_t)elfcorehdr_size, _addr);
+#endif /* CONFIG_PROC_VMCORE */
+}
+
 #ifdef CONFIG_KEXEC
 /*
  * Parse passed crashkernel parameter and reserve corresponding memory
@@ -499,6 +548,9 @@ static void __init bootmem_init(void)
/* Check and reserve memory occupied by initrd */
mips_reserve_initrd_mem();
 
+   /* Reserve memory for elfcorehdr */
+   mips_reserve_elfcorehdr();
+
/* Parse crashkernel parameter */
mips_parse_crashkernel();
 
@@ -719,37 +771,6 @@ static void __init bootmem_init(void)
  * initialization hook for anything else was introduced.
  */
 
-#ifdef CONFIG_PROC_VMCORE
-unsigned long setup_elfcorehdr, setup_elfcorehdr_size;
-static int __init early_parse_elfcorehdr(char *p)
-{
-   int i;
-
-   setup_elfcorehdr = memparse(p, );
-
-   for (i = 0; i < boot_mem_map.nr_map; i++) {
-   unsigned long start = boot_mem_map.map[i].addr;
-   unsigned long end = (boot_mem_map.map[i].addr +
-boot_mem_map.map[i].size);
-   if (setup_elfcorehdr >= start && setup_elfcorehdr < end) {
-   /*
-* Reserve from the elf core header to the end of
-* the memory segment, that should all be kdump
-* reserved memory.
-*/
-   setup_elfcorehdr_size = end - setup_elfcorehdr;
-   break;
-   }
-   }
-   /*
-* If we don't find it in the memory map, then we shouldn't
-* have to worry about it, as the new kernel won't use it.
-*/
-   return 0;
-}
-early_param("elfcorehdr", early_parse_elfcorehdr);
-#endif
-
 static void __init arch_mem_addpart(phys_addr_t mem, phys_addr_t end, int type)
 {
phys_addr_t size;
@@ -833,14 +854,6 @@ static void __init arch_mem_init(char **cmdline_p)
parse_early_param();
 
bootmem_init();
-#ifdef CONFIG_PROC_VMCORE
-   if (setup_elfcorehdr && setup_elfcorehdr_size) {
-   printk(KERN_INFO "kdump reserved memory at %lx-%lx\n",
-  setup_elfcorehdr, setup_elfcorehdr_size);
-   reserve_bootmem(setup_elfcorehdr, setup_elfcorehdr_size,
-   BOOTMEM_DEFAULT);
-   }
-#endif
 
device_tree_init();
sparse_init();
-- 
2.6.6

[PATCH 08/21] MIPS memblock: Move kernel parameters parser into individual method

[Serge Semin]

Main memory initialization method looks messy with cmd line parser
built-in. So it's better for readability to put it into a separated
method.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 arch/mips/kernel/setup.c | 87 --
 1 file changed, 48 insertions(+), 39 deletions(-)

diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 52205fb..9c1a60d 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -231,6 +231,51 @@ static void __init print_memory_map(void)
 }
 
 /*
+ * Parse passed cmdline
+ */
+#define USE_PROM_CMDLINE   IS_ENABLED(CONFIG_MIPS_CMDLINE_FROM_BOOTLOADER)
+#define USE_DTB_CMDLINEIS_ENABLED(CONFIG_MIPS_CMDLINE_FROM_DTB)
+#define EXTEND_WITH_PROM   IS_ENABLED(CONFIG_MIPS_CMDLINE_EXTEND)
+#define BUILTIN_EXTEND_WITH_PROM   \
+   IS_ENABLED(CONFIG_MIPS_CMDLINE_BUILTIN_EXTEND)
+
+static void __init mips_parse_param(char **cmdline_p)
+{
+#if defined(CONFIG_CMDLINE_BOOL) && defined(CONFIG_CMDLINE_OVERRIDE)
+   strlcpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE);
+#else
+   if ((USE_PROM_CMDLINE && arcs_cmdline[0]) ||
+   (USE_DTB_CMDLINE && !boot_command_line[0]))
+   strlcpy(boot_command_line, arcs_cmdline, COMMAND_LINE_SIZE);
+
+   if (EXTEND_WITH_PROM && arcs_cmdline[0]) {
+   if (boot_command_line[0])
+   strlcat(boot_command_line, " ", COMMAND_LINE_SIZE);
+   strlcat(boot_command_line, arcs_cmdline, COMMAND_LINE_SIZE);
+   }
+
+#if defined(CONFIG_CMDLINE_BOOL)
+   if (builtin_cmdline[0]) {
+   if (boot_command_line[0])
+   strlcat(boot_command_line, " ", COMMAND_LINE_SIZE);
+   strlcat(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE);
+   }
+
+   if (BUILTIN_EXTEND_WITH_PROM && arcs_cmdline[0]) {
+   if (boot_command_line[0])
+   strlcat(boot_command_line, " ", COMMAND_LINE_SIZE);
+   strlcat(boot_command_line, arcs_cmdline, COMMAND_LINE_SIZE);
+   }
+#endif
+#endif
+   strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE);
+
+   *cmdline_p = command_line;
+
+   parse_early_param();
+}
+
+/*
  * Parse "mem=size@start" parameter rewriting a defined memory map
  * We look for mem=size@start, where start and size are "value[KkMm]"
  */
@@ -790,12 +835,6 @@ static void __init arch_mem_addpart(phys_addr_t mem, 
phys_addr_t end, int type)
add_memory_region(mem, size, type);
 }
 
-#define USE_PROM_CMDLINE   IS_ENABLED(CONFIG_MIPS_CMDLINE_FROM_BOOTLOADER)
-#define USE_DTB_CMDLINEIS_ENABLED(CONFIG_MIPS_CMDLINE_FROM_DTB)
-#define EXTEND_WITH_PROM   IS_ENABLED(CONFIG_MIPS_CMDLINE_DTB_EXTEND)
-#define BUILTIN_EXTEND_WITH_PROM   \
-   IS_ENABLED(CONFIG_MIPS_CMDLINE_BUILTIN_EXTEND)
-
 static void __init arch_mem_init(char **cmdline_p)
 {
struct memblock_region *reg;
@@ -804,6 +843,9 @@ static void __init arch_mem_init(char **cmdline_p)
/* call board setup routine */
plat_mem_setup();
 
+   /* Parse passed parameters */
+   mips_parse_param(cmdline_p);
+
/*
 * Make sure all kernel memory is in the maps.  The "UP" and
 * "DOWN" are opposite for initdata since if it crosses over
@@ -820,39 +862,6 @@ static void __init arch_mem_init(char **cmdline_p)
pr_info("Determined physical RAM map:\n");
print_memory_map();
 
-#if defined(CONFIG_CMDLINE_BOOL) && defined(CONFIG_CMDLINE_OVERRIDE)
-   strlcpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE);
-#else
-   if ((USE_PROM_CMDLINE && arcs_cmdline[0]) ||
-   (USE_DTB_CMDLINE && !boot_command_line[0]))
-   strlcpy(boot_command_line, arcs_cmdline, COMMAND_LINE_SIZE);
-
-   if (EXTEND_WITH_PROM && arcs_cmdline[0]) {
-   if (boot_command_line[0])
-   strlcat(boot_command_line, " ", COMMAND_LINE_SIZE);
-   strlcat(boot_command_line, arcs_cmdline, COMMAND_LINE_SIZE);
-   }
-
-#if defined(CONFIG_CMDLINE_BOOL)
-   if (builtin_cmdline[0]) {
-   if (boot_command_line[0])
-   strlcat(boot_command_line, " ", COMMAND_LINE_SIZE);
-   strlcat(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE);
-   }
-
-   if (BUILTIN_EXTEND_WITH_PROM && arcs_cmdline[0]) {
-   if (boot_command_line[0])
-   strlcat(boot_command_line, " ", COMMAND_LINE_SIZE);
-   strlcat(boot_command_line, arcs_cmdline, COMMAND_LINE_SIZE);
-   }
-#endif
-#endif
-   strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE);
-
-   *cmdline_p = command_line;
-
-   parse_early_param();
-
bootmem_init();
 
device_tree_init();
-- 
2.6.6

[PATCH 09/21] MIPS memblock: Move kernel memory reservation to individual method

[Serge Semin]

The whole kernel text/data/bss must be reserved to prevent sudden
kernel crashes, for instance, due to unexpected non-zero default static
variables initializations.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 arch/mips/kernel/setup.c | 117 +++--
 1 file changed, 59 insertions(+), 58 deletions(-)

diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 9c1a60d..e746793 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -472,6 +472,62 @@ static void __init mips_reserve_initrd_mem(void) { }
 #endif
 
 /*
+ * Reserve kernel code and data within memblock allocator
+ */
+static void __init mips_reserve_kernel_mem(void)
+{
+   phys_addr_t start, size;
+
+   /*
+* Add kernel _text, _data, _bss, __init*, upto __end sections to
+* boot_mem_map and memblock. We must reserve all of them!
+*/
+   start = __pa_symbol(&_text);
+   size = __pa_symbol(&_end) - start;
+   add_memory_region(start, size, BOOT_MEM_RAM);
+   /*
+* It needs to be reserved within memblock as well. It's ok if memory
+* has already been reserved with previous method
+*/
+   memblock_reserve(start, size);
+
+   /* Reserve nosave region for hibernation */
+   start = __pa_symbol(&__nosave_begin);
+   size = __pa_symbol(&__nosave_end) - start;
+   add_memory_region(start, size, BOOT_MEM_RAM);
+   memblock_reserve(start, size);
+
+   /* Initialize some init_mm fieldis. We may not need this? */
+   init_mm.start_code = (unsigned long)&_text;
+   init_mm.end_code = (unsigned long)&_etext;
+   init_mm.end_data = (unsigned long)&_edata;
+   init_mm.brk = (unsigned long)&_end;
+
+   /*
+* The kernel reserves all memory below its _end symbol as bootmem,
+* but the kernel may now be at a much higher address. The memory
+* between the original and new locations may be returned to the system.
+*/
+#ifdef CONFIG_RELOCATABLE
+   if (__pa_symbol(&_text) > __pa_symbol(VMLINUX_LOAD_ADDRESS)) {
+   phys_addr_t offset;
+   extern void show_kernel_relocation(const char *level);
+
+   offset = __pa_symbol(_text) - __pa_symbol(VMLINUX_LOAD_ADDRESS);
+   memblock_free(__pa_symbol(VMLINUX_LOAD_ADDRESS), offset);
+
+#if defined(CONFIG_DEBUG_KERNEL) && defined(CONFIG_DEBUG_INFO)
+   /*
+* This information is necessary when debugging the kernel
+* But is a security vulnerability otherwise!
+*/
+   show_kernel_relocation(KERN_INFO);
+#endif
+   }
+#endif
+}
+
+/*
  * Reserve memory occupied by elfcorehdr
  */
 static void __init mips_reserve_elfcorehdr(void)
@@ -590,6 +646,9 @@ static void __init bootmem_init(void)
unsigned long bootmap_size;
int i;
 
+   /* Reserve kernel code/data memory */
+   mips_reserve_kernel_mem();
+
/* Check and reserve memory occupied by initrd */
mips_reserve_initrd_mem();
 
@@ -766,29 +825,6 @@ static void __init bootmem_init(void)
 * Reserve the bootmap memory.
 */
reserve_bootmem(PFN_PHYS(mapstart), bootmap_size, BOOTMEM_DEFAULT);
-
-#ifdef CONFIG_RELOCATABLE
-   /*
-* The kernel reserves all memory below its _end symbol as bootmem,
-* but the kernel may now be at a much higher address. The memory
-* between the original and new locations may be returned to the system.
-*/
-   if (__pa_symbol(_text) > __pa_symbol(VMLINUX_LOAD_ADDRESS)) {
-   unsigned long offset;
-   extern void show_kernel_relocation(const char *level);
-
-   offset = __pa_symbol(_text) - __pa_symbol(VMLINUX_LOAD_ADDRESS);
-   free_bootmem(__pa_symbol(VMLINUX_LOAD_ADDRESS), offset);
-
-#if defined(CONFIG_DEBUG_KERNEL) && defined(CONFIG_DEBUG_INFO)
-   /*
-* This information is necessary when debugging the kernel
-* But is a security vulnerability otherwise!
-*/
-   show_kernel_relocation(KERN_INFO);
-#endif
-   }
-#endif
 }
 
 #endif /* CONFIG_SGI_IP27 */
@@ -816,25 +852,6 @@ static void __init bootmem_init(void)
  * initialization hook for anything else was introduced.
  */
 
-static void __init arch_mem_addpart(phys_addr_t mem, phys_addr_t end, int type)
-{
-   phys_addr_t size;
-   int i;
-
-   size = end - mem;
-   if (!size)
-   return;
-
-   /* Make sure it is in the boot_mem_map */
-   for (i = 0; i < boot_mem_map.nr_map; i++) {
-   if (mem >= boot_mem_map.map[i].addr &&
-   mem < (boot_mem_map.map[i].addr +
-  boot_mem_map.map[i].size))
-   return;
-   }
-   add_memory_region(mem, size, type);
-}
-
 sta

[PATCH 20/21] MIPS memblock: Add free low memory test method call

[Serge Semin]

Right after all the necessary reservations are done, free memory
regions can be tested if it is activated with "memtest" parameter.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 arch/mips/mm/init.c | 6 ++
 1 file changed, 6 insertions(+)

diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 35e7ba8..ccc0e96 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -452,6 +452,12 @@ void maar_init(void)
 
 void __init paging_init(void)
 {
+   /*
+* Test low memory registered within memblock. The method shall test
+* valid and free memory only
+*/
+   early_memtest(PFN_PHYS(min_low_pfn), PFN_PHYS(max_low_pfn));
+
/* Initialize page tables */
pagetable_init();
 
-- 
2.6.6

[PATCH 21/21] MIPS memblock: Deactivate old bootmem allocator

[Serge Semin]

Memblock allocator can be successfully used from now for early
memory management.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 arch/mips/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 2ef1e2d..527f2fe 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -40,9 +40,9 @@ config MIPS
select HAVE_ARCH_JUMP_LABEL
select ARCH_WANT_IPC_PARSE_VERSION
select IRQ_FORCED_THREADING
+   select NO_BOOTMEM
select HAVE_MEMBLOCK
select HAVE_MEMBLOCK_NODE_MAP
-   select ARCH_DISCARD_MEMBLOCK
select GENERIC_SMP_IDLE_THREAD
select BUILDTIME_EXTABLE_SORT
select GENERIC_CLOCKEVENTS
-- 
2.6.6

[PATCH 04/21] MIPS memblock: Alter user-defined memory parameter parser

[Serge Semin]

Both new memblock and boot_mem_map subsystems need to be fully
cleared before a new memory region is added. So the early parser is
correspondingly modified.

Signed-off-by: Serge Semin <fancer.lan...@gmail.com>
---
 arch/mips/kernel/setup.c | 67 +-
 1 file changed, 37 insertions(+), 30 deletions(-)

diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 9da6f8a..789aafe 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -229,6 +229,43 @@ static void __init print_memory_map(void)
 }
 
 /*
+ * Parse "mem=size@start" parameter rewriting a defined memory map
+ * We look for mem=size@start, where start and size are "value[KkMm]"
+ */
+static int __init early_parse_mem(char *p)
+{
+   static int usermem;
+   phys_addr_t start, size;
+
+   start = PHYS_OFFSET;
+   size = memparse(p, );
+   if (*p == '@')
+   start = memparse(p + 1, );
+
+   /*
+* If a user specifies memory size, we blow away any automatically
+* generated regions.
+*/
+   if (usermem == 0) {
+   phys_addr_t ram_start = memblock_start_of_DRAM();
+   phys_addr_t ram_end = memblock_end_of_DRAM() - ram_start;
+
+   pr_notice("Discard memory layout %pa - %pa",
+ _start, _end);
+
+   memblock_remove(ram_start, ram_end - ram_start);
+   boot_mem_map.nr_map = 0;
+   usermem = 1;
+   }
+   pr_notice("Add userdefined memory region %08zx @ %pa",
+ (size_t)size, );
+
+   add_memory_region(start, size, BOOT_MEM_RAM);
+   return 0;
+}
+early_param("mem", early_parse_mem);
+
+/*
  * Manage initrd
  */
 #ifdef CONFIG_BLK_DEV_INITRD
@@ -613,31 +650,6 @@ static void __init bootmem_init(void)
  * initialization hook for anything else was introduced.
  */
 
-static int usermem __initdata;
-
-static int __init early_parse_mem(char *p)
-{
-   phys_addr_t start, size;
-
-   /*
-* If a user specifies memory size, we
-* blow away any automatically generated
-* size.
-*/
-   if (usermem == 0) {
-   boot_mem_map.nr_map = 0;
-   usermem = 1;
-   }
-   start = 0;
-   size = memparse(p, );
-   if (*p == '@')
-   start = memparse(p + 1, );
-
-   add_memory_region(start, size, BOOT_MEM_RAM);
-   return 0;
-}
-early_param("mem", early_parse_mem);
-
 #ifdef CONFIG_PROC_VMCORE
 unsigned long setup_elfcorehdr, setup_elfcorehdr_size;
 static int __init early_parse_elfcorehdr(char *p)
@@ -797,11 +809,6 @@ static void __init arch_mem_init(char **cmdline_p)
 
parse_early_param();
 
-   if (usermem) {
-   pr_info("User-defined physical RAM map:\n");
-   print_memory_map();
-   }
-
bootmem_init();
 #ifdef CONFIG_PROC_VMCORE
if (setup_elfcorehdr && setup_elfcorehdr_size) {
-- 
2.6.6