Re: [patch V2 46/46] irqchip: Add IMS (Interrupt Message Storm) driver - NOT FOR MERGING

2020-08-31 Thread Jason Gunthorpe
On Wed, Aug 26, 2020 at 01:17:14PM +0200, Thomas Gleixner wrote:
> + * ims_queue_info - Information to create an IMS queue domain
> + * @queue_lock:  Callback which informs the device driver that
> + *   an interrupt management operation starts.
> + * @queue_sync_unlock:   Callback which informs the device driver that an
> + *   interrupt management operation ends.
> +
> + * @queue_get_shadow:   Callback to retrieve te shadow storage for a MSI
> + *   entry associated to a queue. The queue is
> + *   identified by the device struct which is used for
> + *   allocating interrupts and the msi entry index.
> + *
> + * @queue_lock() and @queue_sync_unlock() are only called for management
> + * operations on a particular interrupt: request, free, enable, disable,
> + * affinity setting.  These functions are never called from atomic context,
> + * like low level interrupt handling code. The purpose of these functions
> + * is to signal the device driver the start and end of an operation which
> + * affects the IMS queue shadow state. @queue_lock() allows the driver to
> + * do preperatory work, e.g. locking. Note, that @queue_lock() has to
> + * preserve the sleepable state on return. That means the driver cannot
> + * disable preemption and (soft)interrupts in @queue_lock and then undo
> + * that operation in @queue_sync_unlock() which restricts the lock types
> + * for eventual serialization of these operations to sleepable locks. Of
> + * course the driver can disable preemption and (soft)interrupts
> + * temporarily for internal work.
> + *
> + * On @queue_sync_unlock() the driver has to check whether the shadow state
> + * changed and issue a command to update the hardware state and wait for
> + * the command to complete. If the command fails or times out then the
> + * driver has to take care of the resulting mess as this is called from
> + * functions which have no return value and none of the callers can deal
> + * with the failure. The lock which is used by the driver to protect a
> + * operation sequence must obviously not be released before the command
> + * completes or fails. Otherwise new operations on the same interrupt line
> + * could take place and change the shadow state before the driver was able
> + * to compose the command.

I haven't looked through everything in detail, but this does look like
it is good for the mlx5 devices. Looked like it was only one small
update to the set_affinity, so not very disruptive?

Thanks,
Jason
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu


[patch V2 46/46] irqchip: Add IMS (Interrupt Message Storm) driver - NOT FOR MERGING

2020-08-26 Thread Thomas Gleixner
Generic IMS irq chips and irq domain implementations for IMS based devices
in both variants:

   - Message store in an array in device memory
   - Message store in system RAM (part of queue memory)

Allocation and freeing of interrupts happens via the generic
msi_domain_alloc/free_irqs() interface. No special purpose IMS magic
required as long as the interrupt domain is stored in the underlying device
struct.



   
Completely untested of course and mostly for illustration and educational
purpose. This should of course be a modular irq chip, but adding that
support is left as an exercise for the people who care about this deeply.

Signed-off-by: Thomas Gleixner 
---
V2: Reworked to handle both devmem arrays and queue based storage.
---
 drivers/irqchip/Kconfig |   19 +
 drivers/irqchip/Makefile|1 
 drivers/irqchip/irq-ims-msi.c   |  343 
 include/linux/irqchip/irq-ims-msi.h |   95 +
 4 files changed, 458 insertions(+)

--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -571,4 +571,23 @@ config LOONGSON_PCH_MSI
help
  Support for the Loongson PCH MSI Controller.
 
+config IMS_MSI
+   depends on PCI
+   select DEVICE_MSI
+   bool
+
+config IMS_MSI_ARRAY
+   bool "IMS Interrupt Message Storm MSI controller for device memory 
storage arrays"
+   select IMS_MSI
+   help
+ Support for IMS Interrupt Message Storm MSI controller
+ with IMS slot storage in a slot array in device memory
+
+config IMS_MSI_QUEUE
+   bool "IMS Interrupt Message Storm MSI controller for IMS queue storage"
+   select IMS_MSI
+   help
+ Support for IMS Interrupt Message Storm MSI controller
+ with IMS slot storage in the queue storage of a device
+
 endmenu
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -111,3 +111,4 @@ obj-$(CONFIG_LOONGSON_HTPIC)+= irq-loo
 obj-$(CONFIG_LOONGSON_HTVEC)   += irq-loongson-htvec.o
 obj-$(CONFIG_LOONGSON_PCH_PIC) += irq-loongson-pch-pic.o
 obj-$(CONFIG_LOONGSON_PCH_MSI) += irq-loongson-pch-msi.o
+obj-$(CONFIG_IMS_MSI)  += irq-ims-msi.o
--- /dev/null
+++ b/drivers/irqchip/irq-ims-msi.c
@@ -0,0 +1,343 @@
+// SPDX-License-Identifier: GPL-2.0
+// (C) Copyright 2020 Thomas Gleixner 
+/*
+ * Shared interrupt chips and irq domains for IMS devices
+ */
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#ifdef CONFIG_IMS_ARRAY
+
+struct ims_array_data {
+   struct ims_array_info   info;
+   unsigned long   map[0];
+};
+
+static void ims_array_mask_irq(struct irq_data *data)
+{
+   struct msi_desc *desc = irq_data_get_msi_desc(data);
+   struct ims_slot __iomem *slot = desc->device_msi.priv_iomem;
+   u32 __iomem *ctrl = >ctrl;
+
+   iowrite32(ioread32(ctrl) & ~IMS_VECTOR_CTRL_UNMASK, ctrl);
+}
+
+static void ims_array_unmask_irq(struct irq_data *data)
+{
+   struct msi_desc *desc = irq_data_get_msi_desc(data);
+   struct ims_slot __iomem *slot = desc->device_msi.priv_iomem;
+   u32 __iomem *ctrl = >ctrl;
+
+   iowrite32(ioread32(ctrl) | IMS_VECTOR_CTRL_UNMASK, ctrl);
+}
+
+static void ims_array_write_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+   struct msi_desc *desc = irq_data_get_msi_desc(data);
+   struct ims_slot __iomem *slot = desc->device_msi.priv_iomem;
+
+   iowrite32(msg->address_lo, >address_lo);
+   iowrite32(msg->address_hi, >address_hi);
+   iowrite32(msg->data, >data);
+}
+
+static const struct irq_chip ims_array_msi_controller = {
+   .name   = "IMS",
+   .irq_mask   = ims_array_mask_irq,
+   .irq_unmask = ims_array_unmask_irq,
+   .irq_write_msi_msg  = ims_array_write_msi_msg,
+   .irq_retrigger  = irq_chip_retrigger_hierarchy,
+   .flags  = IRQCHIP_SKIP_SET_WAKE,
+};
+
+static void ims_array_reset_slot(struct ims_slot __iomem *slot)
+{
+   iowrite32(0, >address_lo);
+   iowrite32(0, >address_hi);
+   iowrite32(0, >data);
+   iowrite32(0, >ctrl);
+}
+
+static void ims_array_free_msi_store(struct irq_domain *domain,
+struct device *dev)
+{
+   struct msi_domain_info *info = domain->host_data;
+   struct ims_array_data *ims = info->data;
+   struct msi_desc *entry;
+
+   for_each_msi_entry(entry, dev) {
+   if (entry->device_msi.priv_iomem) {
+   clear_bit(entry->device_msi.hwirq, ims->map);
+   ims_array_reset_slot(entry->device_msi.priv_iomem);
+   entry->device_msi.priv_iomem =