CXL VH RAS handling requires a path for the AER driver to hand off CXL protocol errors to cxl_core for logging and recovery before PCIe AER recovery tears down the device. Add drivers/pci/pcie/aer_cxl_vh.c to implement this handoff via a kfifo-backed work item.
Introduce is_aer_internal_error() to identify CXL protocol errors from AER internal error status bits across both correctable and uncorrectable severities. Introduce is_cxl_error() to gate the VH kfifo path. Introduce struct cxl_proto_err_work_data to carry the error source PCI device and severity through the kfifo. Encapsulate the kfifo, per-producer spinlock, registration rwsem, and work pointer in struct cxl_proto_err_kfifo. Initialize the embedded kfifo via INIT_KFIFO() from a subsys_initcall so its metadata is populated before any producer or consumer runs. Introduce cxl_forward_error() to enqueue a CXL protocol error. A reference is taken on the PCI device; the consumer releases it via for_each_cxl_proto_err(). On enqueue failure the reference is released immediately, the error is dropped, and the consumer is scheduled to drain existing entries. A subsequent patch wires cxl_forward_error() into handle_error_source() where correctable and uncorrectable status clearing is left to pci_aer_handle_error(). Introduce cxl_proto_err_flush() to synchronously wait for the consumer worker to drain the kfifo. A subsequent patch wires this into handle_error_source() for UCE events so the CXL plane completes error handling and panic policy before pci_aer_handle_error() drives PCIe recovery. Introduce cxl_register_proto_err_work() and cxl_unregister_proto_err_work() for cxl_core to register and deregister its work handler. On unregistration, pending kfifo entries are drained and their pdev references released before cancel_work_sync() runs. Export these and for_each_cxl_proto_err() via EXPORT_SYMBOL_FOR_MODULES restricted to cxl_core. Protect the work pointer with a rwsem to correctly serialize registration, deregistration, enqueue, and dequeue against concurrent AER IRQ threads. Serialize concurrent kfifo writers with a spinlock. Add MAINTAINERS entries for aer_cxl_vh.c and aer_cxl_rch.c under the CXL entry so CXL maintainers are CC'd on changes to the AER-CXL bridging code. Co-developed-by: Dan Williams <[email protected]> Signed-off-by: Dan Williams <[email protected]> Signed-off-by: Terry Bowman <[email protected]> --- Changes in v17->v18: - Remove correctable status clear from cxl_forward_error(); the AER core clears all status bits via pci_aer_handle_error() info->status writeback - Schedule consumer on kfifo overflow so existing entries can be drained Changes in v16->v17: - Reword "kfifo semaphore" to "kfifo spinlock" to match fifo_lock. - Defer the handle_error_source() is_cxl_error() switch to the patch that registers the kfifo consumer to keep each commit bisect-safe. - Rename rwsema to rwsem - Change CPER exports to use EXPORT_SYMBOL_FOR_MODULES. - Add work cancel function. - Replace kfifo_put() with kfifo_in_spinlocked() for multiple producers - Add fifo_lock spinlock for concurrent producer serialisation - Initialize the embedded kfifo with INIT_KFIFO() in a subsys_initcall so kfifo->mask, ->esize and ->data are set before first use. - Clear PCI_ERR_COR_STATUS in cxl_forward_error() after enqueue so the device is acked for correctable events even when the consumer drops the event. Uncorrectable status is left for cxl_do_recovery() to clear after recovery completes, mirroring the AER core convention. - WARN on double-registration in cxl_register_proto_err_work() to make an unintended second consumer visible at runtime. - Add direct rwsem.h, cleanup.h and workqueue.h includes for symbols used in aer_cxl_vh.c - Add MAINTAINERS entries for drivers/pci/pcie/aer_cxl_*.c - Update message --- MAINTAINERS | 2 + drivers/pci/pcie/Makefile | 1 + drivers/pci/pcie/aer.c | 10 -- drivers/pci/pcie/aer_cxl_vh.c | 221 ++++++++++++++++++++++++++++++++++ drivers/pci/pcie/portdrv.h | 6 + include/linux/aer.h | 24 ++++ 6 files changed, 254 insertions(+), 10 deletions(-) create mode 100644 drivers/pci/pcie/aer_cxl_vh.c diff --git a/MAINTAINERS b/MAINTAINERS index 806bd2d80d153..39007aa90b20f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6526,6 +6526,8 @@ S: Maintained F: Documentation/driver-api/cxl F: Documentation/userspace-api/fwctl/fwctl-cxl.rst F: drivers/cxl/ +F: drivers/pci/pcie/aer_cxl_rch.c +F: drivers/pci/pcie/aer_cxl_vh.c F: include/cxl/ F: include/uapi/linux/cxl_mem.h F: tools/testing/cxl/ diff --git a/drivers/pci/pcie/Makefile b/drivers/pci/pcie/Makefile index b0b43a18c304b..62d3d3c69a5df 100644 --- a/drivers/pci/pcie/Makefile +++ b/drivers/pci/pcie/Makefile @@ -9,6 +9,7 @@ obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o bwctrl.o obj-y += aspm.o obj-$(CONFIG_PCIEAER) += aer.o err.o tlp.o obj-$(CONFIG_CXL_RAS) += aer_cxl_rch.o +obj-$(CONFIG_CXL_RAS) += aer_cxl_vh.o obj-$(CONFIG_PCIEAER_INJECT) += aer_inject.o obj-$(CONFIG_PCIE_PME) += pme.o obj-$(CONFIG_PCIE_DPC) += dpc.o diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index c4fd9c0b2a548..c5bce25df51cb 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -1150,16 +1150,6 @@ void pci_aer_unmask_internal_errors(struct pci_dev *dev) */ EXPORT_SYMBOL_FOR_MODULES(pci_aer_unmask_internal_errors, "cxl_core"); -#ifdef CONFIG_CXL_RAS -bool is_aer_internal_error(struct aer_err_info *info) -{ - if (info->severity == AER_CORRECTABLE) - return info->status & PCI_ERR_COR_INTERNAL; - - return info->status & PCI_ERR_UNC_INTN; -} -#endif - /** * pci_aer_handle_error - handle logging error into an event log * @dev: pointer to pci_dev data structure of error source device diff --git a/drivers/pci/pcie/aer_cxl_vh.c b/drivers/pci/pcie/aer_cxl_vh.c new file mode 100644 index 0000000000000..93bed07936100 --- /dev/null +++ b/drivers/pci/pcie/aer_cxl_vh.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2026 AMD Corporation. All rights reserved. */ + +#include <linux/aer.h> +#include <linux/atomic.h> +#include <linux/cleanup.h> +#include <linux/init.h> +#include <linux/kfifo.h> +#include <linux/rwsem.h> +#include <linux/wait_bit.h> +#include <linux/workqueue.h> +#include "../pci.h" +#include "portdrv.h" + +#define CXL_ERROR_SOURCES_MAX 128 + +struct cxl_proto_err_kfifo { + struct work_struct *work; + void (*flush)(void); + struct rw_semaphore rwsem; + spinlock_t fifo_lock; + atomic_t flush_inflight; + DECLARE_KFIFO(fifo, struct cxl_proto_err_work_data, + CXL_ERROR_SOURCES_MAX); +}; + +static struct cxl_proto_err_kfifo cxl_proto_err_kfifo = { + .rwsem = __RWSEM_INITIALIZER(cxl_proto_err_kfifo.rwsem), + .fifo_lock = __SPIN_LOCK_UNLOCKED(cxl_proto_err_kfifo.fifo_lock), +}; + +static int __init cxl_proto_err_kfifo_init(void) +{ + INIT_KFIFO(cxl_proto_err_kfifo.fifo); + return 0; +} +subsys_initcall(cxl_proto_err_kfifo_init); + +bool is_aer_internal_error(struct aer_err_info *info) +{ + if (info->severity == AER_CORRECTABLE) + return info->status & PCI_ERR_COR_INTERNAL; + + return info->status & PCI_ERR_UNC_INTN; +} + +bool is_cxl_error(struct pci_dev *pdev, struct aer_err_info *info) +{ + if (!info || !info->is_cxl) + return false; + + if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ENDPOINT) + return false; + + return is_aer_internal_error(info); +} + +/** + * cxl_forward_error - Forward a CXL protocol error to the CXL subsystem via kfifo + * @pdev: PCI device that reported the AER error + * @info: AER error info containing severity and status + * + * Producer side of the AER-CXL kfifo. Enqueues a CXL protocol error work + * item and schedules the consumer workqueue. Takes a reference on @pdev + * that the consumer releases after handling. + * + * Return: true if the caller must flush the kfifo before AER recovery, + * false if no CXL error handling was initiated due to early return on + * error. + */ +bool cxl_forward_error(struct pci_dev *pdev, struct aer_err_info *info) +{ + struct cxl_proto_err_work_data wd = { + .severity = info->severity, + .pdev = pdev, + }; + + guard(rwsem_read)(&cxl_proto_err_kfifo.rwsem); + + if (!cxl_proto_err_kfifo.work) { + dev_err_ratelimited(&pdev->dev, "AER-CXL kfifo reader not registered\n"); + return false; + } + + /* + * Reference discipline: the AER caller (handle_error_source()) + * holds a ref on @pdev for the duration of this call and releases + * it on return. Take a fresh ref here so the pdev stays live while + * queued in the kfifo; the consumer (for_each_cxl_proto_err()) + * drops that ref after handling. On enqueue failure below, drop + * the ref we just took to avoid a leak. + */ + pci_dev_get(pdev); + + /* Serialize concurrent kfifo writers: multiple AER threaded IRQs */ + if (!kfifo_in_spinlocked(&cxl_proto_err_kfifo.fifo, &wd, 1, + &cxl_proto_err_kfifo.fifo_lock)) { + /* Dropped; no panic - UCE unconfirmed without RAS read */ + dev_err_ratelimited(&pdev->dev, "AER-CXL kfifo add failed\n"); + pci_dev_put(pdev); + schedule_work(cxl_proto_err_kfifo.work); + return true; + } + + schedule_work(cxl_proto_err_kfifo.work); + return true; +} + +void cxl_register_proto_err_work(struct work_struct *work, + void (*flush)(void)) +{ + guard(rwsem_write)(&cxl_proto_err_kfifo.rwsem); + + /* + * Warn on double-registration to surface driver bugs (e.g. missing + * cxl_unregister_proto_err_work() on module exit) + */ + if (WARN(cxl_proto_err_kfifo.work, + "AER-CXL kfifo consumer already registered\n")) + return; + cxl_proto_err_kfifo.work = work; + cxl_proto_err_kfifo.flush = flush; +} +EXPORT_SYMBOL_FOR_MODULES(cxl_register_proto_err_work, "cxl_core"); + +static struct work_struct *cancel_cxl_proto_err(void) +{ + struct work_struct *work; + struct cxl_proto_err_work_data wd; + + guard(rwsem_write)(&cxl_proto_err_kfifo.rwsem); + work = cxl_proto_err_kfifo.work; + cxl_proto_err_kfifo.work = NULL; + cxl_proto_err_kfifo.flush = NULL; + + /* rwsem_write excludes all producers; fifo_lock not needed */ + while (kfifo_get(&cxl_proto_err_kfifo.fifo, &wd)) { + dev_err_ratelimited(&wd.pdev->dev, + "AER-CXL error report canceled\n"); + pci_dev_put(wd.pdev); + } + return work; +} + +void cxl_unregister_proto_err_work(void) +{ + struct work_struct *work; + + lockdep_assert_not_held(&cxl_proto_err_kfifo.rwsem); + + work = cancel_cxl_proto_err(); + + /* Wait for any in-flight cxl_proto_err_flush() calls to complete */ + wait_var_event(&cxl_proto_err_kfifo.flush_inflight, + atomic_read(&cxl_proto_err_kfifo.flush_inflight) == 0); + + if (work) + cancel_work_sync(work); +} +EXPORT_SYMBOL_FOR_MODULES(cxl_unregister_proto_err_work, "cxl_core"); + +/** + * for_each_cxl_proto_err - Call a function for each kfifo work item + * + * Single-consumer invariant: this function is only called from + * cxl_proto_err_work_fn() via a single DECLARE_WORK. + * + * Holds rwsem_read internally; fn() must not call cxl_register_proto_err_work() + * or cxl_unregister_proto_err_work(). + */ +void for_each_cxl_proto_err(struct cxl_proto_err_work_data *wd, + cxl_proto_err_fn_t fn) +{ + guard(rwsem_read)(&cxl_proto_err_kfifo.rwsem); + while (kfifo_get(&cxl_proto_err_kfifo.fifo, wd)) { + fn(wd); + pci_dev_put(wd->pdev); + } +} +EXPORT_SYMBOL_FOR_MODULES(for_each_cxl_proto_err, "cxl_core"); + +/** + * cxl_proto_err_flush - drain pending AER-CXL kfifo work synchronously + * + * Wait for the consumer worker to finish processing all entries + * currently in the kfifo. Used by handle_error_source() for UCE so + * the CXL plane can read CXL RAS, apply panic policy, and clear CXL + * state before pci_aer_handle_error() drives PCIe recovery. + * + * Snapshots the flush callback under rwsem_read and releases the rwsem + * before calling it. This avoids holding rwsem_read across flush_work(), + * which would deadlock via the rwsem HANDOFF mechanism when a concurrent + * rwsem_write waiter (cxl_unregister_proto_err_work) blocks new readers + * including the worker's for_each_cxl_proto_err() rwsem_read acquisition. + * + * The flush_inflight counter prevents cxl_core module unload while a + * flush is in progress outside the rwsem. The counter is incremented + * under rwsem_read (mutually exclusive with the rwsem_write in + * cancel_cxl_proto_err() that NULLs the flush pointer) and decremented + * after the flush completes. cxl_unregister_proto_err_work() waits for + * the counter to reach zero before proceeding with cancel_work_sync(). + * + * For correctable events the consumer can run asynchronously; AER + * does not need to call this helper for AER_CORRECTABLE. + */ +void cxl_proto_err_flush(void) +{ + void (*flush)(void); + + scoped_guard(rwsem_read, &cxl_proto_err_kfifo.rwsem) { + flush = cxl_proto_err_kfifo.flush; + if (flush) + atomic_inc(&cxl_proto_err_kfifo.flush_inflight); + } + + if (flush) { + flush(); + if (atomic_dec_and_test(&cxl_proto_err_kfifo.flush_inflight)) + wake_up_var(&cxl_proto_err_kfifo.flush_inflight); + } +} diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h index cc58bf2f2c844..fd203010877bf 100644 --- a/drivers/pci/pcie/portdrv.h +++ b/drivers/pci/pcie/portdrv.h @@ -130,9 +130,15 @@ struct aer_err_info; bool is_aer_internal_error(struct aer_err_info *info); void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info); void cxl_rch_enable_rcec(struct pci_dev *rcec); +bool is_cxl_error(struct pci_dev *pdev, struct aer_err_info *info); +bool cxl_forward_error(struct pci_dev *pdev, struct aer_err_info *info); +void cxl_proto_err_flush(void); #else static inline bool is_aer_internal_error(struct aer_err_info *info) { return false; } static inline void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info) { } static inline void cxl_rch_enable_rcec(struct pci_dev *rcec) { } +static inline bool is_cxl_error(struct pci_dev *pdev, struct aer_err_info *info) { return false; } +static inline bool cxl_forward_error(struct pci_dev *pdev, struct aer_err_info *info) { return false; } +static inline void cxl_proto_err_flush(void) { } #endif /* CONFIG_CXL_RAS */ #endif /* _PORTDRV_H_ */ diff --git a/include/linux/aer.h b/include/linux/aer.h index df0f5c382286f..8eba3192e2d15 100644 --- a/include/linux/aer.h +++ b/include/linux/aer.h @@ -25,6 +25,7 @@ #define PCIE_STD_MAX_TLP_HEADERLOG (PCIE_STD_NUM_TLP_HEADERLOG + 10) struct pci_dev; +struct work_struct; struct pcie_tlp_log { union { @@ -66,6 +67,29 @@ static inline int pcie_aer_is_native(struct pci_dev *dev) { return 0; } static inline void pci_aer_unmask_internal_errors(struct pci_dev *dev) { } #endif +#ifdef CONFIG_CXL_RAS +/** + * struct cxl_proto_err_work_data - Error information used in CXL error handling + * @pdev: PCI device detecting the error + * @severity: AER severity + */ +struct cxl_proto_err_work_data { + struct pci_dev *pdev; + int severity; +}; + +/** + * Callback for processing a CXL protocol error from the AER-CXL kfifo. + */ +typedef void (*cxl_proto_err_fn_t)(struct cxl_proto_err_work_data *wd); + +void cxl_register_proto_err_work(struct work_struct *work, + void (*flush)(void)); +void for_each_cxl_proto_err(struct cxl_proto_err_work_data *wd, + cxl_proto_err_fn_t fn); +void cxl_unregister_proto_err_work(void); +#endif + void pci_print_aer(struct pci_dev *dev, int aer_severity, struct aer_capability_regs *aer); int cper_severity_to_aer(int cper_severity); -- 2.34.1
