On Wed, Oct 07, 2026 at 01:16:39PM +0800, Anisa Su wrote:
> From: Ira Weiny <[email protected]>
>
> Dynamic Capacity Devices (DCD) support extent change notifications
> through the event log mechanism. The interrupt mailbox commands were
> extended in CXL 3.1 to support these notifications. Firmware can't
> configure DCD events to be FW controlled but can retain control of
> memory events.
>
> Configure DCD event log interrupts on devices supporting dynamic
> capacity. Disable DCD if interrupts are not supported.
>
> The DCD event log is not drained here. cxl_event_drain_mask() reports
> only the logs the driver can service, and the DCD bit is added by
> the patch introducing DCD event handling. Until then a DCD interrupt
> wakes the event thread, which finds no log it owns and returns.
>
> Care is taken to preserve the interrupt policy set by the FW if FW first
> has been selected by the BIOS.
>
> Based on an original patch by Navneet Singh.
>
> Signed-off-by: Ira Weiny <[email protected]>
> Co-developed-by: Anisa Su <[email protected]>
> Signed-off-by: Anisa Su <[email protected]>
>
> ---
> Changes:
> 1. cxl.h: add CXLDEV_EVENT_STATUS_STANDARD_LOGS for the four logs whose
> ownership follows native_cxl_error, and use it in
> cxl_event_drain_mask() and the cxl_test mock instead of spelling the
> mask out in both places.
> [Dave, Richard, Jonathan]
> 2. On platforms where BIOS owns the event logs, a failed Get/Set Event
> Interrupt Policy disables DCD instead of failing probe. Before DCD
> this path never reached the mailbox there; now DCD is the only
> reason it does, so its failure costs DCD, not the memdev.
> [Sashiko]
> ---
> drivers/cxl/cxl.h | 14 ++--
> drivers/cxl/cxlmem.h | 2 +
> drivers/cxl/pci.c | 143 +++++++++++++++++++++++++++++------
> tools/testing/cxl/test/mem.c | 2 +-
> 4 files changed, 133 insertions(+), 28 deletions(-)
>
> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
> index cab8ce39f465..15356d25369d 100644
> --- a/drivers/cxl/cxl.h
> +++ b/drivers/cxl/cxl.h
> @@ -192,11 +192,15 @@ static inline int ways_to_eiw(unsigned int ways, u8
> *eiw)
> #define CXLDEV_EVENT_STATUS_WARN BIT(1)
> #define CXLDEV_EVENT_STATUS_FAIL BIT(2)
> #define CXLDEV_EVENT_STATUS_FATAL BIT(3)
> -
> -#define CXLDEV_EVENT_STATUS_ALL (CXLDEV_EVENT_STATUS_INFO | \
> - CXLDEV_EVENT_STATUS_WARN | \
> - CXLDEV_EVENT_STATUS_FAIL | \
> - CXLDEV_EVENT_STATUS_FATAL)
> +#define CXLDEV_EVENT_STATUS_DCD BIT(4)
> +/*
> + * Logs whose ownership follows native_cxl_error; the DCD log is always
> + * OS owned.
> + */
> +#define CXLDEV_EVENT_STATUS_STANDARD_LOGS (CXLDEV_EVENT_STATUS_INFO | \
> + CXLDEV_EVENT_STATUS_WARN | \
> + CXLDEV_EVENT_STATUS_FAIL | \
> + CXLDEV_EVENT_STATUS_FATAL)
>
> /* CXL rev 3.0 section 8.2.9.2.4; Table 8-52 */
> #define CXLDEV_EVENT_INT_MODE_MASK GENMASK(1, 0)
> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> index 9fd5af2e41d4..9106affd7c81 100644
> --- a/drivers/cxl/cxlmem.h
> +++ b/drivers/cxl/cxlmem.h
> @@ -240,7 +240,9 @@ struct cxl_event_interrupt_policy {
> u8 warn_settings;
> u8 failure_settings;
> u8 fatal_settings;
> + u8 dcd_settings;
> } __packed;
> +#define CXL_EVENT_INT_POLICY_BASE_SIZE 4 /* info, warn, failure, fatal */
>
> /**
> * struct cxl_event_state - Event log driver state
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 7b754af2168d..d2bed92be30d 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -510,11 +510,22 @@ static bool cxl_alloc_irq_vectors(struct pci_dev *pdev)
> return true;
> }
>
> +/* Event logs the driver drains: standard logs when native_cxl */
> +static u32 cxl_event_drain_mask(struct pci_host_bridge *host_bridge)
> +{
> + if (host_bridge->native_cxl_error)
> + return CXLDEV_EVENT_STATUS_STANDARD_LOGS;
> + return 0;
> +}
> +
> static irqreturn_t cxl_event_thread(int irq, void *id)
> {
> struct cxl_dev_id *dev_id = id;
> struct cxl_dev_state *cxlds = dev_id->cxlds;
> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
> + struct pci_host_bridge *host_bridge =
> + pci_find_host_bridge(to_pci_dev(cxlds->dev)->bus);
> + u32 mask = cxl_event_drain_mask(host_bridge);
> u32 status;
>
> do {
> @@ -523,8 +534,8 @@ static irqreturn_t cxl_event_thread(int irq, void *id)
> * ignore the reserved upper 32 bits
> */
> status = readl(cxlds->regs.status +
> CXLDEV_DEV_EVENT_STATUS_OFFSET);
> - /* Ignore logs unknown to the driver */
> - status &= CXLDEV_EVENT_STATUS_ALL;
> + /* Ignore logs unknown to the driver or owned by BIOS */
> + status &= mask;
> if (!status)
> break;
> cxl_mem_get_event_records(mds, status);
> @@ -551,42 +562,62 @@ static int cxl_event_req_irq(struct cxl_dev_state
> *cxlds, u8 setting)
> }
>
> static int cxl_event_get_int_policy(struct cxl_memdev_state *mds,
> - struct cxl_event_interrupt_policy *policy)
> + struct cxl_event_interrupt_policy *policy,
> + size_t *policy_size)
> {
> struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct cxl_mbox_cmd mbox_cmd = {
> .opcode = CXL_MBOX_OP_GET_EVT_INT_POLICY,
> .payload_out = policy,
> .size_out = sizeof(*policy),
> + /* CXL 2.0 firmware omits dcd_settings; accept the shorter
> reply */
> + .min_out = CXL_EVENT_INT_POLICY_BASE_SIZE,
> };
> int rc;
>
> rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
> - if (rc < 0)
> + if (rc < 0) {
> dev_err(mds->cxlds.dev,
> "Failed to get event interrupt policy : %d", rc);
> + return rc;
> + }
>
> + if (policy_size)
> + *policy_size = mbox_cmd.size_out;
> return rc;
> }
>
> static int cxl_event_config_msgnums(struct cxl_memdev_state *mds,
> - struct cxl_event_interrupt_policy *policy)
> + struct cxl_event_interrupt_policy *policy,
> + bool native_cxl, size_t policy_size)
> {
> struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
> struct cxl_mbox_cmd mbox_cmd;
> int rc;
>
> - *policy = (struct cxl_event_interrupt_policy) {
> - .info_settings = CXL_INT_MSI_MSIX,
> - .warn_settings = CXL_INT_MSI_MSIX,
> - .failure_settings = CXL_INT_MSI_MSIX,
> - .fatal_settings = CXL_INT_MSI_MSIX,
> - };
> + /* memory event policy is left if FW has control */
> + if (native_cxl) {
> + *policy = (struct cxl_event_interrupt_policy) {
> + .info_settings = CXL_INT_MSI_MSIX,
> + .warn_settings = CXL_INT_MSI_MSIX,
> + .failure_settings = CXL_INT_MSI_MSIX,
> + .fatal_settings = CXL_INT_MSI_MSIX,
> + .dcd_settings = 0,
> + };
> + }
> +
> + /*
> + * A CXL 3.0+ device can carry dcd_settings field without DCD command
> + * support, so size the request by the device's policy_size and only
> + * enable the DCD interrupt when DCD commands are supported.
> + */
> + if (cxl_dcd_supported(mds))
> + policy->dcd_settings = CXL_INT_MSI_MSIX;
>
> mbox_cmd = (struct cxl_mbox_cmd) {
> .opcode = CXL_MBOX_OP_SET_EVT_INT_POLICY,
> .payload_in = policy,
> - .size_in = sizeof(*policy),
> + .size_in = policy_size,
> };
>
> rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
> @@ -597,7 +628,7 @@ static int cxl_event_config_msgnums(struct
> cxl_memdev_state *mds,
> }
>
> /* Retrieve final interrupt settings */
> - return cxl_event_get_int_policy(mds, policy);
> + return cxl_event_get_int_policy(mds, policy, NULL);
> }
>
> static int cxl_event_irqsetup(struct cxl_memdev_state *mds,
> @@ -633,6 +664,30 @@ static int cxl_event_irqsetup(struct cxl_memdev_state
> *mds,
> return 0;
> }
>
> +static int cxl_irqsetup(struct cxl_memdev_state *mds,
> + struct cxl_event_interrupt_policy *policy,
> + bool native_cxl)
> +{
> + struct cxl_dev_state *cxlds = &mds->cxlds;
> + int rc;
> +
> + if (native_cxl) {
> + rc = cxl_event_irqsetup(mds, policy);
> + if (rc)
> + return rc;
> + }
> +
> + if (cxl_dcd_supported(mds)) {
> + rc = cxl_event_req_irq(cxlds, policy->dcd_settings);
> + if (rc) {
> + dev_err(cxlds->dev, "Failed to get interrupt for DCD
> event log\n");
> + cxl_disable_dcd(mds);
Hi Anisa,
If I didn't understand wrong, the interrupt has devicce half and host half.
When cxl_event_req_irq() fails for DCD log, cxl_irqsetup() disable DCD and
return 0, but still dcd_settings = MSI/MSI-X, so the device keeps signalling
a vector with no handler while driver consider DCD off.
Should we match the state together ?
Best regards,
Richard Cheng.
> + }
> + }
> +
> + return 0;
> +}
> +
> static bool cxl_event_int_is_fw(u8 setting)
> {
> u8 mode = FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting);
> @@ -651,46 +706,90 @@ static bool cxl_event_fw_owns_mem_logs(struct
> cxl_event_interrupt_policy *policy
> static int cxl_event_config(struct pci_host_bridge *host_bridge,
> struct cxl_memdev_state *mds, bool irq_avail)
> {
> - struct cxl_event_interrupt_policy policy;
> + struct cxl_event_interrupt_policy policy = { 0 };
> + bool native_cxl = host_bridge->native_cxl_error;
> + size_t policy_size;
> + u32 mask;
> int rc;
>
> /*
> * When BIOS maintains CXL error reporting control, it will process
> * event records. Only one agent can do so.
> + *
> + * If BIOS has control of events and DCD is not supported skip event
> + * configuration.
> */
> - if (!host_bridge->native_cxl_error)
> + if (!native_cxl && !cxl_dcd_supported(mds))
> return 0;
>
> if (!irq_avail) {
> dev_info(mds->cxlds.dev, "No interrupt support, disable event
> processing.\n");
> + if (cxl_dcd_supported(mds)) {
> + dev_info(mds->cxlds.dev, "DCD requires interrupts,
> disable DCD\n");
> + cxl_disable_dcd(mds);
> + }
> return 0;
> }
>
> - rc = cxl_event_get_int_policy(mds, &policy);
> + rc = cxl_event_get_int_policy(mds, &policy, &policy_size);
> if (rc)
> - return rc;
> + goto err;
> +
> + /*
> + * dcd_settings cannot be sent to a device whose policy is too short
> + * to hold it, so the device would never accept the setting
> + * cxl_irqsetup() arms the DCD interrupt with.
> + */
> + if (cxl_dcd_supported(mds) && policy_size < sizeof(policy)) {
> + dev_warn(mds->cxlds.dev,
> + "DCD supported but interrupt policy is only %zu
> bytes\n",
> + policy_size);
> + cxl_disable_dcd(mds);
> + /* DCD was the only reason to be here */
> + if (!native_cxl)
> + return 0;
> + }
>
> - if (cxl_event_fw_owns_mem_logs(&policy)) {
> + if (native_cxl && cxl_event_fw_owns_mem_logs(&policy)) {
> dev_err(mds->cxlds.dev,
> "FW still in control of Event Logs despite _OSC
> settings\n");
> return -EBUSY;
> }
>
> - rc = cxl_event_config_msgnums(mds, &policy);
> + rc = cxl_event_config_msgnums(mds, &policy, native_cxl, policy_size);
> if (rc)
> - return rc;
> + goto err;
>
> rc = cxl_mem_alloc_event_buf(mds);
> if (rc)
> return rc;
>
> - rc = cxl_event_irqsetup(mds, &policy);
> + rc = cxl_irqsetup(mds, &policy, native_cxl);
> if (rc)
> return rc;
>
> - cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL);
> + mask = cxl_event_drain_mask(host_bridge);
> + if (mask)
> + cxl_mem_get_event_records(mds, mask);
> +
> + dev_dbg(mds->cxlds.dev, "Event config : %s DCD %s\n",
> + native_cxl ? "OS" : "BIOS",
> + cxl_dcd_supported(mds) ? "supported" : "not supported");
>
> return 0;
> +
> +err:
> + /*
> + * Failed to Get/Set interrupt policy for DCD while BIOS owns the
> + * other logs. Disable DCD and continue.
> + */
> + if (!native_cxl) {
> + dev_warn(mds->cxlds.dev,
> + "Event interrupt policy failed (%d), disabling DCD\n",
> rc);
> + cxl_disable_dcd(mds);
> + return 0;
> + }
> + return rc;
> }
>
> static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds)
> diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c
> index 7b756000a1a6..c740372bdaaa 100644
> --- a/tools/testing/cxl/test/mem.c
> +++ b/tools/testing/cxl/test/mem.c
> @@ -1818,7 +1818,7 @@ static int cxl_mock_mem_probe(struct platform_device
> *pdev)
> if (rc)
> dev_dbg(dev, "No CXL FWCTL setup\n");
>
> - cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL);
> + cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_STANDARD_LOGS);
> cxl_mock_test_feat_init(mdata);
>
> return 0;
> --
> 2.43.0
>
