On 22-02-02 14:10:14, Jonathan Cameron wrote:
> From: Ben Widawsky <ben.widaw...@intel.com>
>
> A CXL memory device (AKA Type 3) is a CXL component that contains some
> combination of volatile and persistent memory. It also implements the
> previously defined mailbox interface as well as the memory device
> firmware interface.
>
> Although the memory device is configured like a normal PCIe device, the
> memory traffic is on an entirely separate bus conceptually (using the
> same physical wires as PCIe, but different protocol).
>
> Once the CXL topology is fully configure and address decoders committed,
> the guest physical address for the memory device is part of a larger
> window which is owned by the platform. The creation of these windows
> is later in this series.
>
> The following example will create a 256M device in a 512M window:
> -object "memory-backend-file,id=cxl-mem1,share,mem-path=cxl-type3,size=512M"
> -device "cxl-type3,bus=rp0,memdev=cxl-mem1,id=cxl-pmem0"
>
> Note: Dropped PCDIMM info interfaces for now. They can be added if
> appropriate at a later date.
>
> Signed-off-by: Ben Widawsky <ben.widaw...@intel.com>
> Signed-off-by: Jonathan Cameron <jonathan.came...@huawei.com>
> ---
> hw/cxl/cxl-mailbox-utils.c | 47 ++++++++++
> hw/mem/Kconfig | 5 ++
> hw/mem/cxl_type3.c | 170 +++++++++++++++++++++++++++++++++++++
> hw/mem/meson.build | 1 +
> include/hw/cxl/cxl.h | 1 +
> include/hw/cxl/cxl_pci.h | 22 +++++
> include/hw/pci/pci_ids.h | 1 +
> 7 files changed, 247 insertions(+)
> create mode 100644 hw/mem/cxl_type3.c
>
> diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
> index 16bb998735..808faec114 100644
> --- a/hw/cxl/cxl-mailbox-utils.c
> +++ b/hw/cxl/cxl-mailbox-utils.c
> @@ -50,6 +50,8 @@ enum {
> LOGS = 0x04,
> #define GET_SUPPORTED 0x0
> #define GET_LOG 0x1
> + IDENTIFY = 0x40,
> + #define MEMORY_DEVICE 0x0
> };
>
> /* 8.2.8.4.5.1 Command Return Codes */
> @@ -216,6 +218,48 @@ static ret_code cmd_logs_get_log(struct cxl_cmd *cmd,
> return CXL_MBOX_SUCCESS;
> }
>
> +/* 8.2.9.5.1.1 */
> +static ret_code cmd_identify_memory_device(struct cxl_cmd *cmd,
> + CXLDeviceState *cxl_dstate,
> + uint16_t *len)
> +{
> + struct {
> + char fw_revision[0x10];
> + uint64_t total_capacity;
> + uint64_t volatile_capacity;
> + uint64_t persistent_capacity;
> + uint64_t partition_align;
> + uint16_t info_event_log_size;
> + uint16_t warning_event_log_size;
> + uint16_t failure_event_log_size;
> + uint16_t fatal_event_log_size;
> + uint32_t lsa_size;
> + uint8_t poison_list_max_mer[3];
> + uint16_t inject_poison_limit;
> + uint8_t poison_caps;
> + uint8_t qos_telemetry_caps;
> + } __attribute__((packed)) *id;
> + _Static_assert(sizeof(*id) == 0x43, "Bad identify size");
> +
> + uint64_t size = cxl_dstate->pmem_size;
> +
> + if (!QEMU_IS_ALIGNED(size, 256 << 20)) {
> + return CXL_MBOX_INTERNAL_ERROR;
> + }
> +
> + id = (void *)cmd->payload;
> + memset(id, 0, sizeof(*id));
> +
> + /* PMEM only */
> + snprintf(id->fw_revision, 0x10, "BWFW VERSION %02d", 0);
> +
> + id->total_capacity = size / (256 << 20);
> + id->persistent_capacity = size / (256 << 20);
> +
> + *len = sizeof(*id);
> + return CXL_MBOX_SUCCESS;
> +}
> +
> #define IMMEDIATE_CONFIG_CHANGE (1 << 1)
> #define IMMEDIATE_POLICY_CHANGE (1 << 3)
> #define IMMEDIATE_LOG_CHANGE (1 << 4)
> @@ -233,8 +277,11 @@ static struct cxl_cmd cxl_cmd_set[256][256] = {
> [TIMESTAMP][SET] = { "TIMESTAMP_SET", cmd_timestamp_set, 8,
> IMMEDIATE_POLICY_CHANGE },
> [LOGS][GET_SUPPORTED] = { "LOGS_GET_SUPPORTED", cmd_logs_get_supported,
> 0, 0 },
> [LOGS][GET_LOG] = { "LOGS_GET_LOG", cmd_logs_get_log, 0x18, 0 },
> + [IDENTIFY][MEMORY_DEVICE] = { "IDENTIFY_MEMORY_DEVICE",
> + cmd_identify_memory_device, 0, 0 },
> };
>
> +
> void cxl_process_mailbox(CXLDeviceState *cxl_dstate)
> {
> uint16_t ret = CXL_MBOX_SUCCESS;
> diff --git a/hw/mem/Kconfig b/hw/mem/Kconfig
> index 03dbb3c7df..73c5ae8ad9 100644
> --- a/hw/mem/Kconfig
> +++ b/hw/mem/Kconfig
> @@ -11,3 +11,8 @@ config NVDIMM
>
> config SPARSE_MEM
> bool
> +
> +config CXL_MEM_DEVICE
> + bool
> + default y if CXL
> + select MEM_DEVICE
> diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
> new file mode 100644
> index 0000000000..c4021d2434
> --- /dev/null
> +++ b/hw/mem/cxl_type3.c
> @@ -0,0 +1,170 @@
> +#include "qemu/osdep.h"
> +#include "qemu/units.h"
> +#include "qemu/error-report.h"
> +#include "hw/mem/memory-device.h"
> +#include "hw/mem/pc-dimm.h"
> +#include "hw/pci/pci.h"
> +#include "hw/qdev-properties.h"
> +#include "qapi/error.h"
> +#include "qemu/log.h"
> +#include "qemu/module.h"
> +#include "qemu/range.h"
> +#include "qemu/rcu.h"
> +#include "sysemu/hostmem.h"
> +#include "hw/cxl/cxl.h"
> +
> +typedef struct cxl_type3_dev {
> + /* Private */
> + PCIDevice parent_obj;
> +
> + /* Properties */
> + uint64_t size;
> + HostMemoryBackend *hostmem;
> +
> + /* State */
> + CXLComponentState cxl_cstate;
> + CXLDeviceState cxl_dstate;
> +} CXLType3Dev;
> +
> +#define CT3(obj) OBJECT_CHECK(CXLType3Dev, (obj), TYPE_CXL_TYPE3_DEV)
> +
> +static void build_dvsecs(CXLType3Dev *ct3d)
> +{
> + CXLComponentState *cxl_cstate = &ct3d->cxl_cstate;
> + uint8_t *dvsec;
> +
> + dvsec = (uint8_t *)&(struct cxl_dvsec_device){
> + .cap = 0x1e,
> + .ctrl = 0x6,
> + .status2 = 0x2,
> + .range1_size_hi = 0,
> +#ifdef SET_PMEM_PADDR
> + .range1_size_lo = (2 << 5) | (2 << 2) | 0x3 | ct3d->size,
> +#else
> + .range1_size_lo = 0x3,
> +#endif
> + .range1_base_hi = 0,
> + .range1_base_lo = 0,
> + };
> + cxl_component_create_dvsec(cxl_cstate, PCIE_CXL_DEVICE_DVSEC_LENGTH,
> + PCIE_CXL_DEVICE_DVSEC,
> + PCIE_CXL2_DEVICE_DVSEC_REVID, dvsec);
> +
> + dvsec = (uint8_t *)&(struct cxl_dvsec_register_locator){
> + .rsvd = 0,
> + .reg0_base_lo = RBI_COMPONENT_REG | CXL_COMPONENT_REG_BAR_IDX,
> + .reg0_base_hi = 0,
> + .reg1_base_lo = RBI_CXL_DEVICE_REG | CXL_DEVICE_REG_BAR_IDX,
> + .reg1_base_hi = 0,
> + };
> + cxl_component_create_dvsec(cxl_cstate, REG_LOC_DVSEC_LENGTH,
> REG_LOC_DVSEC,
> + REG_LOC_DVSEC_REVID, dvsec);
> +}
> +
> +static void cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
> +{
> + MemoryRegion *mr;
> +
> + if (!ct3d->hostmem) {
> + error_setg(errp, "memdev property must be set");
> + return;
> + }
This check can go away with the below.
> +
> + mr = host_memory_backend_get_memory(ct3d->hostmem);
> + if (!mr) {
> + error_setg(errp, "memdev property must be set");
> + return;
> + }
> + memory_region_set_nonvolatile(mr, true);
> + memory_region_set_enabled(mr, true);
> + host_memory_backend_set_mapped(ct3d->hostmem, true);
> + ct3d->cxl_dstate.pmem_size = ct3d->hostmem->size;
> +}
> +
> +
> +static void ct3_realize(PCIDevice *pci_dev, Error **errp)
> +{
> + CXLType3Dev *ct3d = CT3(pci_dev);
> + CXLComponentState *cxl_cstate = &ct3d->cxl_cstate;
> + ComponentRegisters *regs = &cxl_cstate->crb;
> + MemoryRegion *mr = ®s->component_registers;
> + uint8_t *pci_conf = pci_dev->config;
> +
> + if (!ct3d->hostmem) {
> + cxl_setup_memory(ct3d, errp);
> + }
I still haven't had time to review, sorry. We did hit this in our driver
development: I think this should be inverted. The original code worked a bit
differently.
if (ct3d->hostmem) {
cxl_setup_memory(ct3d, errp);
}
> +
> + pci_config_set_prog_interface(pci_conf, 0x10);
> + pci_config_set_class(pci_conf, PCI_CLASS_MEMORY_CXL);
> +
> + pcie_endpoint_cap_init(pci_dev, 0x80);
> + cxl_cstate->dvsec_offset = 0x100;
> +
> + ct3d->cxl_cstate.pdev = pci_dev;
> + build_dvsecs(ct3d);
> +
> + cxl_component_register_block_init(OBJECT(pci_dev), cxl_cstate,
> + TYPE_CXL_TYPE3_DEV);
> +
> + pci_register_bar(
> + pci_dev, CXL_COMPONENT_REG_BAR_IDX,
> + PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64, mr);
> +
> + cxl_device_register_block_init(OBJECT(pci_dev), &ct3d->cxl_dstate);
> + pci_register_bar(pci_dev, CXL_DEVICE_REG_BAR_IDX,
> + PCI_BASE_ADDRESS_SPACE_MEMORY |
> + PCI_BASE_ADDRESS_MEM_TYPE_64,
> + &ct3d->cxl_dstate.device_registers);
> +}
> +
> +static void ct3d_reset(DeviceState *dev)
> +{
> + CXLType3Dev *ct3d = CT3(dev);
> + uint32_t *reg_state = ct3d->cxl_cstate.crb.cache_mem_registers;
> +
> + cxl_component_register_init_common(reg_state, CXL2_TYPE3_DEVICE);
> + cxl_device_register_init_common(&ct3d->cxl_dstate);
> +}
> +
> +static Property ct3_props[] = {
> + DEFINE_PROP_SIZE("size", CXLType3Dev, size, -1),
> + DEFINE_PROP_LINK("memdev", CXLType3Dev, hostmem, TYPE_MEMORY_BACKEND,
> + HostMemoryBackend *),
> + DEFINE_PROP_END_OF_LIST(),
> +};
> +
> +static void ct3_class_init(ObjectClass *oc, void *data)
> +{
> + DeviceClass *dc = DEVICE_CLASS(oc);
> + PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
> +
> + pc->realize = ct3_realize;
> + pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
> + pc->vendor_id = PCI_VENDOR_ID_INTEL;
> + pc->device_id = 0xd93; /* LVF for now */
> + pc->revision = 1;
> +
> + set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
> + dc->desc = "CXL PMEM Device (Type 3)";
> + dc->reset = ct3d_reset;
> + device_class_set_props(dc, ct3_props);
> +}
> +
> +static const TypeInfo ct3d_info = {
> + .name = TYPE_CXL_TYPE3_DEV,
> + .parent = TYPE_PCI_DEVICE,
> + .class_init = ct3_class_init,
> + .instance_size = sizeof(CXLType3Dev),
> + .interfaces = (InterfaceInfo[]) {
> + { INTERFACE_CXL_DEVICE },
> + { INTERFACE_PCIE_DEVICE },
> + {}
> + },
> +};
> +
> +static void ct3d_registers(void)
> +{
> + type_register_static(&ct3d_info);
> +}
> +
> +type_init(ct3d_registers);
> diff --git a/hw/mem/meson.build b/hw/mem/meson.build
> index 82f86d117e..609b2b36fc 100644
> --- a/hw/mem/meson.build
> +++ b/hw/mem/meson.build
> @@ -3,6 +3,7 @@ mem_ss.add(files('memory-device.c'))
> mem_ss.add(when: 'CONFIG_DIMM', if_true: files('pc-dimm.c'))
> mem_ss.add(when: 'CONFIG_NPCM7XX', if_true: files('npcm7xx_mc.c'))
> mem_ss.add(when: 'CONFIG_NVDIMM', if_true: files('nvdimm.c'))
> +mem_ss.add(when: 'CONFIG_CXL_MEM_DEVICE', if_true: files('cxl_type3.c'))
>
> softmmu_ss.add_all(when: 'CONFIG_MEM_DEVICE', if_true: mem_ss)
>
> diff --git a/include/hw/cxl/cxl.h b/include/hw/cxl/cxl.h
> index 75e5bf71e1..6889362230 100644
> --- a/include/hw/cxl/cxl.h
> +++ b/include/hw/cxl/cxl.h
> @@ -17,6 +17,7 @@
> #define CXL_COMPONENT_REG_BAR_IDX 0
> #define CXL_DEVICE_REG_BAR_IDX 2
>
> +#define TYPE_CXL_TYPE3_DEV "cxl-type3"
> #define CXL_WINDOW_MAX 10
>
> typedef struct CXLState {
> diff --git a/include/hw/cxl/cxl_pci.h b/include/hw/cxl/cxl_pci.h
> index 40c7329afe..e8235b10cc 100644
> --- a/include/hw/cxl/cxl_pci.h
> +++ b/include/hw/cxl/cxl_pci.h
> @@ -64,6 +64,28 @@ _Static_assert(sizeof(struct dvsec_header) == 10,
> * CXL 2.0 Downstream Port: 3, 4, 7, 8
> */
>
> +/* CXL 2.0 - 8.1.3 (ID 0001) */
> +struct cxl_dvsec_device {
> + struct dvsec_header hdr;
> + uint16_t cap;
> + uint16_t ctrl;
> + uint16_t status;
> + uint16_t ctrl2;
> + uint16_t status2;
> + uint16_t lock;
> + uint16_t cap2;
> + uint32_t range1_size_hi;
> + uint32_t range1_size_lo;
> + uint32_t range1_base_hi;
> + uint32_t range1_base_lo;
> + uint32_t range2_size_hi;
> + uint32_t range2_size_lo;
> + uint32_t range2_base_hi;
> + uint32_t range2_base_lo;
> +};
> +_Static_assert(sizeof(struct cxl_dvsec_device) == 0x38,
> + "dvsec device size incorrect");
> +
> /* CXL 2.0 - 8.1.5 (ID 0003) */
> struct cxl_dvsec_port_extensions {
> struct dvsec_header hdr;
> diff --git a/include/hw/pci/pci_ids.h b/include/hw/pci/pci_ids.h
> index 11abe22d46..898083b86f 100644
> --- a/include/hw/pci/pci_ids.h
> +++ b/include/hw/pci/pci_ids.h
> @@ -53,6 +53,7 @@
> #define PCI_BASE_CLASS_MEMORY 0x05
> #define PCI_CLASS_MEMORY_RAM 0x0500
> #define PCI_CLASS_MEMORY_FLASH 0x0501
> +#define PCI_CLASS_MEMORY_CXL 0x0502
> #define PCI_CLASS_MEMORY_OTHER 0x0580
>
> #define PCI_BASE_CLASS_BRIDGE 0x06
> --
> 2.32.0
>
