Re: [PATCH v3 8/9] hw/cxl/events: Add qmp interfaces to add/release dynamic capacity extents
On Tue, Feb 13, 2024 at 05:44:05PM +, Jonathan Cameron wrote: > > diff --git a/qapi/cxl.json b/qapi/cxl.json > > index 8cc4c72fa9..6b631f64f1 100644 > > --- a/qapi/cxl.json > > +++ b/qapi/cxl.json > > @@ -25,7 +25,8 @@ > >'data': ['informational', > > 'warning', > > 'failure', > > - 'fatal'] > > + 'fatal', > > + 'dyncap'] > > } > Needs a docs update. > > Upstream QEMU seems to have gained some stricter checks > so this just broke my build after a rebase. > > Jonathan Thanks. Updated. FYI. The new version is completed. https://lore.kernel.org/linux-cxl/ZcuyZ0Nwq31z8YIr@debian/T/#m07b4b4586e2f421a617f08a002b196d932a88966 Thanks, Fan
Re: [PATCH v3 8/9] hw/cxl/events: Add qmp interfaces to add/release dynamic capacity extents
> diff --git a/qapi/cxl.json b/qapi/cxl.json > index 8cc4c72fa9..6b631f64f1 100644 > --- a/qapi/cxl.json > +++ b/qapi/cxl.json > @@ -25,7 +25,8 @@ >'data': ['informational', > 'warning', > 'failure', > - 'fatal'] > + 'fatal', > + 'dyncap'] > } Needs a docs update. Upstream QEMU seems to have gained some stricter checks so this just broke my build after a rebase. Jonathan
Re: [PATCH v3 8/9] hw/cxl/events: Add qmp interfaces to add/release dynamic capacity extents
> > > #endif
> > > diff --git a/include/hw/cxl/cxl_events.h b/include/hw/cxl/cxl_events.h
> > > index d778487b7e..4f8cb3215d 100644
> > > --- a/include/hw/cxl/cxl_events.h
> > > +++ b/include/hw/cxl/cxl_events.h
> > > @@ -166,4 +166,19 @@ typedef struct CXLEventMemoryModule {
> > > uint8_t reserved[0x3d];
> > > } QEMU_PACKED CXLEventMemoryModule;
> > >
> > > +/*
> > > + * CXL r3.0 section Table 8-47: Dynamic Capacity Event Record
> > > + * All fields little endian.
> > > + */
> > > +typedef struct CXLEventDynamicCapacity {
> > > +CXLEventRecordHdr hdr;
> > > +uint8_t type;
> > > +uint8_t reserved1;
> > > +uint16_t host_id;
> > > +uint8_t updated_region_id;
> > > +uint8_t reserved2[3];
> > > +uint8_t dynamic_capacity_extent[0x28]; /* defined in cxl_device.h */
> > >
> >
> > Can't we use that definition here?
>
> REPLY:
>
> I leave it as it is to avoid include cxl_device.h to cxl_extent.h.
>
> Do you think we need to include the file and use the definition here?
I don't feel strongly either way.
Jonathan
>
> Fan
Re: [PATCH v3 8/9] hw/cxl/events: Add qmp interfaces to add/release dynamic capacity extents
On Wed, Jan 24, 2024 at 04:50:04PM +, Jonathan Cameron wrote:
> On Tue, 7 Nov 2023 10:07:12 -0800
> nifan@gmail.com wrote:
>
> > From: Fan Ni
> >
> > Since fabric manager emulation is not supported yet, the change implements
> > the functions to add/release dynamic capacity extents as QMP interfaces.
> >
> > Note: we block any FM issued extent release request if the exact extent
> > does not exist in the extent list of the device. We will loose the
> > restriction later once we have partial release support in the kernel.
> >
> > 1. Add dynamic capacity extents:
> >
> > For example, the command to add two continuous extents (each 128MiB long)
> > to region 0 (starting at DPA offset 0) looks like below:
> >
> > { "execute": "qmp_capabilities" }
> >
> > { "execute": "cxl-add-dynamic-capacity",
> > "arguments": {
> > "path": "/machine/peripheral/cxl-dcd0",
> > "region-id": 0,
> > "extents": [
> > {
> > "dpa": 0,
> > "len": 128
> > },
> > {
> > "dpa": 128,
> > "len": 128
> > }
> > ]
> > }
> > }
> >
> > 2. Release dynamic capacity extents:
> >
> > For example, the command to release an extent of size 128MiB from region 0
> > (DPA offset 128MiB) look like below:
> >
> > { "execute": "cxl-release-dynamic-capacity",
> > "arguments": {
> > "path": "/machine/peripheral/cxl-dcd0",
> > "region-id": 0,
> > "extents": [
> > {
> > "dpa": 128,
> > "len": 128
> > }
> > ]
> > }
> > }
> >
> > Signed-off-by: Fan Ni
> A few more comment and found some answers to previous comments. I should have
> read the whole thing first :(
Hi Jonathan,
One reply to your comment is inlined. Search REPLY to locate it.
>
> > ---
> > hw/cxl/cxl-mailbox-utils.c | 25 +++-
> > hw/mem/cxl_type3.c | 225 +++-
> > hw/mem/cxl_type3_stubs.c| 14 +++
> > include/hw/cxl/cxl_device.h | 8 +-
> > include/hw/cxl/cxl_events.h | 15 +++
> > qapi/cxl.json | 60 +-
> > 6 files changed, 338 insertions(+), 9 deletions(-)
> >
> > diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
> > index 9f788b03b6..8e6a98753a 100644
> > --- a/hw/cxl/cxl-mailbox-utils.c
> > +++ b/hw/cxl/cxl-mailbox-utils.c
> > @@ -1362,7 +1362,7 @@ static CXLRetCode cmd_dcd_get_dyn_cap_ext_list(const
> > struct cxl_cmd *cmd,
> > * Check whether any bit between addr[nr, nr+size) is set,
> > * return true if any bit is set, otherwise return false
> > */
> > -static bool test_any_bits_set(const unsigned long *addr, int nr, int size)
> > +bool test_any_bits_set(const unsigned long *addr, int nr, int size)
> > {
> > unsigned long res = find_next_bit(addr, size + nr, nr);
> >
> > @@ -1400,7 +1400,7 @@ CXLDCDRegion *cxl_find_dc_region(CXLType3Dev *ct3d,
> > uint64_t dpa, uint64_t len)
> > return NULL;
> > }
> >
> > -static void cxl_insert_extent_to_extent_list(CXLDCDExtentList *list,
> > +void cxl_insert_extent_to_extent_list(CXLDCDExtentList *list,
> > uint64_t dpa,
> > uint64_t len,
> > uint8_t *tag,
> > @@ -1538,15 +1538,28 @@ static CXLRetCode cmd_dcd_add_dyn_cap_rsp(const
> > struct cxl_cmd *cmd,
> > }
> > }
> >
> > -/*
> > - * TODO: add a pending extent list based on event log record and
> > - * verify the input response
> > - */
>
> Ahah. I should have read on :) Ignore comments on previous agreeing
> that such a list was needed.
>
> > +QTAILQ_FOREACH(ent, >dc.extents_pending_to_add, node) {
> > +if (ent->start_dpa <= dpa &&
> > +dpa + len <= ent->start_dpa + ent->len) {
> > +break;
> > +}
> > +}
> > +if (ent) {
> > +QTAILQ_REMOVE(>dc.extents_pending_to_add, ent, node);
> > +g_free(ent);
> > +} else {
> > +return CXL_MBOX_INVALID_PA;
> > +}
> Flip to simplify logic
>
>if (!end) {
> return CXL_MBOX_INVALID_PA;
>}
>
> QTAILQ...
>
> >
> > cxl_insert_extent_to_extent_list(extent_list, dpa, len, NULL, 0);
> > ct3d->dc.total_extent_count += 1;
> > }
> >
> > +/*
> > + * TODO: extents_pending_to_add needs to be cleared so the extents not
> > + * accepted can be reclaimed base on spec r3.0: 8.2.9.8.9.3
> > + */
> > +
> > return CXL_MBOX_SUCCESS;
> > }
> >
> > diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
> > index 482329a499..43cea3d818 100644
> > --- a/hw/mem/cxl_type3.c
> > +++ b/hw/mem/cxl_type3.c
> > @@ -813,6 +813,7 @@ static int cxl_create_dc_regions(CXLType3Dev *ct3d)
> > region_base += region->len;
> > }
> > QTAILQ_INIT(>dc.extents);
> > +
Re: [PATCH v3 8/9] hw/cxl/events: Add qmp interfaces to add/release dynamic capacity extents
On Tue, 7 Nov 2023 10:07:12 -0800
nifan@gmail.com wrote:
> From: Fan Ni
>
> Since fabric manager emulation is not supported yet, the change implements
> the functions to add/release dynamic capacity extents as QMP interfaces.
>
> Note: we block any FM issued extent release request if the exact extent
> does not exist in the extent list of the device. We will loose the
> restriction later once we have partial release support in the kernel.
>
> 1. Add dynamic capacity extents:
>
> For example, the command to add two continuous extents (each 128MiB long)
> to region 0 (starting at DPA offset 0) looks like below:
>
> { "execute": "qmp_capabilities" }
>
> { "execute": "cxl-add-dynamic-capacity",
> "arguments": {
> "path": "/machine/peripheral/cxl-dcd0",
> "region-id": 0,
> "extents": [
> {
> "dpa": 0,
> "len": 128
> },
> {
> "dpa": 128,
> "len": 128
> }
> ]
> }
> }
>
> 2. Release dynamic capacity extents:
>
> For example, the command to release an extent of size 128MiB from region 0
> (DPA offset 128MiB) look like below:
>
> { "execute": "cxl-release-dynamic-capacity",
> "arguments": {
> "path": "/machine/peripheral/cxl-dcd0",
> "region-id": 0,
> "extents": [
> {
> "dpa": 128,
> "len": 128
> }
> ]
> }
> }
>
> Signed-off-by: Fan Ni
A few more comment and found some answers to previous comments. I should have
read the whole thing first :(
> ---
> hw/cxl/cxl-mailbox-utils.c | 25 +++-
> hw/mem/cxl_type3.c | 225 +++-
> hw/mem/cxl_type3_stubs.c| 14 +++
> include/hw/cxl/cxl_device.h | 8 +-
> include/hw/cxl/cxl_events.h | 15 +++
> qapi/cxl.json | 60 +-
> 6 files changed, 338 insertions(+), 9 deletions(-)
>
> diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
> index 9f788b03b6..8e6a98753a 100644
> --- a/hw/cxl/cxl-mailbox-utils.c
> +++ b/hw/cxl/cxl-mailbox-utils.c
> @@ -1362,7 +1362,7 @@ static CXLRetCode cmd_dcd_get_dyn_cap_ext_list(const
> struct cxl_cmd *cmd,
> * Check whether any bit between addr[nr, nr+size) is set,
> * return true if any bit is set, otherwise return false
> */
> -static bool test_any_bits_set(const unsigned long *addr, int nr, int size)
> +bool test_any_bits_set(const unsigned long *addr, int nr, int size)
> {
> unsigned long res = find_next_bit(addr, size + nr, nr);
>
> @@ -1400,7 +1400,7 @@ CXLDCDRegion *cxl_find_dc_region(CXLType3Dev *ct3d,
> uint64_t dpa, uint64_t len)
> return NULL;
> }
>
> -static void cxl_insert_extent_to_extent_list(CXLDCDExtentList *list,
> +void cxl_insert_extent_to_extent_list(CXLDCDExtentList *list,
> uint64_t dpa,
> uint64_t len,
> uint8_t *tag,
> @@ -1538,15 +1538,28 @@ static CXLRetCode cmd_dcd_add_dyn_cap_rsp(const
> struct cxl_cmd *cmd,
> }
> }
>
> -/*
> - * TODO: add a pending extent list based on event log record and
> - * verify the input response
> - */
Ahah. I should have read on :) Ignore comments on previous agreeing
that such a list was needed.
> +QTAILQ_FOREACH(ent, >dc.extents_pending_to_add, node) {
> +if (ent->start_dpa <= dpa &&
> +dpa + len <= ent->start_dpa + ent->len) {
> +break;
> +}
> +}
> +if (ent) {
> +QTAILQ_REMOVE(>dc.extents_pending_to_add, ent, node);
> +g_free(ent);
> +} else {
> +return CXL_MBOX_INVALID_PA;
> +}
Flip to simplify logic
if (!end) {
return CXL_MBOX_INVALID_PA;
}
QTAILQ...
>
> cxl_insert_extent_to_extent_list(extent_list, dpa, len, NULL, 0);
> ct3d->dc.total_extent_count += 1;
> }
>
> +/*
> + * TODO: extents_pending_to_add needs to be cleared so the extents not
> + * accepted can be reclaimed base on spec r3.0: 8.2.9.8.9.3
> + */
> +
> return CXL_MBOX_SUCCESS;
> }
>
> diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
> index 482329a499..43cea3d818 100644
> --- a/hw/mem/cxl_type3.c
> +++ b/hw/mem/cxl_type3.c
> @@ -813,6 +813,7 @@ static int cxl_create_dc_regions(CXLType3Dev *ct3d)
> region_base += region->len;
> }
> QTAILQ_INIT(>dc.extents);
> +QTAILQ_INIT(>dc.extents_pending_to_add);
>
> return 0;
> }
> @@ -1616,7 +1617,8 @@ static int ct3d_qmp_cxl_event_log_enc(CxlEventLog log)
> return CXL_EVENT_TYPE_FAIL;
> case CXL_EVENT_LOG_FATAL:
> return CXL_EVENT_TYPE_FATAL;
> -/* DCD not yet supported */
> +case CXL_EVENT_LOG_DYNCAP:
> +return CXL_EVENT_TYPE_DYNAMIC_CAP;
> default:
> return -EINVAL;
> }
> @@ -1867,6 +1869,227 @@
[PATCH v3 8/9] hw/cxl/events: Add qmp interfaces to add/release dynamic capacity extents
From: Fan Ni
Since fabric manager emulation is not supported yet, the change implements
the functions to add/release dynamic capacity extents as QMP interfaces.
Note: we block any FM issued extent release request if the exact extent
does not exist in the extent list of the device. We will loose the
restriction later once we have partial release support in the kernel.
1. Add dynamic capacity extents:
For example, the command to add two continuous extents (each 128MiB long)
to region 0 (starting at DPA offset 0) looks like below:
{ "execute": "qmp_capabilities" }
{ "execute": "cxl-add-dynamic-capacity",
"arguments": {
"path": "/machine/peripheral/cxl-dcd0",
"region-id": 0,
"extents": [
{
"dpa": 0,
"len": 128
},
{
"dpa": 128,
"len": 128
}
]
}
}
2. Release dynamic capacity extents:
For example, the command to release an extent of size 128MiB from region 0
(DPA offset 128MiB) look like below:
{ "execute": "cxl-release-dynamic-capacity",
"arguments": {
"path": "/machine/peripheral/cxl-dcd0",
"region-id": 0,
"extents": [
{
"dpa": 128,
"len": 128
}
]
}
}
Signed-off-by: Fan Ni
---
hw/cxl/cxl-mailbox-utils.c | 25 +++-
hw/mem/cxl_type3.c | 225 +++-
hw/mem/cxl_type3_stubs.c| 14 +++
include/hw/cxl/cxl_device.h | 8 +-
include/hw/cxl/cxl_events.h | 15 +++
qapi/cxl.json | 60 +-
6 files changed, 338 insertions(+), 9 deletions(-)
diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
index 9f788b03b6..8e6a98753a 100644
--- a/hw/cxl/cxl-mailbox-utils.c
+++ b/hw/cxl/cxl-mailbox-utils.c
@@ -1362,7 +1362,7 @@ static CXLRetCode cmd_dcd_get_dyn_cap_ext_list(const
struct cxl_cmd *cmd,
* Check whether any bit between addr[nr, nr+size) is set,
* return true if any bit is set, otherwise return false
*/
-static bool test_any_bits_set(const unsigned long *addr, int nr, int size)
+bool test_any_bits_set(const unsigned long *addr, int nr, int size)
{
unsigned long res = find_next_bit(addr, size + nr, nr);
@@ -1400,7 +1400,7 @@ CXLDCDRegion *cxl_find_dc_region(CXLType3Dev *ct3d,
uint64_t dpa, uint64_t len)
return NULL;
}
-static void cxl_insert_extent_to_extent_list(CXLDCDExtentList *list,
+void cxl_insert_extent_to_extent_list(CXLDCDExtentList *list,
uint64_t dpa,
uint64_t len,
uint8_t *tag,
@@ -1538,15 +1538,28 @@ static CXLRetCode cmd_dcd_add_dyn_cap_rsp(const struct
cxl_cmd *cmd,
}
}
-/*
- * TODO: add a pending extent list based on event log record and
- * verify the input response
- */
+QTAILQ_FOREACH(ent, >dc.extents_pending_to_add, node) {
+if (ent->start_dpa <= dpa &&
+dpa + len <= ent->start_dpa + ent->len) {
+break;
+}
+}
+if (ent) {
+QTAILQ_REMOVE(>dc.extents_pending_to_add, ent, node);
+g_free(ent);
+} else {
+return CXL_MBOX_INVALID_PA;
+}
cxl_insert_extent_to_extent_list(extent_list, dpa, len, NULL, 0);
ct3d->dc.total_extent_count += 1;
}
+/*
+ * TODO: extents_pending_to_add needs to be cleared so the extents not
+ * accepted can be reclaimed base on spec r3.0: 8.2.9.8.9.3
+ */
+
return CXL_MBOX_SUCCESS;
}
diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index 482329a499..43cea3d818 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -813,6 +813,7 @@ static int cxl_create_dc_regions(CXLType3Dev *ct3d)
region_base += region->len;
}
QTAILQ_INIT(>dc.extents);
+QTAILQ_INIT(>dc.extents_pending_to_add);
return 0;
}
@@ -1616,7 +1617,8 @@ static int ct3d_qmp_cxl_event_log_enc(CxlEventLog log)
return CXL_EVENT_TYPE_FAIL;
case CXL_EVENT_LOG_FATAL:
return CXL_EVENT_TYPE_FATAL;
-/* DCD not yet supported */
+case CXL_EVENT_LOG_DYNCAP:
+return CXL_EVENT_TYPE_DYNAMIC_CAP;
default:
return -EINVAL;
}
@@ -1867,6 +1869,227 @@ void qmp_cxl_inject_memory_module_event(const char
*path, CxlEventLog log,
}
}
+/* CXL r3.0 Table 8-47: Dynanic Capacity Event Record */
+static const QemuUUID dynamic_capacity_uuid = {
+.data = UUID(0xca95afa7, 0xf183, 0x4018, 0x8c, 0x2f,
+ 0x95, 0x26, 0x8e, 0x10, 0x1a, 0x2a),
+};
+
+typedef enum CXLDCEventType {
+DC_EVENT_ADD_CAPACITY = 0x0,
+DC_EVENT_RELEASE_CAPACITY = 0x1,
+DC_EVENT_FORCED_RELEASE_CAPACITY = 0x2,
+DC_EVENT_REGION_CONFIG_UPDATED = 0x3,
+DC_EVENT_ADD_CAPACITY_RSP = 0x4,
+DC_EVENT_CAPACITY_RELEASED = 0x5,
+DC_EVENT_NUM
+} CXLDCEventType;
+
+/*
+ * Check whether the exact extent exists in the list
+ *
