Re: [PATCH 2/3] vfio/ap: Remove pointless apdev variable
Reviewed-by: Tony Krowiak
On 10/8/23 22:20, Zhenzhong Duan wrote:
No need to double-cast, call VFIO_AP_DEVICE() on DeviceState.
No functional changes.
Signed-off-by: Zhenzhong Duan
---
hw/vfio/ap.c | 9 +++--
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 22e564f4f7..e083a19eac 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -156,8 +156,7 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
{
int ret;
Error *err = NULL;
-APDevice *apdev = AP_DEVICE(dev);
-VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
+VFIOAPDevice *vapdev = VFIO_AP_DEVICE(dev);
VFIODevice *vbasedev = >vdev;
vbasedev->name = g_path_get_basename(vbasedev->sysfsdev);
@@ -195,8 +194,7 @@ error:
static void vfio_ap_unrealize(DeviceState *dev)
{
-APDevice *apdev = AP_DEVICE(dev);
-VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
+VFIOAPDevice *vapdev = VFIO_AP_DEVICE(dev);
vfio_ap_unregister_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX);
vfio_detach_device(>vdev);
@@ -211,8 +209,7 @@ static Property vfio_ap_properties[] = {
static void vfio_ap_reset(DeviceState *dev)
{
int ret;
-APDevice *apdev = AP_DEVICE(dev);
-VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
+VFIOAPDevice *vapdev = VFIO_AP_DEVICE(dev);
ret = ioctl(vapdev->vdev.fd, VFIO_DEVICE_RESET);
if (ret) {
[PATCH v2 2/2] s390x/ap: Wire up the device request notifier interface
Let's wire up the device request notifier interface to handle device unplug
requests for AP.
Signed-off-by: Tony Krowiak
Link:
https://lore.kernel.org/qemu-devel/20230530225544.280031-1-akrow...@linux.ibm.com/
---
hw/vfio/ap.c | 113 +++
1 file changed, 113 insertions(+)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index e0dd561e85a3..6e21d1da5a70 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -18,6 +18,8 @@
#include "hw/vfio/vfio-common.h"
#include "hw/s390x/ap-device.h"
#include "qemu/error-report.h"
+#include "qemu/event_notifier.h"
+#include "qemu/main-loop.h"
#include "qemu/module.h"
#include "qemu/option.h"
#include "qemu/config-file.h"
@@ -33,6 +35,7 @@
struct VFIOAPDevice {
APDevice apdev;
VFIODevice vdev;
+EventNotifier req_notifier;
};
OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
@@ -84,10 +87,110 @@ static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev,
Error **errp)
return vfio_get_group(groupid, _space_memory, errp);
}
+static void vfio_ap_req_notifier_handler(void *opaque)
+{
+VFIOAPDevice *vapdev = opaque;
+Error *err = NULL;
+
+if (!event_notifier_test_and_clear(>req_notifier)) {
+return;
+}
+
+qdev_unplug(DEVICE(vapdev), );
+
+if (err) {
+warn_reportf_err(err, VFIO_MSG_PREFIX, vapdev->vdev.name);
+}
+}
+
+static void vfio_ap_register_irq_notifier(VFIOAPDevice *vapdev,
+ unsigned int irq, Error **errp)
+{
+int fd;
+size_t argsz;
+IOHandler *fd_read;
+EventNotifier *notifier;
+struct vfio_irq_info *irq_info;
+VFIODevice *vdev = >vdev;
+
+switch (irq) {
+case VFIO_AP_REQ_IRQ_INDEX:
+notifier = >req_notifier;
+fd_read = vfio_ap_req_notifier_handler;
+break;
+default:
+error_setg(errp, "vfio: Unsupported device irq(%d)", irq);
+return;
+}
+
+if (vdev->num_irqs < irq + 1) {
+error_setg(errp, "vfio: IRQ %u not available (number of irqs %u)",
+ irq, vdev->num_irqs);
+return;
+}
+
+argsz = sizeof(*irq_info);
+irq_info = g_malloc0(argsz);
+irq_info->index = irq;
+irq_info->argsz = argsz;
+
+if (ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO,
+ irq_info) < 0 || irq_info->count < 1) {
+error_setg_errno(errp, errno, "vfio: Error getting irq info");
+goto out_free_info;
+}
+
+if (event_notifier_init(notifier, 0)) {
+error_setg_errno(errp, errno,
+ "vfio: Unable to init event notifier for irq (%d)",
+ irq);
+goto out_free_info;
+}
+
+fd = event_notifier_get_fd(notifier);
+qemu_set_fd_handler(fd, fd_read, NULL, vapdev);
+
+if (vfio_set_irq_signaling(vdev, irq, 0, VFIO_IRQ_SET_ACTION_TRIGGER, fd,
+ errp)) {
+qemu_set_fd_handler(fd, NULL, NULL, vapdev);
+event_notifier_cleanup(notifier);
+}
+
+out_free_info:
+g_free(irq_info);
+
+}
+
+static void vfio_ap_unregister_irq_notifier(VFIOAPDevice *vapdev,
+unsigned int irq)
+{
+Error *err = NULL;
+EventNotifier *notifier;
+
+switch (irq) {
+case VFIO_AP_REQ_IRQ_INDEX:
+notifier = >req_notifier;
+break;
+default:
+error_report("vfio: Unsupported device irq(%d)", irq);
+return;
+}
+
+if (vfio_set_irq_signaling(>vdev, irq, 0,
+ VFIO_IRQ_SET_ACTION_TRIGGER, -1, )) {
+warn_reportf_err(err, VFIO_MSG_PREFIX, vapdev->vdev.name);
+}
+
+qemu_set_fd_handler(event_notifier_get_fd(notifier),
+NULL, NULL, vapdev);
+event_notifier_cleanup(notifier);
+}
+
static void vfio_ap_realize(DeviceState *dev, Error **errp)
{
int ret;
char *mdevid;
+Error *err = NULL;
VFIOGroup *vfio_group;
APDevice *apdev = AP_DEVICE(dev);
VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
@@ -116,6 +219,15 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
goto out_get_dev_err;
}
+vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, );
+if (err) {
+/*
+ * Report this error, but do not make it a failing condition.
+ * Lack of this IRQ in the host does not prevent normal operation.
+ */
+error_report_err(err);
+}
+
return;
out_get_dev_err:
@@ -129,6 +241,7 @@ static void vfio_ap_unrealize(DeviceState *dev)
VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
VFIOGroup *group = vapdev->vdev.group;
+vfio_ap_unregister_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX);
vfio_ap_put_device(vapdev);
vfio_put_group(group);
}
--
2.31.1
[PATCH v2 0/2] s390x/ap: fix hang when mdev attached to guest is removed
When a user attempts to remove a vfio-ap mediated device attached to a guest, the operation hangs until the mdev's fd is closed by the guest (i.e., the guest is shut down). This patch series provides userspace (i.e., qemu) code to handle device unplug requests for AP. When notified that the mdev is being removed, the handler will unplug the device, thus avoiding the hang condition. Link: https://lore.kernel.org/r/20230530223538.279198-4-akrow...@linux.ibm.com Tony Krowiak (2): linux-headers: Update with vfio_ap IRQ index mapping s390x/ap: Wire up the device request notifier interface hw/vfio/ap.c | 113 ++ include/standard-headers/linux/const.h| 2 +- include/standard-headers/linux/virtio_blk.h | 18 +-- .../standard-headers/linux/virtio_config.h| 6 + include/standard-headers/linux/virtio_net.h | 1 + linux-headers/asm-arm64/kvm.h | 33 + linux-headers/asm-riscv/kvm.h | 53 +++- linux-headers/asm-riscv/unistd.h | 9 ++ linux-headers/asm-s390/unistd_32.h| 1 + linux-headers/asm-s390/unistd_64.h| 1 + linux-headers/asm-x86/kvm.h | 3 + linux-headers/linux/const.h | 2 +- linux-headers/linux/kvm.h | 12 +- linux-headers/linux/psp-sev.h | 7 ++ linux-headers/linux/userfaultfd.h | 17 ++- linux-headers/linux/vfio.h| 9 ++ 16 files changed, 271 insertions(+), 16 deletions(-) -- 2.31.1
[PATCH v2 1/2] linux-headers: Update with vfio_ap IRQ index mapping
Note: This is a placeholder patch that includes unmerged uapi changes.
Signed-off-by: Tony Krowiak
Link:
https://lore.kernel.org/qemu-devel/20230530225544.280031-1-akrow...@linux.ibm.com/
---
include/standard-headers/linux/const.h| 2 +-
include/standard-headers/linux/virtio_blk.h | 18 +++
.../standard-headers/linux/virtio_config.h| 6 +++
include/standard-headers/linux/virtio_net.h | 1 +
linux-headers/asm-arm64/kvm.h | 33
linux-headers/asm-riscv/kvm.h | 53 ++-
linux-headers/asm-riscv/unistd.h | 9
linux-headers/asm-s390/unistd_32.h| 1 +
linux-headers/asm-s390/unistd_64.h| 1 +
linux-headers/asm-x86/kvm.h | 3 ++
linux-headers/linux/const.h | 2 +-
linux-headers/linux/kvm.h | 12 +++--
linux-headers/linux/psp-sev.h | 7 +++
linux-headers/linux/userfaultfd.h | 17 +-
linux-headers/linux/vfio.h| 9
15 files changed, 158 insertions(+), 16 deletions(-)
diff --git a/include/standard-headers/linux/const.h
b/include/standard-headers/linux/const.h
index 5e4898725168..1eb84b5087f8 100644
--- a/include/standard-headers/linux/const.h
+++ b/include/standard-headers/linux/const.h
@@ -28,7 +28,7 @@
#define _BITUL(x) (_UL(1) << (x))
#define _BITULL(x) (_ULL(1) << (x))
-#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (typeof(x))(a) -
1)
+#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x,
(__typeof__(x))(a) - 1)
#define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask))
#define __KERNEL_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
diff --git a/include/standard-headers/linux/virtio_blk.h
b/include/standard-headers/linux/virtio_blk.h
index 7155b1a4701b..d7be3cf5e42f 100644
--- a/include/standard-headers/linux/virtio_blk.h
+++ b/include/standard-headers/linux/virtio_blk.h
@@ -138,11 +138,11 @@ struct virtio_blk_config {
/* Zoned block device characteristics (if VIRTIO_BLK_F_ZONED) */
struct virtio_blk_zoned_characteristics {
- uint32_t zone_sectors;
- uint32_t max_open_zones;
- uint32_t max_active_zones;
- uint32_t max_append_sectors;
- uint32_t write_granularity;
+ __virtio32 zone_sectors;
+ __virtio32 max_open_zones;
+ __virtio32 max_active_zones;
+ __virtio32 max_append_sectors;
+ __virtio32 write_granularity;
uint8_t model;
uint8_t unused2[3];
} zoned;
@@ -239,11 +239,11 @@ struct virtio_blk_outhdr {
*/
struct virtio_blk_zone_descriptor {
/* Zone capacity */
- uint64_t z_cap;
+ __virtio64 z_cap;
/* The starting sector of the zone */
- uint64_t z_start;
+ __virtio64 z_start;
/* Zone write pointer position in sectors */
- uint64_t z_wp;
+ __virtio64 z_wp;
/* Zone type */
uint8_t z_type;
/* Zone state */
@@ -252,7 +252,7 @@ struct virtio_blk_zone_descriptor {
};
struct virtio_blk_zone_report {
- uint64_t nr_zones;
+ __virtio64 nr_zones;
uint8_t reserved[56];
struct virtio_blk_zone_descriptor zones[];
};
diff --git a/include/standard-headers/linux/virtio_config.h
b/include/standard-headers/linux/virtio_config.h
index 965ee6ae237e..8a7d0dc8b007 100644
--- a/include/standard-headers/linux/virtio_config.h
+++ b/include/standard-headers/linux/virtio_config.h
@@ -97,6 +97,12 @@
*/
#define VIRTIO_F_SR_IOV37
+/*
+ * This feature indicates that the driver passes extra data (besides
+ * identifying the virtqueue) in its device notifications.
+ */
+#define VIRTIO_F_NOTIFICATION_DATA 38
+
/*
* This feature indicates that the driver can reset a queue individually.
*/
diff --git a/include/standard-headers/linux/virtio_net.h
b/include/standard-headers/linux/virtio_net.h
index c0e797067aae..2325485f2ca8 100644
--- a/include/standard-headers/linux/virtio_net.h
+++ b/include/standard-headers/linux/virtio_net.h
@@ -61,6 +61,7 @@
#define VIRTIO_NET_F_GUEST_USO655 /* Guest can handle USOv6 in. */
#define VIRTIO_NET_F_HOST_USO 56 /* Host can handle USO in. */
#define VIRTIO_NET_F_HASH_REPORT 57 /* Supports hash report */
+#define VIRTIO_NET_F_GUEST_HDRLEN 59 /* Guest provides the exact hdr_len
value. */
#define VIRTIO_NET_F_RSS 60/* Supports RSS RX steering */
#define VIRTIO_NET_F_RSC_EXT 61/* extended coalescing info */
#define VIRTIO_NET_F_STANDBY 62/* Act as standby for another device
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
index d7e7bb885e20..38e5957526c2 100644
--- a/linux-headers/asm-arm64/kvm.h
+++ b/linux-headers/asm-arm64/kvm.h
@@ -198,6 +198,15 @@ struct kvm_arm_copy_mte_tags
[PATCH 0/2] s390x/ap: fix hang when mdev attached to guest is removed
When a user attempts to remove a vfio-ap mediated device attached to a guest, the operation hangs until the mdev's fd is closed by the guest (i.e., the guest is shut down). This patch series provides userspace (i.e., qemu) code to handle device unplug requests for AP. When notified that the mdev is being removed, the handler will unplug the device, thus avoiding the hang condition. Tony Krowiak (2): linux-headers: Update with vfio_ap IRQ index mapping s390x/ap: Wire up the device request notifier interface hw/vfio/ap.c | 113 + linux-headers/linux/vfio.h | 9 +++ 2 files changed, 122 insertions(+) -- 2.31.1
[PATCH 1/2] linux-headers: Update with vfio_ap IRQ index mapping
Signed-off-by: Tony Krowiak
---
linux-headers/linux/vfio.h | 9 +
1 file changed, 9 insertions(+)
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 4a534edbdcba..2658fda219e8 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -646,6 +646,15 @@ enum {
VFIO_CCW_NUM_IRQS
};
+/*
+ * The vfio-ap bus driver makes use of the following IRQ index mapping.
+ * Unimplemented IRQ types return a count of zero.
+ */
+enum {
+ VFIO_AP_REQ_IRQ_INDEX,
+ VFIO_AP_NUM_IRQS
+};
+
/**
* VFIO_DEVICE_GET_PCI_HOT_RESET_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 12,
* struct vfio_pci_hot_reset_info)
--
2.31.1
[PATCH 2/2] s390x/ap: Wire up the device request notifier interface
Let's wire up the device request notifier interface to handle device unplug
requests for AP.
Signed-off-by: Tony Krowiak
---
hw/vfio/ap.c | 113 +++
1 file changed, 113 insertions(+)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index e0dd561e85a3..6e21d1da5a70 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -18,6 +18,8 @@
#include "hw/vfio/vfio-common.h"
#include "hw/s390x/ap-device.h"
#include "qemu/error-report.h"
+#include "qemu/event_notifier.h"
+#include "qemu/main-loop.h"
#include "qemu/module.h"
#include "qemu/option.h"
#include "qemu/config-file.h"
@@ -33,6 +35,7 @@
struct VFIOAPDevice {
APDevice apdev;
VFIODevice vdev;
+EventNotifier req_notifier;
};
OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
@@ -84,10 +87,110 @@ static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev,
Error **errp)
return vfio_get_group(groupid, _space_memory, errp);
}
+static void vfio_ap_req_notifier_handler(void *opaque)
+{
+VFIOAPDevice *vapdev = opaque;
+Error *err = NULL;
+
+if (!event_notifier_test_and_clear(>req_notifier)) {
+return;
+}
+
+qdev_unplug(DEVICE(vapdev), );
+
+if (err) {
+warn_reportf_err(err, VFIO_MSG_PREFIX, vapdev->vdev.name);
+}
+}
+
+static void vfio_ap_register_irq_notifier(VFIOAPDevice *vapdev,
+ unsigned int irq, Error **errp)
+{
+int fd;
+size_t argsz;
+IOHandler *fd_read;
+EventNotifier *notifier;
+struct vfio_irq_info *irq_info;
+VFIODevice *vdev = >vdev;
+
+switch (irq) {
+case VFIO_AP_REQ_IRQ_INDEX:
+notifier = >req_notifier;
+fd_read = vfio_ap_req_notifier_handler;
+break;
+default:
+error_setg(errp, "vfio: Unsupported device irq(%d)", irq);
+return;
+}
+
+if (vdev->num_irqs < irq + 1) {
+error_setg(errp, "vfio: IRQ %u not available (number of irqs %u)",
+ irq, vdev->num_irqs);
+return;
+}
+
+argsz = sizeof(*irq_info);
+irq_info = g_malloc0(argsz);
+irq_info->index = irq;
+irq_info->argsz = argsz;
+
+if (ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO,
+ irq_info) < 0 || irq_info->count < 1) {
+error_setg_errno(errp, errno, "vfio: Error getting irq info");
+goto out_free_info;
+}
+
+if (event_notifier_init(notifier, 0)) {
+error_setg_errno(errp, errno,
+ "vfio: Unable to init event notifier for irq (%d)",
+ irq);
+goto out_free_info;
+}
+
+fd = event_notifier_get_fd(notifier);
+qemu_set_fd_handler(fd, fd_read, NULL, vapdev);
+
+if (vfio_set_irq_signaling(vdev, irq, 0, VFIO_IRQ_SET_ACTION_TRIGGER, fd,
+ errp)) {
+qemu_set_fd_handler(fd, NULL, NULL, vapdev);
+event_notifier_cleanup(notifier);
+}
+
+out_free_info:
+g_free(irq_info);
+
+}
+
+static void vfio_ap_unregister_irq_notifier(VFIOAPDevice *vapdev,
+unsigned int irq)
+{
+Error *err = NULL;
+EventNotifier *notifier;
+
+switch (irq) {
+case VFIO_AP_REQ_IRQ_INDEX:
+notifier = >req_notifier;
+break;
+default:
+error_report("vfio: Unsupported device irq(%d)", irq);
+return;
+}
+
+if (vfio_set_irq_signaling(>vdev, irq, 0,
+ VFIO_IRQ_SET_ACTION_TRIGGER, -1, )) {
+warn_reportf_err(err, VFIO_MSG_PREFIX, vapdev->vdev.name);
+}
+
+qemu_set_fd_handler(event_notifier_get_fd(notifier),
+NULL, NULL, vapdev);
+event_notifier_cleanup(notifier);
+}
+
static void vfio_ap_realize(DeviceState *dev, Error **errp)
{
int ret;
char *mdevid;
+Error *err = NULL;
VFIOGroup *vfio_group;
APDevice *apdev = AP_DEVICE(dev);
VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
@@ -116,6 +219,15 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
goto out_get_dev_err;
}
+vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, );
+if (err) {
+/*
+ * Report this error, but do not make it a failing condition.
+ * Lack of this IRQ in the host does not prevent normal operation.
+ */
+error_report_err(err);
+}
+
return;
out_get_dev_err:
@@ -129,6 +241,7 @@ static void vfio_ap_unrealize(DeviceState *dev)
VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
VFIOGroup *group = vapdev->vdev.group;
+vfio_ap_unregister_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX);
vfio_ap_put_device(vapdev);
vfio_put_group(group);
}
--
2.31.1
Re: [PATCH v4 02/21] vfio: Convert to ram_block_discard_disable()
On 6/10/20 7:54 AM, David Hildenbrand wrote:
VFIO is (except devices without a physical IOMMU or some mediated devices)
incompatible with discarding of RAM. The kernel will pin basically all VM
memory. Let's convert to ram_block_discard_disable(), which can now
fail, in contrast to qemu_balloon_inhibit().
Leave "x-balloon-allowed" named as it is for now.
Cc: Cornelia Huck
Cc: Alex Williamson
Cc: Christian Borntraeger
Cc: Tony Krowiak
Cc: Halil Pasic
Cc: Pierre Morel
Cc: Eric Farman
Signed-off-by: David Hildenbrand
See my two minor comments, other than that:
Reviewed-by: Tony Krowiak
---
hw/vfio/ap.c | 10 +++
hw/vfio/ccw.c | 11
hw/vfio/common.c | 53 +++
hw/vfio/pci.c | 6 ++--
include/hw/vfio/vfio-common.h | 4 +--
5 files changed, 45 insertions(+), 39 deletions(-)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 95564c17ed..d0b1bc7581 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -105,12 +105,12 @@ static void vfio_ap_realize(DeviceState *dev, Error
**errp)
vapdev->vdev.dev = dev;
/*
- * vfio-ap devices operate in a way compatible with
- * memory ballooning, as no pages are pinned in the host.
- * This needs to be set before vfio_get_device() for vfio common to
- * handle the balloon inhibitor.
+ * vfio-ap devices operate in a way compatible discarding of memory in
s/compatible discarding/compatible with discarding/?
+ * RAM blocks, as no pages are pinned in the host. This needs to be
+ * set before vfio_get_device() for vfio common to handle
+ * ram_block_discard_disable().
*/
-vapdev->vdev.balloon_allowed = true;
+vapdev->vdev.ram_block_discard_allowed = true;
ret = vfio_get_device(vfio_group, mdevid, >vdev, errp);
if (ret) {
diff --git a/hw/vfio/ccw.c b/hw/vfio/ccw.c
index 63406184d2..82857f1615 100644
--- a/hw/vfio/ccw.c
+++ b/hw/vfio/ccw.c
@@ -418,12 +418,13 @@ static void vfio_ccw_get_device(VFIOGroup *group,
VFIOCCWDevice *vcdev,
/*
* All vfio-ccw devices are believed to operate in a way compatible with
- * memory ballooning, ie. pages pinned in the host are in the current
- * working set of the guest driver and therefore never overlap with pages
- * available to the guest balloon driver. This needs to be set before
- * vfio_get_device() for vfio common to handle the balloon inhibitor.
+ * discarding of memory in RAM blocks, ie. pages pinned in the host are
+ * in the current working set of the guest driver and therefore never
+ * overlap e.g., with pages available to the guest balloon driver. This
+ * needs to be set before vfio_get_device() for vfio common to handle
+ * ram_block_discard_disable().
*/
-vcdev->vdev.balloon_allowed = true;
+vcdev->vdev.ram_block_discard_allowed = true;
if (vfio_get_device(group, vcdev->cdev.mdevid, >vdev, errp)) {
goto out_err;
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 0b3593b3c0..33357140b8 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -33,7 +33,6 @@
#include "qemu/error-report.h"
#include "qemu/main-loop.h"
#include "qemu/range.h"
-#include "sysemu/balloon.h"
#include "sysemu/kvm.h"
#include "sysemu/reset.h"
#include "trace.h"
@@ -1215,31 +1214,36 @@ static int vfio_connect_container(VFIOGroup *group,
AddressSpace *as,
space = vfio_get_address_space(as);
/*
- * VFIO is currently incompatible with memory ballooning insofar as the
+ * VFIO is currently incompatible with discarding of RAM insofar as the
* madvise to purge (zap) the page from QEMU's address space does not
* interact with the memory API and therefore leaves stale virtual to
* physical mappings in the IOMMU if the page was previously pinned. We
- * therefore add a balloon inhibit for each group added to a container,
+ * therefore set discarding broken for each group added to a container,
* whether the container is used individually or shared. This provides
* us with options to allow devices within a group to opt-in and allow
- * ballooning, so long as it is done consistently for a group (for instance
+ * discarding, so long as it is done consistently for a group (for instance
* if the device is an mdev device where it is known that the host vendor
* driver will never pin pages outside of the working set of the guest
- * driver, which would thus not be ballooning candidates).
+ * driver, which would thus not be discarding candidates).
*
* The first opportunity to induce pinning occurs here where we attempt to
* attach the group to existing containers within the AddressSpace. If
any
- * pages are already zapped from the virtual
Re: [Qemu-devel] [PATCH v3 0/2] s390x/vfio-ap: hot plug/unplug vfio-ap device
On 2/25/19 7:42 AM, Cornelia Huck wrote: On Mon, 18 Feb 2019 14:49:55 -0500 Tony Krowiak wrote: This patch series introduces hot plug/unplug of a vfio-ap device. To hot plug a vfio-ap device, the QEMU device_add function may be used: (qemu) device_add vfio-ap,sysfsdev=$path-to-mdev Where $path-to-mdev is the absolute path to the mediated matrix device to which AP resources to be used by the guest have been assigned. A vfio-ap device can be hot plugged only if: 1. A vfio-ap device has not been attached to the ap-bus (only one is allowed per guest) either via the QEMU command line or a prior hot plug action. 2. The guest was started with the CPU model feature for AP enabled (e.g., -cpu host,ap=on). To hot unplug a vfio-ap device, the QEMU device-del function may be used: (qemu) device_del vfio-ap,sysfsdev=$path-to-mdev Where $path-to-mdev is the absolute path to the mediated matrix device specified when the vfio-ap device was attached to the virtual machine's ap-bus. A vfio-ap device can be hot unplugged only if: 1. A vfio-ap device has been attached to the virtual machine's ap-abus either via the QEMU command line or a prior hot plug action. 2. The guest was started with the CPU model feature for AP enabled (e.g., -cpu host,ap=on). Tony Krowiak (2): s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device s390x/vfio-ap: document hot plug/unplug of vfio-ap device docs/vfio-ap.txt | 58 +++- hw/s390x/ap-bridge.c | 12 ++- hw/vfio/ap.c | 2 +- 3 files changed, 65 insertions(+), 7 deletions(-) In the meantime, the signature of qbus_set_hotplug_handler() has changed to take an Object instead of a DeviceState, but I just went ahead and did that trivial change myself. Thanks, applied. Thanks Connie.
Re: [Qemu-devel] [PATCH v3 2/2] s390x/vfio-ap: document hot plug/unplug of vfio-ap device
On 2/21/19 11:32 AM, Cornelia Huck wrote: On Thu, 21 Feb 2019 11:26:53 -0500 Tony Krowiak wrote: On 2/20/19 10:51 AM, Cornelia Huck wrote: On Mon, 18 Feb 2019 14:49:57 -0500 Tony Krowiak wrote: Let's update the vfio-ap.txt document to include the hot plug/unplug support introduced in this patch set. s/ introduced in this patch set// Nobody will know about this patch set when they look at the commit in the future :) Good point, I'll remove it. I can do that while applying if no other feedback comes in. Okay, sounds good to me. Signed-off-by: Tony Krowiak --- docs/vfio-ap.txt | 61 +++- 1 file changed, 56 insertions(+), 5 deletions(-) Looks good to me, but would not mind a R-b before queuing.
Re: [Qemu-devel] [PATCH v3] qdev/core: fix qbus_is_full()
On 12/17/18 10:57 AM, Tony Krowiak wrote:
Ping!! I'm wondering who might be responsible for merging this fix?
The qbus_is_full(BusState *bus) function (qdev_monitor.c) compares the max_index
value of the BusState structure with the max_dev value of the BusClass structure
to determine whether the maximum number of children has been reached for the
bus. The problem is, the max_index field of the BusState structure does not
necessarily reflect the number of devices that have been plugged into
the bus.
Whenever a child device is plugged into the bus, the bus's max_index value is
assigned to the child device and then incremented. If the child is subsequently
unplugged, the value of the max_index does not change and no longer reflects the
number of children.
When the bus's max_index value reaches the maximum number of devices
allowed for the bus (i.e., the max_dev field in the BusClass structure),
attempts to plug another device will be rejected claiming that the bus is
full -- even if the bus is actually empty.
To resolve the problem, a new 'num_children' field is being added to the
BusState structure to keep track of the number of children plugged into the
bus. It will be incremented when a child is plugged, and decremented when a
child is unplugged.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Reviewed-by: Halil Pasic
---
hw/core/qdev.c | 3 +++
include/hw/qdev-core.h | 1 +
qdev-monitor.c | 2 +-
3 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 6b3cc55b27c2..956923f33520 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState
*child)
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
+bus->num_children--;
+
/* This gives back ownership of kid->child back to us. */
object_property_del(OBJECT(bus), name, NULL);
object_unref(OBJECT(kid->child));
@@ -73,6 +75,7 @@ static void bus_add_child(BusState *bus, DeviceState *child)
char name[32];
BusChild *kid = g_malloc0(sizeof(*kid));
+bus->num_children++;
kid->index = bus->max_index++;
kid->child = child;
object_ref(OBJECT(kid->child));
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index a24d0dd566e3..521f0a947ead 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -206,6 +206,7 @@ struct BusState {
HotplugHandler *hotplug_handler;
int max_index;
bool realized;
+int num_children;
QTAILQ_HEAD(ChildrenHead, BusChild) children;
QLIST_ENTRY(BusState) sibling;
};
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 07147c63bf8b..45a8ba49644c 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -414,7 +414,7 @@ static DeviceState *qbus_find_dev(BusState *bus, char *elem)
static inline bool qbus_is_full(BusState *bus)
{
BusClass *bus_class = BUS_GET_CLASS(bus);
-return bus_class->max_dev && bus->max_index >= bus_class->max_dev;
+return bus_class->max_dev && bus->num_children >= bus_class->max_dev;
}
/*
Re: [Qemu-devel] [PATCH v3 2/2] s390x/vfio-ap: document hot plug/unplug of vfio-ap device
On 2/20/19 10:51 AM, Cornelia Huck wrote: On Mon, 18 Feb 2019 14:49:57 -0500 Tony Krowiak wrote: Let's update the vfio-ap.txt document to include the hot plug/unplug support introduced in this patch set. s/ introduced in this patch set// Nobody will know about this patch set when they look at the commit in the future :) Good point, I'll remove it. Signed-off-by: Tony Krowiak --- docs/vfio-ap.txt | 61 +++- 1 file changed, 56 insertions(+), 5 deletions(-) Looks good to me, but would not mind a R-b before queuing.
[Qemu-devel] [PATCH v3 0/2] s390x/vfio-ap: hot plug/unplug vfio-ap device
This patch series introduces hot plug/unplug of a vfio-ap device. To hot plug a vfio-ap device, the QEMU device_add function may be used: (qemu) device_add vfio-ap,sysfsdev=$path-to-mdev Where $path-to-mdev is the absolute path to the mediated matrix device to which AP resources to be used by the guest have been assigned. A vfio-ap device can be hot plugged only if: 1. A vfio-ap device has not been attached to the ap-bus (only one is allowed per guest) either via the QEMU command line or a prior hot plug action. 2. The guest was started with the CPU model feature for AP enabled (e.g., -cpu host,ap=on). To hot unplug a vfio-ap device, the QEMU device-del function may be used: (qemu) device_del vfio-ap,sysfsdev=$path-to-mdev Where $path-to-mdev is the absolute path to the mediated matrix device specified when the vfio-ap device was attached to the virtual machine's ap-bus. A vfio-ap device can be hot unplugged only if: 1. A vfio-ap device has been attached to the virtual machine's ap-abus either via the QEMU command line or a prior hot plug action. 2. The guest was started with the CPU model feature for AP enabled (e.g., -cpu host,ap=on). Tony Krowiak (2): s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device s390x/vfio-ap: document hot plug/unplug of vfio-ap device docs/vfio-ap.txt | 58 +++- hw/s390x/ap-bridge.c | 12 ++- hw/vfio/ap.c | 2 +- 3 files changed, 65 insertions(+), 7 deletions(-) -- Changes: v2==>v3 * Split documentation changes into a separate patch * Grammatical changes recommended by Connie 2.7.4
[Qemu-devel] [PATCH v3 1/2] s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device
Introduces hot plug/unplug support for the vfio-ap device.
To hot plug a vfio-ap device using the QEMU device_add command:
(qemu) device_add vfio-ap,sysfsdev=$path-to-mdev
Where $path-to-mdev is the absolute path to the mediated matrix device
to which AP resources to be used by the guest have been assigned.
A vfio-ap device can be hot plugged only if:
1. A vfio-ap device has not been attached to the virtual machine's ap-bus
via the QEMU command line or a prior hot plug action.
2. The guest was started with the CPU model feature for AP enabled
(e.g., -cpu host,ap=on)
To hot unplug a vfio-ap device using the QEMU device_del command:
(qemu) device_del vfio-ap,sysfsdev=$path-to-mdev
Where $path-to-mdev is the absolute path to the mediated matrix device
specified when the vfio-ap device was attached to the virtual machine's
ap-bus.
A vfio-ap device can be hot unplugged only if:
1. A vfio-ap device has been attached to the virtual machine's ap-bus
via the QEMU command line or a prior hot plug action.
2. The guest was started with the CPU model feature for AP enabled
(e.g., -cpu host,ap=on)
Please note that a hot plug handler is not necessary for the vfio-ap device
because the AP matrix configuration for the guest is performed by the
kernel device driver when the vfio-ap device is realized. The vfio-ap device
represents a VFIO mediated device created in the host sysfs for use by a guest.
The mdev device is configured with an AP matrix (i.e., adapters and domains) via
its sysfs attribute interfaces prior to starting the guest or plugging a vfio-ap
device in. When the device is realized, a file descriptor is opened on the mdev
device which results in a callback to the vfio_ap kernel device driver. The
device driver then configures the AP matrix in the guest's SIE state description
from the AP matrix assigned via the mdev device's sysfs interfaces. The AP
devices will be created for the guest when the AP bus running on the guest
subsequently performs its periodic scan for AP devices.
The qdev_simple_device_unplug_cb() callback function is used for the same
reaons; namely, the vfio_ap kernel device driver will perform the AP resource
de-configuration for the guest when the vfio-ap device is unplugged. When the
vfio-ap device is unrealized, the mdev device file descriptor is closed which
results in a callback to the vfio_ap kernel device driver. The device driver
then clears the AP matrix configuration in the guest's SIE state description
and resets all of the affected queues. The AP devices created for the guest
will be removed when the AP bus running on the guest subsequently performs
its periodic scan and finds there are no longer any AP resources assigned to the
guest.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Reviewed-by: David Hildenbrand
Reviewed-by: Halil Pasic
Tested-by: Pierre Morel
---
hw/s390x/ap-bridge.c | 12 +++-
hw/vfio/ap.c | 2 +-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
index 3795d30dd7c9..25a03412fcb9 100644
--- a/hw/s390x/ap-bridge.c
+++ b/hw/s390x/ap-bridge.c
@@ -39,6 +39,7 @@ static const TypeInfo ap_bus_info = {
void s390_init_ap(void)
{
DeviceState *dev;
+BusState *bus;
/* If no AP instructions then no need for AP bridge */
if (!s390_has_feat(S390_FEAT_AP)) {
@@ -52,13 +53,18 @@ void s390_init_ap(void)
qdev_init_nofail(dev);
/* Create bus on bridge device */
-qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+bus = qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+
+/* Enable hotplugging */
+qbus_set_hotplug_handler(bus, dev, _abort);
}
static void ap_bridge_class_init(ObjectClass *oc, void *data)
{
DeviceClass *dc = DEVICE_CLASS(oc);
+HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
+hc->unplug = qdev_simple_device_unplug_cb;
set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
}
@@ -67,6 +73,10 @@ static const TypeInfo ap_bridge_info = {
.parent= TYPE_SYS_BUS_DEVICE,
.instance_size = 0,
.class_init= ap_bridge_class_init,
+.interfaces = (InterfaceInfo[]) {
+{ TYPE_HOTPLUG_HANDLER },
+{ }
+}
};
static void ap_register(void)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 6166ccd47a4a..d8b79ebe53ae 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -169,7 +169,7 @@ static void vfio_ap_class_init(ObjectClass *klass, void
*data)
set_bit(DEVICE_CATEGORY_MISC, dc->categories);
dc->realize = vfio_ap_realize;
dc->unrealize = vfio_ap_unrealize;
-dc->hotpluggable = false;
+dc->hotpluggable = true;
dc->reset = vfio_ap_reset;
dc->bus_type = TYPE_AP_BUS;
}
--
2.7.4
[Qemu-devel] [PATCH v3 2/2] s390x/vfio-ap: document hot plug/unplug of vfio-ap device
Let's update the vfio-ap.txt document to include the hot plug/unplug support introduced in this patch set. Signed-off-by: Tony Krowiak --- docs/vfio-ap.txt | 61 +++- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/docs/vfio-ap.txt b/docs/vfio-ap.txt index 12339684cd52..8cd060a01e10 100644 --- a/docs/vfio-ap.txt +++ b/docs/vfio-ap.txt @@ -440,8 +440,7 @@ unassign_control_domain 'unassign_domain' file. This may be done multiple times to unassign more than one control domain. -Notes: Hot plug/unplug is not currently supported for mediated AP matrix -devices, so no changes to the AP matrix will be allowed while a guest using +Notes: No changes to the AP matrix will be allowed while a guest using the mediated matrix device is running. Attempts to assign an adapter, domain or control domain will be rejected and an error (EBUSY) returned. @@ -562,6 +561,54 @@ facilities: for guest usage, no AP devices can be made accessible to a guest started without APFT installed. +Hot plug a vfio-ap device into a running guest: +== +Only one vfio-ap device can be attached to the virtual machine's ap-bus, so a +vfio-ap device can be hot plugged if and only if no vfio-ap device is attached +to the bus already, whether via the QEMU command line or a prior hot plug +action. + +To hot plug a vfio-ap device, use the QEMU device_add command: + +(qemu) device_add vfio-ap,sysfsdev="$path-to-mdev" + +Where the '$path-to-mdev' value specifies the absolute path to a mediated +device to which AP resources to be used by the guest have been assigned. + +Note that on Linux guests, the AP devices will be created in the +/sys/bus/ap/devices directory when the AP bus subsequently performs its periodic +scan, so there may be a short delay before the AP devices are accessible on the +guest. + +The command will fail if: + +* A vfio-ap device has already been attached to the virtual machine's ap-bus. + +* The CPU model features for controlling guest access to AP facilities are not + enabled (see 'CPU model features' subsection in the previous section). + +Hot unplug a vfio-ap device from a running guest: + +A vfio-ap device can be unplugged from a running KVM guest if a vfio-ap device +has been attached to the virtual machine's ap-bus via the QEMU command line +or a prior hot plug action. + +To hot unplug a vfio-ap device, use the QEMU device_del command: + +(qemu) device_del vfio-ap,sysfsdev="$path-to-mdev" + +Where $path-to-mdev is the same as the path specified when the vfio-ap +device was attached to the virtual machine's ap-bus. + +On a Linux guest, the AP devices will be removed from the /sys/bus/ap/devices +directory on the guest when the AP bus subsequently performs its periodic scan, +so there may be a short delay before the AP devices are no longer accessible by +the guest. + +The command will fail if the $path-to-mdev specified on the device_del command +does not match the value specified when the vfio-ap device was attached to +the virtual machine's ap-bus. + Example: Configure AP Matrixes for Three Linux Guests: = Let's now provide an example to illustrate how KVM guests may be given @@ -819,7 +866,11 @@ Limitations assigned lest the host be given access to the private data of the AP queue device, such as a private key configured specifically for the guest. -* Dynamically modifying the AP matrix for a running guest (which would amount to - hot(un)plug of AP devices for the guest) is currently not supported +* Dynamically assigning AP resources to or unassigning AP resources from a + mediated matrix device - see 'Configuring an AP matrix for a linux guest' + section above - while a running guest is using it is currently not supported. -* Live guest migration is not supported for guests using AP devices. +* Live guest migration is not supported for guests using AP devices. If a guest + is using AP devices, the vfio-ap device configured for the guest must be + unplugged before migrating the guest (see 'Hot unplug a vfio-ap device from a + running guest' section above. -- 2.7.4
Re: [Qemu-devel] [PATCH v2 0/2] s390x/vfio-ap: hot plug/unplug vfio-ap device
On 2/18/19 12:59 PM, Tony Krowiak wrote: PLEASE IGNORE THIS PATCH SERIES. A v3 VERSION IS FORTHCOMING. VERSION v2 has already been posted and reviewed. This patch series introduces hot plug/unplug of a vfio-ap device. A vfio-ap device can be hot plugged only if: 1. The guest does not yet have a vfio-ap device (only one is allowed per guest) 2. The guest was started with the following CPU model features enabled: * ap=on * apft=on To hot plug a vfio-ap device, the QEMU device_add function may be used: (qemu) device_add vfio-ap,sysfsdev=$path-to-mdev Where $path-to-mdev is the absolute path to the mediated matrix device to be used to configure the guest's AP device matrix. A vfio-ap device can be hot unplugged only if: 1. The guest was started with a vfio-ap device configured for it: -device vfio-ap,sysfsdev=$path-to-mdev 2. The guest was started with the following CPU model features enabled: * ap=on Tony Krowiak (2): s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device s390x/vfio-ap: document hot plug/unplug of vfio-ap device docs/vfio-ap.txt | 58 +++- hw/s390x/ap-bridge.c | 12 ++- hw/vfio/ap.c | 2 +- 3 files changed, 65 insertions(+), 7 deletions(-)
[Qemu-devel] [PATCH v2 1/2] s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device
Introduces hot plug/unplug support for the vfio-ap device. Note that only one
vfio-ap device can be attached to the ap-bus, so a vfio-ap device can only be
hot plugged if the '-device vfio-ap,sysfsdev=$path_to_mdev' option is not
specified on the QEMU command line.
Please note that a hot plug handler is not necessary for the vfio-ap device
because the AP matrix configuration for the guest is performed by the
kernel device driver when the vfio-ap device is realized. The vfio-ap device
represents a VFIO mediated device created in the host sysfs for use by a guest.
The mdev device is configured with an AP matrix (i.e., adapters and domains) via
its sysfs attribute interfaces prior to starting the guest or plugging a vfio-ap
device in. When the device is realized, a file descriptor is opened on the mdev
device which results in a callback to the vfio_ap kernel device driver. The
device driver then configures the AP matrix in the guest's SIE state description
from the AP matrix assigned via the mdev device's sysfs interfaces. The AP
devices will be created for the guest when the AP bus running on the guest
subsequently performs its periodic scan for AP devices.
The qdev_simple_device_unplug_cb() callback function is used for the same
reaons; namely, the vfio_ap kernel device driver will perform the AP resource
de-configuration for the guest when the vfio-ap device is unplugged. When the
vfio-ap device is unrealized, the mdev device file descriptor is closed which
results in a callback to the vfio_ap kernel device driver. The device driver
then clears the AP matrix configuration in the guest's SIE state description
and resets all of the affected queues. The AP devices created for the guest
will be removed when the AP bus running on the guest subsequently performs
its periodic scan and finds there are no longer any AP resources assigned to the
guest.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Reviewed-by: David Hildenbrand
Reviewed-by: Halil Pasic
Tested-by: Pierre Morel
---
hw/s390x/ap-bridge.c | 12 +++-
hw/vfio/ap.c | 2 +-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
index 3795d30dd7c9..25a03412fcb9 100644
--- a/hw/s390x/ap-bridge.c
+++ b/hw/s390x/ap-bridge.c
@@ -39,6 +39,7 @@ static const TypeInfo ap_bus_info = {
void s390_init_ap(void)
{
DeviceState *dev;
+BusState *bus;
/* If no AP instructions then no need for AP bridge */
if (!s390_has_feat(S390_FEAT_AP)) {
@@ -52,13 +53,18 @@ void s390_init_ap(void)
qdev_init_nofail(dev);
/* Create bus on bridge device */
-qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+bus = qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+
+/* Enable hotplugging */
+qbus_set_hotplug_handler(bus, dev, _abort);
}
static void ap_bridge_class_init(ObjectClass *oc, void *data)
{
DeviceClass *dc = DEVICE_CLASS(oc);
+HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
+hc->unplug = qdev_simple_device_unplug_cb;
set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
}
@@ -67,6 +73,10 @@ static const TypeInfo ap_bridge_info = {
.parent= TYPE_SYS_BUS_DEVICE,
.instance_size = 0,
.class_init= ap_bridge_class_init,
+.interfaces = (InterfaceInfo[]) {
+{ TYPE_HOTPLUG_HANDLER },
+{ }
+}
};
static void ap_register(void)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 6166ccd47a4a..d8b79ebe53ae 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -169,7 +169,7 @@ static void vfio_ap_class_init(ObjectClass *klass, void
*data)
set_bit(DEVICE_CATEGORY_MISC, dc->categories);
dc->realize = vfio_ap_realize;
dc->unrealize = vfio_ap_unrealize;
-dc->hotpluggable = false;
+dc->hotpluggable = true;
dc->reset = vfio_ap_reset;
dc->bus_type = TYPE_AP_BUS;
}
--
2.7.4
[Qemu-devel] [PATCH v2 2/2] s390x/vfio-ap: document hot plug/unplug of vfio-ap device
Let's update the vfio-ap.txt document to include the hot plug/unplug support introduced in this patch set. Signed-off-by: Tony Krowiak --- docs/vfio-ap.txt | 58 +++- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/docs/vfio-ap.txt b/docs/vfio-ap.txt index 12339684cd52..fae40f218620 100644 --- a/docs/vfio-ap.txt +++ b/docs/vfio-ap.txt @@ -440,8 +440,7 @@ unassign_control_domain 'unassign_domain' file. This may be done multiple times to unassign more than one control domain. -Notes: Hot plug/unplug is not currently supported for mediated AP matrix -devices, so no changes to the AP matrix will be allowed while a guest using +Notes: No changes to the AP matrix will be allowed while a guest using the mediated matrix device is running. Attempts to assign an adapter, domain or control domain will be rejected and an error (EBUSY) returned. @@ -562,6 +561,51 @@ facilities: for guest usage, no AP devices can be made accessible to a guest started without APFT installed. +Hot plug a vfio-ap device into a running guest: +== +Only one vfio-ap device can be attached to the guest's ap-bus, so a vfio-ap +device can be hot plugged if and only if the +'-device vfio-ap,sysfsdev=$path-to-mdev' option was NOT specified on the QEMU +command line when the guest was started. + +To hot plug a vfio-ap device, use the QEMU device_add command: + +(qemu) device_add vfio-ap,sysfsdev="$path-to-mdev" + +Where the '$path-to-mdev' value specifies the absolute path to a mediated +device configured with an AP matrix identifying the AP resources assigned +to the guest. + +The AP devices will be created in the /sys/bus/ap/devices directory on the +guest when the AP bus subsequently performs its periodic scan, so there may be +a short delay before the AP devices are accessible on the guest. + +The command will fail if: + +* The KVM guest was started with the '-device vfio-ap,sysfs=$path-to-mdev' + QEMU command line option. + +* The CPU model features for controlling guest access to AP facilities are not + enabled (see 'CPU model features' subsection in the previous section). + +Hot unplug a vfio-ap device from a running guest: + +A vfio-ap device can be unplugged from a running KVM guest if the +'-device vfio-ap,sysfsdev=$path-to-mdev' option was specified on the QEMU +command line when the guest was started. + +To hot unplug a vfio-ap device, use the QEMU device_del command: + +(qemu) device_del vfio-ap,sysfsdev="$path-to-mdev" + +The AP devices will be removed from the /sys/bus/ap/devices directory on the +guest when the AP bus subsequently performs its periodic scan, so there may be +a short delay before the AP devices are no longer accessible by the guest. + +The command will fail if the $path-to-mdev specified on the device_del command +does not match the value specified on the '-device vfio-ap,sysfs=$path-to-mdev' +QEMU command line used to start the guest. + Example: Configure AP Matrixes for Three Linux Guests: = Let's now provide an example to illustrate how KVM guests may be given @@ -819,7 +863,11 @@ Limitations assigned lest the host be given access to the private data of the AP queue device, such as a private key configured specifically for the guest. -* Dynamically modifying the AP matrix for a running guest (which would amount to - hot(un)plug of AP devices for the guest) is currently not supported +* Dynamically assigning AP resources to or unassigning AP resources from a + mediated matrix device - see 'Configuring an AP matrix for a linux guest' + section above - while a running guest is using it is currently not supported. -* Live guest migration is not supported for guests using AP devices. +* Live guest migration is not supported for guests using AP devices. If a guest + is using AP devices, the vfio-ap device configured for the guest must be + unplugged before migrating the guest (see 'Hot unplug a vfio-ap device from a + running guest' section above. -- 2.7.4
[Qemu-devel] [PATCH v2 0/2] s390x/vfio-ap: hot plug/unplug vfio-ap device
This patch series introduces hot plug/unplug of a vfio-ap device. A vfio-ap device can be hot plugged only if: 1. The guest does not yet have a vfio-ap device (only one is allowed per guest) 2. The guest was started with the following CPU model features enabled: * ap=on * apft=on To hot plug a vfio-ap device, the QEMU device_add function may be used: (qemu) device_add vfio-ap,sysfsdev=$path-to-mdev Where $path-to-mdev is the absolute path to the mediated matrix device to be used to configure the guest's AP device matrix. A vfio-ap device can be hot unplugged only if: 1. The guest was started with a vfio-ap device configured for it: -device vfio-ap,sysfsdev=$path-to-mdev 2. The guest was started with the following CPU model features enabled: * ap=on Tony Krowiak (2): s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device s390x/vfio-ap: document hot plug/unplug of vfio-ap device docs/vfio-ap.txt | 58 +++- hw/s390x/ap-bridge.c | 12 ++- hw/vfio/ap.c | 2 +- 3 files changed, 65 insertions(+), 7 deletions(-) -- 2.7.4
Re: [Qemu-devel] [PATCH v3] qdev/core: fix qbus_is_full()
On 2/6/19 3:34 AM, Igor Mammedov wrote: On Mon, 28 Jan 2019 15:35:09 -0500 Tony Krowiak wrote: ping! Who is the maintainer responsible for merging this?
Re: [Qemu-devel] [PATCH v3] qdev/core: fix qbus_is_full()
On 12/17/18 10:57 AM, Tony Krowiak wrote:
The qbus_is_full(BusState *bus) function (qdev_monitor.c) compares the max_index
value of the BusState structure with the max_dev value of the BusClass structure
to determine whether the maximum number of children has been reached for the
bus. The problem is, the max_index field of the BusState structure does not
necessarily reflect the number of devices that have been plugged into
the bus.
Whenever a child device is plugged into the bus, the bus's max_index value is
assigned to the child device and then incremented. If the child is subsequently
unplugged, the value of the max_index does not change and no longer reflects the
number of children.
When the bus's max_index value reaches the maximum number of devices
allowed for the bus (i.e., the max_dev field in the BusClass structure),
attempts to plug another device will be rejected claiming that the bus is
full -- even if the bus is actually empty.
To resolve the problem, a new 'num_children' field is being added to the
BusState structure to keep track of the number of children plugged into the
bus. It will be incremented when a child is plugged, and decremented when a
child is unplugged.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Reviewed-by: Halil Pasic
---
hw/core/qdev.c | 3 +++
include/hw/qdev-core.h | 1 +
qdev-monitor.c | 2 +-
3 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 6b3cc55b27c2..956923f33520 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState
*child)
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
+bus->num_children--;
+
/* This gives back ownership of kid->child back to us. */
object_property_del(OBJECT(bus), name, NULL);
object_unref(OBJECT(kid->child));
@@ -73,6 +75,7 @@ static void bus_add_child(BusState *bus, DeviceState *child)
char name[32];
BusChild *kid = g_malloc0(sizeof(*kid));
+bus->num_children++;
kid->index = bus->max_index++;
kid->child = child;
object_ref(OBJECT(kid->child));
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index a24d0dd566e3..521f0a947ead 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -206,6 +206,7 @@ struct BusState {
HotplugHandler *hotplug_handler;
int max_index;
bool realized;
+int num_children;
QTAILQ_HEAD(ChildrenHead, BusChild) children;
QLIST_ENTRY(BusState) sibling;
};
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 07147c63bf8b..45a8ba49644c 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -414,7 +414,7 @@ static DeviceState *qbus_find_dev(BusState *bus, char *elem)
static inline bool qbus_is_full(BusState *bus)
{
BusClass *bus_class = BUS_GET_CLASS(bus);
-return bus_class->max_dev && bus->max_index >= bus_class->max_dev;
+return bus_class->max_dev && bus->num_children >= bus_class->max_dev;
}
/*
Just checking back on this one. Do we want to merge this patch and deal
with the max_index issue in another patch, in this patch, or not at all?
[Qemu-devel] [PATCH v2] s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device
Introduces hot plug/unplug support for the vfio-ap device. Note that only one vfio-ap device can be attached to the ap-bus, so a vfio-ap device can only be hot plugged if the '-device vfio-ap,sysfsdev=$path_to_mdev' option is not specified on the QEMU command line. Please note that a hot plug handler is not necessary for the vfio-ap device because the AP matrix configuration for the guest is performed by the kernel device driver when the vfio-ap device is realized. The vfio-ap device represents a VFIO mediated device created in the host sysfs for use by a guest. The mdev device is configured with an AP matrix (i.e., adapters and domains) via its sysfs attribute interfaces prior to starting the guest or plugging a vfio-ap device in. When the device is realized, a file descriptor is opened on the mdev device which results in a callback to the vfio_ap kernel device driver. The device driver then configures the AP matrix in the guest's SIE state description from the AP matrix assigned via the mdev device's sysfs interfaces. The AP devices will be created for the guest when the AP bus running on the guest subsequently performs its periodic scan for AP devices. The qdev_simple_device_unplug_cb() callback function is used for the same reaons; namely, the vfio_ap kernel device driver will perform the AP resource de-configuration for the guest when the vfio-ap device is unplugged. When the vfio-ap device is unrealized, the mdev device file descriptor is closed which results in a callback to the vfio_ap kernel device driver. The device driver then clears the AP matrix configuration in the guest's SIE state description and resets all of the affected queues. The AP devices created for the guest will be removed when the AP bus running on the guest subsequently performs its periodic scan and finds there are no longer any AP resources assigned to the guest. Signed-off-by: Tony Krowiak Reviewed-by: Pierre Morel Reviewed-by: David Hildenbrand Reviewed-by: Halil Pasic Tested-by: Pierre Morel --- docs/vfio-ap.txt | 58 +++- hw/s390x/ap-bridge.c | 12 ++- hw/vfio/ap.c | 2 +- 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/docs/vfio-ap.txt b/docs/vfio-ap.txt index 12339684cd52..fae40f218620 100644 --- a/docs/vfio-ap.txt +++ b/docs/vfio-ap.txt @@ -440,8 +440,7 @@ unassign_control_domain 'unassign_domain' file. This may be done multiple times to unassign more than one control domain. -Notes: Hot plug/unplug is not currently supported for mediated AP matrix -devices, so no changes to the AP matrix will be allowed while a guest using +Notes: No changes to the AP matrix will be allowed while a guest using the mediated matrix device is running. Attempts to assign an adapter, domain or control domain will be rejected and an error (EBUSY) returned. @@ -562,6 +561,51 @@ facilities: for guest usage, no AP devices can be made accessible to a guest started without APFT installed. +Hot plug a vfio-ap device into a running guest: +== +Only one vfio-ap device can be attached to the guest's ap-bus, so a vfio-ap +device can be hot plugged if and only if the +'-device vfio-ap,sysfsdev=$path-to-mdev' option was NOT specified on the QEMU +command line when the guest was started. + +To hot plug a vfio-ap device, use the QEMU device_add command: + +(qemu) device_add vfio-ap,sysfsdev="$path-to-mdev" + +Where the '$path-to-mdev' value specifies the absolute path to a mediated +device configured with an AP matrix identifying the AP resources assigned +to the guest. + +The AP devices will be created in the /sys/bus/ap/devices directory on the +guest when the AP bus subsequently performs its periodic scan, so there may be +a short delay before the AP devices are accessible on the guest. + +The command will fail if: + +* The KVM guest was started with the '-device vfio-ap,sysfs=$path-to-mdev' + QEMU command line option. + +* The CPU model features for controlling guest access to AP facilities are not + enabled (see 'CPU model features' subsection in the previous section). + +Hot unplug a vfio-ap device from a running guest: + +A vfio-ap device can be unplugged from a running KVM guest if the +'-device vfio-ap,sysfsdev=$path-to-mdev' option was specified on the QEMU +command line when the guest was started. + +To hot unplug a vfio-ap device, use the QEMU device_del command: + +(qemu) device_del vfio-ap,sysfsdev="$path-to-mdev" + +The AP devices will be removed from the /sys/bus/ap/devices directory on the +guest when the AP bus subsequently performs its periodic scan, so there may be +a short delay before the AP devices are no longer accessible by the guest. + +The command will fail if the $path-to-mdev specified on the device_del command +does not match the value speci
Re: [Qemu-devel] [PATCH] s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device
On 1/14/19 9:16 AM, Cornelia Huck wrote: On Thu, 10 Jan 2019 11:22:48 -0500 Tony Krowiak wrote: Can you extend this patch description a little bit, including what we discussed here? Maybe a short comment that explains why qdev_simple_device_unplug_cb() is appropriate as well (i.e. hits that closing the mdev's fd is what triggers the cleanup of the actual resources)? I personally go log digging only once I get desperate. I go digging if I can't find a public document on how it works ;) Which reminds me, I will also need to update the docs/vfio-ap.txt document. So, I assume there will be a v2 with some documentation changes? I'm sorry Connie, I got sidetracked with some other priorities and missed this response. I will post a v2 with doc changes.
Re: [Qemu-devel] [PATCH] s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device
On 1/9/19 12:28 PM, David Hildenbrand wrote:
On 09.01.19 18:13, Halil Pasic wrote:
On Wed, 9 Jan 2019 17:37:49 +0100
David Hildenbrand wrote:
On 09.01.19 17:27, Tony Krowiak wrote:
On 1/9/19 6:30 AM, Cornelia Huck wrote:
On Tue, 8 Jan 2019 23:13:39 +0100
David Hildenbrand wrote:
On 08.01.19 20:52, Tony Krowiak wrote:
On 1/8/19 11:09 AM, David Hildenbrand wrote:
On 08.01.19 17:01, Tony Krowiak wrote:
Introduces hot plug/unplug support for the vfio-ap device. Note that only one
vfio-ap device can be attached to the ap-bus, so a vfio-ap device can only be
hot plugged if the '-device vfio-ap,sysfsdev=$path_to_mdev' option is not
specified on the QEMU command line.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Tested-by: Pierre Morel
---
hw/s390x/ap-bridge.c | 12 +++-
hw/vfio/ap.c | 2 +-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
index 3795d30dd7c9..25a03412fcb9 100644
--- a/hw/s390x/ap-bridge.c
+++ b/hw/s390x/ap-bridge.c
@@ -39,6 +39,7 @@ static const TypeInfo ap_bus_info = {
void s390_init_ap(void)
{
DeviceState *dev;
+BusState *bus;
/* If no AP instructions then no need for AP bridge */
if (!s390_has_feat(S390_FEAT_AP)) {
@@ -52,13 +53,18 @@ void s390_init_ap(void)
qdev_init_nofail(dev);
/* Create bus on bridge device */
-qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+bus = qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+
+/* Enable hotplugging */
+qbus_set_hotplug_handler(bus, dev, _abort);
}
static void ap_bridge_class_init(ObjectClass *oc, void *data)
{
DeviceClass *dc = DEVICE_CLASS(oc);
+HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
+hc->unplug = qdev_simple_device_unplug_cb;
confused, why is there no plug action?
You will find this is the case for several devices that are hot
pluggable.
Usually missing hotplug handlers are an indication of legacy code ;)
The plug callback is invoked after the device is
attached to the bus and after the device is realized. Not having
a hot plug callback does not preclude hot plugging of a device.
The hotplug handler is there to
1. Assign resources (e.g. ids etc)
2. Notify the system (e.g. hotplug interrupt)
In legacy code (e.g. PCI) such stuff is usually still located in the
realize function (where it doesn't belong anymore but factoring out is
hard ...)
Looking at hw/vfio/ap.c:realize(), there isn't really anything in there.
So I assume that
1. No resources have to be assigned (for vfio-ap, I guess the IDs and
such are implicit)
That's my understanding as well. The interesting stuff will be
configured on kernel level before the device is even handed to QEMU for
consumption.
The vfio-ap device represents a VFIO mdev device. AP resources - i.e.,
adapters, domains and control domains - are assigned to the mdev device
via its sysfs interfaces. This is all handled by the vfio_ap kernel
driver before a guest can use the mdev device. As part of vfio-ap device
realization, a file descriptor is opened on the mdev device. When the
mdev device's fd is opened, a callback is invoked on the vfio_ap kernel
device driver. The device driver then updates the guest's AP matrix
configuration based on the configuration specified via the mdev
device's sysfs interfaces.
(Would be nice to hint at that in the patch description.)
2. No notification will happen. How will the guest know that a new AP
adapter is available?
My understanding is that hotplugging the matrix device will make the
guest go from "no adapters/domains are available" to "some
adapters/domains are available" (and unplug will do the reverse). I.e.,
no hot(un)plugging of individual queues (which would need to be done on
the kernel level, and is tricky IIRC.)
I'm not sure what the architectured options for notifying the guest are
(I dimly recall some kind of "AP configuration has changed event").
IIRC, the Linux guest driver scans for new queues periodically. Does it
even do that if no queues are available to start with?
The AP bus - in this case, running in the guest - does a periodic scan
for AP devices. The bus relies on an AP instruction that queries the
AP configuration information. When the guest's AP matrix is updated -
see description of mdev device fd open processing above - the query
will provide the newly configured AP matrix and the bus will create
the adapter and queue devices on the guest. Consequently, there is
nothing to do in a hot plug handler. If you'd like, I'd be more than
happy to include a hot plug handler that does some logging (or nothing
at all) so it doesn't look like legacy code ;)
Hehe, no it's fine for me.
Can you extend this patch description a little bit, including what we
discussed here?
Maybe a short comment that explains why qdev_simple_device_unplug_cb()
is appropriate as well (i.e. hits t
Re: [Qemu-devel] [PATCH v3] qdev/core: fix qbus_is_full()
On 1/9/19 12:35 PM, Halil Pasic wrote: On Wed, 9 Jan 2019 10:36:11 -0500 Tony Krowiak wrote: On 1/9/19 5:14 AM, Cornelia Huck wrote: On Tue, 8 Jan 2019 15:34:37 -0500 Tony Krowiak wrote: On 1/8/19 12:06 PM, Cornelia Huck wrote: On Tue, 8 Jan 2019 17:50:21 +0100 Halil Pasic wrote: On Tue, 8 Jan 2019 17:31:13 +0100 Cornelia Huck wrote: On Tue, 8 Jan 2019 11:08:56 -0500 Tony Krowiak wrote: On 12/17/18 10:57 AM, Tony Krowiak wrote: The qbus_is_full(BusState *bus) function (qdev_monitor.c) compares the max_index value of the BusState structure with the max_dev value of the BusClass structure to determine whether the maximum number of children has been reached for the bus. The problem is, the max_index field of the BusState structure does not necessarily reflect the number of devices that have been plugged into the bus. Whenever a child device is plugged into the bus, the bus's max_index value is assigned to the child device and then incremented. If the child is subsequently unplugged, the value of the max_index does not change and no longer reflects the number of children. When the bus's max_index value reaches the maximum number of devices allowed for the bus (i.e., the max_dev field in the BusClass structure), attempts to plug another device will be rejected claiming that the bus is full -- even if the bus is actually empty. To resolve the problem, a new 'num_children' field is being added to the BusState structure to keep track of the number of children plugged into the bus. It will be incremented when a child is plugged, and decremented when a child is unplugged. Signed-off-by: Tony Krowiak Reviewed-by: Pierre Morel Reviewed-by: Halil Pasic --- hw/core/qdev.c | 3 +++ include/hw/qdev-core.h | 1 + qdev-monitor.c | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) Ping ... I could not determine who the maintainer is for the three files listed above. I checked the MAINTAINERS file and the prologue of each individual file. Can someone please tell me who is responsible for merging these changes? Any additional review comments? diff --git a/hw/core/qdev.c b/hw/core/qdev.c index 6b3cc55b27c2..956923f33520 100644 --- a/hw/core/qdev.c +++ b/hw/core/qdev.c @@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState *child) snprintf(name, sizeof(name), "child[%d]", kid->index); QTAILQ_REMOVE(>children, kid, sibling); +bus->num_children--; + /* This gives back ownership of kid->child back to us. */ object_property_del(OBJECT(bus), name, NULL); object_unref(OBJECT(kid->child)); @@ -73,6 +75,7 @@ static void bus_add_child(BusState *bus, DeviceState *child) char name[32]; BusChild *kid = g_malloc0(sizeof(*kid)); +bus->num_children++; kid->index = bus->max_index++; Hm... I'm wondering what happens for insane numbers of hotplugging operations here? (Preexisting problem for busses without limit, but busses with a limit could now run into that as well.) How does this patch change things? I mean bus->num_children gets decremented on unplug. We don't stop anymore if max_index >= max_dev, which means that we can now trigger that even if max_dev != 0. I guess I am missing your point. If max_dev == 0, then there is nothing stopping an insane number of hot plug operations; either before this patch, or with this patch. With the patch, once the number of children hot plugged reaches max_dev, the qbus_is_full function will return false and no more children can be plugged. If a child device is unplugged, the num_children - which counts the number of children attached to the bus - will be decremented, so it always reflects the number of children added to the bus. Besides, checking max_index against max_dev is erroneous, because max_index is incremented every time a child device is plugged and is never decremented. It really operates as more of a uinique identifier than a counter and is also used to create a unique property name when the child device is linked to the bus as a property (see bus_add_child function in hw/core/qdev.c). Checking num_children against max_dev is the right thing to do, no argument here. However, max_index continues to be incremented even if devices have been unplugged again. That means it can overflow, as it is never bound by the max_dev constraint. This has been a problem before for busses with an unrestricted number of devices before, but with your patch, the problem is now triggerable for all busses. Not a problem with your patch, but we might want to look into making max_index overflow/wraparound save. I see your point. It does beg the question, what are the chances that max_index reaches INT_MAX? In the interest of making this code more bullet proof, I suppose it is something that should be deal
Re: [Qemu-devel] [PATCH] s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device
On 1/9/19 6:30 AM, Cornelia Huck wrote:
On Tue, 8 Jan 2019 23:13:39 +0100
David Hildenbrand wrote:
On 08.01.19 20:52, Tony Krowiak wrote:
On 1/8/19 11:09 AM, David Hildenbrand wrote:
On 08.01.19 17:01, Tony Krowiak wrote:
Introduces hot plug/unplug support for the vfio-ap device. Note that only one
vfio-ap device can be attached to the ap-bus, so a vfio-ap device can only be
hot plugged if the '-device vfio-ap,sysfsdev=$path_to_mdev' option is not
specified on the QEMU command line.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Tested-by: Pierre Morel
---
hw/s390x/ap-bridge.c | 12 +++-
hw/vfio/ap.c | 2 +-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
index 3795d30dd7c9..25a03412fcb9 100644
--- a/hw/s390x/ap-bridge.c
+++ b/hw/s390x/ap-bridge.c
@@ -39,6 +39,7 @@ static const TypeInfo ap_bus_info = {
void s390_init_ap(void)
{
DeviceState *dev;
+BusState *bus;
/* If no AP instructions then no need for AP bridge */
if (!s390_has_feat(S390_FEAT_AP)) {
@@ -52,13 +53,18 @@ void s390_init_ap(void)
qdev_init_nofail(dev);
/* Create bus on bridge device */
-qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+bus = qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+
+/* Enable hotplugging */
+qbus_set_hotplug_handler(bus, dev, _abort);
}
static void ap_bridge_class_init(ObjectClass *oc, void *data)
{
DeviceClass *dc = DEVICE_CLASS(oc);
+HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
+hc->unplug = qdev_simple_device_unplug_cb;
confused, why is there no plug action?
You will find this is the case for several devices that are hot
pluggable.
Usually missing hotplug handlers are an indication of legacy code ;)
The plug callback is invoked after the device is
attached to the bus and after the device is realized. Not having
a hot plug callback does not preclude hot plugging of a device.
The hotplug handler is there to
1. Assign resources (e.g. ids etc)
2. Notify the system (e.g. hotplug interrupt)
In legacy code (e.g. PCI) such stuff is usually still located in the
realize function (where it doesn't belong anymore but factoring out is
hard ...)
Looking at hw/vfio/ap.c:realize(), there isn't really anything in there.
So I assume that
1. No resources have to be assigned (for vfio-ap, I guess the IDs and
such are implicit)
That's my understanding as well. The interesting stuff will be
configured on kernel level before the device is even handed to QEMU for
consumption.
The vfio-ap device represents a VFIO mdev device. AP resources - i.e.,
adapters, domains and control domains - are assigned to the mdev device
via its sysfs interfaces. This is all handled by the vfio_ap kernel
driver before a guest can use the mdev device. As part of vfio-ap device
realization, a file descriptor is opened on the mdev device. When the
mdev device's fd is opened, a callback is invoked on the vfio_ap kernel
device driver. The device driver then updates the guest's AP matrix
configuration based on the configuration specified via the mdev
device's sysfs interfaces.
(Would be nice to hint at that in the patch description.)
2. No notification will happen. How will the guest know that a new AP
adapter is available?
My understanding is that hotplugging the matrix device will make the
guest go from "no adapters/domains are available" to "some
adapters/domains are available" (and unplug will do the reverse). I.e.,
no hot(un)plugging of individual queues (which would need to be done on
the kernel level, and is tricky IIRC.)
I'm not sure what the architectured options for notifying the guest are
(I dimly recall some kind of "AP configuration has changed event").
IIRC, the Linux guest driver scans for new queues periodically. Does it
even do that if no queues are available to start with?
The AP bus - in this case, running in the guest - does a periodic scan
for AP devices. The bus relies on an AP instruction that queries the
AP configuration information. When the guest's AP matrix is updated -
see description of mdev device fd open processing above - the query
will provide the newly configured AP matrix and the bus will create
the adapter and queue devices on the guest. Consequently, there is
nothing to do in a hot plug handler. If you'd like, I'd be more than
happy to include a hot plug handler that does some logging (or nothing
at all) so it doesn't look like legacy code ;)
I presume the purpose of the callback is to provide an opportunity
to do perform any additional processing that may be required to prepare
the device for use. In the case of the vfio-ap device, there is nothing
to do once the device is plugged.
When removing the device, is it really a forced removal? ("simply rip it
out without telling the guest")
When the vfio-ap device is unrealized, the mdev devi
Re: [Qemu-devel] [PATCH v3] qdev/core: fix qbus_is_full()
On 1/9/19 5:14 AM, Cornelia Huck wrote: On Tue, 8 Jan 2019 15:34:37 -0500 Tony Krowiak wrote: On 1/8/19 12:06 PM, Cornelia Huck wrote: On Tue, 8 Jan 2019 17:50:21 +0100 Halil Pasic wrote: On Tue, 8 Jan 2019 17:31:13 +0100 Cornelia Huck wrote: On Tue, 8 Jan 2019 11:08:56 -0500 Tony Krowiak wrote: On 12/17/18 10:57 AM, Tony Krowiak wrote: The qbus_is_full(BusState *bus) function (qdev_monitor.c) compares the max_index value of the BusState structure with the max_dev value of the BusClass structure to determine whether the maximum number of children has been reached for the bus. The problem is, the max_index field of the BusState structure does not necessarily reflect the number of devices that have been plugged into the bus. Whenever a child device is plugged into the bus, the bus's max_index value is assigned to the child device and then incremented. If the child is subsequently unplugged, the value of the max_index does not change and no longer reflects the number of children. When the bus's max_index value reaches the maximum number of devices allowed for the bus (i.e., the max_dev field in the BusClass structure), attempts to plug another device will be rejected claiming that the bus is full -- even if the bus is actually empty. To resolve the problem, a new 'num_children' field is being added to the BusState structure to keep track of the number of children plugged into the bus. It will be incremented when a child is plugged, and decremented when a child is unplugged. Signed-off-by: Tony Krowiak Reviewed-by: Pierre Morel Reviewed-by: Halil Pasic --- hw/core/qdev.c | 3 +++ include/hw/qdev-core.h | 1 + qdev-monitor.c | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) Ping ... I could not determine who the maintainer is for the three files listed above. I checked the MAINTAINERS file and the prologue of each individual file. Can someone please tell me who is responsible for merging these changes? Any additional review comments? diff --git a/hw/core/qdev.c b/hw/core/qdev.c index 6b3cc55b27c2..956923f33520 100644 --- a/hw/core/qdev.c +++ b/hw/core/qdev.c @@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState *child) snprintf(name, sizeof(name), "child[%d]", kid->index); QTAILQ_REMOVE(>children, kid, sibling); +bus->num_children--; + /* This gives back ownership of kid->child back to us. */ object_property_del(OBJECT(bus), name, NULL); object_unref(OBJECT(kid->child)); @@ -73,6 +75,7 @@ static void bus_add_child(BusState *bus, DeviceState *child) char name[32]; BusChild *kid = g_malloc0(sizeof(*kid)); +bus->num_children++; kid->index = bus->max_index++; Hm... I'm wondering what happens for insane numbers of hotplugging operations here? (Preexisting problem for busses without limit, but busses with a limit could now run into that as well.) How does this patch change things? I mean bus->num_children gets decremented on unplug. We don't stop anymore if max_index >= max_dev, which means that we can now trigger that even if max_dev != 0. I guess I am missing your point. If max_dev == 0, then there is nothing stopping an insane number of hot plug operations; either before this patch, or with this patch. With the patch, once the number of children hot plugged reaches max_dev, the qbus_is_full function will return false and no more children can be plugged. If a child device is unplugged, the num_children - which counts the number of children attached to the bus - will be decremented, so it always reflects the number of children added to the bus. Besides, checking max_index against max_dev is erroneous, because max_index is incremented every time a child device is plugged and is never decremented. It really operates as more of a uinique identifier than a counter and is also used to create a unique property name when the child device is linked to the bus as a property (see bus_add_child function in hw/core/qdev.c). Checking num_children against max_dev is the right thing to do, no argument here. However, max_index continues to be incremented even if devices have been unplugged again. That means it can overflow, as it is never bound by the max_dev constraint. This has been a problem before for busses with an unrestricted number of devices before, but with your patch, the problem is now triggerable for all busses. Not a problem with your patch, but we might want to look into making max_index overflow/wraparound save. I see your point. It does beg the question, what are the chances that max_index reaches INT_MAX? In the interest of making this code more bullet proof, I suppose it is something that should be dealt with. A search reveals that max_index is used in only two places: It is used to set the index for a child of the bus
Re: [Qemu-devel] [PATCH v3] qdev/core: fix qbus_is_full()
On 1/8/19 12:06 PM, Cornelia Huck wrote:
On Tue, 8 Jan 2019 17:50:21 +0100
Halil Pasic wrote:
On Tue, 8 Jan 2019 17:31:13 +0100
Cornelia Huck wrote:
On Tue, 8 Jan 2019 11:08:56 -0500
Tony Krowiak wrote:
On 12/17/18 10:57 AM, Tony Krowiak wrote:
The qbus_is_full(BusState *bus) function (qdev_monitor.c) compares the max_index
value of the BusState structure with the max_dev value of the BusClass structure
to determine whether the maximum number of children has been reached for the
bus. The problem is, the max_index field of the BusState structure does not
necessarily reflect the number of devices that have been plugged into
the bus.
Whenever a child device is plugged into the bus, the bus's max_index value is
assigned to the child device and then incremented. If the child is subsequently
unplugged, the value of the max_index does not change and no longer reflects the
number of children.
When the bus's max_index value reaches the maximum number of devices
allowed for the bus (i.e., the max_dev field in the BusClass structure),
attempts to plug another device will be rejected claiming that the bus is
full -- even if the bus is actually empty.
To resolve the problem, a new 'num_children' field is being added to the
BusState structure to keep track of the number of children plugged into the
bus. It will be incremented when a child is plugged, and decremented when a
child is unplugged.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Reviewed-by: Halil Pasic
---
hw/core/qdev.c | 3 +++
include/hw/qdev-core.h | 1 +
qdev-monitor.c | 2 +-
3 files changed, 5 insertions(+), 1 deletion(-)
Ping ...
I could not determine who the maintainer is for the three files
listed above. I checked the MAINTAINERS file and the prologue of each
individual file. Can someone please tell me who is responsible
for merging these changes? Any additional review comments?
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 6b3cc55b27c2..956923f33520 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState
*child)
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
+bus->num_children--;
+
/* This gives back ownership of kid->child back to us. */
object_property_del(OBJECT(bus), name, NULL);
object_unref(OBJECT(kid->child));
@@ -73,6 +75,7 @@ static void bus_add_child(BusState *bus, DeviceState *child)
char name[32];
BusChild *kid = g_malloc0(sizeof(*kid));
+bus->num_children++;
kid->index = bus->max_index++;
Hm... I'm wondering what happens for insane numbers of hotplugging
operations here?
(Preexisting problem for busses without limit, but busses with a limit
could now run into that as well.)
How does this patch change things? I mean bus->num_children gets
decremented on unplug.
We don't stop anymore if max_index >= max_dev, which means that we can
now trigger that even if max_dev != 0.
I guess I am missing your point. If max_dev == 0, then there is nothing
stopping an insane number of hot plug operations; either before this
patch, or with this patch. With the patch, once the number of children
hot plugged reaches max_dev, the qbus_is_full function will return false
and no more children can be plugged. If a child device is unplugged,
the num_children - which counts the number of children attached to the
bus - will be decremented, so it always reflects the number of children
added to the bus. Besides, checking max_index against max_dev
is erroneous, because max_index is incremented every time a child device
is plugged and is never decremented. It really operates as more of a
uinique identifier than a counter and is also used to create a unique
property name when the child device is linked to the bus as a property
(see bus_add_child function in hw/core/qdev.c).
Regards,
Halil
kid->child = child;
object_ref(OBJECT(kid->child));
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index a24d0dd566e3..521f0a947ead 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -206,6 +206,7 @@ struct BusState {
HotplugHandler *hotplug_handler;
int max_index;
bool realized;
+int num_children;
QTAILQ_HEAD(ChildrenHead, BusChild) children;
QLIST_ENTRY(BusState) sibling;
};
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 07147c63bf8b..45a8ba49644c 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -414,7 +414,7 @@ static DeviceState *qbus_find_dev(BusState *bus, char *elem)
static inline bool qbus_is_full(BusState *bus)
{
BusClass *bus_class = BUS_GET_CLASS(bus);
-return bus_class->max_dev && bus->max_index >= bus_class->max_dev;
+return bus_class->max_dev && bus->num_chi
Re: [Qemu-devel] [PATCH] s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device
On 1/8/19 11:09 AM, David Hildenbrand wrote:
On 08.01.19 17:01, Tony Krowiak wrote:
Introduces hot plug/unplug support for the vfio-ap device. Note that only one
vfio-ap device can be attached to the ap-bus, so a vfio-ap device can only be
hot plugged if the '-device vfio-ap,sysfsdev=$path_to_mdev' option is not
specified on the QEMU command line.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Tested-by: Pierre Morel
---
hw/s390x/ap-bridge.c | 12 +++-
hw/vfio/ap.c | 2 +-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
index 3795d30dd7c9..25a03412fcb9 100644
--- a/hw/s390x/ap-bridge.c
+++ b/hw/s390x/ap-bridge.c
@@ -39,6 +39,7 @@ static const TypeInfo ap_bus_info = {
void s390_init_ap(void)
{
DeviceState *dev;
+BusState *bus;
/* If no AP instructions then no need for AP bridge */
if (!s390_has_feat(S390_FEAT_AP)) {
@@ -52,13 +53,18 @@ void s390_init_ap(void)
qdev_init_nofail(dev);
/* Create bus on bridge device */
-qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+bus = qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+
+/* Enable hotplugging */
+qbus_set_hotplug_handler(bus, dev, _abort);
}
static void ap_bridge_class_init(ObjectClass *oc, void *data)
{
DeviceClass *dc = DEVICE_CLASS(oc);
+HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
+hc->unplug = qdev_simple_device_unplug_cb;
confused, why is there no plug action?
You will find this is the case for several devices that are hot
pluggable. The plug callback is invoked after the device is
attached to the bus and after the device is realized. Not having
a hot plug callback does not preclude hot plugging of a device.
I presume the purpose of the callback is to provide an opportunity
to do perform any additional processing that may be required to prepare
the device for use. In the case of the vfio-ap device, there is nothing
to do once the device is plugged.
set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
}
@@ -67,6 +73,10 @@ static const TypeInfo ap_bridge_info = {
.parent= TYPE_SYS_BUS_DEVICE,
.instance_size = 0,
.class_init= ap_bridge_class_init,
+.interfaces = (InterfaceInfo[]) {
+{ TYPE_HOTPLUG_HANDLER },
+{ }
+}
};
static void ap_register(void)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 6166ccd47a4a..d8b79ebe53ae 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -169,7 +169,7 @@ static void vfio_ap_class_init(ObjectClass *klass, void
*data)
set_bit(DEVICE_CATEGORY_MISC, dc->categories);
dc->realize = vfio_ap_realize;
dc->unrealize = vfio_ap_unrealize;
-dc->hotpluggable = false;
+dc->hotpluggable = true;
dc->reset = vfio_ap_reset;
dc->bus_type = TYPE_AP_BUS;
}
Re: [Qemu-devel] [PATCH] s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device
On 1/8/19 11:01 AM, Tony Krowiak wrote: Introduces hot plug/unplug support for the vfio-ap device. Note that only one vfio-ap device can be attached to the ap-bus, so a vfio-ap device can only be hot plugged if the '-device vfio-ap,sysfsdev=$path_to_mdev' option is not specified on the QEMU command line. Signed-off-by: Tony Krowiak Reviewed-by: Pierre Morel Tested-by: Pierre Morel --- hw/s390x/ap-bridge.c | 12 +++- hw/vfio/ap.c | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) Please note: During testing of this patch I discovered a bug whereby an attempt to plug in a vfio-ap device subsequent to an unplug of the device will result in the following error: error: Failed to attach device from /root/conf/vfio-ap.xml error: internal error: unable to execute QEMU command 'device_add': No 'ap-bus' bus found for device 'vfio-ap' I wrote a patch for this bug: Message ID: <1545062250-7573-1-git-send-email-akrow...@linux.ibm.com>
Re: [Qemu-devel] [PATCH v3] qdev/core: fix qbus_is_full()
On 12/17/18 10:57 AM, Tony Krowiak wrote:
The qbus_is_full(BusState *bus) function (qdev_monitor.c) compares the max_index
value of the BusState structure with the max_dev value of the BusClass structure
to determine whether the maximum number of children has been reached for the
bus. The problem is, the max_index field of the BusState structure does not
necessarily reflect the number of devices that have been plugged into
the bus.
Whenever a child device is plugged into the bus, the bus's max_index value is
assigned to the child device and then incremented. If the child is subsequently
unplugged, the value of the max_index does not change and no longer reflects the
number of children.
When the bus's max_index value reaches the maximum number of devices
allowed for the bus (i.e., the max_dev field in the BusClass structure),
attempts to plug another device will be rejected claiming that the bus is
full -- even if the bus is actually empty.
To resolve the problem, a new 'num_children' field is being added to the
BusState structure to keep track of the number of children plugged into the
bus. It will be incremented when a child is plugged, and decremented when a
child is unplugged.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Reviewed-by: Halil Pasic
---
hw/core/qdev.c | 3 +++
include/hw/qdev-core.h | 1 +
qdev-monitor.c | 2 +-
3 files changed, 5 insertions(+), 1 deletion(-)
Ping ...
I could not determine who the maintainer is for the three files
listed above. I checked the MAINTAINERS file and the prologue of each
individual file. Can someone please tell me who is responsible
for merging these changes? Any additional review comments?
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 6b3cc55b27c2..956923f33520 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState
*child)
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
+bus->num_children--;
+
/* This gives back ownership of kid->child back to us. */
object_property_del(OBJECT(bus), name, NULL);
object_unref(OBJECT(kid->child));
@@ -73,6 +75,7 @@ static void bus_add_child(BusState *bus, DeviceState *child)
char name[32];
BusChild *kid = g_malloc0(sizeof(*kid));
+bus->num_children++;
kid->index = bus->max_index++;
kid->child = child;
object_ref(OBJECT(kid->child));
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index a24d0dd566e3..521f0a947ead 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -206,6 +206,7 @@ struct BusState {
HotplugHandler *hotplug_handler;
int max_index;
bool realized;
+int num_children;
QTAILQ_HEAD(ChildrenHead, BusChild) children;
QLIST_ENTRY(BusState) sibling;
};
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 07147c63bf8b..45a8ba49644c 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -414,7 +414,7 @@ static DeviceState *qbus_find_dev(BusState *bus, char *elem)
static inline bool qbus_is_full(BusState *bus)
{
BusClass *bus_class = BUS_GET_CLASS(bus);
-return bus_class->max_dev && bus->max_index >= bus_class->max_dev;
+return bus_class->max_dev && bus->num_children >= bus_class->max_dev;
}
/*
[Qemu-devel] [PATCH] s390x/vfio-ap: Implement hot plug/unplug of vfio-ap device
Introduces hot plug/unplug support for the vfio-ap device. Note that only one
vfio-ap device can be attached to the ap-bus, so a vfio-ap device can only be
hot plugged if the '-device vfio-ap,sysfsdev=$path_to_mdev' option is not
specified on the QEMU command line.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Tested-by: Pierre Morel
---
hw/s390x/ap-bridge.c | 12 +++-
hw/vfio/ap.c | 2 +-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
index 3795d30dd7c9..25a03412fcb9 100644
--- a/hw/s390x/ap-bridge.c
+++ b/hw/s390x/ap-bridge.c
@@ -39,6 +39,7 @@ static const TypeInfo ap_bus_info = {
void s390_init_ap(void)
{
DeviceState *dev;
+BusState *bus;
/* If no AP instructions then no need for AP bridge */
if (!s390_has_feat(S390_FEAT_AP)) {
@@ -52,13 +53,18 @@ void s390_init_ap(void)
qdev_init_nofail(dev);
/* Create bus on bridge device */
-qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+bus = qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+
+/* Enable hotplugging */
+qbus_set_hotplug_handler(bus, dev, _abort);
}
static void ap_bridge_class_init(ObjectClass *oc, void *data)
{
DeviceClass *dc = DEVICE_CLASS(oc);
+HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
+hc->unplug = qdev_simple_device_unplug_cb;
set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
}
@@ -67,6 +73,10 @@ static const TypeInfo ap_bridge_info = {
.parent= TYPE_SYS_BUS_DEVICE,
.instance_size = 0,
.class_init= ap_bridge_class_init,
+.interfaces = (InterfaceInfo[]) {
+{ TYPE_HOTPLUG_HANDLER },
+{ }
+}
};
static void ap_register(void)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 6166ccd47a4a..d8b79ebe53ae 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -169,7 +169,7 @@ static void vfio_ap_class_init(ObjectClass *klass, void
*data)
set_bit(DEVICE_CATEGORY_MISC, dc->categories);
dc->realize = vfio_ap_realize;
dc->unrealize = vfio_ap_unrealize;
-dc->hotpluggable = false;
+dc->hotpluggable = true;
dc->reset = vfio_ap_reset;
dc->bus_type = TYPE_AP_BUS;
}
--
2.7.4
[Qemu-devel] [PATCH v3] qdev/core: fix qbus_is_full()
The qbus_is_full(BusState *bus) function (qdev_monitor.c) compares the max_index
value of the BusState structure with the max_dev value of the BusClass structure
to determine whether the maximum number of children has been reached for the
bus. The problem is, the max_index field of the BusState structure does not
necessarily reflect the number of devices that have been plugged into
the bus.
Whenever a child device is plugged into the bus, the bus's max_index value is
assigned to the child device and then incremented. If the child is subsequently
unplugged, the value of the max_index does not change and no longer reflects the
number of children.
When the bus's max_index value reaches the maximum number of devices
allowed for the bus (i.e., the max_dev field in the BusClass structure),
attempts to plug another device will be rejected claiming that the bus is
full -- even if the bus is actually empty.
To resolve the problem, a new 'num_children' field is being added to the
BusState structure to keep track of the number of children plugged into the
bus. It will be incremented when a child is plugged, and decremented when a
child is unplugged.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Reviewed-by: Halil Pasic
---
hw/core/qdev.c | 3 +++
include/hw/qdev-core.h | 1 +
qdev-monitor.c | 2 +-
3 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 6b3cc55b27c2..956923f33520 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState
*child)
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
+bus->num_children--;
+
/* This gives back ownership of kid->child back to us. */
object_property_del(OBJECT(bus), name, NULL);
object_unref(OBJECT(kid->child));
@@ -73,6 +75,7 @@ static void bus_add_child(BusState *bus, DeviceState *child)
char name[32];
BusChild *kid = g_malloc0(sizeof(*kid));
+bus->num_children++;
kid->index = bus->max_index++;
kid->child = child;
object_ref(OBJECT(kid->child));
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index a24d0dd566e3..521f0a947ead 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -206,6 +206,7 @@ struct BusState {
HotplugHandler *hotplug_handler;
int max_index;
bool realized;
+int num_children;
QTAILQ_HEAD(ChildrenHead, BusChild) children;
QLIST_ENTRY(BusState) sibling;
};
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 07147c63bf8b..45a8ba49644c 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -414,7 +414,7 @@ static DeviceState *qbus_find_dev(BusState *bus, char *elem)
static inline bool qbus_is_full(BusState *bus)
{
BusClass *bus_class = BUS_GET_CLASS(bus);
-return bus_class->max_dev && bus->max_index >= bus_class->max_dev;
+return bus_class->max_dev && bus->num_children >= bus_class->max_dev;
}
/*
--
2.7.4
Re: [Qemu-devel] [PATCH v2] qdev/core: Can not replug device on bus that allows one device
On 12/14/18 7:09 AM, Halil Pasic wrote:
On Thu, 13 Dec 2018 11:26:42 -0500
Tony Krowiak wrote:
If the maximum number of devices allowed on a bus is 1 and a device
which is plugged into the bus is subsequently unplugged, attempting to replug
the device fails with error "Bus 'xxx' does not support hotplugging".
The "error" is detected in the qbus_is_full(BusState *bus) function
(qdev_monitor.c) because bus->max_index >= bus_class->max_dev. The
root of the problem is that the bus->max_index is not decremented when a device
is unplugged from the bus. This patch fixes that problem.
As Pierre has pointed out, the commit message is stale and inaccurate.
Subject could be better as well. I mean the problem is not limited to
buses that allow only one device. Each bus that can get full looses
capacity with every plug/unplug op pair. With that fixed:
Reviewed-by: Halil Pasic
I'll make the changes recommended by you and Pierre.
Signed-off-by: Tony Krowiak
---
hw/core/qdev.c | 3 +++
include/hw/qdev-core.h | 1 +
qdev-monitor.c | 2 +-
3 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 6b3cc55b27c2..956923f33520 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState
*child)
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
+bus->num_children--;
+
/* This gives back ownership of kid->child back to us. */
object_property_del(OBJECT(bus), name, NULL);
object_unref(OBJECT(kid->child));
@@ -73,6 +75,7 @@ static void bus_add_child(BusState *bus, DeviceState *child)
char name[32];
BusChild *kid = g_malloc0(sizeof(*kid));
+bus->num_children++;
kid->index = bus->max_index++;
kid->child = child;
object_ref(OBJECT(kid->child));
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index a24d0dd566e3..521f0a947ead 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -206,6 +206,7 @@ struct BusState {
HotplugHandler *hotplug_handler;
int max_index;
bool realized;
+int num_children;
QTAILQ_HEAD(ChildrenHead, BusChild) children;
QLIST_ENTRY(BusState) sibling;
};
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 07147c63bf8b..45a8ba49644c 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -414,7 +414,7 @@ static DeviceState *qbus_find_dev(BusState *bus, char *elem)
static inline bool qbus_is_full(BusState *bus)
{
BusClass *bus_class = BUS_GET_CLASS(bus);
-return bus_class->max_dev && bus->max_index >= bus_class->max_dev;
+return bus_class->max_dev && bus->num_children >= bus_class->max_dev;
}
/*
Re: [Qemu-devel] [PATCH v2] qdev/core: Can not replug device on bus that allows one device
On 12/14/18 4:16 AM, Pierre Morel wrote:
On 13/12/2018 17:26, Tony Krowiak wrote:
If the maximum number of devices allowed on a bus is 1 and a device
which is plugged into the bus is subsequently unplugged, attempting to
replug
the device fails with error "Bus 'xxx' does not support hotplugging".
The "error" is detected in the qbus_is_full(BusState *bus) function
(qdev_monitor.c) because bus->max_index >= bus_class->max_dev. The
root of the problem is that the bus->max_index is not decremented when
a device
is unplugged from the bus. This patch fixes that problem.
I think that the root of the problem is that the max_index is not the
right measure to determine the number of devices currently plugged.
With the comment amended in this direction:
Good point, I should have changed that when I refactored this fix.
Reviewed-by: Pierre Morel
Signed-off-by: Tony Krowiak
---
 hw/core/qdev.c | 3 +++
 include/hw/qdev-core.h | 1 +
 qdev-monitor.c | 2 +-
 3 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 6b3cc55b27c2..956923f33520 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus,
DeviceState *child)
 snprintf(name, sizeof(name), "child[%d]", kid->index);
 QTAILQ_REMOVE(>children, kid, sibling);
+Â Â Â bus->num_children--;
+
 /* This gives back ownership of kid->child back to us. */
 object_property_del(OBJECT(bus), name, NULL);
 object_unref(OBJECT(kid->child));
@@ -73,6 +75,7 @@ static void bus_add_child(BusState *bus, DeviceState
*child)
 char name[32];
 BusChild *kid = g_malloc0(sizeof(*kid));
+Â Â Â bus->num_children++;
 kid->index = bus->max_index++;
 kid->child = child;
 object_ref(OBJECT(kid->child));
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index a24d0dd566e3..521f0a947ead 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -206,6 +206,7 @@ struct BusState {
 HotplugHandler *hotplug_handler;
 int max_index;
 bool realized;
+Â Â Â int num_children;
 QTAILQ_HEAD(ChildrenHead, BusChild) children;
 QLIST_ENTRY(BusState) sibling;
 };
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 07147c63bf8b..45a8ba49644c 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -414,7 +414,7 @@ static DeviceState *qbus_find_dev(BusState *bus,
char *elem)
 static inline bool qbus_is_full(BusState *bus)
 {
 BusClass *bus_class = BUS_GET_CLASS(bus);
-Â Â Â return bus_class->max_dev && bus->max_index >= bus_class->max_dev;
+Â Â Â return bus_class->max_dev && bus->num_children >=
bus_class->max_dev;
 }
 /*
[Qemu-devel] [PATCH v2] qdev/core: Can not replug device on bus that allows one device
If the maximum number of devices allowed on a bus is 1 and a device
which is plugged into the bus is subsequently unplugged, attempting to replug
the device fails with error "Bus 'xxx' does not support hotplugging".
The "error" is detected in the qbus_is_full(BusState *bus) function
(qdev_monitor.c) because bus->max_index >= bus_class->max_dev. The
root of the problem is that the bus->max_index is not decremented when a device
is unplugged from the bus. This patch fixes that problem.
Signed-off-by: Tony Krowiak
---
hw/core/qdev.c | 3 +++
include/hw/qdev-core.h | 1 +
qdev-monitor.c | 2 +-
3 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 6b3cc55b27c2..956923f33520 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState
*child)
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
+bus->num_children--;
+
/* This gives back ownership of kid->child back to us. */
object_property_del(OBJECT(bus), name, NULL);
object_unref(OBJECT(kid->child));
@@ -73,6 +75,7 @@ static void bus_add_child(BusState *bus, DeviceState *child)
char name[32];
BusChild *kid = g_malloc0(sizeof(*kid));
+bus->num_children++;
kid->index = bus->max_index++;
kid->child = child;
object_ref(OBJECT(kid->child));
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index a24d0dd566e3..521f0a947ead 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -206,6 +206,7 @@ struct BusState {
HotplugHandler *hotplug_handler;
int max_index;
bool realized;
+int num_children;
QTAILQ_HEAD(ChildrenHead, BusChild) children;
QLIST_ENTRY(BusState) sibling;
};
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 07147c63bf8b..45a8ba49644c 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -414,7 +414,7 @@ static DeviceState *qbus_find_dev(BusState *bus, char *elem)
static inline bool qbus_is_full(BusState *bus)
{
BusClass *bus_class = BUS_GET_CLASS(bus);
-return bus_class->max_dev && bus->max_index >= bus_class->max_dev;
+return bus_class->max_dev && bus->num_children >= bus_class->max_dev;
}
/*
--
2.7.4
Re: [Qemu-devel] [PATCH] qdev/core: Can not replug device on bus that allows one device
On 12/13/18 8:03 AM, Halil Pasic wrote:
On Thu, 13 Dec 2018 13:09:59 +0100
Igor Mammedov wrote:
On Tue, 11 Dec 2018 14:41:00 -0500
Tony Krowiak wrote:
On 12/11/18 10:23 AM, Igor Mammedov wrote:
On Mon, 10 Dec 2018 14:14:14 -0500
Tony Krowiak wrote:
If the maximum number of devices allowed on a bus is 1 and a device
which is plugged into the bus is subsequently unplugged, attempting to replug
the device fails with error "Bus 'xxx' does not support hotplugging".
The "error" is detected in the qbus_is_full(BusState *bus) function
(qdev_monitor.c) because bus->max_index >= bus_class->max_dev. The
root of the problem is that the bus->max_index is not decremented when a device
is unplugged from the bus. This patch fixes that problem.
Signed-off-by: Tony Krowiak
---
hw/core/qdev.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 6b3cc55b27c2..b35b0bf27925 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState
*child)
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
+bus->max_index--;
that might cause problem when bus is allowed to has more than 1 device
and unplugged device is not the last one.
In that case bus_add_child() might create a child with duplicate name.
I see what you are saying. The max_index is assigned to the child device
index in the bus_add_child() function. The child device index is used to
generate a unique name for the child device. The generated name is then
used to link the child as a property to the bus. When the child is
removed from the bus, the name is regenerated from the child device
index and the named property is . It would therefore appear that the
real purpose of the max_index is to generate indexes for children of
the bus thus ensuring each child has a unique index. In other words,
the max_index value is only tangentially connected to indexing the list
of children.
This results in a disconnect between the usage of the max_index value
when adding and removing a child from the bus, and the check in the
qbus_is_full() function which compares the max_index to the maximum
number of devices allowed on the bus. If a child has been removed from
the bus, the max_index value does not indicate whether the bus is
full, it only specifies the index to be assigned to the next child to be
added to the bus.
To resolve this problem, I propose the following:
Add the following field to struct BusState (include/hw/qdev_core.h):
struct BusState {
Object obj;
DeviceState *parent;
char *name;
HotplugHandler *hotplug_handler;
int max_index;
bool realized;
+int num_children;
QTAILQ_HEAD(ChildrenHead, BusChild) children;
QLIST_ENTRY(BusState) sibling;
};
Add the following lines of code to the add/remove child functions in
hw/core/qdev.c:
static void bus_add_child(BusState *bus, DeviceState *child)
{
char name[32];
BusChild *kid = g_malloc0(sizeof(*kid));
kid->index = bus->max_index++;
kid->child = child;
object_ref(OBJECT(kid->child));
QTAILQ_INSERT_HEAD(>children, kid, sibling);
/* This transfers ownership of kid->child to the property. */
snprintf(name, sizeof(name), "child[%d]", kid->index);
object_property_add_link(OBJECT(bus), name,
object_get_typename(OBJECT(child)),
(Object **)>child,
NULL, /* read-only property */
0, /* return ownership on prop deletion */
NULL);
+bus->num_children++;
}
static void bus_remove_child(BusState *bus, DeviceState *child)
{
BusChild *kid;
QTAILQ_FOREACH(kid, >children, sibling) {
if (kid->child == child) {
char name[32];
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
/* This gives back ownership of kid->child back to us. */
object_property_del(OBJECT(bus), name, NULL);
object_unref(OBJECT(kid->child));
g_free(kid);
+ bus->num_children--;
return;
}
}
}
Change the line of code in the qbus_is_full() function in
qdev_monitor.c:
static inline bool qbus_is_full(BusState *bus)
{
BusClass *bus_class = BUS_GET_CLASS(bus);
-return bus_class->max_dev &&
- bus->max_index >= bus_class->max_dev;
+return bus_class->max_dev &&
+ bus->num_children >= bus_class->max_dev;
}
looks good to me
[...]
I agree, the second proposal looks reasonable. Can you send a proper
patch so I can r-b it?
will do
Halil
Re: [Qemu-devel] [PATCH] qdev/core: Can not replug device on bus that allows one device
On 12/11/18 10:23 AM, Igor Mammedov wrote:
On Mon, 10 Dec 2018 14:14:14 -0500
Tony Krowiak wrote:
If the maximum number of devices allowed on a bus is 1 and a device
which is plugged into the bus is subsequently unplugged, attempting to replug
the device fails with error "Bus 'xxx' does not support hotplugging".
The "error" is detected in the qbus_is_full(BusState *bus) function
(qdev_monitor.c) because bus->max_index >= bus_class->max_dev. The
root of the problem is that the bus->max_index is not decremented when a device
is unplugged from the bus. This patch fixes that problem.
Signed-off-by: Tony Krowiak
---
hw/core/qdev.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 6b3cc55b27c2..b35b0bf27925 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState
*child)
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
+bus->max_index--;
that might cause problem when bus is allowed to has more than 1 device
and unplugged device is not the last one.
In that case bus_add_child() might create a child with duplicate name.
I see what you are saying. The max_index is assigned to the child device
index in the bus_add_child() function. The child device index is used to
generate a unique name for the child device. The generated name is then
used to link the child as a property to the bus. When the child is
removed from the bus, the name is regenerated from the child device
index and the named property is . It would therefore appear that the
real purpose of the max_index is to generate indexes for children of
the bus thus ensuring each child has a unique index. In other words,
the max_index value is only tangentially connected to indexing the list
of children.
This results in a disconnect between the usage of the max_index value
when adding and removing a child from the bus, and the check in the
qbus_is_full() function which compares the max_index to the maximum
number of devices allowed on the bus. If a child has been removed from
the bus, the max_index value does not indicate whether the bus is
full, it only specifies the index to be assigned to the next child to be
added to the bus.
To resolve this problem, I propose the following:
Add the following field to struct BusState (include/hw/qdev_core.h):
struct BusState {
Object obj;
DeviceState *parent;
char *name;
HotplugHandler *hotplug_handler;
int max_index;
bool realized;
+int num_children;
QTAILQ_HEAD(ChildrenHead, BusChild) children;
QLIST_ENTRY(BusState) sibling;
};
Add the following lines of code to the add/remove child functions in
hw/core/qdev.c:
static void bus_add_child(BusState *bus, DeviceState *child)
{
char name[32];
BusChild *kid = g_malloc0(sizeof(*kid));
kid->index = bus->max_index++;
kid->child = child;
object_ref(OBJECT(kid->child));
QTAILQ_INSERT_HEAD(>children, kid, sibling);
/* This transfers ownership of kid->child to the property. */
snprintf(name, sizeof(name), "child[%d]", kid->index);
object_property_add_link(OBJECT(bus), name,
object_get_typename(OBJECT(child)),
(Object **)>child,
NULL, /* read-only property */
0, /* return ownership on prop deletion */
NULL);
+bus->num_children++;
}
static void bus_remove_child(BusState *bus, DeviceState *child)
{
BusChild *kid;
QTAILQ_FOREACH(kid, >children, sibling) {
if (kid->child == child) {
char name[32];
snprintf(name, sizeof(name), "child[%d]", kid->index);
QTAILQ_REMOVE(>children, kid, sibling);
/* This gives back ownership of kid->child back to us. */
object_property_del(OBJECT(bus), name, NULL);
object_unref(OBJECT(kid->child));
g_free(kid);
+ bus->num_children--;
return;
}
}
}
Change the line of code in the qbus_is_full() function in
qdev_monitor.c:
static inline bool qbus_is_full(BusState *bus)
{
BusClass *bus_class = BUS_GET_CLASS(bus);
-return bus_class->max_dev &&
- bus->max_index >= bus_class->max_dev;
+return bus_class->max_dev &&
+ bus->num_children >= bus_class->max_dev;
}
+
/* This gives back ownership of kid->child back to us. */
object_property_del(OBJECT(bus), name, NULL);
object_unref(OBJECT(kid->child));
[Qemu-devel] [PATCH] qdev/core: Can not replug device on bus that allows one device
If the maximum number of devices allowed on a bus is 1 and a device which is plugged into the bus is subsequently unplugged, attempting to replug the device fails with error "Bus 'xxx' does not support hotplugging". The "error" is detected in the qbus_is_full(BusState *bus) function (qdev_monitor.c) because bus->max_index >= bus_class->max_dev. The root of the problem is that the bus->max_index is not decremented when a device is unplugged from the bus. This patch fixes that problem. Signed-off-by: Tony Krowiak --- hw/core/qdev.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hw/core/qdev.c b/hw/core/qdev.c index 6b3cc55b27c2..b35b0bf27925 100644 --- a/hw/core/qdev.c +++ b/hw/core/qdev.c @@ -59,6 +59,8 @@ static void bus_remove_child(BusState *bus, DeviceState *child) snprintf(name, sizeof(name), "child[%d]", kid->index); QTAILQ_REMOVE(>children, kid, sibling); +bus->max_index--; + /* This gives back ownership of kid->child back to us. */ object_property_del(OBJECT(bus), name, NULL); object_unref(OBJECT(kid->child)); -- 2.7.4
Re: [Qemu-devel] [PATCH RFC] vfio-ap: flag as compatible with balloon
On 12/5/18 9:51 AM, Cornelia Huck wrote:
vfio-ap devices do not pin any pages in the host. Therefore, they
are belived to be compatible with memory ballooning.
Flag them as compatible, so both vfio-ap and a balloon can be
used simultaneously.
Signed-off-by: Cornelia Huck
---
As briefly discussed on IRC. RFC as I do not have easy access to
hardware I can test this with.
---
hw/vfio/ap.c | 8
1 file changed, 8 insertions(+)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 65de952f44..3bf48eed28 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -104,6 +104,14 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
vapdev->vdev.name = g_strdup_printf("%s", mdevid);
vapdev->vdev.dev = dev;
+/*
+ * vfio-ap devices are believed to operate in a way compatible with
+ * memory ballooning, as no pages are pinned in the host.
+ * This needs to be set before vfio_get_device() for vfio common to
+ * handle the balloon inhibitor.
+ */
+vapdev->vdev.balloon_allowed = true;
+
ret = vfio_get_device(vfio_group, mdevid, >vdev, _err);
if (ret) {
goto out_get_dev_err;
Tested-by: Tony Krowiak
Re: [Qemu-devel] [PATCH RFC] vfio-ap: flag as compatible with balloon
On 12/6/18 7:48 AM, Cornelia Huck wrote:
On Thu, 6 Dec 2018 13:32:39 +0100
Halil Pasic wrote:
On Thu, 6 Dec 2018 09:28:34 +0100
David Hildenbrand wrote:
On 05.12.18 18:25, Christian Borntraeger wrote:
On 05.12.2018 17:45, Cornelia Huck wrote:
On Wed, 5 Dec 2018 17:38:22 +0100
David Hildenbrand wrote:
On 05.12.18 15:51, Cornelia Huck wrote:
vfio-ap devices do not pin any pages in the host. Therefore, they
are belived to be compatible with memory ballooning.
Flag them as compatible, so both vfio-ap and a balloon can be
used simultaneously.
Signed-off-by: Cornelia Huck
---
As briefly discussed on IRC. RFC as I do not have easy access to
hardware I can test this with.
---
hw/vfio/ap.c | 8
1 file changed, 8 insertions(+)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 65de952f44..3bf48eed28 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -104,6 +104,14 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
vapdev->vdev.name = g_strdup_printf("%s", mdevid);
vapdev->vdev.dev = dev;
+/*
+ * vfio-ap devices are believed to operate in a way compatible with
+ * memory ballooning, as no pages are pinned in the host.
+ * This needs to be set before vfio_get_device() for vfio common to
+ * handle the balloon inhibitor.
+ */
+vapdev->vdev.balloon_allowed = true;
+
ret = vfio_get_device(vfio_group, mdevid, >vdev, _err);
if (ret) {
goto out_get_dev_err;
What happens if this ever changes? Shouldn't we have an API to at least
check what the vfio device can guarantee?
"are believed to operate" doesn't sound like guarantees to me :)
I would actually remove that comment or fix it. We either know or we dont.
In the way vfio-works I see no reason to disallow balloon. Even if the guest
does
something wrong (e.g. crypto I/O on freed pages) the host would handle that the
same as it would for normal page accesses. From a host point of view the crypto
instructions are just CISC instructions with load/store semantics.
As long as vfio-ap does not and will never pin pages (and keep them
pinned), we are fine. I don't know about the details, but if vfio-ap
really just issues a synchronous instruction for us, we are fine.
I agree with Christian. That comment is best removed.
What about s/believed to operate/operate/?
The second part of the comment is still useful, I believe.
@Tony, I guess you should have the most elaborate test setup. Can you give
this some testing just in case?
Actual testing would be great :)
Will do.
It's the same for ccw :)
As a matter of fact, I don't like that comment.
Do you have a suggestion for rewording it?
Regards,
Halil
While such an API definitely sounds like a good idea, it is probably
overkill to introduce it for this case (do we envision changing the way
vfio-ap operates in the future to make that statement non-true?)
agreed.
Re: [Qemu-devel] [PATCH v2 2/6] s390x/vfio: ap: Use the APdevice as a child of the APBus
On 11/30/18 4:31 AM, Pierre Morel wrote:
On 29/11/2018 21:42, Tony Krowiak wrote:
On 11/22/18 11:35 AM, Pierre Morel wrote:
Two good reasons to use the base device as a child of the
AP BUS:
- We can easily find the device without traversing the qtree.
- In case we have different APdevice instantiation, VFIO with
  interception or emulation, we will need the APDevice as
  a parent device.
Signed-off-by: Pierre Morel
---
 hw/s390x/ap-device.c | 22 ++
 hw/vfio/ap.c | 16 ++--
 include/hw/s390x/ap-device.h | 2 ++
 3 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/hw/s390x/ap-device.c b/hw/s390x/ap-device.c
index f5ac8db..554d5aa 100644
--- a/hw/s390x/ap-device.c
+++ b/hw/s390x/ap-device.c
@@ -11,13 +11,35 @@
 #include "qemu/module.h"
 #include "qapi/error.h"
 #include "hw/qdev.h"
+#include "hw/s390x/ap-bridge.h"
 #include "hw/s390x/ap-device.h"
+APDevice *s390_get_ap(void)
How about ap_get_device(void)?
Yes, keep same conventions.
+{
+Â Â Â static DeviceState *apb;
Why static if you call s390_get_ap_bridge()
to retrieve it without first checking for NULL?
static are initialized as NULL.
Yes, but down below, you call s390_get_ap_bridge() to set apb
regardless of whether apb is NULL or not. It makes no sense to
declare is as static here. It is also redundant because the
s390_get_ap_bridge() function already caches it in a static
variable.
+Â Â Â BusState *bus;
+Â Â Â BusChild *child;
+Â Â Â static APDevice *ap;
+
+Â Â Â if (ap) {
+Â Â Â return ap;
+Â Â Â }
+
+Â Â Â apb = s390_get_ap_bridge();
+Â Â Â /* We have only a single child on the BUS */
+Â Â Â bus = qdev_get_child_bus(apb, TYPE_AP_BUS
+Â Â Â child = QTAILQ_FIRST(>children);
+Â Â Â assert(child != NULL);
+Â Â Â ap = DO_UPCAST(APDevice, parent_obj, child->child);
I received a comment from Thomas Huth in Message ID
<2291104a-4cbf-e4fd-3496-fa0910beb...@redhat.com>
that DO_UPCAST should be avoided in new code. You should
create an AP_DEVICE macro for this in ap-device.h
Thanks I will do.
Regards,
Pierre
Re: [Qemu-devel] [PATCH v2 6/6] s390x/vfio: ap: Implementing AP Queue Interrupt Control
On 11/22/18 11:35 AM, Pierre Morel wrote:
We intercept the PQAP(AQIC) instruction and transform
the guest's AQIC command parameters for the host AQIC
parameters.
Doing this we use the standard adapter interface to provide
the adapter NIB, indicator and ISC.
We define new structures, APCard and APQueue to keep track of
the ISC, route and indicator address and we add an array of
APCards in the APDevice.
We call the VFIO ioctl to set or clear the interruption
according to the "i" bit of the parameter.
Signed-off-by: Pierre Morel
---
hw/vfio/ap.c | 117 ++-
include/hw/s390x/ap-device.h | 63 +++
target/s390x/kvm.c | 31
3 files changed, 210 insertions(+), 1 deletion(-)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 94e5a1a..20d2e5f 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -4,6 +4,7 @@
* Copyright 2018 IBM Corp.
* Author(s): Tony Krowiak
*Halil Pasic
+ *Pierre Morel
*
* This work is licensed under the terms of the GNU GPL, version 2 or (at
* your option) any later version. See the COPYING file in the top-level
@@ -14,7 +15,6 @@
#include
#include "qemu/osdep.h"
#include "qapi/error.h"
-#include "hw/sysbus.h"
#include "hw/vfio/vfio.h"
#include "hw/vfio/vfio-common.h"
#include "hw/s390x/ap-device.h"
@@ -27,6 +27,8 @@
#include "sysemu/sysemu.h"
#include "hw/s390x/ap-bridge.h"
#include "exec/address-spaces.h"
+#include "hw/s390x/s390_flic.h"
+#include "hw/s390x/css.h"
#define VFIO_AP_DEVICE_TYPE "vfio-ap"
@@ -35,6 +37,117 @@ typedef struct VFIOAPDevice {
VFIODevice vdev;
} VFIOAPDevice;
+static uint32_t ap_pqap_clear_irq(VFIODevice *vdev, APQueue *apq)
+{
+struct vfio_ap_aqic param;
+uint32_t retval;
+
+param.apqn = apq->apqn;
+param.isc = apq->isc;
+param.argsz = sizeof(param);
+
+retval = ioctl(vdev->fd, VFIO_AP_CLEAR_IRQ, );
+switch (retval) {
+case 0:/* Fall through and return the instruction status */
Unnecessary comment, we can see the code is going to fall through and
return status. Sorry, I know its a nit.
+release_indicator(>routes.adapter, apq->nib);
+case -EIO: /* The case the PQAP instruction failed with status */
We know it's a case statement, so leave off the "The case the".
+return param.status;
+default: /* The case the ioctl call failed without isuing instruction */
Same here.
+break;
+}
+return ap_reg_set_status(AP_RC_INVALID_ADDR);
+}
+
+static uint32_t ap_pqap_set_irq(VFIODevice *vdev, APQueue *apq, uint64_t g_nib)
+{
+struct vfio_ap_aqic param;
+uint32_t retval;
+uint32_t id;
+
+id = css_get_adapter_id(CSS_IO_ADAPTER_AP, apq->isc);
+if (id == -1) {
+return ap_reg_set_status(AP_RC_INVALID_ADDR);
+}
+apq->routes.adapter.adapter_id = id;
+apq->nib = get_indicator(ldq_p(_nib), 8);
+
+retval = map_indicator(>routes.adapter, apq->nib);
+if (retval) {
+return ap_reg_set_status(AP_RC_INVALID_ADDR);
+}
+
+param.apqn = apq->apqn;
+param.isc = apq->isc;
+param.nib = g_nib;
+param.adapter_id = id;
+param.argsz = sizeof(param);
+
+retval = ioctl(vdev->fd, VFIO_AP_SET_IRQ, );
+switch (retval) {
+case -EIO: /* The case the PQAP instruction failed with status */
We know it's a case statement, so leave off the "The case the".
+release_indicator(>routes.adapter, apq->nib);
+case 0:/* Fall through and return the instruction status */
Unnecessary comment, the code speaks for itself.
+return param.status;
+default: /* The case the ioctl call failed without isuing instruction */
We know it's a case statement, so leave off the "The case the".
+break;
+}
+release_indicator(>routes.adapter, apq->nib);
+return ap_reg_set_status(AP_RC_INVALID_ADDR);
+}
+
+static int ap_pqap_aqic(CPUS390XState *env)
+{
+uint64_t g_nib = env->regs[2];
+uint8_t apid = ap_reg_get_apid(env->regs[0]);
+uint8_t apqi = ap_reg_get_apqi(env->regs[0]);
+uint32_t retval;
+APDevice *ap = s390_get_ap();
To be consistent with the naming convention in the rest of
this file, can you name this variable 'apdev'?
+VFIODevice *vdev;
+VFIOAPDevice *ap_vdev;
To be consistent with the naming convention in the rest of
this file, can you name this variable 'vapdev'?
+APQueue *apq;
+
+ap_vdev = DO_UPCAST(VFIOAPDevice, apdev, ap);
Where is 'apdev' defined/initialized?
+vdev = _vdev->vdev;
+apq = >card[apid].queue[apqi];
+apq->isc = ap_reg_get_isc(env->regs[1]);
+apq->apqn = (apid << 8) | apqi;
+
+if
Re: [Qemu-devel] [PATCH v2 4/6] s390x/cpumodel: Set up CPU model for AQIC interception
On 11/22/18 11:35 AM, Pierre Morel wrote:
A new CPU model facilities is introduced to support AP devices
interruption interception for a KVM guest.
CPU model facility:
The S390_FEAT_AP_QUEUE_INTERRUPT_CONTROL, CPU facility indicates
whether AP interruption interception is available to the guest.
This feature will be enabled only if the AP instructions are
available on the linux host and AQIC facility is installed on
the host.
This feature must be turned on from userspace to intercept AP
instructions on the KVM guest. The QEMU command line to turn
this feature on looks something like this:
qemu-system-s390x ... -cpu xxx,aqci=on ...
Signed-off-by: Pierre Morel
---
target/s390x/cpu_features.c | 1 +
target/s390x/cpu_features_def.h | 1 +
target/s390x/cpu_models.c | 1 +
target/s390x/gen-features.c | 1 +
4 files changed, 4 insertions(+)
diff --git a/target/s390x/cpu_features.c b/target/s390x/cpu_features.c
index 60cfeba..c464abf 100644
--- a/target/s390x/cpu_features.c
+++ b/target/s390x/cpu_features.c
@@ -84,6 +84,7 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT("sema", S390_FEAT_TYPE_STFL, 59, "Semaphore-assist facility"),
FEAT_INIT("tsi", S390_FEAT_TYPE_STFL, 60, "Time-slice Instrumentation
facility"),
FEAT_INIT("ri", S390_FEAT_TYPE_STFL, 64, "CPU runtime-instrumentation
facility"),
+FEAT_INIT("aqic", S390_FEAT_TYPE_STFL, 65, "AP-Queue interruption Control
facility"),
FEAT_INIT("zpci", S390_FEAT_TYPE_STFL, 69, "z/PCI facility"),
FEAT_INIT("aen", S390_FEAT_TYPE_STFL, 71,
"General-purpose-adapter-event-notification facility"),
FEAT_INIT("ais", S390_FEAT_TYPE_STFL, 72,
"General-purpose-adapter-interruption-suppression facility"),
diff --git a/target/s390x/cpu_features_def.h b/target/s390x/cpu_features_def.h
index 5fc7e7b..3f22780 100644
--- a/target/s390x/cpu_features_def.h
+++ b/target/s390x/cpu_features_def.h
@@ -72,6 +72,7 @@ typedef enum {
S390_FEAT_SEMAPHORE_ASSIST,
S390_FEAT_TIME_SLICE_INSTRUMENTATION,
S390_FEAT_RUNTIME_INSTRUMENTATION,
+S390_FEAT_AP_QUEUE_INTERRUPT_CONTROL,
S390_FEAT_ZPCI,
S390_FEAT_ADAPTER_EVENT_NOTIFICATION,
S390_FEAT_ADAPTER_INT_SUPPRESSION,
diff --git a/target/s390x/cpu_models.c b/target/s390x/cpu_models.c
index 7c253ff..6b5e94b 100644
--- a/target/s390x/cpu_models.c
+++ b/target/s390x/cpu_models.c
@@ -788,6 +788,7 @@ static void check_consistency(const S390CPUModel *model)
{ S390_FEAT_SIE_KSS, S390_FEAT_SIE_F2 },
{ S390_FEAT_AP_QUERY_CONFIG_INFO, S390_FEAT_AP },
{ S390_FEAT_AP_FACILITIES_TEST, S390_FEAT_AP },
+{ S390_FEAT_AP_QUEUE_INTERRUPT_CONTROL, S390_FEAT_AP },
};
int i;
diff --git a/target/s390x/gen-features.c b/target/s390x/gen-features.c
index 70015ea..b051221 100644
--- a/target/s390x/gen-features.c
+++ b/target/s390x/gen-features.c
@@ -448,6 +448,7 @@ static uint16_t full_GEN12_GA1[] = {
S390_FEAT_EDAT_2,
S390_FEAT_SIDE_EFFECT_ACCESS_ESOP2,
S390_FEAT_AP_QUERY_CONFIG_INFO,
+S390_FEAT_AP_QUEUE_INTERRUPT_CONTROL,
S390_FEAT_AP_FACILITIES_TEST,
S390_FEAT_AP,
};
Reviewed-by: Tony Krowiak
Re: [Qemu-devel] [PATCH v2 1/6] s390x/vfio: ap: Finding the AP bridge
On 11/22/18 11:35 AM, Pierre Morel wrote:
In the case we will enter QEMU through interception of instructions,
we will need to retrieve the AP devices.
The base device is the AP bridge.
Let us implement a way to retrieve the AP Bridge from qtree.
Signed-off-by: Pierre Morel
---
hw/s390x/ap-bridge.c | 12
include/hw/s390x/ap-bridge.h | 1 +
2 files changed, 13 insertions(+)
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
index 3795d30..831ad5d 100644
--- a/hw/s390x/ap-bridge.c
+++ b/hw/s390x/ap-bridge.c
@@ -14,6 +14,18 @@
#include "hw/s390x/ap-bridge.h"
#include "cpu.h"
+DeviceState *s390_get_ap_bridge(void)
+{
+static DeviceState *apb;
Since you are caching a reference to the bridge after
retrieving it, why not make apb a global scope variable
and initialize it in s390_init_ap() when the bridge is
created. You can then declare it as an extern in the
ap-bridge.h header file and you eliminate the need for
this function. If you do make it a global var, I would
name it ap_bridge;
+
+if (!apb) {
+apb = DEVICE(object_resolve_path(TYPE_AP_BRIDGE, NULL));
+assert(apb != NULL);
+}
+
+return apb;
+}
+
static char *ap_bus_get_dev_path(DeviceState *dev)
{
/* at most one */
diff --git a/include/hw/s390x/ap-bridge.h b/include/hw/s390x/ap-bridge.h
index 470e439..dacb877 100644
--- a/include/hw/s390x/ap-bridge.h
+++ b/include/hw/s390x/ap-bridge.h
@@ -15,5 +15,6 @@
#define TYPE_AP_BUS "ap-bus"
void s390_init_ap(void);
+DeviceState *s390_get_ap_bridge(void);
#endif
Re: [Qemu-devel] [PATCH v2 2/6] s390x/vfio: ap: Use the APdevice as a child of the APBus
On 11/22/18 11:35 AM, Pierre Morel wrote:
Two good reasons to use the base device as a child of the
AP BUS:
- We can easily find the device without traversing the qtree.
- In case we have different APdevice instantiation, VFIO with
interception or emulation, we will need the APDevice as
a parent device.
Signed-off-by: Pierre Morel
---
hw/s390x/ap-device.c | 22 ++
hw/vfio/ap.c | 16 ++--
include/hw/s390x/ap-device.h | 2 ++
3 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/hw/s390x/ap-device.c b/hw/s390x/ap-device.c
index f5ac8db..554d5aa 100644
--- a/hw/s390x/ap-device.c
+++ b/hw/s390x/ap-device.c
@@ -11,13 +11,35 @@
#include "qemu/module.h"
#include "qapi/error.h"
#include "hw/qdev.h"
+#include "hw/s390x/ap-bridge.h"
#include "hw/s390x/ap-device.h"
+APDevice *s390_get_ap(void)
How about ap_get_device(void)?
+{
+static DeviceState *apb;
Why static if you call s390_get_ap_bridge()
to retrieve it without first checking for NULL?
+BusState *bus;
+BusChild *child;
+static APDevice *ap;
+
+if (ap) {
+return ap;
+}
+
+apb = s390_get_ap_bridge();
+/* We have only a single child on the BUS */
+bus = qdev_get_child_bus(apb, TYPE_AP_BUS
+child = QTAILQ_FIRST(>children);
+assert(child != NULL);
+ap = DO_UPCAST(APDevice, parent_obj, child->child);
I received a comment from Thomas Huth in Message ID
<2291104a-4cbf-e4fd-3496-fa0910beb...@redhat.com>
that DO_UPCAST should be avoided in new code. You should
create an AP_DEVICE macro for this in ap-device.h
+return ap;
+}
+
static void ap_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
dc->desc = "AP device class";
+dc->bus_type = TYPE_AP_BUS;
dc->hotpluggable = false;
}
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 65de952..94e5a1a 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -35,9 +35,6 @@ typedef struct VFIOAPDevice {See my comment above about
DO_UPCAST
VFIODevice vdev;
} VFIOAPDevice;
-#define VFIO_AP_DEVICE(obj) \
-OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE)
-
static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
{
vdev->needs_reset = false;
@@ -90,8 +87,8 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
char *mdevid;
Error *local_err = NULL;
VFIOGroup *vfio_group;
-APDevice *apdev = AP_DEVICE(dev);
-VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
+APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);See my comment
above about DO_UPCAST
+VFIOAPDevice *vapdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
See my comment above about DO_UPCAST.
vfio_group = vfio_ap_get_group(vapdev, _err);
if (!vfio_group) {
@@ -120,8 +117,8 @@ out_err:
static void vfio_ap_unrealize(DeviceState *dev, Error **errp)
{
-APDevice *apdev = AP_DEVICE(dev);
-VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
+APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
+VFIOAPDevice *vapdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
See my comment above about DO_UPCAST
VFIOGroup *group = vapdev->vdev.group;
vfio_ap_put_device(vapdev);
@@ -136,8 +133,8 @@ static Property vfio_ap_properties[] = {
static void vfio_ap_reset(DeviceState *dev)
{
int ret;
-APDevice *apdev = AP_DEVICE(dev);
-VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
FYI, these macros were created in response to Thomas Huth's
comments about DO_UPCAST.
+APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
+VFIOAPDevice *vapdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
ret = ioctl(vapdev->vdev.fd, VFIO_DEVICE_RESET);
if (ret) {
@@ -163,7 +160,6 @@ static void vfio_ap_class_init(ObjectClass *klass, void
*data)
dc->unrealize = vfio_ap_unrealize;
dc->hotpluggable = false;
dc->reset = vfio_ap_reset;
-dc->bus_type = TYPE_AP_BUS;
}
static const TypeInfo vfio_ap_info = {
diff --git a/include/hw/s390x/ap-device.h b/include/hw/s390x/ap-device.h
index 765e908..5f3c840 100644
--- a/include/hw/s390x/ap-device.h
+++ b/include/hw/s390x/ap-device.h
@@ -19,4 +19,6 @@ typedef struct APDevice {
#define AP_DEVICE(obj) \
OBJECT_CHECK(APDevice, (obj), AP_DEVICE_TYPE)
+APDevice *s390_get_ap(void);
How about APDevice *ap_get_device(void)?
+
#endif /* HW_S390X_AP_DEVICE_H */
Re: [Qemu-devel] [PATCH v2 2/6] s390x/vfio: ap: Use the APdevice as a child of the APBus
On 11/29/18 6:45 AM, Cornelia Huck wrote:
On Thu, 22 Nov 2018 17:35:51 +0100
Pierre Morel wrote:
Two good reasons to use the base device as a child of the
AP BUS:
- We can easily find the device without traversing the qtree.
- In case we have different APdevice instantiation, VFIO with
interception or emulation, we will need the APDevice as
a parent device.
Signed-off-by: Pierre Morel
---
hw/s390x/ap-device.c | 22 ++
hw/vfio/ap.c | 16 ++--
include/hw/s390x/ap-device.h | 2 ++
3 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/hw/s390x/ap-device.c b/hw/s390x/ap-device.c
index f5ac8db..554d5aa 100644
--- a/hw/s390x/ap-device.c
+++ b/hw/s390x/ap-device.c
@@ -11,13 +11,35 @@
#include "qemu/module.h"
#include "qapi/error.h"
#include "hw/qdev.h"
+#include "hw/s390x/ap-bridge.h"
#include "hw/s390x/ap-device.h"
+APDevice *s390_get_ap(void)
+{
+static DeviceState *apb;
+BusState *bus;
+BusChild *child;
+static APDevice *ap;
+
+if (ap) {
+return ap;
+}
+
+apb = s390_get_ap_bridge();
+/* We have only a single child on the BUS */
So, there'll never a mixed environment? Or would that have a 'hybrid'
ap device?
It is not possible to have interpretation and interception. I suppose
one could mix emulated and virtual AP devices, but that seems highly
unlikely; in fact, I think it is highly unlikely that emulation is ever
implemented.
+bus = qdev_get_child_bus(apb, TYPE_AP_BUS);
+child = QTAILQ_FIRST(>children);
+assert(child != NULL);
+ap = DO_UPCAST(APDevice, parent_obj, child->child);
+return ap;
+}
+
static void ap_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
dc->desc = "AP device class";
+dc->bus_type = TYPE_AP_BUS;
dc->hotpluggable = false;
}
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 65de952..94e5a1a 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -35,9 +35,6 @@ typedef struct VFIOAPDevice {
VFIODevice vdev;
} VFIOAPDevice;
-#define VFIO_AP_DEVICE(obj) \
-OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE)
Hm?
I received a comment from Thomas Huth in Message ID
<2291104a-4cbf-e4fd-3496-fa0910beb...@redhat.com>
that DO_UPCAST should be avoided in new code. This macro
should probably be restored and an AP_DEVICE() macro added.
-
static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
{
vdev->needs_reset = false;
Re: [Qemu-devel] [PATCH] s390x/vfio-ap: report correct error
On 10/16/2018 07:58 AM, Cornelia Huck wrote:
If ioctl(..., VFIO_DEVICE_RESET) fails, we want to report errno
instead of ret (which is always -1 on error).
Fixes Coverity issue CID 1396176.
Reported-by: Peter Maydell
Signed-off-by: Cornelia Huck
---
hw/vfio/ap.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 3962bb74e5..65de952f44 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -142,7 +142,7 @@ static void vfio_ap_reset(DeviceState *dev)
ret = ioctl(vapdev->vdev.fd, VFIO_DEVICE_RESET);
if (ret) {
error_report("%s: failed to reset %s device: %s", __func__,
- vapdev->vdev.name, strerror(ret));
+ vapdev->vdev.name, strerror(errno));
}
}
Reviewed-by: Tony Krowiak
[Qemu-devel] [PATCH v11 5/6] s390x/vfio: ap: Introduce VFIO AP device
Introduces a VFIO based AP device. The device is defined via
the QEMU command line by specifying:
-device vfio-ap,sysfsdev=
There may be only one vfio-ap device configured for a guest.
The mediated matrix device is created by the VFIO AP device
driver by writing a UUID to a sysfs attribute file (see
docs/vfio-ap.txt). The mediated matrix device will be named
after the UUID. Symbolic links to the $uuid are created in
many places, so the path to the mediated matrix device $uuid
can be specified in any of the following ways:
/sys/devices/vfio_ap/matrix/$uuid
/sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
/sys/bus/mdev/devices/$uuid
/sys/bus/mdev/drivers/vfio_mdev/$uuid
When the vfio-ap device is realized, it acquires and opens the
VFIO iommu group to which the mediated matrix device is
bound. This causes a VFIO group notification event to be
signaled. The vfio_ap device driver's group notification
handler will get called at which time the device driver
will configure the the AP devices to which the guest will
be granted access.
Signed-off-by: Tony Krowiak
Tested-by: Pierre Morel
Acked-by: Halil Pasic
Tested-by: Pierre Morel
Tested-by: Christian Borntraeger
---
MAINTAINERS | 2 +
default-configs/s390x-softmmu.mak | 1 +
hw/vfio/Makefile.objs | 1 +
hw/vfio/ap.c | 179 ++
include/hw/vfio/vfio-common.h | 1 +
5 files changed, 184 insertions(+)
create mode 100644 hw/vfio/ap.c
diff --git a/MAINTAINERS b/MAINTAINERS
index b9984242820b..2730757b4482 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -88,6 +88,7 @@ F: hw/char/terminal3270.c
F: hw/intc/s390_flic.c
F: hw/intc/s390_flic_kvm.c
F: hw/s390x/
+F: hw/vfio/ap.c
F: hw/vfio/ccw.c
F: hw/watchdog/wdt_diag288.c
F: include/hw/s390x/
@@ -1217,6 +1218,7 @@ F: hw/s390x/ap-device.c
F: hw/s390x/ap-bridge.c
F: include/hw/s390x/ap-device.h
F: include/hw/s390x/ap-bridge.h
+F: hw/vfio/ap.c
L: qemu-s3...@nongnu.org
vhost
diff --git a/default-configs/s390x-softmmu.mak
b/default-configs/s390x-softmmu.mak
index d6b67d50f0e4..5eef37592451 100644
--- a/default-configs/s390x-softmmu.mak
+++ b/default-configs/s390x-softmmu.mak
@@ -7,3 +7,4 @@ CONFIG_S390_FLIC=y
CONFIG_S390_FLIC_KVM=$(CONFIG_KVM)
CONFIG_VFIO_CCW=$(CONFIG_LINUX)
CONFIG_WDT_DIAG288=y
+CONFIG_VFIO_AP=$(CONFIG_LINUX)
diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs
index a2e7a0a7cf02..8b3f664d85f7 100644
--- a/hw/vfio/Makefile.objs
+++ b/hw/vfio/Makefile.objs
@@ -6,4 +6,5 @@ obj-$(CONFIG_SOFTMMU) += platform.o
obj-$(CONFIG_VFIO_XGMAC) += calxeda-xgmac.o
obj-$(CONFIG_VFIO_AMD_XGBE) += amd-xgbe.o
obj-$(CONFIG_SOFTMMU) += spapr.o
+obj-$(CONFIG_VFIO_AP) += ap.o
endif
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
new file mode 100644
index ..8ec9ad7d4d90
--- /dev/null
+++ b/hw/vfio/ap.c
@@ -0,0 +1,179 @@
+/*
+ * VFIO based AP matrix device assignment
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *Halil Pasic
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include
+#include
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "hw/vfio/vfio.h"
+#include "hw/vfio/vfio-common.h"
+#include "hw/s390x/ap-device.h"
+#include "qemu/error-report.h"
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "cpu.h"
+#include "kvm_s390x.h"
+#include "sysemu/sysemu.h"
+#include "hw/s390x/ap-bridge.h"
+#include "exec/address-spaces.h"
+
+#define VFIO_AP_DEVICE_TYPE "vfio-ap"
+
+typedef struct VFIOAPDevice {
+APDevice apdev;
+VFIODevice vdev;
+} VFIOAPDevice;
+
+#define VFIO_AP_DEVICE(obj) \
+OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE)
+
+static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
+{
+vdev->needs_reset = false;
+}
+
+/*
+ * We don't need vfio_hot_reset_multi and vfio_eoi operations for
+ * vfio-ap device now.
+ */
+struct VFIODeviceOps vfio_ap_ops = {
+.vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
+};
+
+static void vfio_ap_put_device(VFIOAPDevice *vapdev)
+{
+g_free(vapdev->vdev.name);
+vfio_put_base_device(>vdev);
+}
+
+static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
+{
+GError *gerror = NULL;
+char *symlink, *group_path;
+int groupid;
+
+symlink = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
+group_path = g_file_read_link(symlink, );
+g_free(symlink);
+
+if (!group_path) {
+error_setg(errp, "%s: no iommu_group found for %s: %s",
+ VFIO_AP_DEVICE_TYPE, vapdev->vdev.sysfs
[Qemu-devel] [PATCH v11 4/6] s390x/ap: base Adjunct Processor (AP) object model
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
Tested-by: Pierre Morel
Acked-by: David Hildenbrand
Reviewed-by: Thomas Huth
Reviewed-by: Halil Pasic
---
MAINTAINERS | 12 ++
hw/s390x/Makefile.objs | 2 +
hw/s390x/ap-bridge.c | 78
hw/s390x/ap-device.c | 38 ++
hw/s390x/s390-virtio-ccw.c | 4 ++
include/hw/s390x/ap-bridge.h | 19 +
include/hw/s390x/ap-device.h | 22 ++
7 files changed, 175 insertions(+)
create mode 100644 hw/s390x/ap-bridge.c
create mode 100644 hw/s390x/ap-device.c
create mode 100644 include/hw/s390x/ap-bridge.h
create mode 100644 include/hw/s390x/ap-device.h
diff --git a/MAINTAINERS b/MAINTAINERS
index 3275cc6bbed1..b9984242820b 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1207,6 +1207,18 @@ F: include/hw/s390x/s390-ccw.h
T: git git://github.com/cohuck/qemu.git s390-next
L: qemu-s3...@nongnu.org
+vfio-ap
+M: Christian Borntraeger
+M: Tony Krowiak
+M: Halil Pasic
+M: Pierre Morel
+S: Supported
+F: hw/s390x/ap-device.c
+F: hw/s390x/ap-bridge.c
+F: include/hw/s390x/ap-device.h
+F: include/hw/s390x/ap-bridge.h
+L: qemu-s3...@nongnu.org
+
vhost
M: Michael S. Tsirkin
S: Supported
diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index 5dbc00ce9b45..49cae2f5c4bd 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -31,3 +31,5 @@ obj-$(CONFIG_TCG) += tod-qemu.o
obj-$(CONFIG_KVM) += s390-skeys-kvm.o
obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
obj-y += s390-ccw.o
+obj-y += ap-device.o
+obj-y += ap-bridge.o
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
new file mode 100644
index ..3795d30dd7c9
--- /dev/null
+++ b/hw/s390x/ap-bridge.c
@@ -0,0 +1,78 @@
+/*
+ * ap bridge
+ *
+ * Copyright 2018 IBM Corp.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "qemu/bitops.h"
+#include "hw/s390x/ap-bridge.h"
+#include "cpu.h"
+
+static char *ap_bus_get_dev_path(DeviceState *dev)
+{
+/* at most one */
+return g_strdup_printf("/1");
+}
+
+static void ap_bus_class_init(ObjectClass *oc, void *data)
+{
+BusClass *k = BUS_CLASS(oc);
+
+k->get_dev_path = ap_bus_get_dev_path;
+/* More than one ap device does not make sense */
+k->max_dev = 1;
+}
+
+static const TypeInfo ap_bus_info = {
+.name = TYPE_AP_BUS,
+.parent = TYPE_BUS,
+.instance_size = 0,
+.class_init = ap_bus_class_init,
+};
+
+void s390_init_ap(void)
+{
+DeviceState *dev;
+
+/* If no AP instructions then no need for AP bridge */
+if (!s390_has_feat(S390_FEAT_AP)) {
+return;
+}
+
+/* Create bridge device */
+dev = qdev_create(NULL, TYPE_AP_BRIDGE);
+object_property_add_child(qdev_get_machine(), TYPE_AP_BRIDGE,
+ OBJECT(dev), NULL);
+qdev_init_nofail(dev);
+
+/* Create bus on bridge device */
+qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+ }
+
+static void ap_bridge_class_init(ObjectClass *oc, void *data)
+{
+DeviceClass *dc = DEVICE_CLASS(oc);
+
+set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+}
+
+static const TypeInfo ap_bridge_info = {
+.name = TYPE_AP_BRIDGE,
+.parent= TYPE_SYS_BUS_DEVICE,
+.instance_size = 0,
+.class_init= ap_bridge_class_init,
+};
+
+static void ap_register(void)
+{
+type_register_static(_bridge_info);
+type_register_static(_bus_info);
+}
+
+type_init(ap_register)
diff --git a/hw/s390x/ap-device.c b/hw/s390x/ap-device.c
new file mode 100644
index ..f5ac8db96826
--- /dev/null
+++ b/hw/s390x/ap-device.c
@@ -0,0 +1,38 @@
+/*
+ * Adjunct Processor (AP) matrix device
+ *
+ * Copyright 2018 IBM Corp.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu/module.h"
+#include "qapi/error.h"
+#include "hw/qdev.h"
+#include "hw/s390x/ap-device.h"
+
+static void ap_class_init(ObjectClass *klass, void *data)
+{
+DeviceClass *dc = DEVICE_CLASS(klass);
+
+dc->desc = "AP device class";
+dc->hotpluggable = false;
+}
+
+static const TypeInfo ap_device_info = {
+.name = AP_DEVICE_TYPE,
+.parent = TYPE_DEVICE,
+.instance_size = sizeof(APDevice),
+.class_size = sizeof(DeviceClass),
+.class_init = ap_class_init,
+.abstract = true,
+};
+
+static void ap_device_register(void)
+{
+type_register_static(_device_info);
+}
+
+type_init(ap_device_register)
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-v
[Qemu-devel] [PATCH v11 6/6] s390: doc: detailed specifications for AP virtualization
This patch provides documentation describing the AP architecture and design concepts behind the virtualization of AP devices. It also includes an example of how to configure AP devices for exclusive use of KVM guests. Signed-off-by: Tony Krowiak Reviewed-by: Pierre Morel Tested-by: Pierre Morel --- MAINTAINERS | 2 + docs/vfio-ap.txt | 825 +++ 2 files changed, 827 insertions(+) create mode 100644 docs/vfio-ap.txt diff --git a/MAINTAINERS b/MAINTAINERS index 2730757b4482..fb81b3a8eb06 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -96,6 +96,7 @@ F: include/hw/watchdog/wdt_diag288.h F: pc-bios/s390-ccw/ F: pc-bios/s390-ccw.img F: target/s390x/ +F: docs/vfio-ap.txt K: ^Subject:.*(?i)s390x? T: git git://github.com/cohuck/qemu.git s390-next L: qemu-s3...@nongnu.org @@ -1219,6 +1220,7 @@ F: hw/s390x/ap-bridge.c F: include/hw/s390x/ap-device.h F: include/hw/s390x/ap-bridge.h F: hw/vfio/ap.c +F: docs/vfio-ap.txt L: qemu-s3...@nongnu.org vhost diff --git a/docs/vfio-ap.txt b/docs/vfio-ap.txt new file mode 100644 index ..12339684cd52 --- /dev/null +++ b/docs/vfio-ap.txt @@ -0,0 +1,825 @@ +Adjunct Processor (AP) Device += + +Contents: += +* Introduction +* AP Architectural Overview +* Start Interpretive Execution (SIE) Instruction +* AP Matrix Configuration on Linux Host +* Starting a Linux Guest Configured with an AP Matrix +* Example: Configure AP Matrices for Three Linux Guests + +Introduction: + +The IBM Adjunct Processor (AP) Cryptographic Facility is comprised +of three AP instructions and from 1 to 256 PCIe cryptographic adapter cards. +These AP devices provide cryptographic functions to all CPUs assigned to a +linux system running in an IBM Z system LPAR. + +On s390x, AP adapter cards are exposed via the AP bus. This document +describes how those cards may be made available to KVM guests using the +VFIO mediated device framework. + +AP Architectural Overview: += +In order understand the terminology used in the rest of this document, let's +start with some definitions: + +* AP adapter + + An AP adapter is an IBM Z adapter card that can perform cryptographic + functions. There can be from 0 to 256 adapters assigned to an LPAR depending + on the machine model. Adapters assigned to the LPAR in which a linux host is + running will be available to the linux host. Each adapter is identified by a + number from 0 to 255; however, the maximum adapter number allowed is + determined by machine model. When installed, an AP adapter is accessed by + AP instructions executed by any CPU. + +* AP domain + + An adapter is partitioned into domains. Each domain can be thought of as + a set of hardware registers for processing AP instructions. An adapter can + hold up to 256 domains; however, the maximum domain number allowed is + determined by machine model. Each domain is identified by a number from 0 to + 255. Domains can be further classified into two types: + +* Usage domains are domains that can be accessed directly to process AP + commands + +* Control domains are domains that are accessed indirectly by AP + commands sent to a usage domain to control or change the domain; for + example, to set a secure private key for the domain. + +* AP Queue + + An AP queue is the means by which an AP command-request message is sent to an + AP usage domain inside a specific AP. An AP queue is identified by a tuple + comprised of an AP adapter ID (APID) and an AP queue index (APQI). The + APQI corresponds to a given usage domain number within the adapter. This tuple + forms an AP Queue Number (APQN) uniquely identifying an AP queue. AP + instructions include a field containing the APQN to identify the AP queue to + which the AP command-request message is to be sent for processing. + +* AP Instructions: + + There are three AP instructions: + + * NQAP: to enqueue an AP command-request message to a queue + * DQAP: to dequeue an AP command-reply message from a queue + * PQAP: to administer the queues + + AP instructions identify the domain that is targeted to process the AP + command; this must be one of the usage domains. An AP command may modify a + domain that is not one of the usage domains, but the modified domain + must be one of the control domains. + +Start Interpretive Execution (SIE) Instruction +== +A KVM guest is started by executing the Start Interpretive Execution (SIE) +instruction. The SIE state description is a control block that contains the +state information for a KVM guest and is supplied as input to the SIE +instruction. The SIE state description contains a satellite control block called +the Crypto Control Block (CRYCB). The CRYCB contains three fields to identify +the adapters, usage domains and control domains assigned to the KVM guest: + +* The AP Mask (APM) field is a bit
[Qemu-devel] [PATCH v11 3/6] s390x/kvm: enable AP instruction interpretation for guest
Let's use the KVM_SET_DEVICE_ATTR ioctl to enable hardware
interpretation of AP instructions executed on the guest.
If the S390_FEAT_AP feature is switched on for the guest,
AP instructions must be interpreted by default; otherwise,
they will be intercepted.
This attribute setting may be overridden by a device. For example,
a device may want to provide AP instructions to the guest (i.e.,
S390_FEAT_AP turned on), but it may want to emulate them. In this
case, the AP instructions executed on the guest must be
intercepted; so when the device is realized, it must disable
interpretation.
Signed-off-by: Tony Krowiak
Tested-by: Pierre Morel
Reviewed-by: David Hildenbrand
Reviewed-by: Thomas Huth
Reviewed-by: Christian Borntraeger
Acked-by: Halil Pasic
---
target/s390x/kvm.c | 19 +++
1 file changed, 19 insertions(+)
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 78d39b34d0b8..2ebf26adfe24 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -2299,11 +2299,26 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model,
Error **errp)
error_setg(errp, "KVM: host CPU model could not be identified");
return;
}
+/* for now, we can only provide the AP feature with HW support */
+if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO,
+KVM_S390_VM_CRYPTO_ENABLE_APIE)) {
+set_bit(S390_FEAT_AP, model->features);
+}
/* strip of features that are not part of the maximum model */
bitmap_and(model->features, model->features, model->def->full_feat,
S390_FEAT_MAX);
}
+static void kvm_s390_configure_apie(bool interpret)
+{
+uint64_t attr = interpret ? KVM_S390_VM_CRYPTO_ENABLE_APIE :
+KVM_S390_VM_CRYPTO_DISABLE_APIE;
+
+if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO, attr)) {
+kvm_s390_set_attr(attr);
+}
+}
+
void kvm_s390_apply_cpu_model(const S390CPUModel *model, Error **errp)
{
struct kvm_s390_vm_cpu_processor prop = {
@@ -2353,6 +2368,10 @@ void kvm_s390_apply_cpu_model(const S390CPUModel *model,
Error **errp)
if (test_bit(S390_FEAT_CMM, model->features)) {
kvm_s390_enable_cmma();
}
+
+if (test_bit(S390_FEAT_AP, model->features)) {
+kvm_s390_configure_apie(true);
+}
}
void kvm_s390_restart_interrupt(S390CPU *cpu)
--
2.19.0.221.g150f307
[Qemu-devel] [PATCH v11 1/6] linux-headers: linux header updates for AP support
Updates the linux header files in preparation for introduction
of the VFIO AP device:
* Added device attributes to the KVM_S390_VM_CRYPTO group
to indicate whether AP instructions are to be interpreted
* Added VFIO device information for AP devices
Signed-off-by: Tony Krowiak
---
linux-headers/asm-s390/kvm.h | 2 ++
linux-headers/linux/vfio.h | 2 ++
2 files changed, 4 insertions(+)
diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index 1ab9901911bf..0265482f8fdf 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -160,6 +160,8 @@ struct kvm_s390_vm_cpu_subfunc {
#define KVM_S390_VM_CRYPTO_ENABLE_DEA_KW 1
#define KVM_S390_VM_CRYPTO_DISABLE_AES_KW 2
#define KVM_S390_VM_CRYPTO_DISABLE_DEA_KW 3
+#define KVM_S390_VM_CRYPTO_ENABLE_APIE 4
+#define KVM_S390_VM_CRYPTO_DISABLE_APIE5
/* kvm attributes for migration mode */
#define KVM_S390_VM_MIGRATION_STOP 0
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 3615a269d378..838919a4c03a 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -200,6 +200,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2)/* vfio-platform device */
#define VFIO_DEVICE_FLAGS_AMBA (1 << 3) /* vfio-amba device */
#define VFIO_DEVICE_FLAGS_CCW (1 << 4)/* vfio-ccw device */
+#define VFIO_DEVICE_FLAGS_AP (1 << 5) /* vfio-ap device */
__u32 num_regions;/* Max region index + 1 */
__u32 num_irqs; /* Max IRQ index + 1 */
};
@@ -215,6 +216,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_API_PLATFORM_STRING"vfio-platform"
#define VFIO_DEVICE_API_AMBA_STRING"vfio-amba"
#define VFIO_DEVICE_API_CCW_STRING "vfio-ccw"
+#define VFIO_DEVICE_API_AP_STRING "vfio-ap"
/**
* VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
--
2.19.0.221.g150f307
[Qemu-devel] [PATCH v11 2/6] s390x/cpumodel: Set up CPU model for AP device support
A new CPU model feature and two new CPU model facilities are
introduced to support AP devices for a KVM guest.
CPU model features:
1. The S390_FEAT_AP CPU model feature indicates whether AP
instructions are available to the guest. This feature will
be enabled only if the AP instructions are available on the
linux host as determined by the availability of the
KVM_S390_VM_CRYPTO_ENABLE_APIE VM attribute which is exposed
by KVM only if the AP instructions are available on the
host.
This feature must be turned on from userspace to execute AP
instructions on the KVM guest. The QEMU command line to turn
this feature on looks something like this:
qemu-system-s390x ... -cpu xxx,ap=on ...
This feature will be supported for zEC12 and newer CPU models.
The feature will not be supported for older models because
there are few older systems on which to test and the older
crypto cards will be going out of service in the relatively
near future.
CPU model facilities:
1. The S390_FEAT_AP_QUERY_CONFIG_INFO feature indicates whether the
AP Query Configuration Information (QCI) facility is available
to the guest as determined by whether the facility is available
on the host. This feature will be exposed by KVM only if the
QCI facility is installed on the host.
2. The S390_FEAT_AP_FACILITY_TEST feature indicates whether the AP
Facility Test (APFT) facility is available to the guest as
determined by whether the facility is available on the host.
This feature will be exposed by KVM only if APFT is installed
on the host.
Signed-off-by: Tony Krowiak
Tested-by: Pierre Morel
Reviewed-by: David Hildenbrand
Reviewed-by: Halil Pasic
Reviewed-by: Christian Borntraeger
---
target/s390x/cpu_features.c | 3 +++
target/s390x/cpu_features_def.h | 3 +++
target/s390x/cpu_models.c | 2 ++
target/s390x/gen-features.c | 3 +++
4 files changed, 11 insertions(+)
diff --git a/target/s390x/cpu_features.c b/target/s390x/cpu_features.c
index 172fb18df718..60cfeba48f4e 100644
--- a/target/s390x/cpu_features.c
+++ b/target/s390x/cpu_features.c
@@ -39,8 +39,10 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT("srs", S390_FEAT_TYPE_STFL, 9, "Sense-running-status facility"),
FEAT_INIT("csske", S390_FEAT_TYPE_STFL, 10, "Conditional-SSKE facility"),
FEAT_INIT("ctop", S390_FEAT_TYPE_STFL, 11, "Configuration-topology
facility"),
+FEAT_INIT("apqci", S390_FEAT_TYPE_STFL, 12, "Query AP Configuration
Information facility"),
FEAT_INIT("ipter", S390_FEAT_TYPE_STFL, 13, "IPTE-range facility"),
FEAT_INIT("nonqks", S390_FEAT_TYPE_STFL, 14, "Nonquiescing key-setting
facility"),
+FEAT_INIT("apft", S390_FEAT_TYPE_STFL, 15, "AP Facilities Test facility"),
FEAT_INIT("etf2", S390_FEAT_TYPE_STFL, 16, "Extended-translation facility
2"),
FEAT_INIT("msa-base", S390_FEAT_TYPE_STFL, 17, "Message-security-assist
facility (excluding subfunctions)"),
FEAT_INIT("ldisp", S390_FEAT_TYPE_STFL, 18, "Long-displacement facility"),
@@ -129,6 +131,7 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT_MISC("dateh2", "DAT-enhancement facility 2"),
FEAT_INIT_MISC("cmm", "Collaborative-memory-management facility"),
+FEAT_INIT_MISC("ap", "AP instructions installed"),
FEAT_INIT("plo-cl", S390_FEAT_TYPE_PLO, 0, "PLO Compare and load (32 bit
in general registers)"),
FEAT_INIT("plo-clg", S390_FEAT_TYPE_PLO, 1, "PLO Compare and load (64 bit
in parameter list)"),
diff --git a/target/s390x/cpu_features_def.h b/target/s390x/cpu_features_def.h
index ac2c947f30a8..5fc7e7bf0116 100644
--- a/target/s390x/cpu_features_def.h
+++ b/target/s390x/cpu_features_def.h
@@ -27,8 +27,10 @@ typedef enum {
S390_FEAT_SENSE_RUNNING_STATUS,
S390_FEAT_CONDITIONAL_SSKE,
S390_FEAT_CONFIGURATION_TOPOLOGY,
+S390_FEAT_AP_QUERY_CONFIG_INFO,
S390_FEAT_IPTE_RANGE,
S390_FEAT_NONQ_KEY_SETTING,
+S390_FEAT_AP_FACILITIES_TEST,
S390_FEAT_EXTENDED_TRANSLATION_2,
S390_FEAT_MSA,
S390_FEAT_LONG_DISPLACEMENT,
@@ -119,6 +121,7 @@ typedef enum {
/* Misc */
S390_FEAT_DAT_ENH_2,
S390_FEAT_CMM,
+S390_FEAT_AP,
/* PLO */
S390_FEAT_PLO_CL,
diff --git a/target/s390x/cpu_models.c b/target/s390x/cpu_models.c
index 265d25c937bb..7c253ff308c5 100644
--- a/target/s390x/cpu_models.c
+++ b/target/s390x/cpu_models.c
@@ -786,6 +786,8 @@ static void check_consistency(const S390CPUModel *model)
{ S390_FEAT_PRNO_TRNG_QRTCR, S390_FEAT_MSA_EXT_5 },
{ S390_FEAT_PRNO_TRNG, S390_FEAT_MSA_EXT_5 },
{ S390_FEAT_SIE_KSS, S390_FEAT_SIE_F2 },
+{ S390_FEAT_AP_QUERY_CON
[Qemu-devel] [PATCH v11 0/6] s390x: vfio-ap: guest dedicated crypto adapters
This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model. See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series. v10 => v11 Change log: = * Replaced DO_UPCAST macros in ap.c * Initializing GError in vfio_ap_get_group() function (BUG) * Updated s390 maintainers with new files in MAINTAINERS * Fixed memory leak in vfio_ap_get_group() function v9 => v10 Change log: * Removed KVM_S390_VM_CPU_FEAT_AP feature from kvm.h * Moved check for KVM_S390_VM_CRYPTO_ENABLE_APIE from patch 2/6 to patch 3/6 * Removed vfio from all function names in ap-bridge.c * Removed unused macros and structure from ap-bridge.h * Removed unused macros from ap-device.h v8 => v9 Change log: === * Removed all references to VFIO in AP bridge and bus * Expose AP feature only if the KVM_S390_VM_CRYPTO_ENABLE_APIE VM attribute is exposed by KVM - i.e., if AP instructions are available on the linux host. * Enable AP interpretation only if AP feature is switched on; no need to disable because it is disabled by default. v7 => v8 Change log: === * Enable SIE interpretation AP instructions if the CPU model feature for AP instructions is turned on for the guest. v6 => v7 Change log; === * Changed email address for Signed-off-by v5 => v6 Change log: === * Added reset handling fo vfio-ap device * Added a bridge/bus to AP device object model - thanks to Halil Pasic v4 => v5 Change log: === * Added MAINTAINERS entries for VFIO AP * Added explanation for why we are only supporting zEC12 and newer CPU models. * Changed CPU model feature qci=on|off to apqci=on|off * Misc. minor changes v3 => v4 Change log: === * Made vfio-ap device unpluggable for now * Renamed command line CPU model feature for QCI: qci=on -> apqci=on * Removed call to KVM_S390_VM_CRYPTO_INTERPRET_AP ioctl - ioctl was removed from kernel and AP instruction interpretation is set from the VFIO device driver * Added check to ensure only one vfio-ap device can be configured per guest * Removed AP instruction interception handlers: AP instructions will be interpreted by default if AP facilities are installed to handle the case where feature ap=on and no vfio-ap device is configured for the guest. Tony Krowiak (6): linux-headers: linux header updates for AP support s390x/cpumodel: Set up CPU model for AP device support s390x/kvm: enable AP instruction interpretation for guest s390x/ap: base Adjunct Processor (AP) object model s390x/vfio: ap: Introduce VFIO AP device s390: doc: detailed specifications for AP virtualization MAINTAINERS | 16 + default-configs/s390x-softmmu.mak | 1 + docs/vfio-ap.txt | 825 ++ hw/s390x/Makefile.objs| 2 + hw/s390x/ap-bridge.c | 78 +++ hw/s390x/ap-device.c | 38 ++ hw/s390x/s390-virtio-ccw.c| 4 + hw/vfio/Makefile.objs | 1 + hw/vfio/ap.c | 179 +++ include/hw/s390x/ap-bridge.h | 19 + include/hw/s390x/ap-device.h | 22 + include/hw/vfio/vfio-common.h | 1 + linux-headers/asm-s390/kvm.h | 2 + linux-headers/linux/vfio.h| 2 + target/s390x/cpu_features.c | 3 + target/s390x/cpu_features_def.h | 3 + target/s390x/cpu_models.c | 2 + target/s390x/gen-features.c | 3 + target/s390x/kvm.c| 19 + 19 files changed, 1220 insertions(+) create mode 100644 docs/vfio-ap.txt create mode 100644 hw/s390x/ap-bridge.c create mode 100644 hw/s390x/ap-device.c create mode 100644 hw/vfio/ap.c create mode 100644 include/hw/s390x/ap-bridge.h create mode 100644 include/hw/s390x/ap-device.h -- 2.19.0.221.g150f307
Re: [Qemu-devel] [PATCH v10 5/6] s390x/vfio: ap: Introduce VFIO AP device
On 10/10/2018 05:21 AM, Cornelia Huck wrote: On Tue, 9 Oct 2018 13:52:25 -0400 Tony Krowiak wrote: diff --git a/MAINTAINERS b/MAINTAINERS index 97e8ed808bc0..29041da69237 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1209,6 +1209,7 @@ F: hw/s390x/ap-device.c F: hw/s390x/ap-bridge.c F: include/hw/s390x/ap-device.h F: include/hw/s390x/ap-bridge.h +F: hw/vfio/ap.c L: qemu-s3...@nongnu.org vhost Can you please also add hw/vfio/ap.c to the overall s390 architecture section in MAINTAINERS? The existing pattern does not catch it (but it does catch the files added in the last patch, so that's ok.) Yes I can.
Re: [Qemu-devel] [qemu-s390x] [PATCH v10 6/6] s390: doc: detailed specifications for AP virtualization
On 10/10/2018 04:14 AM, Thomas Huth wrote:
On 2018-10-09 19:52, Tony Krowiak wrote:
This patch provides documentation describing the AP architecture and
design concepts behind the virtualization of AP devices. It also
includes an example of how to configure AP devices for exclusive
use of KVM guests.
Signed-off-by: Tony Krowiak
Reviewed-by: Pierre Morel
Tested-by: Pierre Morel
---
MAINTAINERS | 1 +
docs/vfio-ap.txt | 825 +++
2 files changed, 826 insertions(+)
create mode 100644 docs/vfio-ap.txt
[...]
+ * Individual bits in the mask can be switched on and off by specifying
+each bit number to be switched in a comma separated list. Each bit
+number string must be prepended with a ('+') or minus ('-') to indicate
+the corresponding bit is to be switched on ('+') or off ('-'). Some
+valid values are:
Nit: My git complains about a superfluous white space after "are:"
I ran checkpatch and my git did not complain, but I'll make sure there
are no whitespaces at the end of the lines.
[...]
+assign_adapter
+ To assign an AP adapter to the mediated matrix device, its APID is written
+ to the 'assign_adapter' file. This may be done multiple times to assign more
+ than one adapter. The APID may be specified using conventional semantics
+ as a decimal, hexidecimal, or octal number. For example, to assign adapters
+ 4, 5 and 16 to a mediated matrix device in decimal, hexidecimal and octal
+ respectively:
In case you respin: I'd still prefer s/hexidecimal/hexadecimal/g in the
whole document (but it's just a nit, so no need to respin just because
of this)
I will be submitting a v11, so I'll go ahead and fix this.
Thomas
Re: [Qemu-devel] [PATCH v10 5/6] s390x/vfio: ap: Introduce VFIO AP device
On 10/10/2018 08:49 AM, Christian Borntraeger wrote:
On 10/10/2018 02:37 PM, Pierre Morel wrote:
On 09/10/2018 19:52, Tony Krowiak wrote:
+static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
+{
+GError *gerror;
+char *symlink, *group_path;
+int groupid;
+
+symlink = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
+group_path = g_file_read_link(symlink, );
hum I oversaw this change, it leads to segfault.
Yes, this was a review feedback from v9 to use the glib function.
You must initialize gerror before use.
The following patch avoid a segmentation fault:
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
index 5543406afc..3b8e9ba6dc 100644
--- a/hw/vfio/ap.c
+++ b/hw/vfio/ap.c
@@ -59,7 +59,7 @@ static void vfio_ap_put_device(VFIOAPDevice *vapdev)
static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
{
-GError *gerror;
+GError *gerror = NULL;
char *symlink, *group_path;
int groupid;
With that fix, series
Tested-by: Christian Borntraeger
Tony, can you fold that fixup from Pierre into your v11?
It is done.
Re: [Qemu-devel] [qemu-s390x] [PATCH v10 5/6] s390x/vfio: ap: Introduce VFIO AP device
On 10/10/2018 04:52 AM, Cornelia Huck wrote:
On Wed, 10 Oct 2018 10:25:22 +0200
Thomas Huth wrote:
On 2018-10-09 19:52, Tony Krowiak wrote:
[...]
+static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
+{
+GError *gerror;
+char *symlink, *group_path;
+int groupid;
+
+symlink = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
+group_path = g_file_read_link(symlink, );
+g_free(symlink);
+
+if (!group_path) {
+error_setg(errp, "%s: no iommu_group found for %s: %s",
+ VFIO_AP_DEVICE_TYPE, vapdev->vdev.sysfsdev,
gerror->message);
+return NULL;
+}
+
+if (sscanf(basename(group_path), "%d", ) != 1) {
+error_setg(errp, "vfio: failed to read %s", group_path);
+return NULL;
+}
+
+return vfio_get_group(groupid, _space_memory, errp);
+}
I think you've got to g_free(group_path) after you don't need it anymore.
+static void vfio_ap_realize(DeviceState *dev, Error **errp)
+{
+int ret;
+char *mdevid;
+Error *local_err = NULL;
+VFIOGroup *vfio_group;
+APDevice *apdev = AP_DEVICE(dev);
+VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
+
+vfio_group = vfio_ap_get_group(vapdev, _err);
+if (!vfio_group) {
+goto out_err;
+}
+
+vapdev->vdev.ops = _ap_ops;
+vapdev->vdev.type = VFIO_DEVICE_TYPE_AP;
+mdevid = basename(vapdev->vdev.sysfsdev);
+vapdev->vdev.name = g_strdup_printf("%s", mdevid);
+vapdev->vdev.dev = dev;
+
+ret = vfio_get_device(vfio_group, mdevid, >vdev, _err);
+if (ret) {
+goto out_get_dev_err;
+}
+
+/* Enable hardware to intepret AP instructions executed on the guest */
+object_property_set_bool(OBJECT(qdev_get_machine()), true, "apie", NULL);
+
+return;
+
+out_get_dev_err:
+vfio_ap_put_device(vapdev);
+vfio_put_group(vfio_group);
+out_err:
+error_propagate(errp, local_err);
+}
+
+static void vfio_ap_unrealize(DeviceState *dev, Error **errp)
+{
+APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
+VFIOAPDevice *vapdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
Didn't you want to remove the DO_UPCASTs ?
+VFIOGroup *group = vapdev->vdev.group;
+
+vfio_ap_put_device(vapdev);
+vfio_put_group(group);
+}
+
+static Property vfio_ap_properties[] = {
+DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+DEFINE_PROP_END_OF_LIST(),
+};
+
+static void vfio_ap_reset(DeviceState *dev)
+{
+int ret;
+APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
+VFIOAPDevice *vapdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
dito
+ret = ioctl(vapdev->vdev.fd, VFIO_DEVICE_RESET);
+if (ret) {
+error_report("%s: failed to reset %s device: %s", __func__,
+ vapdev->vdev.name, strerror(ret));
+}
+}
OK, this is now a bit too much stuff all in all to change while
applying...
Tony, can you please do a respin with these changes and the other
things people have noticed? Thanks!
I will have v11 posted today.
Re: [Qemu-devel] [qemu-s390x] [PATCH v10 5/6] s390x/vfio: ap: Introduce VFIO AP device
On 10/10/2018 04:25 AM, Thomas Huth wrote:
On 2018-10-09 19:52, Tony Krowiak wrote:
Introduces a VFIO based AP device. The device is defined via
the QEMU command line by specifying:
-device vfio-ap,sysfsdev=
There may be only one vfio-ap device configured for a guest.
The mediated matrix device is created by the VFIO AP device
driver by writing a UUID to a sysfs attribute file (see
docs/vfio-ap.txt). The mediated matrix device will be named
after the UUID. Symbolic links to the $uuid are created in
many places, so the path to the mediated matrix device $uuid
can be specified in any of the following ways:
/sys/devices/vfio_ap/matrix/$uuid
/sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
/sys/bus/mdev/devices/$uuid
/sys/bus/mdev/drivers/vfio_mdev/$uuid
When the vfio-ap device is realized, it acquires and opens the
VFIO iommu group to which the mediated matrix device is
bound. This causes a VFIO group notification event to be
signaled. The vfio_ap device driver's group notification
handler will get called at which time the device driver
will configure the the AP devices to which the guest will
be granted access.
Signed-off-by: Tony Krowiak
Tested-by: Pierre Morel
---
[...]
+static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
+{
+GError *gerror;
+char *symlink, *group_path;
+int groupid;
+
+symlink = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
+group_path = g_file_read_link(symlink, );
+g_free(symlink);
+
+if (!group_path) {
+error_setg(errp, "%s: no iommu_group found for %s: %s",
+ VFIO_AP_DEVICE_TYPE, vapdev->vdev.sysfsdev,
gerror->message);
+return NULL;
+}
+
+if (sscanf(basename(group_path), "%d", ) != 1) {
+error_setg(errp, "vfio: failed to read %s", group_path);
+return NULL;
+}
+
+return vfio_get_group(groupid, _space_memory, errp);
+}
I think you've got to g_free(group_path) after you don't need it anymore.
Right you are.
+static void vfio_ap_realize(DeviceState *dev, Error **errp)
+{
+int ret;
+char *mdevid;
+Error *local_err = NULL;
+VFIOGroup *vfio_group;
+APDevice *apdev = AP_DEVICE(dev);
+VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
+
+vfio_group = vfio_ap_get_group(vapdev, _err);
+if (!vfio_group) {
+goto out_err;
+}
+
+vapdev->vdev.ops = _ap_ops;
+vapdev->vdev.type = VFIO_DEVICE_TYPE_AP;
+mdevid = basename(vapdev->vdev.sysfsdev);
+vapdev->vdev.name = g_strdup_printf("%s", mdevid);
+vapdev->vdev.dev = dev;
+
+ret = vfio_get_device(vfio_group, mdevid, >vdev, _err);
+if (ret) {
+goto out_get_dev_err;
+}
+
+/* Enable hardware to intepret AP instructions executed on the guest */
+object_property_set_bool(OBJECT(qdev_get_machine()), true, "apie", NULL);
+
+return;
+
+out_get_dev_err:
+vfio_ap_put_device(vapdev);
+vfio_put_group(vfio_group);
+out_err:
+error_propagate(errp, local_err);
+}
+
+static void vfio_ap_unrealize(DeviceState *dev, Error **errp)
+{
+APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
+VFIOAPDevice *vapdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
Didn't you want to remove the DO_UPCASTs ?
oops, missed these fixed them in the realize function, but
missed this one and the next will fix
+VFIOGroup *group = vapdev->vdev.group;
+
+vfio_ap_put_device(vapdev);
+vfio_put_group(group);
+}
+
+static Property vfio_ap_properties[] = {
+DEFINE_PROP_STRING("sysfsdev", VFIOAPDevice, vdev.sysfsdev),
+DEFINE_PROP_END_OF_LIST(),
+};
+
+static void vfio_ap_reset(DeviceState *dev)
+{
+int ret;
+APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
+VFIOAPDevice *vapdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
dito
ditto
+ret = ioctl(vapdev->vdev.fd, VFIO_DEVICE_RESET);
+if (ret) {
+error_report("%s: failed to reset %s device: %s", __func__,
+ vapdev->vdev.name, strerror(ret));
+}
+}
Thomas
Re: [Qemu-devel] [PATCH v10 5/6] s390x/vfio: ap: Introduce VFIO AP device
On 10/09/2018 03:51 PM, David Hildenbrand wrote:
+static void vfio_ap_realize(DeviceState *dev, Error **errp)
+{
+int ret;
+char *mdevid;
+Error *local_err = NULL;
+VFIOGroup *vfio_group;
+APDevice *apdev = AP_DEVICE(dev);
+VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
+
+vfio_group = vfio_ap_get_group(vapdev, _err);
+if (!vfio_group) {
+goto out_err;
+}
+
+vapdev->vdev.ops = _ap_ops;
+vapdev->vdev.type = VFIO_DEVICE_TYPE_AP;
+mdevid = basename(vapdev->vdev.sysfsdev);
+vapdev->vdev.name = g_strdup_printf("%s", mdevid);
+vapdev->vdev.dev = dev;
+
+ret = vfio_get_device(vfio_group, mdevid, >vdev, _err);
+if (ret) {
+goto out_get_dev_err;
+}
+
+/* Enable hardware to intepret AP instructions executed on the guest */
+object_property_set_bool(OBJECT(qdev_get_machine()), true, "apie", NULL);
+
I commented on the old version that this is wrong (if I am not starting
to lose my memory). This has to go. (there is no such property, this
will simply report an error we ignore)
(can most probably be fixed when applying)
I don't recall whether you mentioned it, but I will fix it, or it can be
fixed when applied Connie?
Re: [Qemu-devel] [PATCH v10 4/6] s390x/ap: base Adjunct Processor (AP) object model
On 10/10/2018 04:14 AM, Cornelia Huck wrote: On Tue, 9 Oct 2018 13:52:24 -0400 Tony Krowiak wrote: diff --git a/hw/s390x/ap-device.c b/hw/s390x/ap-device.c new file mode 100644 index ..fb6e35024c82 --- /dev/null +++ b/hw/s390x/ap-device.c @@ -0,0 +1,39 @@ +/* + * Adjunct Processor (AP) matrix device + * + * Copyright 2018 IBM Corp. + * Author(s): Tony Krowiak Do you also want this to use the vnet-less form instead? + * + * This work is licensed under the terms of the GNU GPL, version 2 or (at + * your option) any later version. See the COPYING file in the top-level + * directory. + */ diff --git a/include/hw/s390x/ap-device.h b/include/hw/s390x/ap-device.h new file mode 100644 index ..4fb3c9ab82f2 --- /dev/null +++ b/include/hw/s390x/ap-device.h @@ -0,0 +1,23 @@ +/* + * Adjunct Processor (AP) matrix device interfaces + * + * Copyright 2018 IBM Corp. + * Author(s): Tony Krowiak Same here. Yes. Shall I just submit a v11?
Re: [Qemu-devel] [PATCH v10 2/6] s390x/cpumodel: Set up CPU model for AP device support
On 10/09/2018 03:48 PM, David Hildenbrand wrote: On 09/10/2018 21:14, Christian Borntraeger wrote: On 10/09/2018 07:52 PM, Tony Krowiak wrote: A new CPU model feature and two new CPU model facilities are introduced to support AP devices for a KVM guest. CPU model features: 1. The S390_FEAT_AP CPU model feature indicates whether AP instructions are available to the guest. This feature will be enabled only if the AP instructions are available on the linux host as determined by the availability of the KVM_S390_VM_CRYPTO_ENABLE_APIE VM attribute which is exposed by KVM only if the AP instructions are available on the host. This feature must be turned on from userspace to execute AP instructions on the KVM guest. The QEMU command line to turn this feature on looks something like this: qemu-system-s390x ... -cpu xxx,ap=on ... This feature will be supported for zEC12 and newer CPU models. The feature will not be supported for older models because there are few older systems on which to test and the older crypto cards will be going out of service in the relatively near future. CPU model facilities: 1. The S390_FEAT_AP_QUERY_CONFIG_INFO feature indicates whether the AP Query Configuration Information (QCI) facility is available to the guest as determined by whether the facility is available on the host. This feature will be exposed by KVM only if the QCI facility is installed on the host. 2. The S390_FEAT_AP_FACILITY_TEST feature indicates whether the AP Facility Test (APFT) facility is available to the guest as determined by whether the facility is available on the host. This feature will be exposed by KVM only if APFT is installed on the host. Signed-off-by: Tony Krowiak Tested-by: Pierre Morel Reviewed-by: Christian Borntraeger and as you have moved the one hunk, Davids RB should still count. Indeed Reviewed-by: David Hildenbrand Got it.
[Qemu-devel] [PATCH v10 4/6] s390x/ap: base Adjunct Processor (AP) object model
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
Tested-by: Pierre Morel
Acked-by: David Hildenbrand
---
MAINTAINERS | 12 ++
hw/s390x/Makefile.objs | 2 +
hw/s390x/ap-bridge.c | 78
hw/s390x/ap-device.c | 39 ++
hw/s390x/s390-virtio-ccw.c | 4 ++
include/hw/s390x/ap-bridge.h | 19 +
include/hw/s390x/ap-device.h | 23 +++
7 files changed, 177 insertions(+)
create mode 100644 hw/s390x/ap-bridge.c
create mode 100644 hw/s390x/ap-device.c
create mode 100644 include/hw/s390x/ap-bridge.h
create mode 100644 include/hw/s390x/ap-device.h
diff --git a/MAINTAINERS b/MAINTAINERS
index d12518c08f10..97e8ed808bc0 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1199,6 +1199,18 @@ F: include/hw/s390x/s390-ccw.h
T: git git://github.com/cohuck/qemu.git s390-next
L: qemu-s3...@nongnu.org
+vfio-ap
+M: Christian Borntraeger
+M: Tony Krowiak
+M: Halil Pasic
+M: Pierre Morel
+S: Supported
+F: hw/s390x/ap-device.c
+F: hw/s390x/ap-bridge.c
+F: include/hw/s390x/ap-device.h
+F: include/hw/s390x/ap-bridge.h
+L: qemu-s3...@nongnu.org
+
vhost
M: Michael S. Tsirkin
S: Supported
diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index 93282f7c593c..add89b150d90 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -20,3 +20,5 @@ obj-$(CONFIG_TCG) += tod-qemu.o
obj-$(CONFIG_KVM) += s390-skeys-kvm.o
obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
obj-y += s390-ccw.o
+obj-y += ap-device.o
+obj-y += ap-bridge.o
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
new file mode 100644
index ..3795d30dd7c9
--- /dev/null
+++ b/hw/s390x/ap-bridge.c
@@ -0,0 +1,78 @@
+/*
+ * ap bridge
+ *
+ * Copyright 2018 IBM Corp.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "qemu/bitops.h"
+#include "hw/s390x/ap-bridge.h"
+#include "cpu.h"
+
+static char *ap_bus_get_dev_path(DeviceState *dev)
+{
+/* at most one */
+return g_strdup_printf("/1");
+}
+
+static void ap_bus_class_init(ObjectClass *oc, void *data)
+{
+BusClass *k = BUS_CLASS(oc);
+
+k->get_dev_path = ap_bus_get_dev_path;
+/* More than one ap device does not make sense */
+k->max_dev = 1;
+}
+
+static const TypeInfo ap_bus_info = {
+.name = TYPE_AP_BUS,
+.parent = TYPE_BUS,
+.instance_size = 0,
+.class_init = ap_bus_class_init,
+};
+
+void s390_init_ap(void)
+{
+DeviceState *dev;
+
+/* If no AP instructions then no need for AP bridge */
+if (!s390_has_feat(S390_FEAT_AP)) {
+return;
+}
+
+/* Create bridge device */
+dev = qdev_create(NULL, TYPE_AP_BRIDGE);
+object_property_add_child(qdev_get_machine(), TYPE_AP_BRIDGE,
+ OBJECT(dev), NULL);
+qdev_init_nofail(dev);
+
+/* Create bus on bridge device */
+qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+ }
+
+static void ap_bridge_class_init(ObjectClass *oc, void *data)
+{
+DeviceClass *dc = DEVICE_CLASS(oc);
+
+set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+}
+
+static const TypeInfo ap_bridge_info = {
+.name = TYPE_AP_BRIDGE,
+.parent= TYPE_SYS_BUS_DEVICE,
+.instance_size = 0,
+.class_init= ap_bridge_class_init,
+};
+
+static void ap_register(void)
+{
+type_register_static(_bridge_info);
+type_register_static(_bus_info);
+}
+
+type_init(ap_register)
diff --git a/hw/s390x/ap-device.c b/hw/s390x/ap-device.c
new file mode 100644
index ..fb6e35024c82
--- /dev/null
+++ b/hw/s390x/ap-device.c
@@ -0,0 +1,39 @@
+/*
+ * Adjunct Processor (AP) matrix device
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu/module.h"
+#include "qapi/error.h"
+#include "hw/qdev.h"
+#include "hw/s390x/ap-device.h"
+
+static void ap_class_init(ObjectClass *klass, void *data)
+{
+DeviceClass *dc = DEVICE_CLASS(klass);
+
+dc->desc = "AP device class";
+dc->hotpluggable = false;
+}
+
+static const TypeInfo ap_device_info = {
+.name = AP_DEVICE_TYPE,
+.parent = TYPE_DEVICE,
+.instance_size = sizeof(APDevice),
+.class_size = sizeof(DeviceClass),
+.class_init = ap_class_init,
+.abstract = true,
+};
+
+static void ap_device_register(void)
+{
+type_register_static(_device_info);
+}
+
+type_init(ap_device_register)
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-v
[Qemu-devel] [PATCH v10 3/6] s390x/kvm: enable AP instruction interpretation for guest
From: Tony Krowiak
Let's use the KVM_SET_DEVICE_ATTR ioctl to enable hardware
interpretation of AP instructions executed on the guest.
If the S390_FEAT_AP feature is switched on for the guest,
AP instructions must be interpreted by default; otherwise,
they will be intercepted.
This attribute setting may be overridden by a device. For example,
a device may want to provide AP instructions to the guest (i.e.,
S390_FEAT_AP turned on), but it may want to emulate them. In this
case, the AP instructions executed on the guest must be
intercepted; so when the device is realized, it must disable
interpretation.
Signed-off-by: Tony Krowiak
Tested-by: Pierre Morel
---
target/s390x/kvm.c | 19 +++
1 file changed, 19 insertions(+)
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 348e8cc5467a..d042deed1af3 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -2291,11 +2291,26 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model,
Error **errp)
error_setg(errp, "KVM: host CPU model could not be identified");
return;
}
+/* for now, we can only provide the AP feature with HW support */
+if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO,
+KVM_S390_VM_CRYPTO_ENABLE_APIE)) {
+set_bit(S390_FEAT_AP, model->features);
+}
/* strip of features that are not part of the maximum model */
bitmap_and(model->features, model->features, model->def->full_feat,
S390_FEAT_MAX);
}
+static void kvm_s390_configure_apie(bool interpret)
+{
+uint64_t attr = interpret ? KVM_S390_VM_CRYPTO_ENABLE_APIE :
+KVM_S390_VM_CRYPTO_DISABLE_APIE;
+
+if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO, attr)) {
+kvm_s390_set_attr(attr);
+}
+}
+
void kvm_s390_apply_cpu_model(const S390CPUModel *model, Error **errp)
{
struct kvm_s390_vm_cpu_processor prop = {
@@ -2345,6 +2360,10 @@ void kvm_s390_apply_cpu_model(const S390CPUModel *model,
Error **errp)
if (test_bit(S390_FEAT_CMM, model->features)) {
kvm_s390_enable_cmma();
}
+
+if (test_bit(S390_FEAT_AP, model->features)) {
+kvm_s390_configure_apie(true);
+}
}
void kvm_s390_restart_interrupt(S390CPU *cpu)
--
2.19.0.221.g150f307
[Qemu-devel] [PATCH v10 1/6] linux-headers: linux header updates for AP support
Updates the linux header files in preparation for introduction
of the VFIO AP device:
* Added device attributes to the KVM_S390_VM_CRYPTO group
to indicate whether AP instructions are to be interpreted
* Added VFIO device information for AP devices
Signed-off-by: Tony Krowiak
---
linux-headers/asm-s390/kvm.h | 2 ++
linux-headers/linux/vfio.h | 2 ++
2 files changed, 4 insertions(+)
diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index 1ab9901911bf..0265482f8fdf 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -160,6 +160,8 @@ struct kvm_s390_vm_cpu_subfunc {
#define KVM_S390_VM_CRYPTO_ENABLE_DEA_KW 1
#define KVM_S390_VM_CRYPTO_DISABLE_AES_KW 2
#define KVM_S390_VM_CRYPTO_DISABLE_DEA_KW 3
+#define KVM_S390_VM_CRYPTO_ENABLE_APIE 4
+#define KVM_S390_VM_CRYPTO_DISABLE_APIE5
/* kvm attributes for migration mode */
#define KVM_S390_VM_MIGRATION_STOP 0
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 3615a269d378..838919a4c03a 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -200,6 +200,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2)/* vfio-platform device */
#define VFIO_DEVICE_FLAGS_AMBA (1 << 3) /* vfio-amba device */
#define VFIO_DEVICE_FLAGS_CCW (1 << 4)/* vfio-ccw device */
+#define VFIO_DEVICE_FLAGS_AP (1 << 5) /* vfio-ap device */
__u32 num_regions;/* Max region index + 1 */
__u32 num_irqs; /* Max IRQ index + 1 */
};
@@ -215,6 +216,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_API_PLATFORM_STRING"vfio-platform"
#define VFIO_DEVICE_API_AMBA_STRING"vfio-amba"
#define VFIO_DEVICE_API_CCW_STRING "vfio-ccw"
+#define VFIO_DEVICE_API_AP_STRING "vfio-ap"
/**
* VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
--
2.19.0.221.g150f307
[Qemu-devel] [PATCH v10 5/6] s390x/vfio: ap: Introduce VFIO AP device
Introduces a VFIO based AP device. The device is defined via
the QEMU command line by specifying:
-device vfio-ap,sysfsdev=
There may be only one vfio-ap device configured for a guest.
The mediated matrix device is created by the VFIO AP device
driver by writing a UUID to a sysfs attribute file (see
docs/vfio-ap.txt). The mediated matrix device will be named
after the UUID. Symbolic links to the $uuid are created in
many places, so the path to the mediated matrix device $uuid
can be specified in any of the following ways:
/sys/devices/vfio_ap/matrix/$uuid
/sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
/sys/bus/mdev/devices/$uuid
/sys/bus/mdev/drivers/vfio_mdev/$uuid
When the vfio-ap device is realized, it acquires and opens the
VFIO iommu group to which the mediated matrix device is
bound. This causes a VFIO group notification event to be
signaled. The vfio_ap device driver's group notification
handler will get called at which time the device driver
will configure the the AP devices to which the guest will
be granted access.
Signed-off-by: Tony Krowiak
Tested-by: Pierre Morel
---
MAINTAINERS | 1 +
default-configs/s390x-softmmu.mak | 1 +
hw/vfio/Makefile.objs | 1 +
hw/vfio/ap.c | 180 ++
include/hw/vfio/vfio-common.h | 1 +
5 files changed, 184 insertions(+)
create mode 100644 hw/vfio/ap.c
diff --git a/MAINTAINERS b/MAINTAINERS
index 97e8ed808bc0..29041da69237 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1209,6 +1209,7 @@ F: hw/s390x/ap-device.c
F: hw/s390x/ap-bridge.c
F: include/hw/s390x/ap-device.h
F: include/hw/s390x/ap-bridge.h
+F: hw/vfio/ap.c
L: qemu-s3...@nongnu.org
vhost
diff --git a/default-configs/s390x-softmmu.mak
b/default-configs/s390x-softmmu.mak
index d6b67d50f0e4..5eef37592451 100644
--- a/default-configs/s390x-softmmu.mak
+++ b/default-configs/s390x-softmmu.mak
@@ -7,3 +7,4 @@ CONFIG_S390_FLIC=y
CONFIG_S390_FLIC_KVM=$(CONFIG_KVM)
CONFIG_VFIO_CCW=$(CONFIG_LINUX)
CONFIG_WDT_DIAG288=y
+CONFIG_VFIO_AP=$(CONFIG_LINUX)
diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs
index a2e7a0a7cf02..8b3f664d85f7 100644
--- a/hw/vfio/Makefile.objs
+++ b/hw/vfio/Makefile.objs
@@ -6,4 +6,5 @@ obj-$(CONFIG_SOFTMMU) += platform.o
obj-$(CONFIG_VFIO_XGMAC) += calxeda-xgmac.o
obj-$(CONFIG_VFIO_AMD_XGBE) += amd-xgbe.o
obj-$(CONFIG_SOFTMMU) += spapr.o
+obj-$(CONFIG_VFIO_AP) += ap.o
endif
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
new file mode 100644
index ..5543406afc58
--- /dev/null
+++ b/hw/vfio/ap.c
@@ -0,0 +1,180 @@
+/*
+ * VFIO based AP matrix device assignment
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *Halil Pasic
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include
+#include
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "hw/vfio/vfio.h"
+#include "hw/vfio/vfio-common.h"
+#include "hw/s390x/ap-device.h"
+#include "qemu/error-report.h"
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "cpu.h"
+#include "kvm_s390x.h"
+#include "sysemu/sysemu.h"
+#include "hw/s390x/ap-bridge.h"
+#include "exec/address-spaces.h"
+
+#define VFIO_AP_DEVICE_TYPE "vfio-ap"
+
+typedef struct VFIOAPDevice {
+APDevice apdev;
+VFIODevice vdev;
+} VFIOAPDevice;
+
+#define VFIO_AP_DEVICE(obj) \
+OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE)
+
+static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
+{
+vdev->needs_reset = false;
+}
+
+/*
+ * We don't need vfio_hot_reset_multi and vfio_eoi operations for
+ * vfio-ap device now.
+ */
+struct VFIODeviceOps vfio_ap_ops = {
+.vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
+};
+
+static void vfio_ap_put_device(VFIOAPDevice *vapdev)
+{
+g_free(vapdev->vdev.name);
+vfio_put_base_device(>vdev);
+}
+
+static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
+{
+GError *gerror;
+char *symlink, *group_path;
+int groupid;
+
+symlink = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
+group_path = g_file_read_link(symlink, );
+g_free(symlink);
+
+if (!group_path) {
+error_setg(errp, "%s: no iommu_group found for %s: %s",
+ VFIO_AP_DEVICE_TYPE, vapdev->vdev.sysfsdev,
gerror->message);
+return NULL;
+}
+
+if (sscanf(basename(group_path), "%d", ) != 1) {
+error_setg(errp, "vfio: failed to read %s", group_path);
+return NULL;
+}
+
+return vfio_get_group(groupid, _space_memory, errp);
+}
+
+
[Qemu-devel] [PATCH v10 6/6] s390: doc: detailed specifications for AP virtualization
This patch provides documentation describing the AP architecture and design concepts behind the virtualization of AP devices. It also includes an example of how to configure AP devices for exclusive use of KVM guests. Signed-off-by: Tony Krowiak Reviewed-by: Pierre Morel Tested-by: Pierre Morel --- MAINTAINERS | 1 + docs/vfio-ap.txt | 825 +++ 2 files changed, 826 insertions(+) create mode 100644 docs/vfio-ap.txt diff --git a/MAINTAINERS b/MAINTAINERS index 29041da69237..b64a12034c2c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1210,6 +1210,7 @@ F: hw/s390x/ap-bridge.c F: include/hw/s390x/ap-device.h F: include/hw/s390x/ap-bridge.h F: hw/vfio/ap.c +F: docs/vfio-ap.txt L: qemu-s3...@nongnu.org vhost diff --git a/docs/vfio-ap.txt b/docs/vfio-ap.txt new file mode 100644 index ..e020f8f9e8e5 --- /dev/null +++ b/docs/vfio-ap.txt @@ -0,0 +1,825 @@ +Adjunct Processor (AP) Device += + +Contents: += +* Introduction +* AP Architectural Overview +* Start Interpretive Execution (SIE) Instruction +* AP Matrix Configuration on Linux Host +* Starting a Linux Guest Configured with an AP Matrix +* Example: Configure AP Matrices for Three Linux Guests + +Introduction: + +The IBM Adjunct Processor (AP) Cryptographic Facility is comprised +of three AP instructions and from 1 to 256 PCIe cryptographic adapter cards. +These AP devices provide cryptographic functions to all CPUs assigned to a +linux system running in an IBM Z system LPAR. + +On s390x, AP adapter cards are exposed via the AP bus. This document +describes how those cards may be made available to KVM guests using the +VFIO mediated device framework. + +AP Architectural Overview: += +In order understand the terminology used in the rest of this document, let's +start with some definitions: + +* AP adapter + + An AP adapter is an IBM Z adapter card that can perform cryptographic + functions. There can be from 0 to 256 adapters assigned to an LPAR depending + on the machine model. Adapters assigned to the LPAR in which a linux host is + running will be available to the linux host. Each adapter is identified by a + number from 0 to 255; however, the maximum adapter number allowed is + determined by machine model. When installed, an AP adapter is accessed by + AP instructions executed by any CPU. + +* AP domain + + An adapter is partitioned into domains. Each domain can be thought of as + a set of hardware registers for processing AP instructions. An adapter can + hold up to 256 domains; however, the maximum domain number allowed is + determined by machine model. Each domain is identified by a number from 0 to + 255. Domains can be further classified into two types: + +* Usage domains are domains that can be accessed directly to process AP + commands + +* Control domains are domains that are accessed indirectly by AP + commands sent to a usage domain to control or change the domain; for + example, to set a secure private key for the domain. + +* AP Queue + + An AP queue is the means by which an AP command-request message is sent to an + AP usage domain inside a specific AP. An AP queue is identified by a tuple + comprised of an AP adapter ID (APID) and an AP queue index (APQI). The + APQI corresponds to a given usage domain number within the adapter. This tuple + forms an AP Queue Number (APQN) uniquely identifying an AP queue. AP + instructions include a field containing the APQN to identify the AP queue to + which the AP command-request message is to be sent for processing. + +* AP Instructions: + + There are three AP instructions: + + * NQAP: to enqueue an AP command-request message to a queue + * DQAP: to dequeue an AP command-reply message from a queue + * PQAP: to administer the queues + + AP instructions identify the domain that is targeted to process the AP + command; this must be one of the usage domains. An AP command may modify a + domain that is not one of the usage domains, but the modified domain + must be one of the control domains. + +Start Interpretive Execution (SIE) Instruction +== +A KVM guest is started by executing the Start Interpretive Execution (SIE) +instruction. The SIE state description is a control block that contains the +state information for a KVM guest and is supplied as input to the SIE +instruction. The SIE state description contains a satellite control block called +the Crypto Control Block (CRYCB). The CRYCB contains three fields to identify +the adapters, usage domains and control domains assigned to the KVM guest: + +* The AP Mask (APM) field is a bit mask that identifies the AP adapters assigned + to the KVM guest. Each bit in the mask, from left to right, corresponds to + an APID from 0-255. If a bit is set, the corresponding adapter is valid for + use by the KVM guest. + +* The AP
[Qemu-devel] [PATCH v10 0/6] s390x: vfio-ap: guest dedicated crypto adapters
From: Tony Krowiak This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model. See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series. v9 => v10 Change log: * Removed KVM_S390_VM_CPU_FEAT_AP feature from kvm.h * Moved check for KVM_S390_VM_CRYPTO_ENABLE_APIE from patch 2/6 to patch 3/6 * Removed vfio from all function names in ap-bridge.c * Removed unused macros and structure from ap-bridge.h * Removed unused macros from ap-device.h v8 => v9 Change log: === * Removed all references to VFIO in AP bridge and bus * Expose AP feature only if the KVM_S390_VM_CRYPTO_ENABLE_APIE VM attribute is exposed by KVM - i.e., if AP instructions are available on the linux host. * Enable AP interpretation only if AP feature is switched on; no need to disable because it is disabled by default. v7 => v8 Change log: === * Enable SIE interpretation AP instructions if the CPU model feature for AP instructions is turned on for the guest. v6 => v7 Change log; === * Changed email address for Signed-off-by v5 => v6 Change log: === * Added reset handling fo vfio-ap device * Added a bridge/bus to AP device object model - thanks to Halil Pasic v4 => v5 Change log: === * Added MAINTAINERS entries for VFIO AP * Added explanation for why we are only supporting zEC12 and newer CPU models. * Changed CPU model feature qci=on|off to apqci=on|off * Misc. minor changes v3 => v4 Change log: === * Made vfio-ap device unpluggable for now * Renamed command line CPU model feature for QCI: qci=on -> apqci=on * Removed call to KVM_S390_VM_CRYPTO_INTERPRET_AP ioctl - ioctl was removed from kernel and AP instruction interpretation is set from the VFIO device driver * Added check to ensure only one vfio-ap device can be configured per guest * Removed AP instruction interception handlers: AP instructions will be interpreted by default if AP facilities are installed to handle the case where feature ap=on and no vfio-ap device is configured for the guest. Tony Krowiak (6): linux-headers: linux header updates for AP support s390x/cpumodel: Set up CPU model for AP device support s390x/kvm: enable AP instruction interpretation for guest s390x/ap: base Adjunct Processor (AP) object model s390x/vfio: ap: Introduce VFIO AP device s390: doc: detailed specifications for AP virtualization MAINTAINERS | 14 + default-configs/s390x-softmmu.mak | 1 + docs/vfio-ap.txt | 825 ++ hw/s390x/Makefile.objs| 2 + hw/s390x/ap-bridge.c | 78 +++ hw/s390x/ap-device.c | 39 ++ hw/s390x/s390-virtio-ccw.c| 4 + hw/vfio/Makefile.objs | 1 + hw/vfio/ap.c | 180 +++ include/hw/s390x/ap-bridge.h | 19 + include/hw/s390x/ap-device.h | 23 + include/hw/vfio/vfio-common.h | 1 + linux-headers/asm-s390/kvm.h | 2 + linux-headers/linux/vfio.h| 2 + target/s390x/cpu_features.c | 3 + target/s390x/cpu_features_def.h | 3 + target/s390x/cpu_models.c | 2 + target/s390x/gen-features.c | 3 + target/s390x/kvm.c| 19 + 19 files changed, 1221 insertions(+) create mode 100644 docs/vfio-ap.txt create mode 100644 hw/s390x/ap-bridge.c create mode 100644 hw/s390x/ap-device.c create mode 100644 hw/vfio/ap.c create mode 100644 include/hw/s390x/ap-bridge.h create mode 100644 include/hw/s390x/ap-device.h -- 2.19.0.221.g150f307
[Qemu-devel] [PATCH v10 2/6] s390x/cpumodel: Set up CPU model for AP device support
A new CPU model feature and two new CPU model facilities are
introduced to support AP devices for a KVM guest.
CPU model features:
1. The S390_FEAT_AP CPU model feature indicates whether AP
instructions are available to the guest. This feature will
be enabled only if the AP instructions are available on the
linux host as determined by the availability of the
KVM_S390_VM_CRYPTO_ENABLE_APIE VM attribute which is exposed
by KVM only if the AP instructions are available on the
host.
This feature must be turned on from userspace to execute AP
instructions on the KVM guest. The QEMU command line to turn
this feature on looks something like this:
qemu-system-s390x ... -cpu xxx,ap=on ...
This feature will be supported for zEC12 and newer CPU models.
The feature will not be supported for older models because
there are few older systems on which to test and the older
crypto cards will be going out of service in the relatively
near future.
CPU model facilities:
1. The S390_FEAT_AP_QUERY_CONFIG_INFO feature indicates whether the
AP Query Configuration Information (QCI) facility is available
to the guest as determined by whether the facility is available
on the host. This feature will be exposed by KVM only if the
QCI facility is installed on the host.
2. The S390_FEAT_AP_FACILITY_TEST feature indicates whether the AP
Facility Test (APFT) facility is available to the guest as
determined by whether the facility is available on the host.
This feature will be exposed by KVM only if APFT is installed
on the host.
Signed-off-by: Tony Krowiak
Tested-by: Pierre Morel
---
target/s390x/cpu_features.c | 3 +++
target/s390x/cpu_features_def.h | 3 +++
target/s390x/cpu_models.c | 2 ++
target/s390x/gen-features.c | 3 +++
4 files changed, 11 insertions(+)
diff --git a/target/s390x/cpu_features.c b/target/s390x/cpu_features.c
index 172fb18df718..60cfeba48f4e 100644
--- a/target/s390x/cpu_features.c
+++ b/target/s390x/cpu_features.c
@@ -39,8 +39,10 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT("srs", S390_FEAT_TYPE_STFL, 9, "Sense-running-status facility"),
FEAT_INIT("csske", S390_FEAT_TYPE_STFL, 10, "Conditional-SSKE facility"),
FEAT_INIT("ctop", S390_FEAT_TYPE_STFL, 11, "Configuration-topology
facility"),
+FEAT_INIT("apqci", S390_FEAT_TYPE_STFL, 12, "Query AP Configuration
Information facility"),
FEAT_INIT("ipter", S390_FEAT_TYPE_STFL, 13, "IPTE-range facility"),
FEAT_INIT("nonqks", S390_FEAT_TYPE_STFL, 14, "Nonquiescing key-setting
facility"),
+FEAT_INIT("apft", S390_FEAT_TYPE_STFL, 15, "AP Facilities Test facility"),
FEAT_INIT("etf2", S390_FEAT_TYPE_STFL, 16, "Extended-translation facility
2"),
FEAT_INIT("msa-base", S390_FEAT_TYPE_STFL, 17, "Message-security-assist
facility (excluding subfunctions)"),
FEAT_INIT("ldisp", S390_FEAT_TYPE_STFL, 18, "Long-displacement facility"),
@@ -129,6 +131,7 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT_MISC("dateh2", "DAT-enhancement facility 2"),
FEAT_INIT_MISC("cmm", "Collaborative-memory-management facility"),
+FEAT_INIT_MISC("ap", "AP instructions installed"),
FEAT_INIT("plo-cl", S390_FEAT_TYPE_PLO, 0, "PLO Compare and load (32 bit
in general registers)"),
FEAT_INIT("plo-clg", S390_FEAT_TYPE_PLO, 1, "PLO Compare and load (64 bit
in parameter list)"),
diff --git a/target/s390x/cpu_features_def.h b/target/s390x/cpu_features_def.h
index ac2c947f30a8..5fc7e7bf0116 100644
--- a/target/s390x/cpu_features_def.h
+++ b/target/s390x/cpu_features_def.h
@@ -27,8 +27,10 @@ typedef enum {
S390_FEAT_SENSE_RUNNING_STATUS,
S390_FEAT_CONDITIONAL_SSKE,
S390_FEAT_CONFIGURATION_TOPOLOGY,
+S390_FEAT_AP_QUERY_CONFIG_INFO,
S390_FEAT_IPTE_RANGE,
S390_FEAT_NONQ_KEY_SETTING,
+S390_FEAT_AP_FACILITIES_TEST,
S390_FEAT_EXTENDED_TRANSLATION_2,
S390_FEAT_MSA,
S390_FEAT_LONG_DISPLACEMENT,
@@ -119,6 +121,7 @@ typedef enum {
/* Misc */
S390_FEAT_DAT_ENH_2,
S390_FEAT_CMM,
+S390_FEAT_AP,
/* PLO */
S390_FEAT_PLO_CL,
diff --git a/target/s390x/cpu_models.c b/target/s390x/cpu_models.c
index 265d25c937bb..7c253ff308c5 100644
--- a/target/s390x/cpu_models.c
+++ b/target/s390x/cpu_models.c
@@ -786,6 +786,8 @@ static void check_consistency(const S390CPUModel *model)
{ S390_FEAT_PRNO_TRNG_QRTCR, S390_FEAT_MSA_EXT_5 },
{ S390_FEAT_PRNO_TRNG, S390_FEAT_MSA_EXT_5 },
{ S390_FEAT_SIE_KSS, S390_FEAT_SIE_F2 },
+{ S390_FEAT_AP_QUERY_CONFIG_INFO, S390_FEAT_AP },
+{ S390_FEAT_AP_FACILITIES_TEST, S390_FEAT_AP
Re: [Qemu-devel] [qemu-s390x] [PATCH v9 4/6] s390x/ap: base Adjunct Processor (AP) object model
On 10/08/2018 10:44 AM, Thomas Huth wrote:
On 2018-10-08 16:20, Tony Krowiak wrote:
On 09/27/2018 08:52 AM, Cornelia Huck wrote:
On Thu, 27 Sep 2018 14:29:01 +0200
Thomas Huth wrote:
On 2018-09-27 00:54, Tony Krowiak wrote:
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
---
+typedef struct APBridge {
+SysBusDevice sysbus_dev;
+bool css_dev_path;
What is this css_dev_path variable good for? I don't see it used in any
of the other patches?
If you don't need it, I think you could get rid of this struct
completely?
Huh, now I remember complaining about it before. Looks like a
copy-and-paste from the css bridge; that variable is used for compat
handling there (and should be ditched here).
+} APBridge;
+
+#define TYPE_AP_BRIDGE "ap-bridge"
+#define AP_BRIDGE(obj) \
+OBJECT_CHECK(APBridge, (obj), TYPE_AP_BRIDGE)
+
+typedef struct APBus {
+BusState parent_obj;
+} APBus;
+
+#define TYPE_AP_BUS "ap-bus"
+#define AP_BUS(obj) \
+ OBJECT_CHECK(APBus, (obj), TYPE_AP_BUS)
I think you could also get rid of AP_BRIDGE(), AP_BUS() and maybe even
struct APBus.
If there's nothing interesting to put in these inherited structures,
probably yes.
+void s390_init_ap(void);
+
+#endif
diff --git a/include/hw/s390x/ap-device.h
b/include/hw/s390x/ap-device.h
new file mode 100644
index ..693df90cc041
--- /dev/null
+++ b/include/hw/s390x/ap-device.h
@@ -0,0 +1,38 @@
+/*
+ * Adjunct Processor (AP) matrix device interfaces
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2
or (at
+ * your option) any later version. See the COPYING file in the
top-level
+ * directory.
+ */
+#ifndef HW_S390X_AP_DEVICE_H
+#define HW_S390X_AP_DEVICE_H
+
+#define AP_DEVICE_TYPE "ap-device"
+
+typedef struct APDevice {
+DeviceState parent_obj;
+} APDevice;
+
+typedef struct APDeviceClass {
+DeviceClass parent_class;
+} APDeviceClass;
+
+static inline APDevice *to_ap_dev(DeviceState *dev)
+{
+return container_of(dev, APDevice, parent_obj);
+}
+
+#define AP_DEVICE(obj) \
+OBJECT_CHECK(APDevice, (obj), AP_DEVICE_TYPE)
+
+#define AP_DEVICE_GET_CLASS(obj) \
+OBJECT_GET_CLASS(APDeviceClass, (obj), AP_DEVICE_TYPE)
+
+#define AP_DEVICE_CLASS(klass) \
+OBJECT_CLASS_CHECK(APDeviceClass, (klass), AP_DEVICE_TYPE)
Do you really need any of these definitions except AP_DEVICE_TYPE ?
Yes, we need AP_DEVICE(obj) and struct APDevice; they are both used in
patch 5/6.
Fine for me, if you replace the DO_UPCAST in patch 5 with AP_DEVICE().
We can probably get rid of AP_DEVICE_GET_CLASS(obj) and
AP_DEVICE_CLASS(klass), but aren't those typically included in all
QOM definitions?
As long as you don't really need them, I'd simply remove them. They can
be added back when some code really needs them.
That is the plan
Thomas
Re: [Qemu-devel] [qemu-s390x] [PATCH v9 4/6] s390x/ap: base Adjunct Processor (AP) object model
On 10/08/2018 10:35 AM, Cornelia Huck wrote:
On Mon, 8 Oct 2018 16:22:27 +0200
David Hildenbrand wrote:
On 08/10/2018 16:20, Tony Krowiak wrote:
On 09/27/2018 08:52 AM, Cornelia Huck wrote:
On Thu, 27 Sep 2018 14:29:01 +0200
Thomas Huth wrote:
On 2018-09-27 00:54, Tony Krowiak wrote:
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
---
+typedef struct APBridge {
+SysBusDevice sysbus_dev;
+bool css_dev_path;
What is this css_dev_path variable good for? I don't see it used in any
of the other patches?
If you don't need it, I think you could get rid of this struct completely?
Huh, now I remember complaining about it before. Looks like a
copy-and-paste from the css bridge; that variable is used for compat
handling there (and should be ditched here).
+} APBridge;
+
+#define TYPE_AP_BRIDGE "ap-bridge"
+#define AP_BRIDGE(obj) \
+OBJECT_CHECK(APBridge, (obj), TYPE_AP_BRIDGE)
+
+typedef struct APBus {
+BusState parent_obj;
+} APBus;
+
+#define TYPE_AP_BUS "ap-bus"
+#define AP_BUS(obj) \
+ OBJECT_CHECK(APBus, (obj), TYPE_AP_BUS)
I think you could also get rid of AP_BRIDGE(), AP_BUS() and maybe even
struct APBus.
If there's nothing interesting to put in these inherited structures,
probably yes.
+void s390_init_ap(void);
+
+#endif
diff --git a/include/hw/s390x/ap-device.h b/include/hw/s390x/ap-device.h
new file mode 100644
index ..693df90cc041
--- /dev/null
+++ b/include/hw/s390x/ap-device.h
@@ -0,0 +1,38 @@
+/*
+ * Adjunct Processor (AP) matrix device interfaces
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#ifndef HW_S390X_AP_DEVICE_H
+#define HW_S390X_AP_DEVICE_H
+
+#define AP_DEVICE_TYPE "ap-device"
+
+typedef struct APDevice {
+DeviceState parent_obj;
+} APDevice;
+
+typedef struct APDeviceClass {
+DeviceClass parent_class;
+} APDeviceClass;
+
+static inline APDevice *to_ap_dev(DeviceState *dev)
+{
+return container_of(dev, APDevice, parent_obj);
+}
+
+#define AP_DEVICE(obj) \
+OBJECT_CHECK(APDevice, (obj), AP_DEVICE_TYPE)
+
+#define AP_DEVICE_GET_CLASS(obj) \
+OBJECT_GET_CLASS(APDeviceClass, (obj), AP_DEVICE_TYPE)
+
+#define AP_DEVICE_CLASS(klass) \
+OBJECT_CLASS_CHECK(APDeviceClass, (klass), AP_DEVICE_TYPE)
Do you really need any of these definitions except AP_DEVICE_TYPE ?
Yes, we need AP_DEVICE(obj) and struct APDevice; they are both used in
patch 5/6. We can probably get rid of AP_DEVICE_GET_CLASS(obj) and
AP_DEVICE_CLASS(klass), but aren't those typically included in all
QOM definitions?
Yes, we usually add all of them although only some might actually be
used. (adding a new device usually looks like filling out a template)
Much of this seems to be boilerplate in this case, and I'm not sure how
much sense it makes. On the plus side, however, it looks like
everything else :)
So, I would merge both a complete version or a
stripped-down-to-the-needed version, unless someone else has a strong
argument.
The 'I would merge both' implies you are asking for two versions, but
the 'or' implies you are asking for one or the other; I'm going to
assume you are asking for one or the other. I'll provide a stripped down
version in v10 which I am planning on posting today.
Re: [Qemu-devel] [qemu-s390x] [PATCH v9 4/6] s390x/ap: base Adjunct Processor (AP) object model
On 09/27/2018 08:52 AM, Cornelia Huck wrote:
On Thu, 27 Sep 2018 14:29:01 +0200
Thomas Huth wrote:
On 2018-09-27 00:54, Tony Krowiak wrote:
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
---
+typedef struct APBridge {
+SysBusDevice sysbus_dev;
+bool css_dev_path;
What is this css_dev_path variable good for? I don't see it used in any
of the other patches?
If you don't need it, I think you could get rid of this struct completely?
Huh, now I remember complaining about it before. Looks like a
copy-and-paste from the css bridge; that variable is used for compat
handling there (and should be ditched here).
+} APBridge;
+
+#define TYPE_AP_BRIDGE "ap-bridge"
+#define AP_BRIDGE(obj) \
+OBJECT_CHECK(APBridge, (obj), TYPE_AP_BRIDGE)
+
+typedef struct APBus {
+BusState parent_obj;
+} APBus;
+
+#define TYPE_AP_BUS "ap-bus"
+#define AP_BUS(obj) \
+ OBJECT_CHECK(APBus, (obj), TYPE_AP_BUS)
I think you could also get rid of AP_BRIDGE(), AP_BUS() and maybe even
struct APBus.
If there's nothing interesting to put in these inherited structures,
probably yes.
+void s390_init_ap(void);
+
+#endif
diff --git a/include/hw/s390x/ap-device.h b/include/hw/s390x/ap-device.h
new file mode 100644
index ..693df90cc041
--- /dev/null
+++ b/include/hw/s390x/ap-device.h
@@ -0,0 +1,38 @@
+/*
+ * Adjunct Processor (AP) matrix device interfaces
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#ifndef HW_S390X_AP_DEVICE_H
+#define HW_S390X_AP_DEVICE_H
+
+#define AP_DEVICE_TYPE "ap-device"
+
+typedef struct APDevice {
+DeviceState parent_obj;
+} APDevice;
+
+typedef struct APDeviceClass {
+DeviceClass parent_class;
+} APDeviceClass;
+
+static inline APDevice *to_ap_dev(DeviceState *dev)
+{
+return container_of(dev, APDevice, parent_obj);
+}
+
+#define AP_DEVICE(obj) \
+OBJECT_CHECK(APDevice, (obj), AP_DEVICE_TYPE)
+
+#define AP_DEVICE_GET_CLASS(obj) \
+OBJECT_GET_CLASS(APDeviceClass, (obj), AP_DEVICE_TYPE)
+
+#define AP_DEVICE_CLASS(klass) \
+OBJECT_CLASS_CHECK(APDeviceClass, (klass), AP_DEVICE_TYPE)
Do you really need any of these definitions except AP_DEVICE_TYPE ?
Yes, we need AP_DEVICE(obj) and struct APDevice; they are both used in
patch 5/6. We can probably get rid of AP_DEVICE_GET_CLASS(obj) and
AP_DEVICE_CLASS(klass), but aren't those typically included in all
QOM definitions?
Same here, I think.
Re: [Qemu-devel] [qemu-s390x] [PATCH v9 4/6] s390x/ap: base Adjunct Processor (AP) object model
On 09/28/2018 08:51 AM, Halil Pasic wrote:
On 09/27/2018 02:52 PM, Cornelia Huck wrote:
On Thu, 27 Sep 2018 14:29:01 +0200
Thomas Huth wrote:
On 2018-09-27 00:54, Tony Krowiak wrote:
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
---
+typedef struct APBridge {
+SysBusDevice sysbus_dev;
+bool css_dev_path;
What is this css_dev_path variable good for? I don't see it used in any
of the other patches?
If you don't need it, I think you could get rid of this struct completely?
Huh, now I remember complaining about it before. Looks like a
copy-and-paste from the css bridge; that variable is used for compat
handling there (and should be ditched here).
Yes, we definitively discussed this before. And yes, it is a copy-paste
error.
+} APBridge;
+
+#define TYPE_AP_BRIDGE "ap-bridge"
+#define AP_BRIDGE(obj) \
+OBJECT_CHECK(APBridge, (obj), TYPE_AP_BRIDGE)
+
+typedef struct APBus {
+BusState parent_obj;
+} APBus;
+
+#define TYPE_AP_BUS "ap-bus"
+#define AP_BUS(obj) \
+ OBJECT_CHECK(APBus, (obj), TYPE_AP_BUS)
I think you could also get rid of AP_BRIDGE(), AP_BUS() and maybe even
struct APBus.
If there's nothing interesting to put in these inherited structures,
probably yes.
I was under impression that this is how inheritance/subtyping is done in
QEMU. Was probably added for the sake of interface completeness.
+void s390_init_ap(void);
+
+#endif
diff --git a/include/hw/s390x/ap-device.h b/include/hw/s390x/ap-device.h
new file mode 100644
index ..693df90cc041
--- /dev/null
+++ b/include/hw/s390x/ap-device.h
@@ -0,0 +1,38 @@
+/*
+ * Adjunct Processor (AP) matrix device interfaces
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#ifndef HW_S390X_AP_DEVICE_H
+#define HW_S390X_AP_DEVICE_H
+
+#define AP_DEVICE_TYPE "ap-device"
+
+typedef struct APDevice {
+DeviceState parent_obj;
+} APDevice;
+
+typedef struct APDeviceClass {
+DeviceClass parent_class;
+} APDeviceClass;
+
+static inline APDevice *to_ap_dev(DeviceState *dev)
+{
+return container_of(dev, APDevice, parent_obj);
+}
+
+#define AP_DEVICE(obj) \
+OBJECT_CHECK(APDevice, (obj), AP_DEVICE_TYPE)
+
+#define AP_DEVICE_GET_CLASS(obj) \
+OBJECT_GET_CLASS(APDeviceClass, (obj), AP_DEVICE_TYPE)
+
+#define AP_DEVICE_CLASS(klass) \
+OBJECT_CLASS_CHECK(APDeviceClass, (klass), AP_DEVICE_TYPE)
Do you really need any of these definitions except AP_DEVICE_TYPE ?
Same here, I think.
I don't think the qom utility/casting macros will ever be used. I'm
fine with removing what is not needed.
Regards,
Halil
Per Thomas's suggestions and Connie's concurrence, I am going to remove
the macros as well as the structures they reference. The ap_bridge.h
will be relegated to defining the bridge and bus types and serve as a
container for AP bridge and bus definitions that might be needed in the
future.
Re: [Qemu-devel] [qemu-s390x] [PATCH v9 5/6] s390x/vfio: ap: Introduce VFIO AP device
On 09/27/2018 09:56 AM, Thomas Huth wrote:
On 2018-09-27 00:54, Tony Krowiak wrote:
Introduces a VFIO based AP device. The device is defined via
the QEMU command line by specifying:
-device vfio-ap,sysfsdev=
There may be only one vfio-ap device configured for a guest.
The mediated matrix device is created by the VFIO AP device
driver by writing a UUID to a sysfs attribute file (see
docs/vfio-ap.txt). The mediated matrix device will be named
after the UUID. Symbolic links to the $uuid are created in
many places, so the path to the mediated matrix device $uuid
can be specified in any of the following ways:
/sys/devices/vfio_ap/matrix/$uuid
/sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
/sys/bus/mdev/devices/$uuid
/sys/bus/mdev/drivers/vfio_mdev/$uuid
When the vfio-ap device is realized, it acquires and opens the
VFIO iommu group to which the mediated matrix device is
bound. This causes a VFIO group notification event to be
signaled. The vfio_ap device driver's group notification
handler will get called at which time the device driver
will configure the the AP devices to which the guest will
be granted access.
Signed-off-by: Tony Krowiak
---
[...]
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
new file mode 100644
index ..429988f23f98
--- /dev/null
+++ b/hw/vfio/ap.c
@@ -0,0 +1,181 @@
+/*
+ * VFIO based AP matrix device assignment
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *Halil Pasic
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include
+#include
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "hw/vfio/vfio.h"
+#include "hw/vfio/vfio-common.h"
+#include "hw/s390x/ap-device.h"
+#include "qemu/error-report.h"
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "cpu.h"
+#include "kvm_s390x.h"
+#include "sysemu/sysemu.h"
+#include "hw/s390x/ap-bridge.h"
+#include "exec/address-spaces.h"
+
+#define VFIO_AP_DEVICE_TYPE "vfio-ap"
+
+typedef struct VFIOAPDevice {
+APDevice apdev;
+VFIODevice vdev;
+} VFIOAPDevice;
+
+static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
+{
+vdev->needs_reset = false;
+}
+
+/*
+ * We don't need vfio_hot_reset_multi and vfio_eoi operations for
+ * vfio-ap device now.
+ */
+struct VFIODeviceOps vfio_ap_ops = {
+.vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
+};
+
+static void vfio_ap_put_device(VFIOAPDevice *vapdev)
+{
+g_free(vapdev->vdev.name);
+vfio_put_base_device(>vdev);
+}
+
+static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
+{
+char *tmp, group_path[PATH_MAX];
+ssize_t len;
+int groupid;
+
+tmp = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
+len = readlink(tmp, group_path, sizeof(group_path));
+g_free(tmp);
+
+if (len <= 0 || len >= sizeof(group_path)) {
+error_setg(errp, "%s: no iommu_group found for %s",
+ VFIO_AP_DEVICE_TYPE, vapdev->vdev.sysfsdev);
+return NULL;
+}
+
+group_path[len] = 0;
You could maybe use g_file_read_link() instead to avoid the ugliness
that is needed around readlink().
I will make that change.
+if (sscanf(basename(group_path), "%d", ) != 1) {
+error_setg(errp, "vfio: failed to read %s", group_path);
+return NULL;
+}
+
+return vfio_get_group(groupid, _space_memory, errp);
+}
+
+static void vfio_ap_realize(DeviceState *dev, Error **errp)
+{
+int ret;
+char *mdevid;
+Error *local_err = NULL;
+VFIOGroup *vfio_group;
+APDevice *apdev = DO_UPCAST(APDevice, parent_obj, dev);
IIRC DO_UPCAST should be avoided in new code. So this is now here the
right place to finally use the AP_DEVICE() macro?
Will do.
+VFIOAPDevice *vapdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
+
+vapdev = DO_UPCAST(VFIOAPDevice, apdev, apdev);
Double assignment to vapdev.
Ah, yes ... it needs to go
+vfio_group = vfio_ap_get_group(vapdev, _err);
+if (!vfio_group) {
+goto out_err;
+}
+
+vapdev->vdev.ops = _ap_ops;
+vapdev->vdev.type = VFIO_DEVICE_TYPE_AP;
+mdevid = basename(vapdev->vdev.sysfsdev);
+vapdev->vdev.name = g_strdup_printf("%s", mdevid);
g_strdup instead of g_strdup_printf should be sufficient here, shouldn't it?
Yes, it is sufficient, I'll change it.
+vapdev->vdev.dev = dev;
+
+ret = vfio_get_device(vfio_group, mdevid, >vdev, _err);
+if (ret) {
+goto out_get_dev_err;
+}
+
+/* Enable hardware to intepret AP instructions executed on the guest */
+object_property_set_bool(OBJECT(qdev_get_machine()), true, "apie", NULL);
+
+return;
+
+out_get_dev_err:
+vfio_ap_put_device(vapdev);
+vfio_put_group(vfio_group);
+out_err:
+error_propagate(errp, local_err);
+}
Thomas
Re: [Qemu-devel] [PATCH v9 0/6] s390x: vfio-ap: guest dedicated crypto adapters
On 09/27/2018 05:28 AM, Cornelia Huck wrote: On Wed, 26 Sep 2018 18:54:34 -0400 Tony Krowiak wrote: From: Tony Krowiak This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model. See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series. This seems to look sane in general. However, before I merge this: - This needs to wait for a proper headers sync, which can only be done after the Linux code hits upstream. Shouldn't be long, I guess (I can queue on a staging branch while waiting.) - As I don't have the hardware, I cannot run more than some generic "verify it does not break anything" testing. So, I'd like to see a Tested-by by someone who actually does have access to the hardware. - I'd like to see some more review/acks from others. If other (minor) things crop up, I'd prefer a respin (I don't want to juggle too many "minor changes"...) I am planning on a v10 series to address issues raised by Thomas, David and you.
Re: [Qemu-devel] [qemu-s390x] [PATCH v9 4/6] s390x/ap: base Adjunct Processor (AP) object model
On 09/27/2018 08:29 AM, Thomas Huth wrote:
On 2018-09-27 00:54, Tony Krowiak wrote:
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
---
[...]
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
new file mode 100644
index ..8564dfa96ee7
--- /dev/null
+++ b/hw/s390x/ap-bridge.c
@@ -0,0 +1,81 @@
+/*
+ * ap bridge
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Halil Pasic
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "qemu/bitops.h"
+#include "hw/s390x/ap-bridge.h"
+#include "cpu.h"
+
+static char *vfio_ap_bus_get_dev_path(DeviceState *dev)
+{
+/* at most one */
+return g_strdup_printf("/1");
+}
+
+static void vfio_ap_bus_class_init(ObjectClass *klass, void *data)
+{
+BusClass *k = BUS_CLASS(klass);
I think calling the variable "oc" (or something similar) instead of
"klass" is prefered nowadays.
I did a search of the entire qemu project and did not find one case
where the parameter to the BUS_CLASS macro was anything but klass. On
the other hand, I also did a search for "bus_class_init(" and was also
unable to to find a single instance of the use of anything other than
klass or class. On the other hand, a search of "class_init(" found that
most instances do us 'oc' for the ObjectClass parameter name. So, I'll
assume that most busses have been around for a long time and that you
are correct. Besides, I don't have a strong opinion about either name,
so I will replace all 'ObjectClass *klass' with 'ObjectClass *oc' in
the rest of this file too.
+k->get_dev_path = vfio_ap_bus_get_dev_path;
+/* More than one vfio-ap device does not make sense */
+k->max_dev = 1;
Would it make sense to set a DEVICE_CATEGORY here, too?
It does not make sense set a DEVICE_CATEGORY here because categories
do not exist for struct BusClass. Were you referring to the bridge
device? The bridge class category is set to DEVICE_CATEGORY_BRIDGE
in the ap_bridge_class_init() function below. If not, then to what
are you referring here?
+}
+
+static const TypeInfo vfio_ap_bus_info = {
+.name = TYPE_AP_BUS,
+.parent = TYPE_BUS,
+.instance_size = sizeof(APBus),
+.class_init = vfio_ap_bus_class_init,
+};
+
+void s390_init_ap(void)
+{
+DeviceState *dev;
+
+/* If no AP instructions then no need for AP bridge */
+if (!s390_has_feat(S390_FEAT_AP)) {
+return;
+}
+
+/* Create bridge device */
+dev = qdev_create(NULL, TYPE_AP_BRIDGE);
+object_property_add_child(qdev_get_machine(), TYPE_AP_BRIDGE,
+ OBJECT(dev), NULL);
+qdev_init_nofail(dev);
+
+/* Create bus on bridge device */
+qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+ }
+
+
+
One empty line should be enough?
I'll fix that
+static void ap_bridge_class_init(ObjectClass *klass, void *data)
+{
+DeviceClass *dc = DEVICE_CLASS(klass);
+
+set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+}
+
+static const TypeInfo ap_bridge_info = {
+.name = TYPE_AP_BRIDGE,
+.parent= TYPE_SYS_BUS_DEVICE,
+.instance_size = sizeof(APBridge),
+.class_init= ap_bridge_class_init,
+};
+
+static void ap_register(void)
+{
+type_register_static(_bridge_info);
+type_register_static(_ap_bus_info);
+}
+
+type_init(ap_register)
[...]
diff --git a/include/hw/s390x/ap-bridge.h b/include/hw/s390x/ap-bridge.h
new file mode 100644
index ..b6ca6ae4ab17
--- /dev/null
+++ b/include/hw/s390x/ap-bridge.h
@@ -0,0 +1,37 @@
+/*
+ * ap bridge
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Halil Pasic
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390X_AP_BRIDGE_H
+#define HW_S390X_AP_BRIDGE_H
+#include "qom/object.h"
+#include "hw/qdev-core.h"
+#include "hw/sysbus.h"
+
+typedef struct APBridge {
+SysBusDevice sysbus_dev;
+bool css_dev_path;
What is this css_dev_path variable good for? I don't see it used in any
of the other patches?
If you don't need it, I think you could get rid of this struct completely?
+} APBridge;
+
+#define TYPE_AP_BRIDGE "ap-bridge"
+#define AP_BRIDGE(obj) \DEVICE_CATEGORY_BRIDGE
+OBJECT_CHECK(APBridge, (obj), TYPE_AP_BRIDGE)
+
+typedef struct APBus {
+BusState parent_obj;
+} APBus;
+
+#define TYPE_AP_BUS "ap-bus"
+#define AP_BUS(obj) \
+ OBJECT_CHECK(APBus, (obj), TYPE_AP_BUS)
I think you could also get rid of AP_BRIDGE(), AP_BUS() and maybe even
struct APBus.
+void s390_init_ap(void);
+
+#end
Re: [Qemu-devel] [qemu-s390x] [PATCH v9 4/6] s390x/ap: base Adjunct Processor (AP) object model
On 09/28/2018 09:57 AM, Halil Pasic wrote:
On 09/27/2018 12:54 AM, Tony Krowiak wrote:
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
[..]
+
+static char *vfio_ap_bus_get_dev_path(DeviceState *dev)
Cover letter states you want to remove vifo_ap references form the bus. Sane
thing to do as this would also be the bus we would like to plug an emulated
ap device into (if these get implemented).
There are a couple of such references that survived around this comment.
I noticed that earlier today as I was reviewing these comments and
already removed the vfio references.
+{
+/* at most one */
+return g_strdup_printf("/1");
+}
+
+static void vfio_ap_bus_class_init(ObjectClass *klass, void *data)
+{
+BusClass *k = BUS_CLASS(klass);
+
+k->get_dev_path = vfio_ap_bus_get_dev_path;
+/* More than one vfio-ap device does not make sense */
+k->max_dev = 1;
+}
+
+static const TypeInfo vfio_ap_bus_info = {
+.name = TYPE_AP_BUS,
+.parent = TYPE_BUS,
+.instance_size = sizeof(APBus),
+.class_init = vfio_ap_bus_class_init,
+};
+
[..]
+static void ap_register(void)
+{
+type_register_static(_bridge_info);
+type_register_static(_ap_bus_info);
+}
+
[..]
Re: [Qemu-devel] [PATCH v9 3/6] s390x/kvm: enable AP instruction interpretation for guest
On 09/27/2018 03:52 AM, David Hildenbrand wrote:
On 27/09/2018 00:54, Tony Krowiak wrote:
From: Tony Krowiak
Let's use the KVM_SET_DEVICE_ATTR ioctl to enable hardware
interpretation of AP instructions executed on the guest.
If the S390_FEAT_AP feature is switched on for the guest,
AP instructions must be interpreted by default; otherwise,
they will be intercepted.
This attribute setting may be overridden by a device. For example,
a device may want to provide AP instructions to the guest (i.e.,
S390_FEAT_AP turned on), but it may want to emulate them. In this
case, the AP instructions executed on the guest must be
intercepted; so when the device is realized, it must disable
interpretation.
Signed-off-by: Tony Krowiak
---
target/s390x/kvm.c | 14 ++
1 file changed, 14 insertions(+)
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 5277acd79a2c..d55d24abfd78 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -2301,6 +2301,16 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model,
Error **errp)
S390_FEAT_MAX);
}
+static void kvm_s390_configure_apie(bool interpret)
+{
+uint64_t attr = interpret ? KVM_S390_VM_CRYPTO_ENABLE_APIE :
+KVM_S390_VM_CRYPTO_DISABLE_APIE;
+
+if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO, attr)) {
+kvm_s390_set_attr(attr);
Wrong indentation, apart from that
Fixed it!
Reviewed-by: David Hildenbrand
+}
+}
+
void kvm_s390_apply_cpu_model(const S390CPUModel *model, Error **errp)
{
struct kvm_s390_vm_cpu_processor prop = {
@@ -2350,6 +2360,10 @@ void kvm_s390_apply_cpu_model(const S390CPUModel *model,
Error **errp)
if (test_bit(S390_FEAT_CMM, model->features)) {
kvm_s390_enable_cmma();
}
+
+if (test_bit(S390_FEAT_AP, model->features)) {
+kvm_s390_configure_apie(true);
+}
}
void kvm_s390_restart_interrupt(S390CPU *cpu)
Re: [Qemu-devel] [PATCH v9 2/6] s390x/cpumodel: Set up CPU model for AP device support
On 09/27/2018 03:50 AM, David Hildenbrand wrote:
On 27/09/2018 00:54, Tony Krowiak wrote:
A new CPU model feature and two new CPU model facilities are
introduced to support AP devices for a KVM guest.
CPU model features:
1. The S390_FEAT_AP CPU model feature indicates whether AP
instructions are available to the guest. This feature will
be enabled only if the AP instructions are available on the
linux host as determined by the availability of the
KVM_S390_VM_CRYPTO_ENABLE_APIE VM attribute which is exposed
by KVM only if the AP instructions are available on the
host.
This feature must be turned on from userspace to execute AP
instructions on the KVM guest. The QEMU command line to turn
this feature on looks something like this:
qemu-system-s390x ... -cpu xxx,ap=on ...
This feature will be supported for zEC12 and newer CPU models.
The feature will not be supported for older models because
there are few older systems on which to test and the older
crypto cards will be going out of service in the relatively
near future.
CPU model facilities:
1. The S390_FEAT_AP_QUERY_CONFIG_INFO feature indicates whether the
AP Query Configuration Information (QCI) facility is available
to the guest as determined by whether the facility is available
on the host. This feature will be exposed by KVM only if the
QCI facility is installed on the host.
2. The S390_FEAT_AP_FACILITY_TEST feature indicates whether the AP
Facility Test (APFT) facility is available to the guest as
determined by whether the facility is available on the host.
This feature will be exposed by KVM only if APFT is installed
on the host.
Signed-off-by: Tony Krowiak
---
target/s390x/cpu_features.c | 3 +++
target/s390x/cpu_features_def.h | 3 +++
target/s390x/cpu_models.c | 2 ++
target/s390x/gen-features.c | 3 +++
target/s390x/kvm.c | 5 +
5 files changed, 16 insertions(+)
diff --git a/target/s390x/cpu_features.c b/target/s390x/cpu_features.c
index 172fb18df718..60cfeba48f4e 100644
--- a/target/s390x/cpu_features.c
+++ b/target/s390x/cpu_features.c
@@ -39,8 +39,10 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT("srs", S390_FEAT_TYPE_STFL, 9, "Sense-running-status facility"),
FEAT_INIT("csske", S390_FEAT_TYPE_STFL, 10, "Conditional-SSKE facility"),
FEAT_INIT("ctop", S390_FEAT_TYPE_STFL, 11, "Configuration-topology
facility"),
+FEAT_INIT("apqci", S390_FEAT_TYPE_STFL, 12, "Query AP Configuration Information
facility"),
FEAT_INIT("ipter", S390_FEAT_TYPE_STFL, 13, "IPTE-range facility"),
FEAT_INIT("nonqks", S390_FEAT_TYPE_STFL, 14, "Nonquiescing key-setting
facility"),
+FEAT_INIT("apft", S390_FEAT_TYPE_STFL, 15, "AP Facilities Test facility"),
FEAT_INIT("etf2", S390_FEAT_TYPE_STFL, 16, "Extended-translation facility
2"),
FEAT_INIT("msa-base", S390_FEAT_TYPE_STFL, 17, "Message-security-assist
facility (excluding subfunctions)"),
FEAT_INIT("ldisp", S390_FEAT_TYPE_STFL, 18, "Long-displacement facility"),
@@ -129,6 +131,7 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT_MISC("dateh2", "DAT-enhancement facility 2"),
FEAT_INIT_MISC("cmm", "Collaborative-memory-management facility"),
+FEAT_INIT_MISC("ap", "AP instructions installed"),
FEAT_INIT("plo-cl", S390_FEAT_TYPE_PLO, 0, "PLO Compare and load (32 bit in general registers)"),
FEAT_INIT("plo-clg", S390_FEAT_TYPE_PLO, 1, "PLO Compare and load (64 bit in
parameter list)"),
diff --git a/target/s390x/cpu_features_def.h b/target/s390x/cpu_features_def.h
index ac2c947f30a8..5fc7e7bf0116 100644
--- a/target/s390x/cpu_features_def.h
+++ b/target/s390x/cpu_features_def.h
@@ -27,8 +27,10 @@ typedef enum {
S390_FEAT_SENSE_RUNNING_STATUS,
S390_FEAT_CONDITIONAL_SSKE,
S390_FEAT_CONFIGURATION_TOPOLOGY,
+S390_FEAT_AP_QUERY_CONFIG_INFO,
S390_FEAT_IPTE_RANGE,
S390_FEAT_NONQ_KEY_SETTING,
+S390_FEAT_AP_FACILITIES_TEST,
S390_FEAT_EXTENDED_TRANSLATION_2,
S390_FEAT_MSA,
S390_FEAT_LONG_DISPLACEMENT,
@@ -119,6 +121,7 @@ typedef enum {
/* Misc */
S390_FEAT_DAT_ENH_2,
S390_FEAT_CMM,
+S390_FEAT_AP,
/* PLO */
S390_FEAT_PLO_CL,
diff --git a/target/s390x/cpu_models.c b/target/s390x/cpu_models.c
index 265d25c937bb..7c253ff308c5 100644
--- a/target/s390x/cpu_models.c
+++ b/target/s390x/cpu_models.c
@@ -786,6 +786,8 @@ static void check_consistency(const S390CPUModel *model)
{ S390_FEAT_PRNO_TRNG_QRTCR, S390_FEAT_MSA_EXT_5 },
{ S390_FEAT_PRNO_TRNG, S390_FEAT_MSA_EXT_5 },
Re: [Qemu-devel] [PATCH v9 1/6] linux-headers: linux header updates for AP support
On 09/27/2018 03:41 AM, David Hildenbrand wrote:
On 27/09/2018 00:54, Tony Krowiak wrote:
Updates the linux header files in preparation for introduction
of the VFIO AP device:
* Added a feature ID to indicate AP facilities are installed
* Added device attributes to the KVM_S390_VM_CRYPTO group
to indicate whether AP instructions are to be interpreted
* Added VFIO device information for AP devices
Signed-off-by: Tony Krowiak
---
linux-headers/asm-s390/kvm.h | 3 +++
linux-headers/linux/vfio.h | 2 ++
2 files changed, 5 insertions(+)
diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index 1ab9901911bf..b4948706468e 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -130,6 +130,7 @@ struct kvm_s390_vm_cpu_machine {
#define KVM_S390_VM_CPU_FEAT_PFMFI11
#define KVM_S390_VM_CPU_FEAT_SIGPIF 12
#define KVM_S390_VM_CPU_FEAT_KSS 13
+#define KVM_S390_VM_CPU_FEAT_AP14
Guess that's a leftover. But should be fixed automatically by proper
header sync later.
Oops
struct kvm_s390_vm_cpu_feat {
__u64 feat[16];
};
@@ -160,6 +161,8 @@ struct kvm_s390_vm_cpu_subfunc {
#define KVM_S390_VM_CRYPTO_ENABLE_DEA_KW 1
#define KVM_S390_VM_CRYPTO_DISABLE_AES_KW 2
#define KVM_S390_VM_CRYPTO_DISABLE_DEA_KW 3
+#define KVM_S390_VM_CRYPTO_ENABLE_APIE 4
+#define KVM_S390_VM_CRYPTO_DISABLE_APIE5
/* kvm attributes for migration mode */
#define KVM_S390_VM_MIGRATION_STOP0
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 3615a269d378..838919a4c03a 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -200,6 +200,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2) /* vfio-platform device */
#define VFIO_DEVICE_FLAGS_AMBA (1 << 3)/* vfio-amba device */
#define VFIO_DEVICE_FLAGS_CCW (1 << 4) /* vfio-ccw device */
+#define VFIO_DEVICE_FLAGS_AP (1 << 5)/* vfio-ap device */
__u32 num_regions;/* Max region index + 1 */
__u32 num_irqs; /* Max IRQ index + 1 */
};
@@ -215,6 +216,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_API_PLATFORM_STRING "vfio-platform"
#define VFIO_DEVICE_API_AMBA_STRING "vfio-amba"
#define VFIO_DEVICE_API_CCW_STRING"vfio-ccw"
+#define VFIO_DEVICE_API_AP_STRING "vfio-ap"
/**
* VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
[Qemu-devel] [PATCH v9 2/6] s390x/cpumodel: Set up CPU model for AP device support
A new CPU model feature and two new CPU model facilities are
introduced to support AP devices for a KVM guest.
CPU model features:
1. The S390_FEAT_AP CPU model feature indicates whether AP
instructions are available to the guest. This feature will
be enabled only if the AP instructions are available on the
linux host as determined by the availability of the
KVM_S390_VM_CRYPTO_ENABLE_APIE VM attribute which is exposed
by KVM only if the AP instructions are available on the
host.
This feature must be turned on from userspace to execute AP
instructions on the KVM guest. The QEMU command line to turn
this feature on looks something like this:
qemu-system-s390x ... -cpu xxx,ap=on ...
This feature will be supported for zEC12 and newer CPU models.
The feature will not be supported for older models because
there are few older systems on which to test and the older
crypto cards will be going out of service in the relatively
near future.
CPU model facilities:
1. The S390_FEAT_AP_QUERY_CONFIG_INFO feature indicates whether the
AP Query Configuration Information (QCI) facility is available
to the guest as determined by whether the facility is available
on the host. This feature will be exposed by KVM only if the
QCI facility is installed on the host.
2. The S390_FEAT_AP_FACILITY_TEST feature indicates whether the AP
Facility Test (APFT) facility is available to the guest as
determined by whether the facility is available on the host.
This feature will be exposed by KVM only if APFT is installed
on the host.
Signed-off-by: Tony Krowiak
---
target/s390x/cpu_features.c | 3 +++
target/s390x/cpu_features_def.h | 3 +++
target/s390x/cpu_models.c | 2 ++
target/s390x/gen-features.c | 3 +++
target/s390x/kvm.c | 5 +
5 files changed, 16 insertions(+)
diff --git a/target/s390x/cpu_features.c b/target/s390x/cpu_features.c
index 172fb18df718..60cfeba48f4e 100644
--- a/target/s390x/cpu_features.c
+++ b/target/s390x/cpu_features.c
@@ -39,8 +39,10 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT("srs", S390_FEAT_TYPE_STFL, 9, "Sense-running-status facility"),
FEAT_INIT("csske", S390_FEAT_TYPE_STFL, 10, "Conditional-SSKE facility"),
FEAT_INIT("ctop", S390_FEAT_TYPE_STFL, 11, "Configuration-topology
facility"),
+FEAT_INIT("apqci", S390_FEAT_TYPE_STFL, 12, "Query AP Configuration
Information facility"),
FEAT_INIT("ipter", S390_FEAT_TYPE_STFL, 13, "IPTE-range facility"),
FEAT_INIT("nonqks", S390_FEAT_TYPE_STFL, 14, "Nonquiescing key-setting
facility"),
+FEAT_INIT("apft", S390_FEAT_TYPE_STFL, 15, "AP Facilities Test facility"),
FEAT_INIT("etf2", S390_FEAT_TYPE_STFL, 16, "Extended-translation facility
2"),
FEAT_INIT("msa-base", S390_FEAT_TYPE_STFL, 17, "Message-security-assist
facility (excluding subfunctions)"),
FEAT_INIT("ldisp", S390_FEAT_TYPE_STFL, 18, "Long-displacement facility"),
@@ -129,6 +131,7 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT_MISC("dateh2", "DAT-enhancement facility 2"),
FEAT_INIT_MISC("cmm", "Collaborative-memory-management facility"),
+FEAT_INIT_MISC("ap", "AP instructions installed"),
FEAT_INIT("plo-cl", S390_FEAT_TYPE_PLO, 0, "PLO Compare and load (32 bit
in general registers)"),
FEAT_INIT("plo-clg", S390_FEAT_TYPE_PLO, 1, "PLO Compare and load (64 bit
in parameter list)"),
diff --git a/target/s390x/cpu_features_def.h b/target/s390x/cpu_features_def.h
index ac2c947f30a8..5fc7e7bf0116 100644
--- a/target/s390x/cpu_features_def.h
+++ b/target/s390x/cpu_features_def.h
@@ -27,8 +27,10 @@ typedef enum {
S390_FEAT_SENSE_RUNNING_STATUS,
S390_FEAT_CONDITIONAL_SSKE,
S390_FEAT_CONFIGURATION_TOPOLOGY,
+S390_FEAT_AP_QUERY_CONFIG_INFO,
S390_FEAT_IPTE_RANGE,
S390_FEAT_NONQ_KEY_SETTING,
+S390_FEAT_AP_FACILITIES_TEST,
S390_FEAT_EXTENDED_TRANSLATION_2,
S390_FEAT_MSA,
S390_FEAT_LONG_DISPLACEMENT,
@@ -119,6 +121,7 @@ typedef enum {
/* Misc */
S390_FEAT_DAT_ENH_2,
S390_FEAT_CMM,
+S390_FEAT_AP,
/* PLO */
S390_FEAT_PLO_CL,
diff --git a/target/s390x/cpu_models.c b/target/s390x/cpu_models.c
index 265d25c937bb..7c253ff308c5 100644
--- a/target/s390x/cpu_models.c
+++ b/target/s390x/cpu_models.c
@@ -786,6 +786,8 @@ static void check_consistency(const S390CPUModel *model)
{ S390_FEAT_PRNO_TRNG_QRTCR, S390_FEAT_MSA_EXT_5 },
{ S390_FEAT_PRNO_TRNG, S390_FEAT_MSA_EXT_5 },
{ S390_FEAT_SIE_KSS, S390_FEAT_SIE_F2 },
+{ S390_FEAT_AP_QUERY_CONFIG_INFO, S390_FEAT_AP },
+{ S390_FEAT_AP_FACILITIES_TEST, S390_FEAT_AP
[Qemu-devel] [PATCH v9 3/6] s390x/kvm: enable AP instruction interpretation for guest
From: Tony Krowiak
Let's use the KVM_SET_DEVICE_ATTR ioctl to enable hardware
interpretation of AP instructions executed on the guest.
If the S390_FEAT_AP feature is switched on for the guest,
AP instructions must be interpreted by default; otherwise,
they will be intercepted.
This attribute setting may be overridden by a device. For example,
a device may want to provide AP instructions to the guest (i.e.,
S390_FEAT_AP turned on), but it may want to emulate them. In this
case, the AP instructions executed on the guest must be
intercepted; so when the device is realized, it must disable
interpretation.
Signed-off-by: Tony Krowiak
---
target/s390x/kvm.c | 14 ++
1 file changed, 14 insertions(+)
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 5277acd79a2c..d55d24abfd78 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -2301,6 +2301,16 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model,
Error **errp)
S390_FEAT_MAX);
}
+static void kvm_s390_configure_apie(bool interpret)
+{
+uint64_t attr = interpret ? KVM_S390_VM_CRYPTO_ENABLE_APIE :
+KVM_S390_VM_CRYPTO_DISABLE_APIE;
+
+if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO, attr)) {
+kvm_s390_set_attr(attr);
+}
+}
+
void kvm_s390_apply_cpu_model(const S390CPUModel *model, Error **errp)
{
struct kvm_s390_vm_cpu_processor prop = {
@@ -2350,6 +2360,10 @@ void kvm_s390_apply_cpu_model(const S390CPUModel *model,
Error **errp)
if (test_bit(S390_FEAT_CMM, model->features)) {
kvm_s390_enable_cmma();
}
+
+if (test_bit(S390_FEAT_AP, model->features)) {
+kvm_s390_configure_apie(true);
+}
}
void kvm_s390_restart_interrupt(S390CPU *cpu)
--
2.19.0.221.g150f307
[Qemu-devel] [PATCH v9 6/6] s390: doc: detailed specifications for AP virtualization
This patch provides documentation describing the AP architecture and design concepts behind the virtualization of AP devices. It also includes an example of how to configure AP devices for exclusive use of KVM guests. Signed-off-by: Tony Krowiak --- MAINTAINERS | 1 + docs/vfio-ap.txt | 787 +++ 2 files changed, 788 insertions(+) create mode 100644 docs/vfio-ap.txt diff --git a/MAINTAINERS b/MAINTAINERS index 29041da69237..b64a12034c2c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1210,6 +1210,7 @@ F: hw/s390x/ap-bridge.c F: include/hw/s390x/ap-device.h F: include/hw/s390x/ap-bridge.h F: hw/vfio/ap.c +F: docs/vfio-ap.txt L: qemu-s3...@nongnu.org vhost diff --git a/docs/vfio-ap.txt b/docs/vfio-ap.txt new file mode 100644 index ..fd17ce48967b --- /dev/null +++ b/docs/vfio-ap.txt @@ -0,0 +1,787 @@ +Adjunct Processor (AP) Device += + +Contents: += +* Introduction +* AP Architectural Overview +* Start Interpretive Execution (SIE) Instruction +* AP Matrix Configuration on Linux Host +* Starting a Linux Guest Configured with an AP Matrix +* Example: Configure AP Matrices for Three Linux Guests + +Introduction: + +The IBM Adjunct Processor (AP) Cryptographic Facility is comprised +of three AP instructions and from 1 to 256 PCIe cryptographic adapter cards. +These AP devices provide cryptographic functions to all CPUs assigned to a +linux system running in an IBM Z system LPAR. + +On s390x, AP adapter cards are exposed via the AP bus. This document +describes how those cards may be made available to KVM guests using the +VFIO mediated device framework. + +AP Architectural Overview: += +In order understand the terminology used in the rest of this document, let's +start with some definitions: + +* AP adapter + + An AP adapter is an IBM Z adapter card that can perform cryptographic + functions. There can be from 0 to 256 adapters assigned to an LPAR; however, + the maximum adapter number allowed is determined by machine model. Adapters + assigned to the LPAR in which a linux host is running will be available to the + linux host. Each adapter is identified by a number from 0 to 255. When + installed, an AP adapter is accessed by AP instructions executed by any CPU. + +* AP domain + + An adapter is partitioned into domains. Each domain can be thought of as + a set of hardware registers for processing AP instructions. An adapter can + hold up to 256 domains; however, the maximum domain number allowed is + determined by machine model. Each domain is identified by a number from 0 to + 255. Domains can be further classified into two types: + +* Usage domains are domains that can be accessed directly to process AP + commands + +* Control domains are domains that are accessed indirectly by AP + commands sent to a usage domain to control or change the domain; for + example, to set a secure private key for the domain. + +* AP Queue + + An AP queue is the means by which an AP command-request message is sent to an + AP usage domain inside a specific AP. An AP queue is identified by a tuple + comprised of an AP adapter ID (APID) and an AP queue index (APQI). The + APQI corresponds to a given usage domain number within the adapter. This tuple + forms an AP Queue Number (APQN) uniquely identifying an AP queue. AP + instructions include a field containing the APQN to identify the AP queue to + which the AP command-request message is to be sent for processing. + +* AP Instructions: + + There are three AP instructions: + + * NQAP: to enqueue an AP command-request message to a queue + * DQAP: to dequeue an AP command-reply message from a queue + * PQAP: to administer the queues + + AP instructions identify the domain that is targeted to process the AP + command; this must be one of the usage domains. An AP command may modify a + domain that is not one of the usage domains, but the modified domain + must be one of the control domains. + +Start Interpretive Execution (SIE) Instruction +== +A KVM guest is started by executing the Start Interpretive Execution (SIE) +instruction. The SIE state description is a control block that contains the +state information for a KVM guest and is supplied as input to the SIE +instruction. The SIE state description contains a satellite control block called +the Crypto Control Block (CRYCB). The CRYCB contains three fields to identify +the adapters, usage domains and control domains assigned to the KVM guest: + +* The AP Mask (APM) field is a bit mask that identifies the AP adapters assigned + to the KVM guest. Each bit in the mask, from most significant to least + significant bit, corresponds to an APID from 0-255. If a bit is set, the + corresponding adapter is valid for use by the KVM guest. + +* The AP Queue Mask (AQM) field is a bit mask identifying the AP
[Qemu-devel] [PATCH v9 4/6] s390x/ap: base Adjunct Processor (AP) object model
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
---
MAINTAINERS | 12 ++
hw/s390x/Makefile.objs | 2 +
hw/s390x/ap-bridge.c | 81
hw/s390x/ap-device.c | 39 +
hw/s390x/s390-virtio-ccw.c | 4 ++
include/hw/s390x/ap-bridge.h | 37
include/hw/s390x/ap-device.h | 38 +
7 files changed, 213 insertions(+)
create mode 100644 hw/s390x/ap-bridge.c
create mode 100644 hw/s390x/ap-device.c
create mode 100644 include/hw/s390x/ap-bridge.h
create mode 100644 include/hw/s390x/ap-device.h
diff --git a/MAINTAINERS b/MAINTAINERS
index d12518c08f10..97e8ed808bc0 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1199,6 +1199,18 @@ F: include/hw/s390x/s390-ccw.h
T: git git://github.com/cohuck/qemu.git s390-next
L: qemu-s3...@nongnu.org
+vfio-ap
+M: Christian Borntraeger
+M: Tony Krowiak
+M: Halil Pasic
+M: Pierre Morel
+S: Supported
+F: hw/s390x/ap-device.c
+F: hw/s390x/ap-bridge.c
+F: include/hw/s390x/ap-device.h
+F: include/hw/s390x/ap-bridge.h
+L: qemu-s3...@nongnu.org
+
vhost
M: Michael S. Tsirkin
S: Supported
diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index 93282f7c593c..add89b150d90 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -20,3 +20,5 @@ obj-$(CONFIG_TCG) += tod-qemu.o
obj-$(CONFIG_KVM) += s390-skeys-kvm.o
obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
obj-y += s390-ccw.o
+obj-y += ap-device.o
+obj-y += ap-bridge.o
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
new file mode 100644
index ..8564dfa96ee7
--- /dev/null
+++ b/hw/s390x/ap-bridge.c
@@ -0,0 +1,81 @@
+/*
+ * ap bridge
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Halil Pasic
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "qemu/bitops.h"
+#include "hw/s390x/ap-bridge.h"
+#include "cpu.h"
+
+static char *vfio_ap_bus_get_dev_path(DeviceState *dev)
+{
+/* at most one */
+return g_strdup_printf("/1");
+}
+
+static void vfio_ap_bus_class_init(ObjectClass *klass, void *data)
+{
+BusClass *k = BUS_CLASS(klass);
+
+k->get_dev_path = vfio_ap_bus_get_dev_path;
+/* More than one vfio-ap device does not make sense */
+k->max_dev = 1;
+}
+
+static const TypeInfo vfio_ap_bus_info = {
+.name = TYPE_AP_BUS,
+.parent = TYPE_BUS,
+.instance_size = sizeof(APBus),
+.class_init = vfio_ap_bus_class_init,
+};
+
+void s390_init_ap(void)
+{
+DeviceState *dev;
+
+/* If no AP instructions then no need for AP bridge */
+if (!s390_has_feat(S390_FEAT_AP)) {
+return;
+}
+
+/* Create bridge device */
+dev = qdev_create(NULL, TYPE_AP_BRIDGE);
+object_property_add_child(qdev_get_machine(), TYPE_AP_BRIDGE,
+ OBJECT(dev), NULL);
+qdev_init_nofail(dev);
+
+/* Create bus on bridge device */
+qbus_create(TYPE_AP_BUS, dev, TYPE_AP_BUS);
+ }
+
+
+
+static void ap_bridge_class_init(ObjectClass *klass, void *data)
+{
+DeviceClass *dc = DEVICE_CLASS(klass);
+
+set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+}
+
+static const TypeInfo ap_bridge_info = {
+.name = TYPE_AP_BRIDGE,
+.parent= TYPE_SYS_BUS_DEVICE,
+.instance_size = sizeof(APBridge),
+.class_init= ap_bridge_class_init,
+};
+
+static void ap_register(void)
+{
+type_register_static(_bridge_info);
+type_register_static(_ap_bus_info);
+}
+
+type_init(ap_register)
diff --git a/hw/s390x/ap-device.c b/hw/s390x/ap-device.c
new file mode 100644
index ..3cd4bae52591
--- /dev/null
+++ b/hw/s390x/ap-device.c
@@ -0,0 +1,39 @@
+/*
+ * Adjunct Processor (AP) matrix device
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu/module.h"
+#include "qapi/error.h"
+#include "hw/qdev.h"
+#include "hw/s390x/ap-device.h"
+
+static void ap_class_init(ObjectClass *klass, void *data)
+{
+DeviceClass *dc = DEVICE_CLASS(klass);
+
+dc->desc = "AP device class";
+dc->hotpluggable = false;
+}
+
+static const TypeInfo ap_device_info = {
+.name = AP_DEVICE_TYPE,
+.parent = TYPE_DEVICE,
+.instance_size = sizeof(APDevice),
+.class_size = sizeof(APDeviceClass),
+.class_init = ap_class_init,
+.abstract = true,
+};
+
+static void ap_device_register(void)
+{
+type_register_static(_device_info);
+}
+
+type_init(ap_device_
[Qemu-devel] [PATCH v9 1/6] linux-headers: linux header updates for AP support
Updates the linux header files in preparation for introduction
of the VFIO AP device:
* Added a feature ID to indicate AP facilities are installed
* Added device attributes to the KVM_S390_VM_CRYPTO group
to indicate whether AP instructions are to be interpreted
* Added VFIO device information for AP devices
Signed-off-by: Tony Krowiak
---
linux-headers/asm-s390/kvm.h | 3 +++
linux-headers/linux/vfio.h | 2 ++
2 files changed, 5 insertions(+)
diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index 1ab9901911bf..b4948706468e 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -130,6 +130,7 @@ struct kvm_s390_vm_cpu_machine {
#define KVM_S390_VM_CPU_FEAT_PFMFI 11
#define KVM_S390_VM_CPU_FEAT_SIGPIF12
#define KVM_S390_VM_CPU_FEAT_KSS 13
+#define KVM_S390_VM_CPU_FEAT_AP14
struct kvm_s390_vm_cpu_feat {
__u64 feat[16];
};
@@ -160,6 +161,8 @@ struct kvm_s390_vm_cpu_subfunc {
#define KVM_S390_VM_CRYPTO_ENABLE_DEA_KW 1
#define KVM_S390_VM_CRYPTO_DISABLE_AES_KW 2
#define KVM_S390_VM_CRYPTO_DISABLE_DEA_KW 3
+#define KVM_S390_VM_CRYPTO_ENABLE_APIE 4
+#define KVM_S390_VM_CRYPTO_DISABLE_APIE5
/* kvm attributes for migration mode */
#define KVM_S390_VM_MIGRATION_STOP 0
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 3615a269d378..838919a4c03a 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -200,6 +200,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2)/* vfio-platform device */
#define VFIO_DEVICE_FLAGS_AMBA (1 << 3) /* vfio-amba device */
#define VFIO_DEVICE_FLAGS_CCW (1 << 4)/* vfio-ccw device */
+#define VFIO_DEVICE_FLAGS_AP (1 << 5) /* vfio-ap device */
__u32 num_regions;/* Max region index + 1 */
__u32 num_irqs; /* Max IRQ index + 1 */
};
@@ -215,6 +216,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_API_PLATFORM_STRING"vfio-platform"
#define VFIO_DEVICE_API_AMBA_STRING"vfio-amba"
#define VFIO_DEVICE_API_CCW_STRING "vfio-ccw"
+#define VFIO_DEVICE_API_AP_STRING "vfio-ap"
/**
* VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
--
2.19.0.221.g150f307
[Qemu-devel] [PATCH v9 0/6] s390x: vfio-ap: guest dedicated crypto adapters
From: Tony Krowiak This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model. See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series. v8 => v9 Change log: === * Removed all references to VFIO in AP bridge and bus * Expose AP feature only if the KVM_S390_VM_CRYPTO_ENABLE_APIE VM attribute is exposed by KVM - i.e., if AP instructions are available on the linux host. * Enable AP interpretation only if AP feature is switched on; no need to disable because it is disabled by default. v7 => v8 Change log: === * Enable SIE interpretation AP instructions if the CPU model feature for AP instructions is turned on for the guest. v6 => v7 Change log; === * Changed email address for Signed-off-by v5 => v6 Change log: === * Added reset handling fo vfio-ap device * Added a bridge/bus to AP device object model - thanks to Halil Pasic v4 => v5 Change log: === * Added MAINTAINERS entries for VFIO AP * Added explanation for why we are only supporting zEC12 and newer CPU models. * Changed CPU model feature qci=on|off to apqci=on|off * Misc. minor changes v3 => v4 Change log: === * Made vfio-ap device unpluggable for now * Renamed command line CPU model feature for QCI: qci=on -> apqci=on * Removed call to KVM_S390_VM_CRYPTO_INTERPRET_AP ioctl - ioctl was removed from kernel and AP instruction interpretation is set from the VFIO device driver * Added check to ensure only one vfio-ap device can be configured per guest * Removed AP instruction interception handlers: AP instructions will be interpreted by default if AP facilities are installed to handle the case where feature ap=on and no vfio-ap device is configured for the guest. Tony Krowiak (6): linux-headers: linux header updates for AP support s390x/cpumodel: Set up CPU model for AP device support s390x/kvm: enable AP instruction interpretation for guest s390x/ap: base Adjunct Processor (AP) object model s390x/vfio: ap: Introduce VFIO AP device s390: doc: detailed specifications for AP virtualization MAINTAINERS | 14 + default-configs/s390x-softmmu.mak | 1 + docs/vfio-ap.txt | 787 ++ hw/s390x/Makefile.objs| 2 + hw/s390x/ap-bridge.c | 81 +++ hw/s390x/ap-device.c | 39 ++ hw/s390x/s390-virtio-ccw.c| 4 + hw/vfio/Makefile.objs | 1 + hw/vfio/ap.c | 181 +++ include/hw/s390x/ap-bridge.h | 37 ++ include/hw/s390x/ap-device.h | 38 ++ include/hw/vfio/vfio-common.h | 1 + linux-headers/asm-s390/kvm.h | 3 + linux-headers/linux/vfio.h| 2 + target/s390x/cpu_features.c | 3 + target/s390x/cpu_features_def.h | 3 + target/s390x/cpu_models.c | 2 + target/s390x/gen-features.c | 3 + target/s390x/kvm.c| 19 + 19 files changed, 1221 insertions(+) create mode 100644 docs/vfio-ap.txt create mode 100644 hw/s390x/ap-bridge.c create mode 100644 hw/s390x/ap-device.c create mode 100644 hw/vfio/ap.c create mode 100644 include/hw/s390x/ap-bridge.h create mode 100644 include/hw/s390x/ap-device.h -- 2.19.0.221.g150f307
[Qemu-devel] [PATCH v9 5/6] s390x/vfio: ap: Introduce VFIO AP device
Introduces a VFIO based AP device. The device is defined via
the QEMU command line by specifying:
-device vfio-ap,sysfsdev=
There may be only one vfio-ap device configured for a guest.
The mediated matrix device is created by the VFIO AP device
driver by writing a UUID to a sysfs attribute file (see
docs/vfio-ap.txt). The mediated matrix device will be named
after the UUID. Symbolic links to the $uuid are created in
many places, so the path to the mediated matrix device $uuid
can be specified in any of the following ways:
/sys/devices/vfio_ap/matrix/$uuid
/sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
/sys/bus/mdev/devices/$uuid
/sys/bus/mdev/drivers/vfio_mdev/$uuid
When the vfio-ap device is realized, it acquires and opens the
VFIO iommu group to which the mediated matrix device is
bound. This causes a VFIO group notification event to be
signaled. The vfio_ap device driver's group notification
handler will get called at which time the device driver
will configure the the AP devices to which the guest will
be granted access.
Signed-off-by: Tony Krowiak
---
MAINTAINERS | 1 +
default-configs/s390x-softmmu.mak | 1 +
hw/vfio/Makefile.objs | 1 +
hw/vfio/ap.c | 181 ++
include/hw/vfio/vfio-common.h | 1 +
5 files changed, 185 insertions(+)
create mode 100644 hw/vfio/ap.c
diff --git a/MAINTAINERS b/MAINTAINERS
index 97e8ed808bc0..29041da69237 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1209,6 +1209,7 @@ F: hw/s390x/ap-device.c
F: hw/s390x/ap-bridge.c
F: include/hw/s390x/ap-device.h
F: include/hw/s390x/ap-bridge.h
+F: hw/vfio/ap.c
L: qemu-s3...@nongnu.org
vhost
diff --git a/default-configs/s390x-softmmu.mak
b/default-configs/s390x-softmmu.mak
index d6b67d50f0e4..5eef37592451 100644
--- a/default-configs/s390x-softmmu.mak
+++ b/default-configs/s390x-softmmu.mak
@@ -7,3 +7,4 @@ CONFIG_S390_FLIC=y
CONFIG_S390_FLIC_KVM=$(CONFIG_KVM)
CONFIG_VFIO_CCW=$(CONFIG_LINUX)
CONFIG_WDT_DIAG288=y
+CONFIG_VFIO_AP=$(CONFIG_LINUX)
diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs
index a2e7a0a7cf02..8b3f664d85f7 100644
--- a/hw/vfio/Makefile.objs
+++ b/hw/vfio/Makefile.objs
@@ -6,4 +6,5 @@ obj-$(CONFIG_SOFTMMU) += platform.o
obj-$(CONFIG_VFIO_XGMAC) += calxeda-xgmac.o
obj-$(CONFIG_VFIO_AMD_XGBE) += amd-xgbe.o
obj-$(CONFIG_SOFTMMU) += spapr.o
+obj-$(CONFIG_VFIO_AP) += ap.o
endif
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
new file mode 100644
index ..429988f23f98
--- /dev/null
+++ b/hw/vfio/ap.c
@@ -0,0 +1,181 @@
+/*
+ * VFIO based AP matrix device assignment
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *Halil Pasic
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include
+#include
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "hw/vfio/vfio.h"
+#include "hw/vfio/vfio-common.h"
+#include "hw/s390x/ap-device.h"
+#include "qemu/error-report.h"
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "cpu.h"
+#include "kvm_s390x.h"
+#include "sysemu/sysemu.h"
+#include "hw/s390x/ap-bridge.h"
+#include "exec/address-spaces.h"
+
+#define VFIO_AP_DEVICE_TYPE "vfio-ap"
+
+typedef struct VFIOAPDevice {
+APDevice apdev;
+VFIODevice vdev;
+} VFIOAPDevice;
+
+static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
+{
+vdev->needs_reset = false;
+}
+
+/*
+ * We don't need vfio_hot_reset_multi and vfio_eoi operations for
+ * vfio-ap device now.
+ */
+struct VFIODeviceOps vfio_ap_ops = {
+.vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
+};
+
+static void vfio_ap_put_device(VFIOAPDevice *vapdev)
+{
+g_free(vapdev->vdev.name);
+vfio_put_base_device(>vdev);
+}
+
+static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
+{
+char *tmp, group_path[PATH_MAX];
+ssize_t len;
+int groupid;
+
+tmp = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
+len = readlink(tmp, group_path, sizeof(group_path));
+g_free(tmp);
+
+if (len <= 0 || len >= sizeof(group_path)) {
+error_setg(errp, "%s: no iommu_group found for %s",
+ VFIO_AP_DEVICE_TYPE, vapdev->vdev.sysfsdev);
+return NULL;
+}
+
+group_path[len] = 0;
+
+if (sscanf(basename(group_path), "%d", ) != 1) {
+error_setg(errp, "vfio: failed to read %s", group_path);
+return NULL;
+}
+
+return vfio_get_group(groupid, _space_memory, errp);
+}
+
+static void vfio_ap_realize(DeviceState *dev, Error **errp)
+{
+int ret;
+char
Re: [Qemu-devel] [qemu-s390x] [PATCH v8 3/6] s390x/kvm: enable/disable AP instruction interpretation for guest
On 09/17/2018 04:43 AM, David Hildenbrand wrote:
Am 12.09.18 um 22:08 schrieb Tony Krowiak:
From: Tony Krowiak
Let's use the KVM_SET_DEVICE_ATTR ioctl to enable or disable
hardware interpretation of AP instructions executed on the guest.
If the S390_FEAT_AP feature is installed, AP instructions will
be interpreted by default; otherwise, they will be intercepted.
This attribute setting may be overridden by a device. For example,
a device may want to provide AP instructions to the guest (i.e.,
S390_FEAT_AP turned on), but it may want to emulate them. In this
case, the AP instructions executed on the guest must be
intercepted; so when the device is realized, it must disable
interpretation.
Signed-off-by: Tony Krowiak
---
target/s390x/kvm.c | 16
1 files changed, 16 insertions(+), 0 deletions(-)
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index c4bd84d..28a3900 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -2297,6 +2297,19 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model,
Error **errp)
S390_FEAT_MAX);
}
+static void kvm_s390_configure_apie(int interpret)
+{
+uint64_t attr = KVM_S390_VM_CRYPTO_DISABLE_APIE;
+
+if (interpret) {
+attr = KVM_S390_VM_CRYPTO_ENABLE_APIE;
+}
+
+if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO, attr)) {
+kvm_s390_set_attr(attr);
+}
+}
+
void kvm_s390_apply_cpu_model(const S390CPUModel *model, Error **errp)
{
struct kvm_s390_vm_cpu_processor prop = {
@@ -2346,6 +2359,9 @@ void kvm_s390_apply_cpu_model(const S390CPUModel *model,
Error **errp)
if (test_bit(S390_FEAT_CMM, model->features)) {
kvm_s390_enable_cmma();
}
+
+/* configure hardware interpretation of AP instructions */
+kvm_s390_configure_apie(test_bit(S390_FEAT_AP, model->features));
}
As it is off as default,
if (test_bit(S390_FEAT_AP, model->features)) {
kvm_s390_configure_apie(true, model->features));
}
will save one ioctl without AP when starting a guest.
As mentioned as replay to patch #2, instead of KVM_S390_VM_CPU_FEAT_AP,
I think we can simply do the following in kvm_s390_get_host_cpu_model()
/* for now, we can only provide the AP feature with HW support */
if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO,
KVM_S390_VM_CRYPTO_ENABLE_APIE)) {
set_bit(S390_FEAT_AP, model->features);
}
Apart from that, looks good to me.
Let me summarize what I think you are suggesting.
For KVM:
1. Get rid of KVM_S390_VM_CPU_FEAT_AP in KVM
2. Make AP instruction interception the default.
3. Provide the KVM_S390_VM_CRYPTO_ENABLE_APIE attribute in KVM if the
AP instructions are installed on the host
For QEMU:
1. In the kvm_s390_get_host_cpu_model(), set the S390_FEAT_AP CPU model
feature bit if the KVM_S390_VM_CRYPTO_ENABLE_APIE attribute is available
in KVM.
2. In the kvm_s390_apply_cpu_model() function, if the S390_FEAT_AP bit is
set in the guest's CPU model (i.e., ap=on), use the KVM_SET_DEVICE_ATTR
ioctl to set the KVM_S390_VM_CRYPTO_ENABLE_APIE attribute. This will
ultimately result in ECA.28 being set to 1 (i.e., interpret AP instructions)
for each vcpu.
3. Down the road, if a QEMU device for emulating AP is used, the
KVM_S390_VM_CRYPTO_DISABLE_APIE attribute can be set to instruct KVM to
intercept AP instructions. This can be done when the device is realized.
I've discussed this with Halil -- Pierre is out until next week. We
are in agreement that while these changes are viable, they result in
a slightly more complicated implementation compared to previous versions (e.g.
kernel v9 QEMU v7), and locks us into a model that may limit design choices
down the road if/when virtual and/or emulated AP devices are implemented.
Having said that, your proposal makes perfect sense under the assumptions that:
1) The default for ECA.28 has to be zero even if the guest is supposed to have
AP instructions installed, and
2) QEMU needs to set ECA.28 via ioctl.
I don't think, however, those assumptions are necessarily justified. What we
get does not expand on what we had in any meaningful way, but forces us to add
the two new KVM attributes (KVM_S390_VM_CRYPTO_ENABLE_APIE and
KVM_S390_VM_CRYPTO_DISABLE_APIE)
which limits our choices for how we implement future enhancements to the AP
virtualization architecture.
Let's recap the previous behavior case by case and compare it with the one
proposed by you:
1) ap=off in the requested model:
--> ECA.28 not set
--> vfio-ap device has no bus to plug into in QEMU
--> QEMU injects operation exceptions
2) ap=on in the requested model
2.1) if host does not have ap instr:
--> KVM_S390_VM_CPU_FEAT_AP not advertised
--> since we don't provide emulation compatibility fails
--> guest does not start
--> ECA.28 do
Re: [Qemu-devel] [PATCH v8 2/6] s390x/cpumodel: Set up CPU model for AP device support
On 09/17/2018 04:19 AM, David Hildenbrand wrote: Am 12.09.18 um 22:08 schrieb Tony Krowiak: A new CPU model feature and two new CPU model facilities are introduced to support AP devices for a KVM guest. CPU model features: 1. The KVM_S390_VM_CPU_FEAT_AP CPU model feature indicates that AP facilities are installed. This feature will be enabled by the kernel only if the AP facilities are installed on the linux host. This feature must be turned on from userspace to access AP devices from the KVM guest. The QEMU command line to turn this feature looks something like this: qemu-system-s390x ... -cpu xxx,ap=on This feature will be supported for zEC12 and newer CPU models. The feature will not be supported for older models due to testability issues. I think we (me, Halil and Pierre) agreed that this should not be a CPU_FEAT but instead some AP_INTERPRETATION toggle, that can be turned on/off dynamically. (CPU_FEAT cannot be changed after CPUs have been created). Can you discuss that with Halil + Pierre? I will do that.
Re: [Qemu-devel] [qemu-s390x] [PATCH v8 4/6] s390x/ap: base Adjunct Processor (AP) object model
On 09/13/2018 02:24 PM, Halil Pasic wrote:
On 09/13/2018 07:15 PM, Tony Krowiak wrote:
On 09/13/2018 01:02 PM, Tony Krowiak wrote:
On 09/13/2018 02:29 AM, Christian Borntraeger wrote:
On 09/13/2018 07:48 AM, Thomas Huth wrote:
On 2018-09-12 22:08, Tony Krowiak wrote:
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
---
[...]
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
new file mode 100644
index 000..4f10425
--- /dev/null
+++ b/hw/s390x/ap-bridge.c
[...]
+void s390_init_ap(void)
+{
+DeviceState *dev;
+
+/* Create bridge device */
+dev = qdev_create(NULL, TYPE_AP_BRIDGE);
+object_property_add_child(qdev_get_machine(), TYPE_AP_BRIDGE,
+ OBJECT(dev), NULL);
+qdev_init_nofail(dev);
+
+/* Create bus on bridge device */
+qbus_create(TYPE_VFIO_AP_BUS, dev, TYPE_VFIO_AP_BUS);
+ }
[...]
+type_init(ap_device_register)
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index f0f7fdc..3c100c2 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -32,6 +32,7 @@
#include "ipl.h"
#include "hw/s390x/s390-virtio-ccw.h"
#include "hw/s390x/css-bridge.h"
+#include "hw/s390x/ap-bridge.h"
#include "migration/register.h"
#include "cpu_models.h"
#include "hw/nmi.h"
@@ -263,6 +264,9 @@ static void ccw_init(MachineState *machine)
/* init the SIGP facility */
s390_init_sigp();
+/* create AP bridge and bus(es) */
+s390_init_ap();
+
Not sure since there is no explicit migration state involved here,
but I
think you cannot simply create the ap-bridge device always
unconditionally, can you? Did you check whether you can ping-pong
migrate an guest that runs on an older version of QEMU to a QEMU that
contains this patch and back? If it does not work, it might be
necessary
to restrict this ap-bridge device for new machine types only, I
think.
@Thomas
I re-checked the back and forth migration using libvirt's
save/restore. It
works, because as you said, there in no migration state. So there are no
missing/unexpected subsections in the migration stream/file.
We have already the cpu features at this point in time. So simply
doing
s390_has_feat(S390_FEAT_AP) in s390_init_ap ?
This seems like a reasonable solution to me. If the S390_FEAT_AP is
not switched
on for the guest, then the vfio-ap device will not be realized;
therefore,
there is no need for a bridge device. I will fix the s390_init_ap()
function such
that the AP bridge device is created only if the S390_FEAT_AP is
switched on for
the guest. That ought to solve the migration issue David pointed
out. Thanks
David.
With that said, creating the ap-bridge only under the condition
s390_has_feat(S390_FEAT_AP) could still be the better option. We
could probably drop the check in vfio_ap_realize(). I'm not sure, so
I'm fine either way.
I am including the check for S390_FEAT_AP in the s390_ap_init() function
as recommended by Christian, and removing it from the vfio_ap_realize()
function because if there is no AP bridge, QEMU terminates with an
error when trying to add a vfio-ap device.
Regards,
Halil
Re: [Qemu-devel] [qemu-s390x] [PATCH v8 4/6] s390x/ap: base Adjunct Processor (AP) object model
On 09/13/2018 01:02 PM, Tony Krowiak wrote:
On 09/13/2018 02:29 AM, Christian Borntraeger wrote:
On 09/13/2018 07:48 AM, Thomas Huth wrote:
On 2018-09-12 22:08, Tony Krowiak wrote:
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
---
[...]
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
new file mode 100644
index 000..4f10425
--- /dev/null
+++ b/hw/s390x/ap-bridge.c
[...]
+void s390_init_ap(void)
+{
+DeviceState *dev;
+
+/* Create bridge device */
+dev = qdev_create(NULL, TYPE_AP_BRIDGE);
+object_property_add_child(qdev_get_machine(), TYPE_AP_BRIDGE,
+ OBJECT(dev), NULL);
+qdev_init_nofail(dev);
+
+/* Create bus on bridge device */
+qbus_create(TYPE_VFIO_AP_BUS, dev, TYPE_VFIO_AP_BUS);
+ }
[...]
+type_init(ap_device_register)
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index f0f7fdc..3c100c2 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -32,6 +32,7 @@
#include "ipl.h"
#include "hw/s390x/s390-virtio-ccw.h"
#include "hw/s390x/css-bridge.h"
+#include "hw/s390x/ap-bridge.h"
#include "migration/register.h"
#include "cpu_models.h"
#include "hw/nmi.h"
@@ -263,6 +264,9 @@ static void ccw_init(MachineState *machine)
/* init the SIGP facility */
s390_init_sigp();
+/* create AP bridge and bus(es) */
+s390_init_ap();
+
Not sure since there is no explicit migration state involved here,
but I
think you cannot simply create the ap-bridge device always
unconditionally, can you? Did you check whether you can ping-pong
migrate an guest that runs on an older version of QEMU to a QEMU that
contains this patch and back? If it does not work, it might be
necessary
to restrict this ap-bridge device for new machine types only, I think.
We have already the cpu features at this point in time. So simply doing
s390_has_feat(S390_FEAT_AP) in s390_init_ap ?
This seems like a reasonable solution to me. If the S390_FEAT_AP is
not switched
on for the guest, then the vfio-ap device will not be realized;
therefore,
there is no need for a bridge device. I will fix the s390_init_ap()
function such
that the AP bridge device is created only if the S390_FEAT_AP is
switched on for
the guest. That ought to solve the migration issue David pointed out.
Thanks
David.
Sorry, meant Thomas.
Re: [Qemu-devel] [qemu-s390x] [PATCH v8 4/6] s390x/ap: base Adjunct Processor (AP) object model
On 09/13/2018 02:29 AM, Christian Borntraeger wrote:
On 09/13/2018 07:48 AM, Thomas Huth wrote:
On 2018-09-12 22:08, Tony Krowiak wrote:
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
---
[...]
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
new file mode 100644
index 000..4f10425
--- /dev/null
+++ b/hw/s390x/ap-bridge.c
[...]
+void s390_init_ap(void)
+{
+DeviceState *dev;
+
+/* Create bridge device */
+dev = qdev_create(NULL, TYPE_AP_BRIDGE);
+object_property_add_child(qdev_get_machine(), TYPE_AP_BRIDGE,
+ OBJECT(dev), NULL);
+qdev_init_nofail(dev);
+
+/* Create bus on bridge device */
+qbus_create(TYPE_VFIO_AP_BUS, dev, TYPE_VFIO_AP_BUS);
+ }
[...]
+type_init(ap_device_register)
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index f0f7fdc..3c100c2 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -32,6 +32,7 @@
#include "ipl.h"
#include "hw/s390x/s390-virtio-ccw.h"
#include "hw/s390x/css-bridge.h"
+#include "hw/s390x/ap-bridge.h"
#include "migration/register.h"
#include "cpu_models.h"
#include "hw/nmi.h"
@@ -263,6 +264,9 @@ static void ccw_init(MachineState *machine)
/* init the SIGP facility */
s390_init_sigp();
+/* create AP bridge and bus(es) */
+s390_init_ap();
+
Not sure since there is no explicit migration state involved here, but I
think you cannot simply create the ap-bridge device always
unconditionally, can you? Did you check whether you can ping-pong
migrate an guest that runs on an older version of QEMU to a QEMU that
contains this patch and back? If it does not work, it might be necessary
to restrict this ap-bridge device for new machine types only, I think.
We have already the cpu features at this point in time. So simply doing
s390_has_feat(S390_FEAT_AP) in s390_init_ap ?
This seems like a reasonable solution to me. If the S390_FEAT_AP is not
switched
on for the guest, then the vfio-ap device will not be realized; therefore,
there is no need for a bridge device. I will fix the s390_init_ap()
function such
that the AP bridge device is created only if the S390_FEAT_AP is
switched on for
the guest. That ought to solve the migration issue David pointed out.
Thanks
David.
Re: [Qemu-devel] [qemu-s390x] [PATCH v8 6/6] s390: doc: detailed specifications for AP virtualization
On 09/13/2018 02:06 AM, Thomas Huth wrote: On 2018-09-12 22:08, Tony Krowiak wrote: This patch provides documentation describing the AP architecture and design concepts behind the virtualization of AP devices. It also includes an example of how to configure AP devices for exclusive use of KVM guests. Signed-off-by: Tony Krowiak --- [...] + To assign an AP adapter to the mediated matrix device, its APID is written + to the 'assign_adapter' file. This may be done multiple times to assign more + than one adapter. The APID may be specified using conventional semantics + as a decimal, hexidecimal, or octal number. For example, to assign adapters + 4, 5 and 16 to a mediated matrix device in decimal, hexidecimal and octal s/hexidecimal/hexadecimal/g ? While some websites say that hexi is an alternative spelling, I have to admit that I've never seen it before, and there are also not that much hits on google when you search for it... [...] +* Live guest migration is not supported for guests using AP devices. \ No newline at end of file Please add a newline at the end of the file. Thomas Okay
Re: [Qemu-devel] [qemu-s390x] [PATCH v8 3/6] s390x/kvm: enable/disable AP instruction interpretation for guest
On 09/13/2018 01:28 AM, Thomas Huth wrote:
On 2018-09-12 22:08, Tony Krowiak wrote:
From: Tony Krowiak
Let's use the KVM_SET_DEVICE_ATTR ioctl to enable or disable
hardware interpretation of AP instructions executed on the guest.
If the S390_FEAT_AP feature is installed, AP instructions will
be interpreted by default; otherwise, they will be intercepted.
This attribute setting may be overridden by a device. For example,
a device may want to provide AP instructions to the guest (i.e.,
S390_FEAT_AP turned on), but it may want to emulate them. In this
case, the AP instructions executed on the guest must be
intercepted; so when the device is realized, it must disable
interpretation.
Signed-off-by: Tony Krowiak
---
target/s390x/kvm.c | 16
1 files changed, 16 insertions(+), 0 deletions(-)
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index c4bd84d..28a3900 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -2297,6 +2297,19 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model,
Error **errp)
S390_FEAT_MAX);
}
+static void kvm_s390_configure_apie(int interpret)
In case you respin, maybe use "bool interpret" instead of "int interpret"?
Thomas
Will do
+{
+uint64_t attr = KVM_S390_VM_CRYPTO_DISABLE_APIE;
+
+if (interpret) {
+attr = KVM_S390_VM_CRYPTO_ENABLE_APIE;
+}
+
+if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO, attr)) {
+kvm_s390_set_attr(attr);
+}
+}
+
void kvm_s390_apply_cpu_model(const S390CPUModel *model, Error **errp)
{
struct kvm_s390_vm_cpu_processor prop = {
@@ -2346,6 +2359,9 @@ void kvm_s390_apply_cpu_model(const S390CPUModel *model,
Error **errp)
if (test_bit(S390_FEAT_CMM, model->features)) {
kvm_s390_enable_cmma();
}
+
+/* configure hardware interpretation of AP instructions */
+kvm_s390_configure_apie(test_bit(S390_FEAT_AP, model->features));
}
void kvm_s390_restart_interrupt(S390CPU *cpu)
[Qemu-devel] [PATCH v8 6/6] s390: doc: detailed specifications for AP virtualization
This patch provides documentation describing the AP architecture and design concepts behind the virtualization of AP devices. It also includes an example of how to configure AP devices for exclusive use of KVM guests. Signed-off-by: Tony Krowiak --- MAINTAINERS |1 + docs/vfio-ap.txt | 785 ++ 2 files changed, 786 insertions(+), 0 deletions(-) create mode 100644 docs/vfio-ap.txt diff --git a/MAINTAINERS b/MAINTAINERS index 29041da..b64a120 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1210,6 +1210,7 @@ F: hw/s390x/ap-bridge.c F: include/hw/s390x/ap-device.h F: include/hw/s390x/ap-bridge.h F: hw/vfio/ap.c +F: docs/vfio-ap.txt L: qemu-s3...@nongnu.org vhost diff --git a/docs/vfio-ap.txt b/docs/vfio-ap.txt new file mode 100644 index 000..4953088 --- /dev/null +++ b/docs/vfio-ap.txt @@ -0,0 +1,785 @@ +Adjunct Processor (AP) Device += + +Contents: += +* Introduction +* AP Architectural Overview +* Start Interpretive Execution (SIE) Instruction +* AP Matrix Configuration on Linux Host +* Starting a Linux Guest Configured with an AP Matrix +* Example: Configure AP Matrices for Three Linux Guests + +Introduction: + +The IBM Adjunct Processor (AP) Cryptographic Facility is comprised +of three AP instructions and from 1 to 256 PCIe cryptographic adapter cards. +These AP devices provide cryptographic functions to all CPUs assigned to a +linux system running in an IBM Z system LPAR. + +On s390x, AP adapter cards are exposed via the AP bus. This document +describes how those cards may be made available to KVM guests using the +VFIO mediated device framework. + +AP Architectural Overview: += +In order understand the terminology used in the rest of this document, let's +start with some definitions: + +* AP adapter + + An AP adapter is an IBM Z adapter card that can perform cryptographic + functions. There can be from 0 to 256 adapters assigned to an LPAR. Adapters + assigned to the LPAR in which a linux host is running will be available to + the linux host. Each adapter is identified by a number from 0 to 255. When + installed, an AP adapter is accessed by AP instructions executed by any CPU. + +* AP domain + + An adapter is partitioned into domains. Each domain can be thought of as + a set of hardware registers for processing AP instructions. An adapter can + hold up to 256 domains. Each domain is identified by a number from 0 to 255. + Domains can be further classified into two types: + +* Usage domains are domains that can be accessed directly to process AP + commands + +* Control domains are domains that are accessed indirectly by AP + commands sent to a usage domain to control or change the domain; for + example, to set a secure private key for the domain. + +* AP Queue + + An AP queue is the means by which an AP command-request message is sent to an + AP usage domain inside a specific AP. An AP queue is identified by a tuple + comprised of an AP adapter ID (APID) and an AP queue index (APQI). The + APQI corresponds to a given usage domain number within the adapter. This tuple + forms an AP Queue Number (APQN) uniquely identifying an AP queue. AP + instructions include a field containing the APQN to identify the AP queue to + which the AP command-request message is to be sent for processing. + +* AP Instructions: + + There are three AP instructions: + + * NQAP: to enqueue an AP command-request message to a queue + * DQAP: to dequeue an AP command-reply message from a queue + * PQAP: to administer the queues + + AP instructions identify the domain that is targeted to process the AP + command; this must be one of the usage domains. An AP command may modify a + domain that is not one of the usage domains, but the modified domain + must be one of the control domains. + +Start Interpretive Execution (SIE) Instruction +== +A KVM guest is started by executing the Start Interpretive Execution (SIE) +instruction. The SIE state description is a control block that contains the +state information for a KVM guest and is supplied as input to the SIE +instruction. The SIE state description contains a satellite control block called +the Crypto Control Block (CRYCB). The CRYCB contains three fields to identify +the adapters, usage domains and control domains assigned to the KVM guest: + +* The AP Mask (APM) field is a bit mask that identifies the AP adapters assigned + to the KVM guest. Each bit in the mask, from most significant to least + significant bit, corresponds to an APID from 0-255. If a bit is set, the + corresponding adapter is valid for use by the KVM guest. + +* The AP Queue Mask (AQM) field is a bit mask identifying the AP usage domains + assigned to the KVM guest. Each bit in the mask, from most significant to + least significant bit, corresponds to an AP queue index
[Qemu-devel] [PATCH v8 5/6] s390x/vfio: ap: Introduce VFIO AP device
Introduces a VFIO based AP device. The device is defined via
the QEMU command line by specifying:
-device vfio-ap,sysfsdev=
There may be only one vfio-ap device configured for a guest.
The mediated matrix device is created by the VFIO AP device
driver by writing a UUID to a sysfs attribute file (see
docs/vfio-ap.txt). The mediated matrix device will be named
after the UUID. Symbolic links to the $uuid are created in
many places, so the path to the mediated matrix device $uuid
can be specified in any of the following ways:
/sys/devices/vfio_ap/matrix/$uuid
/sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid
/sys/bus/mdev/devices/$uuid
/sys/bus/mdev/drivers/vfio_mdev/$uuid
When the vfio-ap device is realized, it acquires and opens the
VFIO iommu group to which the mediated matrix device is
bound. This causes a VFIO group notification event to be
signaled. The vfio_ap device driver's group notification
handler will get called at which time the device driver
will configure the the AP devices to which the guest will
be granted access.
Signed-off-by: Tony Krowiak
---
MAINTAINERS |1 +
default-configs/s390x-softmmu.mak |1 +
hw/vfio/Makefile.objs |1 +
hw/vfio/ap.c | 186 +
include/hw/vfio/vfio-common.h |1 +
5 files changed, 190 insertions(+), 0 deletions(-)
create mode 100644 hw/vfio/ap.c
diff --git a/MAINTAINERS b/MAINTAINERS
index 97e8ed8..29041da 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1209,6 +1209,7 @@ F: hw/s390x/ap-device.c
F: hw/s390x/ap-bridge.c
F: include/hw/s390x/ap-device.h
F: include/hw/s390x/ap-bridge.h
+F: hw/vfio/ap.c
L: qemu-s3...@nongnu.org
vhost
diff --git a/default-configs/s390x-softmmu.mak
b/default-configs/s390x-softmmu.mak
index d6b67d5..5eef375 100644
--- a/default-configs/s390x-softmmu.mak
+++ b/default-configs/s390x-softmmu.mak
@@ -7,3 +7,4 @@ CONFIG_S390_FLIC=y
CONFIG_S390_FLIC_KVM=$(CONFIG_KVM)
CONFIG_VFIO_CCW=$(CONFIG_LINUX)
CONFIG_WDT_DIAG288=y
+CONFIG_VFIO_AP=$(CONFIG_LINUX)
diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs
index a2e7a0a..8b3f664 100644
--- a/hw/vfio/Makefile.objs
+++ b/hw/vfio/Makefile.objs
@@ -6,4 +6,5 @@ obj-$(CONFIG_SOFTMMU) += platform.o
obj-$(CONFIG_VFIO_XGMAC) += calxeda-xgmac.o
obj-$(CONFIG_VFIO_AMD_XGBE) += amd-xgbe.o
obj-$(CONFIG_SOFTMMU) += spapr.o
+obj-$(CONFIG_VFIO_AP) += ap.o
endif
diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c
new file mode 100644
index 000..ae8d79c
--- /dev/null
+++ b/hw/vfio/ap.c
@@ -0,0 +1,186 @@
+/*
+ * VFIO based AP matrix device assignment
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *Halil Pasic
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include
+#include
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "hw/vfio/vfio.h"
+#include "hw/vfio/vfio-common.h"
+#include "hw/s390x/ap-device.h"
+#include "qemu/error-report.h"
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "cpu.h"
+#include "kvm_s390x.h"
+#include "sysemu/sysemu.h"
+#include "hw/s390x/ap-bridge.h"
+#include "exec/address-spaces.h"
+
+#define VFIO_AP_DEVICE_TYPE "vfio-ap"
+
+typedef struct VFIOAPDevice {
+APDevice apdev;
+VFIODevice vdev;
+} VFIOAPDevice;
+
+static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
+{
+vdev->needs_reset = false;
+}
+
+/*
+ * We don't need vfio_hot_reset_multi and vfio_eoi operations for
+ * vfio-ap device now.
+ */
+struct VFIODeviceOps vfio_ap_ops = {
+.vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
+};
+
+static void vfio_ap_put_device(VFIOAPDevice *vapdev)
+{
+g_free(vapdev->vdev.name);
+vfio_put_base_device(>vdev);
+}
+
+static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
+{
+char *tmp, group_path[PATH_MAX];
+ssize_t len;
+int groupid;
+
+tmp = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
+len = readlink(tmp, group_path, sizeof(group_path));
+g_free(tmp);
+
+if (len <= 0 || len >= sizeof(group_path)) {
+error_setg(errp, "%s: no iommu_group found for %s",
+ VFIO_AP_DEVICE_TYPE, vapdev->vdev.sysfsdev);
+return NULL;
+}
+
+group_path[len] = 0;
+
+if (sscanf(basename(group_path), "%d", ) != 1) {
+error_setg(errp, "vfio: failed to read %s", group_path);
+return NULL;
+}
+
+return vfio_get_group(groupid, _space_memory, errp);
+}
+
+static void vfio_ap_realize(DeviceState *dev, Error **errp)
+{
+int ret;
+char *mdevid;
+
[Qemu-devel] [PATCH v8 2/6] s390x/cpumodel: Set up CPU model for AP device support
A new CPU model feature and two new CPU model facilities are
introduced to support AP devices for a KVM guest.
CPU model features:
1. The KVM_S390_VM_CPU_FEAT_AP CPU model feature indicates that
AP facilities are installed. This feature will be enabled by
the kernel only if the AP facilities are installed on the linux
host. This feature must be turned on from userspace to access
AP devices from the KVM guest. The QEMU command line to turn
this feature looks something like this:
qemu-system-s390x ... -cpu xxx,ap=on
This feature will be supported for zEC12 and newer CPU models.
The feature will not be supported for older models due to
testability issues.
CPU model facilities:
1. The S390_FEAT_AP_QUERY_CONFIG_INFO feature indicates the AP Query
Configuration Information (QCI) facility is installed. This feature
will be enabled by the kernel only if the QCI is installed on
the host.
2. The S390_FEAT_AP_FACILITY_TEST feature indicates that the AP
Facility Test (APFT) facility is installed. This feature will
be enabled by the kernel only if the APFT facility is installed
on the host.
Signed-off-by: Tony Krowiak
---
target/s390x/cpu_features.c |3 +++
target/s390x/cpu_features_def.h |3 +++
target/s390x/cpu_models.c |2 ++
target/s390x/gen-features.c |3 +++
target/s390x/kvm.c |1 +
5 files changed, 12 insertions(+), 0 deletions(-)
diff --git a/target/s390x/cpu_features.c b/target/s390x/cpu_features.c
index 172fb18..60cfeba 100644
--- a/target/s390x/cpu_features.c
+++ b/target/s390x/cpu_features.c
@@ -39,8 +39,10 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT("srs", S390_FEAT_TYPE_STFL, 9, "Sense-running-status facility"),
FEAT_INIT("csske", S390_FEAT_TYPE_STFL, 10, "Conditional-SSKE facility"),
FEAT_INIT("ctop", S390_FEAT_TYPE_STFL, 11, "Configuration-topology
facility"),
+FEAT_INIT("apqci", S390_FEAT_TYPE_STFL, 12, "Query AP Configuration
Information facility"),
FEAT_INIT("ipter", S390_FEAT_TYPE_STFL, 13, "IPTE-range facility"),
FEAT_INIT("nonqks", S390_FEAT_TYPE_STFL, 14, "Nonquiescing key-setting
facility"),
+FEAT_INIT("apft", S390_FEAT_TYPE_STFL, 15, "AP Facilities Test facility"),
FEAT_INIT("etf2", S390_FEAT_TYPE_STFL, 16, "Extended-translation facility
2"),
FEAT_INIT("msa-base", S390_FEAT_TYPE_STFL, 17, "Message-security-assist
facility (excluding subfunctions)"),
FEAT_INIT("ldisp", S390_FEAT_TYPE_STFL, 18, "Long-displacement facility"),
@@ -129,6 +131,7 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT_MISC("dateh2", "DAT-enhancement facility 2"),
FEAT_INIT_MISC("cmm", "Collaborative-memory-management facility"),
+FEAT_INIT_MISC("ap", "AP instructions installed"),
FEAT_INIT("plo-cl", S390_FEAT_TYPE_PLO, 0, "PLO Compare and load (32 bit
in general registers)"),
FEAT_INIT("plo-clg", S390_FEAT_TYPE_PLO, 1, "PLO Compare and load (64 bit
in parameter list)"),
diff --git a/target/s390x/cpu_features_def.h b/target/s390x/cpu_features_def.h
index ac2c947..5fc7e7b 100644
--- a/target/s390x/cpu_features_def.h
+++ b/target/s390x/cpu_features_def.h
@@ -27,8 +27,10 @@ typedef enum {
S390_FEAT_SENSE_RUNNING_STATUS,
S390_FEAT_CONDITIONAL_SSKE,
S390_FEAT_CONFIGURATION_TOPOLOGY,
+S390_FEAT_AP_QUERY_CONFIG_INFO,
S390_FEAT_IPTE_RANGE,
S390_FEAT_NONQ_KEY_SETTING,
+S390_FEAT_AP_FACILITIES_TEST,
S390_FEAT_EXTENDED_TRANSLATION_2,
S390_FEAT_MSA,
S390_FEAT_LONG_DISPLACEMENT,
@@ -119,6 +121,7 @@ typedef enum {
/* Misc */
S390_FEAT_DAT_ENH_2,
S390_FEAT_CMM,
+S390_FEAT_AP,
/* PLO */
S390_FEAT_PLO_CL,
diff --git a/target/s390x/cpu_models.c b/target/s390x/cpu_models.c
index 265d25c..7c253ff 100644
--- a/target/s390x/cpu_models.c
+++ b/target/s390x/cpu_models.c
@@ -786,6 +786,8 @@ static void check_consistency(const S390CPUModel *model)
{ S390_FEAT_PRNO_TRNG_QRTCR, S390_FEAT_MSA_EXT_5 },
{ S390_FEAT_PRNO_TRNG, S390_FEAT_MSA_EXT_5 },
{ S390_FEAT_SIE_KSS, S390_FEAT_SIE_F2 },
+{ S390_FEAT_AP_QUERY_CONFIG_INFO, S390_FEAT_AP },
+{ S390_FEAT_AP_FACILITIES_TEST, S390_FEAT_AP },
};
int i;
diff --git a/target/s390x/gen-features.c b/target/s390x/gen-features.c
index 384b61c..70015ea 100644
--- a/target/s390x/gen-features.c
+++ b/target/s390x/gen-features.c
@@ -447,6 +447,9 @@ static uint16_t full_GEN12_GA1[] = {
S390_FEAT_ADAPTER_INT_SUPPRESSION,
S390_FEAT_EDAT_2,
S390_FEAT_SIDE_EFFECT_ACCESS_ESOP2,
+S390_FEAT_AP_QUERY_CONFIG_INFO,
+S390_FEAT_AP_FACILITIES_TEST,
+
[Qemu-devel] [PATCH v8 4/6] s390x/ap: base Adjunct Processor (AP) object model
From: Tony Krowiak
Introduces the base object model for virtualizing AP devices.
Signed-off-by: Tony Krowiak
---
MAINTAINERS | 12 +++
hw/s390x/Makefile.objs |2 +
hw/s390x/ap-bridge.c | 76 ++
hw/s390x/ap-device.c | 39 +
hw/s390x/s390-virtio-ccw.c |4 ++
include/hw/s390x/ap-bridge.h | 37
include/hw/s390x/ap-device.h | 38 +
7 files changed, 208 insertions(+), 0 deletions(-)
create mode 100644 hw/s390x/ap-bridge.c
create mode 100644 hw/s390x/ap-device.c
create mode 100644 include/hw/s390x/ap-bridge.h
create mode 100644 include/hw/s390x/ap-device.h
diff --git a/MAINTAINERS b/MAINTAINERS
index d12518c..97e8ed8 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1199,6 +1199,18 @@ F: include/hw/s390x/s390-ccw.h
T: git git://github.com/cohuck/qemu.git s390-next
L: qemu-s3...@nongnu.org
+vfio-ap
+M: Christian Borntraeger
+M: Tony Krowiak
+M: Halil Pasic
+M: Pierre Morel
+S: Supported
+F: hw/s390x/ap-device.c
+F: hw/s390x/ap-bridge.c
+F: include/hw/s390x/ap-device.h
+F: include/hw/s390x/ap-bridge.h
+L: qemu-s3...@nongnu.org
+
vhost
M: Michael S. Tsirkin
S: Supported
diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index 93282f7..add89b1 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -20,3 +20,5 @@ obj-$(CONFIG_TCG) += tod-qemu.o
obj-$(CONFIG_KVM) += s390-skeys-kvm.o
obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
obj-y += s390-ccw.o
+obj-y += ap-device.o
+obj-y += ap-bridge.o
diff --git a/hw/s390x/ap-bridge.c b/hw/s390x/ap-bridge.c
new file mode 100644
index 000..4f10425
--- /dev/null
+++ b/hw/s390x/ap-bridge.c
@@ -0,0 +1,76 @@
+/*
+ * ap bridge
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Halil Pasic
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "qemu/bitops.h"
+#include "hw/s390x/ap-bridge.h"
+#include "cpu.h"
+
+static char *vfio_ap_bus_get_dev_path(DeviceState *dev)
+{
+/* at most one */
+return g_strdup_printf("/1");
+}
+
+static void vfio_ap_bus_class_init(ObjectClass *klass, void *data)
+{
+BusClass *k = BUS_CLASS(klass);
+
+k->get_dev_path = vfio_ap_bus_get_dev_path;
+/* More than one vfio-ap device does not make sense */
+k->max_dev = 1;
+}
+
+static const TypeInfo vfio_ap_bus_info = {
+.name = TYPE_VFIO_AP_BUS,
+.parent = TYPE_BUS,
+.instance_size = sizeof(VFIOAPBus),
+.class_init = vfio_ap_bus_class_init,
+};
+
+void s390_init_ap(void)
+{
+DeviceState *dev;
+
+/* Create bridge device */
+dev = qdev_create(NULL, TYPE_AP_BRIDGE);
+object_property_add_child(qdev_get_machine(), TYPE_AP_BRIDGE,
+ OBJECT(dev), NULL);
+qdev_init_nofail(dev);
+
+/* Create bus on bridge device */
+qbus_create(TYPE_VFIO_AP_BUS, dev, TYPE_VFIO_AP_BUS);
+ }
+
+
+
+static void ap_bridge_class_init(ObjectClass *klass, void *data)
+{
+DeviceClass *dc = DEVICE_CLASS(klass);
+
+set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+}
+
+static const TypeInfo ap_bridge_info = {
+.name = TYPE_AP_BRIDGE,
+.parent= TYPE_SYS_BUS_DEVICE,
+.instance_size = sizeof(APBridge),
+.class_init= ap_bridge_class_init,
+};
+
+static void ap_register(void)
+{
+type_register_static(_bridge_info);
+type_register_static(_ap_bus_info);
+}
+
+type_init(ap_register)
diff --git a/hw/s390x/ap-device.c b/hw/s390x/ap-device.c
new file mode 100644
index 000..3cd4bae
--- /dev/null
+++ b/hw/s390x/ap-device.c
@@ -0,0 +1,39 @@
+/*
+ * Adjunct Processor (AP) matrix device
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu/module.h"
+#include "qapi/error.h"
+#include "hw/qdev.h"
+#include "hw/s390x/ap-device.h"
+
+static void ap_class_init(ObjectClass *klass, void *data)
+{
+DeviceClass *dc = DEVICE_CLASS(klass);
+
+dc->desc = "AP device class";
+dc->hotpluggable = false;
+}
+
+static const TypeInfo ap_device_info = {
+.name = AP_DEVICE_TYPE,
+.parent = TYPE_DEVICE,
+.instance_size = sizeof(APDevice),
+.class_size = sizeof(APDeviceClass),
+.class_init = ap_class_init,
+.abstract = true,
+};
+
+static void ap_device_register(void)
+{
+type_register_static(_device_info);
+}
+
+type_init(ap_device_register)
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index f0f7fdc..3c100c2 1
[Qemu-devel] [PATCH v8 1/6] linux-headers: linux header updates for AP support
Updates the linux header files in preparation for introduction
of the VFIO AP device:
* Added a feature ID to indicate AP facilities are installed
* Added device attributes to the KVM_S390_VM_CRYPTO group
to indicate whether AP instructions are to be interpreted
* Added VFIO device information for AP devices
Signed-off-by: Tony Krowiak
---
linux-headers/asm-s390/kvm.h |3 +++
linux-headers/linux/vfio.h |2 ++
2 files changed, 5 insertions(+), 0 deletions(-)
diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index 1ab9901..b494870 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -130,6 +130,7 @@ struct kvm_s390_vm_cpu_machine {
#define KVM_S390_VM_CPU_FEAT_PFMFI 11
#define KVM_S390_VM_CPU_FEAT_SIGPIF12
#define KVM_S390_VM_CPU_FEAT_KSS 13
+#define KVM_S390_VM_CPU_FEAT_AP14
struct kvm_s390_vm_cpu_feat {
__u64 feat[16];
};
@@ -160,6 +161,8 @@ struct kvm_s390_vm_cpu_subfunc {
#define KVM_S390_VM_CRYPTO_ENABLE_DEA_KW 1
#define KVM_S390_VM_CRYPTO_DISABLE_AES_KW 2
#define KVM_S390_VM_CRYPTO_DISABLE_DEA_KW 3
+#define KVM_S390_VM_CRYPTO_ENABLE_APIE 4
+#define KVM_S390_VM_CRYPTO_DISABLE_APIE5
/* kvm attributes for migration mode */
#define KVM_S390_VM_MIGRATION_STOP 0
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 3615a26..838919a 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -200,6 +200,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2)/* vfio-platform device */
#define VFIO_DEVICE_FLAGS_AMBA (1 << 3) /* vfio-amba device */
#define VFIO_DEVICE_FLAGS_CCW (1 << 4)/* vfio-ccw device */
+#define VFIO_DEVICE_FLAGS_AP (1 << 5) /* vfio-ap device */
__u32 num_regions;/* Max region index + 1 */
__u32 num_irqs; /* Max IRQ index + 1 */
};
@@ -215,6 +216,7 @@ struct vfio_device_info {
#define VFIO_DEVICE_API_PLATFORM_STRING"vfio-platform"
#define VFIO_DEVICE_API_AMBA_STRING"vfio-amba"
#define VFIO_DEVICE_API_CCW_STRING "vfio-ccw"
+#define VFIO_DEVICE_API_AP_STRING "vfio-ap"
/**
* VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
--
1.7.1
