On Thu, 22 Oct 2020 16:42:07 +0530
Kirti Wankhede <kwankh...@nvidia.com> wrote:
> Added amount of bytes transferred to the VM at destination by all VFIO
> devices
>
> Signed-off-by: Kirti Wankhede <kwankh...@nvidia.com>
> Reviewed-by: Dr. David Alan Gilbert <dgilb...@redhat.com>
> ---
> hw/vfio/common.c | 20 ++++++++++++++++++++
> hw/vfio/migration.c | 10 ++++++++++
> include/qemu/vfio-helpers.h | 3 +++
> migration/migration.c | 14 ++++++++++++++
> monitor/hmp-cmds.c | 6 ++++++
> qapi/migration.json | 17 +++++++++++++++++
> 6 files changed, 70 insertions(+)
>
> diff --git a/hw/vfio/common.c b/hw/vfio/common.c
> index 9c879e5c0f62..8d0758eda9fa 100644
> --- a/hw/vfio/common.c
> +++ b/hw/vfio/common.c
> @@ -39,6 +39,7 @@
> #include "trace.h"
> #include "qapi/error.h"
> #include "migration/migration.h"
> +#include "qemu/vfio-helpers.h"
>
> VFIOGroupList vfio_group_list =
> QLIST_HEAD_INITIALIZER(vfio_group_list);
> @@ -292,6 +293,25 @@ const MemoryRegionOps vfio_region_ops = {
> * Device state interfaces
> */
>
> +bool vfio_mig_active(void)
> +{
> + VFIOGroup *group;
> + VFIODevice *vbasedev;
> +
> + if (QLIST_EMPTY(&vfio_group_list)) {
> + return false;
> + }
> +
> + QLIST_FOREACH(group, &vfio_group_list, next) {
> + QLIST_FOREACH(vbasedev, &group->device_list, next) {
> + if (vbasedev->migration_blocker) {
> + return false;
> + }
> + }
> + }
> + return true;
> +}
> +
> static bool vfio_devices_all_stopped_and_saving(VFIOContainer *container)
> {
> VFIOGroup *group;
> diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
> index 77ee60a43ea5..b23e21c6de2b 100644
> --- a/hw/vfio/migration.c
> +++ b/hw/vfio/migration.c
> @@ -28,6 +28,7 @@
> #include "pci.h"
> #include "trace.h"
> #include "hw/hw.h"
> +#include "qemu/vfio-helpers.h"
>
> /*
> * Flags to be used as unique delimiters for VFIO devices in the migration
> @@ -45,6 +46,8 @@
> #define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL)
> #define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL)
>
> +static int64_t bytes_transferred;
> +
> static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int count,
> off_t off, bool iswrite)
> {
> @@ -255,6 +258,7 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice
> *vbasedev, uint64_t *size)
> *size = data_size;
> }
>
> + bytes_transferred += data_size;
> return ret;
> }
>
> @@ -776,6 +780,7 @@ static void vfio_migration_state_notifier(Notifier
> *notifier, void *data)
> case MIGRATION_STATUS_CANCELLING:
> case MIGRATION_STATUS_CANCELLED:
> case MIGRATION_STATUS_FAILED:
> + bytes_transferred = 0;
> ret = vfio_migration_set_state(vbasedev,
> ~(VFIO_DEVICE_STATE_SAVING |
> VFIO_DEVICE_STATE_RESUMING),
> VFIO_DEVICE_STATE_RUNNING);
> @@ -862,6 +867,11 @@ err:
>
> /* ---------------------------------------------------------------------- */
>
> +int64_t vfio_mig_bytes_transferred(void)
> +{
> + return bytes_transferred;
> +}
> +
> int vfio_migration_probe(VFIODevice *vbasedev, Error **errp)
> {
> VFIOContainer *container = vbasedev->group->container;
> diff --git a/include/qemu/vfio-helpers.h b/include/qemu/vfio-helpers.h
> index 4491c8e1a6e9..7f7a46e6ef2d 100644
> --- a/include/qemu/vfio-helpers.h
> +++ b/include/qemu/vfio-helpers.h
> @@ -29,4 +29,7 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index,
> void *bar,
> int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e,
> int irq_type, Error **errp);
>
> +bool vfio_mig_active(void);
> +int64_t vfio_mig_bytes_transferred(void);
> +
> #endif
I don't think vfio-helpers is the right place for this, this header is
specifically for using util/vfio-helpers.c. Would
include/hw/vfio/vfio-common.h work?
> diff --git a/migration/migration.c b/migration/migration.c
> index 0575ecb37953..8b2865d25ef4 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -56,6 +56,7 @@
> #include "net/announce.h"
> #include "qemu/queue.h"
> #include "multifd.h"
> +#include "qemu/vfio-helpers.h"
>
> #define MAX_THROTTLE (128 << 20) /* Migration transfer speed
> throttling */
>
> @@ -1002,6 +1003,17 @@ static void populate_disk_info(MigrationInfo *info)
> }
> }
>
> +static void populate_vfio_info(MigrationInfo *info)
> +{
> +#ifdef CONFIG_LINUX
Use CONFIG_VFIO? I get a build failure on qemu-system-avr
/usr/bin/ld: /tmp/tmp.3QbqxgbENl/build/../migration/migration.c:1012:
undefined reference to `vfio_mig_bytes_transferred'. Thanks,
Alex
> + if (vfio_mig_active()) {
> + info->has_vfio = true;
> + info->vfio = g_malloc0(sizeof(*info->vfio));
> + info->vfio->transferred = vfio_mig_bytes_transferred();
> + }
> +#endif
> +}
> +
> static void fill_source_migration_info(MigrationInfo *info)
> {
> MigrationState *s = migrate_get_current();
> @@ -1026,6 +1038,7 @@ static void fill_source_migration_info(MigrationInfo
> *info)
> populate_time_info(info, s);
> populate_ram_info(info, s);
> populate_disk_info(info);
> + populate_vfio_info(info);
> break;
> case MIGRATION_STATUS_COLO:
> info->has_status = true;
> @@ -1034,6 +1047,7 @@ static void fill_source_migration_info(MigrationInfo
> *info)
> case MIGRATION_STATUS_COMPLETED:
> populate_time_info(info, s);
> populate_ram_info(info, s);
> + populate_vfio_info(info);
> break;
> case MIGRATION_STATUS_FAILED:
> info->has_status = true;
> diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
> index 9789f4277f50..56e9bad33d94 100644
> --- a/monitor/hmp-cmds.c
> +++ b/monitor/hmp-cmds.c
> @@ -357,6 +357,12 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict)
> }
> monitor_printf(mon, "]\n");
> }
> +
> + if (info->has_vfio) {
> + monitor_printf(mon, "vfio device transferred: %" PRIu64 " kbytes\n",
> + info->vfio->transferred >> 10);
> + }
> +
> qapi_free_MigrationInfo(info);
> }
>
> diff --git a/qapi/migration.json b/qapi/migration.json
> index a5da513c9e05..3c7582052725 100644
> --- a/qapi/migration.json
> +++ b/qapi/migration.json
> @@ -147,6 +147,18 @@
> 'active', 'postcopy-active', 'postcopy-paused',
> 'postcopy-recover', 'completed', 'failed', 'colo',
> 'pre-switchover', 'device', 'wait-unplug' ] }
> +##
> +# @VfioStats:
> +#
> +# Detailed VFIO devices migration statistics
> +#
> +# @transferred: amount of bytes transferred to the target VM by VFIO devices
> +#
> +# Since: 5.2
> +#
> +##
> +{ 'struct': 'VfioStats',
> + 'data': {'transferred': 'int' } }
>
> ##
> # @MigrationInfo:
> @@ -208,11 +220,16 @@
> #
> # @socket-address: Only used for tcp, to know what the real port is (Since
> 4.0)
> #
> +# @vfio: @VfioStats containing detailed VFIO devices migration statistics,
> +# only returned if VFIO device is present, migration is supported by
> all
> +# VFIO devices and status is 'active' or 'completed' (since 5.2)
> +#
> # Since: 0.14.0
> ##
> { 'struct': 'MigrationInfo',
> 'data': {'*status': 'MigrationStatus', '*ram': 'MigrationStats',
> '*disk': 'MigrationStats',
> + '*vfio': 'VfioStats',
> '*xbzrle-cache': 'XBZRLECacheStats',
> '*total-time': 'int',
> '*expected-downtime': 'int',
