Re: [PATCH v4 6/9] migration: wire up support for snapshot device selection

2020-09-21 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> Modify load_snapshot/save_snapshot to accept the device list and vmstate
> node name parameters previously added to the block layer.
> 
> Signed-off-by: Daniel P. Berrangé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  include/migration/snapshot.h | 12 ++--
>  migration/savevm.c   | 24 ++--
>  monitor/hmp-cmds.c   |  4 ++--
>  replay/replay-snapshot.c |  4 ++--
>  softmmu/vl.c |  2 +-
>  5 files changed, 29 insertions(+), 17 deletions(-)
> 
> diff --git a/include/migration/snapshot.h b/include/migration/snapshot.h
> index d7db1174ef..b2c72e0a1b 100644
> --- a/include/migration/snapshot.h
> +++ b/include/migration/snapshot.h
> @@ -15,7 +15,15 @@
>  #ifndef QEMU_MIGRATION_SNAPSHOT_H
>  #define QEMU_MIGRATION_SNAPSHOT_H
>  
> -int save_snapshot(const char *name, bool overwrite, Error **errp);
> -int load_snapshot(const char *name, Error **errp);
> +#include "qapi/qapi-builtin-types.h"
> +
> +int save_snapshot(const char *name, bool overwrite,
> +  const char *vmstate,
> +  bool has_devices, strList *devices,
> +  Error **errp);
> +int load_snapshot(const char *name,
> +  const char *vmstate,
> +  bool has_devices, strList *devices,
> +  Error **errp);
>  
>  #endif
> diff --git a/migration/savevm.c b/migration/savevm.c
> index 2025e3e579..b3d2ce7045 100644
> --- a/migration/savevm.c
> +++ b/migration/savevm.c
> @@ -43,6 +43,8 @@
>  #include "qapi/error.h"
>  #include "qapi/qapi-commands-migration.h"
>  #include "qapi/qapi-commands-misc.h"
> +#include "qapi/clone-visitor.h"
> +#include "qapi/qapi-builtin-visit.h"
>  #include "qapi/qmp/qerror.h"
>  #include "qemu/error-report.h"
>  #include "sysemu/cpus.h"
> @@ -2658,7 +2660,8 @@ int qemu_load_device_state(QEMUFile *f)
>  return 0;
>  }
>  
> -int save_snapshot(const char *name, bool overwrite, Error **errp)
> +int save_snapshot(const char *name, bool overwrite, const char *vmstate,
> +  bool has_devices, strList *devices, Error **errp)
>  {
>  BlockDriverState *bs;
>  QEMUSnapshotInfo sn1, *sn = , old_sn1, *old_sn = _sn1;
> @@ -2680,18 +2683,18 @@ int save_snapshot(const char *name, bool overwrite, 
> Error **errp)
>  return ret;
>  }
>  
> -if (!bdrv_all_can_snapshot(false, NULL, errp)) {
> +if (!bdrv_all_can_snapshot(has_devices, devices, errp)) {
>  return ret;
>  }
>  
>  /* Delete old snapshots of the same name */
>  if (name && overwrite) {
> -if (bdrv_all_delete_snapshot(name, false, NULL, errp) < 0) {
> +if (bdrv_all_delete_snapshot(name, has_devices, devices, errp) < 0) {
>  return ret;
>  }
>  }
>  
> -bs = bdrv_all_find_vmstate_bs(NULL, false, NULL, errp);
> +bs = bdrv_all_find_vmstate_bs(vmstate, has_devices, devices, errp);
>  if (bs == NULL) {
>  return ret;
>  }
> @@ -2757,7 +2760,7 @@ int save_snapshot(const char *name, bool overwrite, 
> Error **errp)
>  aio_context_release(aio_context);
>  aio_context = NULL;
>  
> -ret = bdrv_all_create_snapshot(sn, bs, vm_state_size, false, NULL, errp);
> +ret = bdrv_all_create_snapshot(sn, bs, vm_state_size, has_devices, 
> devices, errp);
>  if (ret < 0) {
>  goto the_end;
>  }
> @@ -2858,7 +2861,8 @@ void qmp_xen_load_devices_state(const char *filename, 
> Error **errp)
>  migration_incoming_state_destroy();
>  }
>  
> -int load_snapshot(const char *name, Error **errp)
> +int load_snapshot(const char *name, const char *vmstate,
> +  bool has_devices, strList *devices, Error **errp)
>  {
>  BlockDriverState *bs_vm_state;
>  QEMUSnapshotInfo sn;
> @@ -2873,15 +2877,15 @@ int load_snapshot(const char *name, Error **errp)
>  return -1;
>  }
>  
> -if (!bdrv_all_can_snapshot(false, NULL, errp)) {
> +if (!bdrv_all_can_snapshot(has_devices, devices, errp)) {
>  return -1;
>  }
> -ret = bdrv_all_find_snapshot(name, false, NULL, errp);
> +ret = bdrv_all_find_snapshot(name, has_devices, devices, errp);
>  if (ret < 0) {
>  return -1;
>  }
>  
> -bs_vm_state = bdrv_all_find_vmstate_bs(NULL, false, NULL, errp);
> +bs_vm_state = bdrv_all_find_vmstate_bs(vmstate, has_devices, devices, 
> errp);
>  if (!bs_vm_state) {
>  return -1;
>  }
> @@ -2902,7 +2906,7 @@ int 

Re: [RFC PATCH 3/6] hw/sd/sdcard: Do not use legal address '0' for INVALID_ADDRESS

2020-09-21 Thread Dr. David Alan Gilbert
* Markus Armbruster (arm...@redhat.com) wrote:
> Philippe Mathieu-Daudé  writes:
> 
> > +Paolo & Kevin.
> >
> > On 9/21/20 10:40 AM, Markus Armbruster wrote:
> >> Philippe Mathieu-Daudé  writes:
> >> 
> >>> As it is legal to WRITE/ERASE the address/block 0,
> >>> change the value of this definition to an illegal
> >>> address: UINT32_MAX.
> >>>
> >>> Signed-off-by: Philippe Mathieu-Daudé 
> >>> ---
> >>> Cc: Dr. David Alan Gilbert 
> >>> Cc: Markus Armbruster 
> >>>
> >>> Same problem I had with the pflash device last year...
> >>> This break migration :(
> >>> What is the best way to do this?
> >> 
> >> Remind me: did we solve the problem with pflash, and if yes, how?
> >
> > No we can't. The best I could do is add a comment and as this
> > is not fixable. See commit aba53a12bd5: ("hw/block/pflash_cfi01:
> > Document use of non-CFI compliant command '0x00'").
> >
> > I now consider the device in maintenance-only
> > mode and won't add any new features.
> >
> > I started working on a new implementation, hoping it can be a
> > drop in replacement. Laszlo still has hope that QEMU pflash
> > device will support sector locking so firmware developers could
> > test upgrading fw in VMs.
> >
> > Back to the SDcard, it might be less critical, so a migration
> > breaking change might be acceptable. I'm only aware of Paolo
> > and Kevin using this device for testing. Not sure of its
> > importance in production.
> 
> Neither am I.
> 
> Which machine types include this device by default?

To me it looks like it's some of the ARM boards.

Dave

> How can a non-default device be added, and to which machine types?
> 
> I gather the fix changes device state incompatibly.  Always, or only in
> certain states?  I'm asking because if device state remains compatible
> most of the time, we might be able use subsection trickery to keep
> migration working most of the time.  Has been done before, I think.
-- 
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v7 10/13] hmp: Add support for coroutine command handlers

2020-09-16 Thread Dr. David Alan Gilbert
* Kevin Wolf (kw...@redhat.com) wrote:
> Often, QMP command handlers are not only called to handle QMP commands,
> but also from a corresponding HMP command handler. In order to give them
> a consistent environment, optionally run HMP command handlers in a
> coroutine, too.
> 
> The implementation is a lot simpler than in QMP because for HMP, we
> still block the VM while the coroutine is running.
> 
> Signed-off-by: Kevin Wolf 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  monitor/monitor-internal.h |  1 +
>  monitor/hmp.c  | 37 -
>  2 files changed, 33 insertions(+), 5 deletions(-)
> 
> diff --git a/monitor/monitor-internal.h b/monitor/monitor-internal.h
> index b55d6df07f..ad2e64be13 100644
> --- a/monitor/monitor-internal.h
> +++ b/monitor/monitor-internal.h
> @@ -74,6 +74,7 @@ typedef struct HMPCommand {
>  const char *help;
>  const char *flags; /* p=preconfig */
>  void (*cmd)(Monitor *mon, const QDict *qdict);
> +bool coroutine;
>  /*
>   * @sub_table is a list of 2nd level of commands. If it does not exist,
>   * cmd should be used. If it exists, sub_table[?].cmd should be
> diff --git a/monitor/hmp.c b/monitor/hmp.c
> index 4b66ca1cd6..b858b0dbde 100644
> --- a/monitor/hmp.c
> +++ b/monitor/hmp.c
> @@ -1056,12 +1056,26 @@ fail:
>  return NULL;
>  }
>  
> +typedef struct HandleHmpCommandCo {
> +Monitor *mon;
> +const HMPCommand *cmd;
> +QDict *qdict;
> +bool done;
> +} HandleHmpCommandCo;
> +
> +static void handle_hmp_command_co(void *opaque)
> +{
> +HandleHmpCommandCo *data = opaque;
> +data->cmd->cmd(data->mon, data->qdict);
> +monitor_set_cur(qemu_coroutine_self(), NULL);
> +data->done = true;
> +}
> +
>  void handle_hmp_command(MonitorHMP *mon, const char *cmdline)
>  {
>  QDict *qdict;
>  const HMPCommand *cmd;
>  const char *cmd_start = cmdline;
> -Monitor *old_mon;
>  
>  trace_handle_hmp_command(mon, cmdline);
>  
> @@ -1080,10 +1094,23 @@ void handle_hmp_command(MonitorHMP *mon, const char 
> *cmdline)
>  return;
>  }
>  
> -/* old_mon is non-NULL when called from qmp_human_monitor_command() */
> -old_mon = monitor_set_cur(qemu_coroutine_self(), >common);
> -cmd->cmd(>common, qdict);
> -monitor_set_cur(qemu_coroutine_self(), old_mon);
> +if (!cmd->coroutine) {
> +/* old_mon is non-NULL when called from qmp_human_monitor_command() 
> */
> +Monitor *old_mon = monitor_set_cur(qemu_coroutine_self(), 
> >common);
> +cmd->cmd(>common, qdict);
> +monitor_set_cur(qemu_coroutine_self(), old_mon);
> +} else {
> +HandleHmpCommandCo data = {
> +.mon = >common,
> +.cmd = cmd,
> +.qdict = qdict,
> +.done = false,
> +};
> +Coroutine *co = qemu_coroutine_create(handle_hmp_command_co, );
> +monitor_set_cur(co, >common);
> +aio_co_enter(qemu_get_aio_context(), co);
> +AIO_WAIT_WHILE(qemu_get_aio_context(), !data.done);
> +}
>  
>  qobject_unref(qdict);
>  }
> -- 
> 2.25.4
> 
-- 
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 0/7] migration: bring improved savevm/loadvm/delvm to QMP

2020-09-11 Thread Dr. David Alan Gilbert
ough to have these new commands.
> 
> The main limitations of this current impl
> 
>  - The snapshot process runs serialized in the main thread. ie QEMU
>guest execution is blocked for the duration. The job framework
>lets us fix this in future without changing the QMP semantics
>exposed to the apps.
> 
>  - Most vmstate loading errors just go to stderr, as they are not
>using Error **errp reporting. Thus the job framework just
>reports a fairly generic message
> 
>  "Error -22 while loading VM state"
> 
>Again this can be fixed later without changing the QMP semantics
>exposed to apps.
> 
> I've done some minimal work in libvirt to start to make use of the new
> commands to validate their functionality, but this isn't finished yet.
> 
> My ultimate goal is to make the GNOME Boxes maintainer happy again by
> having internal snapshots work with OVMF:
> 
>   https://gitlab.gnome.org/GNOME/gnome-boxes/-/commit/c486da262f6566326fbcb5e=
> f45c5f64048f16a6e
> 
> Changed in v3:
> 
>  - Schedule a bottom half to escape from coroutine context in
>the jobs. This is needed because the locking in the snapshot
>code goes horribly wrong when run from a background coroutine
>instead of the main event thread.
> 
>  - Re-factor way we iterate over devices, so that we correctly
>report non-existant devices passed by the user over QMP.
> 
>  - Add QAPI docs notes about limitations wrt vmstate error
>reporting (it all goes to stderr not an Error **errp)
>so QMP only gets a fairly generic error message currently.
> 
>  - Add I/O test to validate many usage scenarios / errors
> 
>  - Add I/O test helpers to handle QMP events with a deterministic
>ordering
> 
>  - Ensure 'delete-snapshot' reports an error if requesting
>delete from devices that don't support snapshot, instead of
>silently succeeding with no erro.
> 
> Changed in v2:
> 
>  - Use new command names "snapshot-{load,save,delete}" to make it
>clear that these are different from the "savevm|loadvm|delvm"
>as they use the Job framework
> 
>  - Use an include list for block devs, not an exclude list
> 
> Daniel P. Berrang=C3=A9 (7):
>   migration: improve error reporting of block driver state name
>   block: push error reporting into bdrv_all_*_snapshot functions
>   migration: stop returning errno from load_snapshot()
>   block: add ability to specify list of blockdevs during snapshot
>   block: allow specifying name of block device for vmstate storage
>   iotests: add support for capturing and matching QMP events
>   migration: introduce snapshot-{save,load,delete} QMP commands
> 
>  block/monitor/block-hmp-cmds.c |   7 +-
>  block/snapshot.c   | 233 ++---
>  include/block/snapshot.h   |  19 +-
>  include/migration/snapshot.h   |  10 +-
>  migration/savevm.c | 258 +++
>  monitor/hmp-cmds.c |  11 +-
>  qapi/job.json  |   9 +-
>  qapi/migration.json| 135 
>  replay/replay-snapshot.c   |   4 +-
>  softmmu/vl.c   |   2 +-
>  tests/qemu-iotests/267.out |  14 +-
>  tests/qemu-iotests/310 | 255 +++
>  tests/qemu-iotests/310.out | 369 +
>  tests/qemu-iotests/common.qemu | 107 +-
>  tests/qemu-iotests/group   |   1 +
>  15 files changed, 1289 insertions(+), 145 deletions(-)
>  create mode 100755 tests/qemu-iotests/310
>  create mode 100644 tests/qemu-iotests/310.out
> 
> --=20
> 2.26.2
> 
-- 
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 17/17] hw/block/nvme: change controller pci id

2020-09-07 Thread Dr. David Alan Gilbert
* Klaus Jensen (i...@irrelevant.dk) wrote:
> On Sep  7 11:37, Dr. David Alan Gilbert wrote:
> > * Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> > > +David in case
> > > 
> > > On 9/4/20 4:19 PM, Klaus Jensen wrote:
> > > > From: Klaus Jensen 
> > > > 
> > > > There are two reasons for changing this:
> > > > 
> > > >   1. The nvme device currently uses an internal Intel device id.
> > > > 
> > > >   2. Since commits "nvme: fix write zeroes offset and count" and "nvme:
> > > >  support multiple namespaces" the controller device no longer has
> > > >  the quirks that the Linux kernel think it has.
> > > > 
> > > >  As the quirks are applied based on pci vendor and device id, change
> > > >  them to get rid of the quirks.
> > > > 
> > > > To keep backward compatibility, add a new 'x-use-intel-id' parameter to
> > > > the nvme device to force use of the Intel vendor and device id. This is
> > > > off by default but add a compat property to set this for 5.1 machines
> > > > and older.
> > > 
> > > So now what happens if you start a 5.1 machine with a recent kernel?
> > > Simply the kernel will use unnecessary quirks, or are there more
> > > changes in behavior?
> > 
> > Seems reasonable to me...but...
> > 
> > > > 
> > > > Signed-off-by: Klaus Jensen 
> > > > Reviewed-by: Keith Busch 
> > > > Reviewed-by: Maxim Levitsky 
> > > > ---
> > > >  hw/block/nvme.c   | 12 ++--
> > > >  hw/block/nvme.h   |  1 +
> > > >  hw/core/machine.c |  1 +
> > > >  3 files changed, 12 insertions(+), 2 deletions(-)
> > > > 
> > > > diff --git a/hw/block/nvme.c b/hw/block/nvme.c
> > > > index 453d3a89d475..8018f8679366 100644
> > > > --- a/hw/block/nvme.c
> > > > +++ b/hw/block/nvme.c
> > > > @@ -2749,6 +2749,15 @@ static void nvme_init_pci(NvmeCtrl *n, PCIDevice 
> > > > *pci_dev, Error **errp)
> > > >  
> > > >  pci_conf[PCI_INTERRUPT_PIN] = 1;
> > > >      pci_config_set_prog_interface(pci_conf, 0x2);
> > > > +
> > > > +if (n->params.use_intel_id) {
> > > > +pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_INTEL);
> > > > +pci_config_set_device_id(pci_conf, 0x5846);
> > 
> > Wasn't that magic number 5845 down there ?
> > 
> 
> Argh! My first version of this just bumbed the intel device id and it
> got left there.
> 
> Good find! Thank you!

It may be best to turn it into a constant in include/hw/pci/pci_ids.h if
it corresponds to some real Intel device.

Dave

-- 
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 17/17] hw/block/nvme: change controller pci id

2020-09-07 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> +David in case
> 
> On 9/4/20 4:19 PM, Klaus Jensen wrote:
> > From: Klaus Jensen 
> > 
> > There are two reasons for changing this:
> > 
> >   1. The nvme device currently uses an internal Intel device id.
> > 
> >   2. Since commits "nvme: fix write zeroes offset and count" and "nvme:
> >  support multiple namespaces" the controller device no longer has
> >  the quirks that the Linux kernel think it has.
> > 
> >  As the quirks are applied based on pci vendor and device id, change
> >  them to get rid of the quirks.
> > 
> > To keep backward compatibility, add a new 'x-use-intel-id' parameter to
> > the nvme device to force use of the Intel vendor and device id. This is
> > off by default but add a compat property to set this for 5.1 machines
> > and older.
> 
> So now what happens if you start a 5.1 machine with a recent kernel?
> Simply the kernel will use unnecessary quirks, or are there more
> changes in behavior?

Seems reasonable to me...but...

> > 
> > Signed-off-by: Klaus Jensen 
> > Reviewed-by: Keith Busch 
> > Reviewed-by: Maxim Levitsky 
> > ---
> >  hw/block/nvme.c   | 12 ++--
> >  hw/block/nvme.h   |  1 +
> >  hw/core/machine.c |  1 +
> >  3 files changed, 12 insertions(+), 2 deletions(-)
> > 
> > diff --git a/hw/block/nvme.c b/hw/block/nvme.c
> > index 453d3a89d475..8018f8679366 100644
> > --- a/hw/block/nvme.c
> > +++ b/hw/block/nvme.c
> > @@ -2749,6 +2749,15 @@ static void nvme_init_pci(NvmeCtrl *n, PCIDevice 
> > *pci_dev, Error **errp)
> >  
> >  pci_conf[PCI_INTERRUPT_PIN] = 1;
> >  pci_config_set_prog_interface(pci_conf, 0x2);
> > +
> > +if (n->params.use_intel_id) {
> > +pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_INTEL);
> > +pci_config_set_device_id(pci_conf, 0x5846);

Wasn't that magic number 5845 down there ?

Dave

> > +} else {
> > +pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_REDHAT);
> > +pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_REDHAT_NVME);
> > +}
> > +
> >  pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
> >  pcie_endpoint_cap_init(pci_dev, 0x80);
> >  
> > @@ -2903,6 +2912,7 @@ static Property nvme_props[] = {
> >  DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3),
> >  DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 
> > 64),
> >  DEFINE_PROP_UINT8("mdts", NvmeCtrl, params.mdts, 7),
> > +DEFINE_PROP_BOOL("x-use-intel-id", NvmeCtrl, params.use_intel_id, 
> > false),
> >  DEFINE_PROP_END_OF_LIST(),
> >  };
> >  
> > @@ -2919,8 +2929,6 @@ static void nvme_class_init(ObjectClass *oc, void 
> > *data)
> >  pc->realize = nvme_realize;
> >  pc->exit = nvme_exit;
> >  pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
> > -pc->vendor_id = PCI_VENDOR_ID_INTEL;
> > -pc->device_id = 0x5845;
> >  pc->revision = 2;
> >  
> >  set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
> > diff --git a/hw/block/nvme.h b/hw/block/nvme.h
> > index 72260f2e8ea9..a734a5e1370d 100644
> > --- a/hw/block/nvme.h
> > +++ b/hw/block/nvme.h
> > @@ -15,6 +15,7 @@ typedef struct NvmeParams {
> >  uint8_t  aerl;
> >  uint32_t aer_max_queued;
> >  uint8_t  mdts;
> > +bool use_intel_id;
> >  } NvmeParams;
> >  
> >  typedef struct NvmeAsyncEvent {
> > diff --git a/hw/core/machine.c b/hw/core/machine.c
> > index ea26d612374d..67990232528c 100644
> > --- a/hw/core/machine.c
> > +++ b/hw/core/machine.c
> > @@ -34,6 +34,7 @@ GlobalProperty hw_compat_5_1[] = {
> >  { "vhost-user-scsi", "num_queues", "1"},
> >  { "virtio-blk-device", "num-queues", "1"},
> >  { "virtio-scsi-device", "num_queues", "1"},
> > +{ "nvme", "x-use-intel-id", "on"},
> >  };
> >  const size_t hw_compat_5_1_len = G_N_ELEMENTS(hw_compat_5_1);
> >  
> > 
> 
-- 
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v7 0/8] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-08-27 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Thu, Aug 27, 2020 at 10:42:46AM +0200, Lukas Straub wrote:
> > On Tue, 18 Aug 2020 14:26:31 +0200
> > Lukas Straub  wrote:
> > 
> > > On Tue, 4 Aug 2020 10:11:22 +0200
> > > Lukas Straub  wrote:
> > > 
> > > > Hello Everyone,
> > > > In many cases, if qemu has a network connection (qmp, migration, 
> > > > chardev, etc.)
> > > > to some other server and that server dies or hangs, qemu hangs too.
> > > > These patches introduce the new 'yank' out-of-band qmp command to 
> > > > recover from
> > > > these kinds of hangs. The different subsystems register callbacks which 
> > > > get
> > > > executed with the yank command. For example the callback can shutdown() 
> > > > a
> > > > socket. This is intended for the colo use-case, but it can be used for 
> > > > other
> > > > things too of course.
> > > > 
> > > > Regards,
> > > > Lukas Straub
> > > > 
> > > > v7:
> > > >  -yank_register_instance now returns error via Error **errp instead of 
> > > > aborting
> > > >  -dropped "chardev/char.c: Check for duplicate id before  creating 
> > > > chardev"
> > > > 
> > > > v6:
> > > >  -add Reviewed-by and Acked-by tags
> > > >  -rebase on master
> > > >  -lots of changes in nbd due to rebase
> > > >  -only take maintainership of util/yank.c and include/qemu/yank.h 
> > > > (Daniel P. Berrangé)
> > > >  -fix a crash discovered by the newly added chardev test
> > > >  -fix the test itself
> > > > 
> > > > v5:
> > > >  -move yank.c to util/
> > > >  -move yank.h to include/qemu/
> > > >  -add license to yank.h
> > > >  -use const char*
> > > >  -nbd: use atomic_store_release and atomic_load_aqcuire
> > > >  -io-channel: ensure thread-safety and document it
> > > >  -add myself as maintainer for yank
> > > > 
> > > > v4:
> > > >  -fix build errors...
> > > > 
> > > > v3:
> > > >  -don't touch softmmu/vl.c, use __contructor__ attribute instead (Paolo 
> > > > Bonzini)
> > > >  -fix build errors
> > > >  -rewrite migration patch so it actually passes all tests
> > > > 
> > > > v2:
> > > >  -don't touch io/ code anymore
> > > >  -always register yank functions
> > > >  -'yank' now takes a list of instances to yank
> > > >  -'query-yank' returns a list of yankable instances
> > > > 
> > > > Lukas Straub (8):
> > > >   Introduce yank feature
> > > >   block/nbd.c: Add yank feature
> > > >   chardev/char-socket.c: Add yank feature
> > > >   migration: Add yank feature
> > > >   io/channel-tls.c: make qio_channel_tls_shutdown thread-safe
> > > >   io: Document thread-safety of qio_channel_shutdown
> > > >   MAINTAINERS: Add myself as maintainer for yank feature
> > > >   tests/test-char.c: Wait for the chardev to connect in
> > > > char_socket_client_dupid_test
> > > > 
> > > >  MAINTAINERS   |   6 ++
> > > >  block/nbd.c   | 129 +++-
> > > >  chardev/char-socket.c |  31 ++
> > > >  include/io/channel.h  |   2 +
> > > >  include/qemu/yank.h   |  80 +++
> > > >  io/channel-tls.c  |   6 +-
> > > >  migration/channel.c   |  12 +++
> > > >  migration/migration.c |  25 -
> > > >  migration/multifd.c   |  10 ++
> > > >  migration/qemu-file-channel.c |   6 ++
> > > >  migration/savevm.c|   6 ++
> > > >  qapi/misc.json|  45 +
> > > >  tests/Makefile.include|   2 +-
> > > >  tests/test-char.c |   1 +
> > > >  util/Makefile.objs|   1 +
> > > >  util/yank.c   | 184 ++++++++++++++
> > > >  16 files changed, 493 insertions(+), 53 deletions(-)
> > > >  create mode 100644 include/qemu/yank.h
> > > >  create mode 100644 util/yank.c
> > > > 
> > > > --
> > > > 2.20.1  
> > > 
> > > Ping...
> > 
> > Ping 2...
> > 
> > Also, can the different subsystems have a look at this and give their ok?
> 
> We need ACKs from the NBD, migration and chardev maintainers, for the
> respective patches, then I think this series is ready for a pull request.

I'm happy from Migration:

Acked-by: Dr. David Alan Gilbert 

> Once acks arrive, I'm happy to send a PULL unless someone else has a
> desire todo it.

Looks like Markus would like a QMP tweak; but other than that I'd also
be happy to take it via migration; whichever is easiest.

Dave

> 
> Regards,
> Daniel
> -- 
> |: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-https://fstop138.berrange.com :|
> |: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|
-- 
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 1/7] migration: improve error reporting of block driver state name

2020-08-27 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> With blockdev, a BlockDriverState may not have a device name,
> so using a node name is required as an alternative.
> 
> Reviewed-by: Dr. David Alan Gilbert 
> Signed-off-by: Daniel P. Berrangé 

Queuing this one by itself since it's useful anyway.

Dave

> ---
>  migration/savevm.c | 12 ++--
>  tests/qemu-iotests/267.out |  4 ++--
>  2 files changed, 8 insertions(+), 8 deletions(-)
> 
> diff --git a/migration/savevm.c b/migration/savevm.c
> index a843d202b5..304d98ff78 100644
> --- a/migration/savevm.c
> +++ b/migration/savevm.c
> @@ -2682,7 +2682,7 @@ int save_snapshot(const char *name, Error **errp)
>  
>  if (!bdrv_all_can_snapshot()) {
>  error_setg(errp, "Device '%s' is writable but does not support "
> -   "snapshots", bdrv_get_device_name(bs));
> +   "snapshots", bdrv_get_device_or_node_name(bs));
>  return ret;
>  }
>  
> @@ -2691,7 +2691,7 @@ int save_snapshot(const char *name, Error **errp)
>  ret = bdrv_all_delete_snapshot(name, , errp);
>  if (ret < 0) {
>  error_prepend(errp, "Error while deleting snapshot on device "
> -  "'%s': ", bdrv_get_device_name(bs1));
> +  "'%s': ", bdrv_get_device_or_node_name(bs1));
>  return ret;
>  }
>  }
> @@ -2766,7 +2766,7 @@ int save_snapshot(const char *name, Error **errp)
>  ret = bdrv_all_create_snapshot(sn, bs, vm_state_size, );
>  if (ret < 0) {
>  error_setg(errp, "Error while creating snapshot on '%s'",
> -   bdrv_get_device_name(bs));
> +   bdrv_get_device_or_node_name(bs));
>  goto the_end;
>  }
>  
> @@ -2884,14 +2884,14 @@ int load_snapshot(const char *name, Error **errp)
>  if (!bdrv_all_can_snapshot()) {
>  error_setg(errp,
> "Device '%s' is writable but does not support snapshots",
> -   bdrv_get_device_name(bs));
> +   bdrv_get_device_or_node_name(bs));
>  return -ENOTSUP;
>  }
>  ret = bdrv_all_find_snapshot(name, );
>  if (ret < 0) {
>  error_setg(errp,
> "Device '%s' does not have the requested snapshot '%s'",
> -   bdrv_get_device_name(bs), name);
> +   bdrv_get_device_or_node_name(bs), name);
>  return ret;
>  }
>  
> @@ -2920,7 +2920,7 @@ int load_snapshot(const char *name, Error **errp)
>  ret = bdrv_all_goto_snapshot(name, , errp);
>  if (ret < 0) {
>  error_prepend(errp, "Could not load snapshot '%s' on '%s': ",
> -  name, bdrv_get_device_name(bs));
> +  name, bdrv_get_device_or_node_name(bs));
>  goto err_drain;
>  }
>  
> diff --git a/tests/qemu-iotests/267.out b/tests/qemu-iotests/267.out
> index d6d80c099f..215902b3ad 100644
> --- a/tests/qemu-iotests/267.out
> +++ b/tests/qemu-iotests/267.out
> @@ -81,11 +81,11 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
>  Testing: -blockdev driver=file,filename=TEST_DIR/t.IMGFMT,node-name=file
>  QEMU X.Y.Z monitor - type 'help' for more information
>  (qemu) savevm snap0
> -Error: Device '' is writable but does not support snapshots
> +Error: Device 'file' is writable but does not support snapshots
>  (qemu) info snapshots
>  No available block device supports snapshots
>  (qemu) loadvm snap0
> -Error: Device '' is writable but does not support snapshots
> +Error: Device 'file' is writable but does not support snapshots
>  (qemu) quit
>  
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
> -- 
> 2.26.2
> 
-- 
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v4 2/4] iotests.py: Add wait_for_runstate()

2020-08-20 Thread Dr. David Alan Gilbert
* Eric Blake (ebl...@redhat.com) wrote:
> On 8/18/20 8:32 AM, Max Reitz wrote:
> > Signed-off-by: Max Reitz 
> > ---
> >   tests/qemu-iotests/iotests.py | 4 
> >   1 file changed, 4 insertions(+)
> 
> Reviewed-by: Eric Blake 
> 
> > 
> > diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
> > index 717b5b652c..ee93cf22db 100644
> > --- a/tests/qemu-iotests/iotests.py
> > +++ b/tests/qemu-iotests/iotests.py
> > @@ -833,6 +833,10 @@ class VM(qtest.QEMUQtestMachine):
> >  'Found node %s under %s (but expected %s)' % \
> >  (node['name'], path, expected_node)
> > +def wait_for_runstate(self, runstate: str) -> None:
> > +while self.qmp('query-status')['return']['status'] != runstate:
> > +time.sleep(0.2)
> 
> This looks like it could inf-loop if we have a bug where the status never
> changes as expected; but I guess CI bots have timeouts at higher layers that
> would detect if such a bug sneaks in.

Although it might be useful to make sure when such a timeout lands, you
know which state you thought you were waiting for.

Dave

> > +
> >   index_re = re.compile(r'([^\[]+)\[([^\]]+)\]')
> >   class QMPTestCase(unittest.TestCase):
> > 
> 
> -- 
> Eric Blake, Principal Software Engineer
> Red Hat, Inc.   +1-919-301-3226
> Virtualization:  qemu.org | libvirt.org
-- 
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v2 3/6] migration: stop returning errno from load_snapshot()

2020-08-12 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> None of the callers care about the errno value since there is a full
> Error object populated. This gives consistency with save_snapshot()
> which already just returns -1.
> 
> Signed-off-by: Daniel P. Berrangé 

Reviewed-by: Dr. David Alan Gilbert 

(Note this is true of snapshots only; in postcopy there are places we
care about the errno for recovery)

> ---
>  migration/savevm.c | 17 -
>  1 file changed, 8 insertions(+), 9 deletions(-)
> 
> diff --git a/migration/savevm.c b/migration/savevm.c
> index 19259ef7c0..6c4d80fc5a 100644
> --- a/migration/savevm.c
> +++ b/migration/savevm.c
> @@ -2843,20 +2843,20 @@ int load_snapshot(const char *name, Error **errp)
>  if (!replay_can_snapshot()) {
>  error_setg(errp, "Record/replay does not allow loading snapshot "
> "right now. Try once more later.");
> -return -EINVAL;
> +return -1;
>  }
>  
>  if (!bdrv_all_can_snapshot(errp)) {
> -return -ENOTSUP;
> +return -1;
>  }
>  ret = bdrv_all_find_snapshot(name, errp);
>  if (ret < 0) {
> -return ret;
> +return -1;
>  }
>  
>  bs_vm_state = bdrv_all_find_vmstate_bs(errp);
>  if (!bs_vm_state) {
> -return -ENOTSUP;
> +return -1;
>  }
>  aio_context = bdrv_get_aio_context(bs_vm_state);
>  
> @@ -2865,11 +2865,11 @@ int load_snapshot(const char *name, Error **errp)
>  ret = bdrv_snapshot_find(bs_vm_state, , name);
>  aio_context_release(aio_context);
>  if (ret < 0) {
> -return ret;
> +return -1;
>  } else if (sn.vm_state_size == 0) {
>  error_setg(errp, "This is a disk-only snapshot. Revert to it "
> " offline using qemu-img");
> -return -EINVAL;
> +return -1;
>  }
>  
>  /* Flush all IO requests so they don't interfere with the new state.  */
> @@ -2884,7 +2884,6 @@ int load_snapshot(const char *name, Error **errp)
>  f = qemu_fopen_bdrv(bs_vm_state, 0);
>  if (!f) {
>  error_setg(errp, "Could not open VM state file");
> -ret = -EINVAL;
>  goto err_drain;
>  }
>  
> @@ -2900,14 +2899,14 @@ int load_snapshot(const char *name, Error **errp)
>  
>  if (ret < 0) {
>  error_setg(errp, "Error %d while loading VM state", ret);
> -    return ret;
> +return -1;
>  }
>  
>  return 0;
>  
>  err_drain:
>  bdrv_drain_all_end();
> -return ret;
> +return -1;
>  }
>  
>  void vmstate_register_ram(MemoryRegion *mr, DeviceState *dev)
> -- 
> 2.26.2
> 
-- 
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v2 1/6] migration: improve error reporting of block driver state name

2020-08-12 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> With blockdev, a BlockDriverState may not have a device name,
> so using a node name is required as an alternative.
> 
> Signed-off-by: Daniel P. Berrangé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  migration/savevm.c | 12 ++--
>  tests/qemu-iotests/267.out |  4 ++--
>  2 files changed, 8 insertions(+), 8 deletions(-)
> 
> diff --git a/migration/savevm.c b/migration/savevm.c
> index 45c9dd9d8a..cffee6cab7 100644
> --- a/migration/savevm.c
> +++ b/migration/savevm.c
> @@ -2655,7 +2655,7 @@ int save_snapshot(const char *name, Error **errp)
>  
>  if (!bdrv_all_can_snapshot()) {
>  error_setg(errp, "Device '%s' is writable but does not support "
> -   "snapshots", bdrv_get_device_name(bs));
> +   "snapshots", bdrv_get_device_or_node_name(bs));
>  return ret;
>  }
>  
> @@ -2664,7 +2664,7 @@ int save_snapshot(const char *name, Error **errp)
>  ret = bdrv_all_delete_snapshot(name, , errp);
>  if (ret < 0) {
>  error_prepend(errp, "Error while deleting snapshot on device "
> -  "'%s': ", bdrv_get_device_name(bs1));
> +  "'%s': ", bdrv_get_device_or_node_name(bs1));
>  return ret;
>  }
>  }
> @@ -2739,7 +2739,7 @@ int save_snapshot(const char *name, Error **errp)
>  ret = bdrv_all_create_snapshot(sn, bs, vm_state_size, );
>  if (ret < 0) {
>  error_setg(errp, "Error while creating snapshot on '%s'",
> -   bdrv_get_device_name(bs));
> +   bdrv_get_device_or_node_name(bs));
>  goto the_end;
>  }
>  
> @@ -2857,14 +2857,14 @@ int load_snapshot(const char *name, Error **errp)
>  if (!bdrv_all_can_snapshot()) {
>  error_setg(errp,
> "Device '%s' is writable but does not support snapshots",
> -   bdrv_get_device_name(bs));
> +   bdrv_get_device_or_node_name(bs));
>  return -ENOTSUP;
>  }
>  ret = bdrv_all_find_snapshot(name, );
>  if (ret < 0) {
>  error_setg(errp,
> "Device '%s' does not have the requested snapshot '%s'",
> -   bdrv_get_device_name(bs), name);
> +   bdrv_get_device_or_node_name(bs), name);
>  return ret;
>  }
>  
> @@ -2893,7 +2893,7 @@ int load_snapshot(const char *name, Error **errp)
>  ret = bdrv_all_goto_snapshot(name, , errp);
>  if (ret < 0) {
>  error_prepend(errp, "Could not load snapshot '%s' on '%s': ",
> -  name, bdrv_get_device_name(bs));
> +  name, bdrv_get_device_or_node_name(bs));
>  goto err_drain;
>  }
>  
> diff --git a/tests/qemu-iotests/267.out b/tests/qemu-iotests/267.out
> index d6d80c099f..215902b3ad 100644
> --- a/tests/qemu-iotests/267.out
> +++ b/tests/qemu-iotests/267.out
> @@ -81,11 +81,11 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
>  Testing: -blockdev driver=file,filename=TEST_DIR/t.IMGFMT,node-name=file
>  QEMU X.Y.Z monitor - type 'help' for more information
>  (qemu) savevm snap0
> -Error: Device '' is writable but does not support snapshots
> +Error: Device 'file' is writable but does not support snapshots
>  (qemu) info snapshots
>  No available block device supports snapshots
>  (qemu) loadvm snap0
> -Error: Device '' is writable but does not support snapshots
> +Error: Device 'file' is writable but does not support snapshots
>  (qemu) quit
>  
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
> -- 
> 2.26.2
> 
-- 
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: cleanups with long-term benefits (was Re: [PATCH] schemas: Add vim modeline)

2020-08-05 Thread Dr. David Alan Gilbert
* Paolo Bonzini (pbonz...@redhat.com) wrote:
> On 05/08/20 09:36, Markus Armbruster wrote:
> > There's also the longer term pain of having to work around git-blame
> > unable to see beyond the flag day.
> 
> Do you really use "git blame" that much?  "git log -S" does more or less
> the same function (in a different way) and is not affected as much by
> large code movement and transformation patches.

I use it a lot!   Following stuff back to find where a change came
from and then asking people.

Dave

> Paolo
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH] schemas: Add vim modeline

2020-07-31 Thread Dr. David Alan Gilbert
* Paolo Bonzini (pbonz...@redhat.com) wrote:
> On 31/07/20 17:26, John Snow wrote:
> > I saw the critique that we still use JSON-ish for the runtime QMP
> > protocol, and moving the QAPI IDL to a standard wouldn't remove all
> > instances of a custom format from our tree.
> 
> Sorry, but "still using JSON" is not a critique that makes any sense.
> 
> 99% of the websites you use daily use JSON as their RPC
> frontend-to-backend language; OpenAPI is essentially JSON over HTTP.
> There must be something good in JSON.

If there is, I've not found it:
a) It's integer definitions are a mess
b) You can't require ordering
c) No two parsers agree with each other

and those are the only ones I've hit in my very limited JSON wrangling.

It's possible attractions are that no one has anything widely used
that's better, and it's easy to use from JS.

But it seems popular to try and find replacements; e.g. Amazon Ion that
landed a few weeks ago (like JSON but...not quite and with a binary
format optionally).

Dave

> Whenever you hear a complaint about "using JSON", it's actually a
> complaint about bindings for your language, which can be a sensible
> critique: gRPC is essentially {protobuf,FlatBuffers} over HTTP/2 plus a
> boatload of language mappings.  Unfortunately C is not one of those
> mappings.
> 
> Paolo
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH-for-5.1? v2 0/2] util/pagesize: Make qemu_real_host_page_size of type size_t

2020-07-30 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Since v1:
> Make QEMU_VMALLOC_ALIGN unsigned in a previous patch

Nah, not for 5.1 - it feels like the type of thing that might on a
really bad day create a really subtle bug.

Dave

> Philippe Mathieu-Daudé (2):
>   qemu/osdep: Make QEMU_VMALLOC_ALIGN unsigned long
>   util/pagesize: Make qemu_real_host_page_size of type size_t
> 
>  include/exec/ram_addr.h  | 4 ++--
>  include/qemu/osdep.h | 6 +++---
>  accel/kvm/kvm-all.c  | 3 ++-
>  block/qcow2-cache.c  | 2 +-
>  exec.c   | 8 
>  hw/ppc/spapr_pci.c   | 2 +-
>  hw/virtio/virtio-mem.c   | 2 +-
>  migration/migration.c| 2 +-
>  migration/postcopy-ram.c | 2 +-
>  monitor/misc.c   | 2 +-
>  util/pagesize.c  | 2 +-
>  11 files changed, 18 insertions(+), 17 deletions(-)
> 
> -- 
> 2.21.3
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 17/21] migration/savevm: don't worry if bitmap migration postcopy failed

2020-07-27 Thread Dr. David Alan Gilbert
* Eric Blake (ebl...@redhat.com) wrote:
> On 7/24/20 3:43 AM, Vladimir Sementsov-Ogievskiy wrote:
> > First, if only bitmaps postcopy enabled (not ram postcopy)
> 
> is enabled (and not ram postcopy),
> 
> > postcopy_pause_incoming crashes on assertion assert(mis->to_src_file).
> 
> on an
> 
> > 
> > And anyway, bitmaps postcopy is not prepared to be somehow recovered.
> > The original idea instead is that if bitmaps postcopy failed, we just
> > loss some bitmaps, which is not critical. So, on failure we just need
> 
> lose
> 
> > to remove unfinished bitmaps and guest should continue execution on
> > destination.
> > 
> > Signed-off-by: Vladimir Sementsov-Ogievskiy 
> > Reviewed-by: Dr. David Alan Gilbert 
> > Reviewed-by: Andrey Shinkevich 
> > ---
> >   migration/savevm.c | 37 -
> >   1 file changed, 32 insertions(+), 5 deletions(-)
> > 
> 
> Definitely a bug fix, but I'd like David's opinion on whether this is still
> 5.1 material (because it is limited to just bitmaps migration, which is
> opt-in) or too risky (because we've already had several releases where it
> was broken, what's one more?).

I think it's OK for 5.1

Dave

> I'm less familiar with the code, so this is weak, but I did read through it
> and nothing jumped out at me, so:
> 
> Reviewed-by: Eric Blake 
> 
> -- 
> Eric Blake, Principal Software Engineer
> Red Hat, Inc.   +1-919-301-3226
> Virtualization:  qemu.org | libvirt.org
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 16/21] migration/block-dirty-bitmap: cancel migration on shutdown

2020-07-27 Thread Dr. David Alan Gilbert
* Vladimir Sementsov-Ogievskiy (vsement...@virtuozzo.com) wrote:
> If target is turned off prior to postcopy finished, target crashes
> because busy bitmaps are found at shutdown.
> Canceling incoming migration helps, as it removes all unfinished (and
> therefore busy) bitmaps.
> 
> Similarly on source we crash in bdrv_close_all which asserts that all
> bdrv states are removed, because bdrv states involved into dirty bitmap
> migration are referenced by it. So, we need to cancel outgoing
> migration as well.
> 
> Signed-off-by: Vladimir Sementsov-Ogievskiy 
> Reviewed-by: Andrey Shinkevich 
> ---
>  migration/migration.h  |  2 ++
>  migration/block-dirty-bitmap.c | 16 
>  migration/migration.c  | 13 +
>  3 files changed, 31 insertions(+)
> 
> diff --git a/migration/migration.h b/migration/migration.h
> index ab20c756f5..6c6a931d0d 100644
> --- a/migration/migration.h
> +++ b/migration/migration.h
> @@ -335,6 +335,8 @@ void migrate_send_rp_recv_bitmap(MigrationIncomingState 
> *mis,
>  void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value);
>  
>  void dirty_bitmap_mig_before_vm_start(void);
> +void dirty_bitmap_mig_cancel_outgoing(void);
> +void dirty_bitmap_mig_cancel_incoming(void);
>  void migrate_add_address(SocketAddress *address);
>  
>  int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque);
> diff --git a/migration/block-dirty-bitmap.c b/migration/block-dirty-bitmap.c
> index c24d4614bf..a198ec7278 100644
> --- a/migration/block-dirty-bitmap.c
> +++ b/migration/block-dirty-bitmap.c
> @@ -657,6 +657,22 @@ static void cancel_incoming_locked(DBMLoadState *s)
>  s->bitmaps = NULL;
>  }
>  
> +void dirty_bitmap_mig_cancel_outgoing(void)
> +{
> +dirty_bitmap_do_save_cleanup(_state.save);
> +}
> +
> +void dirty_bitmap_mig_cancel_incoming(void)
> +{
> +DBMLoadState *s = _state.load;
> +
> +qemu_mutex_lock(>lock);
> +
> +cancel_incoming_locked(s);
> +
> +qemu_mutex_unlock(>lock);
> +}
> +
>  static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s)
>  {
>  GSList *item;
> diff --git a/migration/migration.c b/migration/migration.c
> index 1c61428988..8fe36339db 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -188,6 +188,19 @@ void migration_shutdown(void)
>   */
>  migrate_fd_cancel(current_migration);
>  object_unref(OBJECT(current_migration));
> +
> +/*
> + * Cancel outgoing migration of dirty bitmaps. It should
> + * at least unref used block nodes.
> + */
> +dirty_bitmap_mig_cancel_outgoing();
> +
> +/*
> + * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
> + * are non-critical data, and their loss never considered as
> + * something serious.
> + */
> +    dirty_bitmap_mig_cancel_incoming();

Are you sure this is the right place to put them - I'm thinking that
perhaps the object_unref of current_migration should still be after
them?

Dave

>  }
>  
>  /* For outgoing */
> -- 
> 2.21.0
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 15/21] migration/block-dirty-bitmap: relax error handling in incoming part

2020-07-27 Thread Dr. David Alan Gilbert
;
> +return -EIO;
> +}
> +
> +if (s->cancelled) {
> +g_free(buf);
> +return 0;
> +}
> +
> +needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap,
> +   first_byte,
> +   nr_bytes);
>  
>  if (needed_size > buf_size ||
>  buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long))
> @@ -667,15 +729,8 @@ static int dirty_bitmap_load_bits(QEMUFile *f, 
> DBMLoadState *s)
>  error_report("Migrated bitmap granularity doesn't "
>   "match the destination bitmap '%s' granularity",
>   bdrv_dirty_bitmap_name(s->bitmap));
> -return -EINVAL;
> -}
> -
> -buf = g_malloc(buf_size);
> -ret = qemu_get_buffer(f, buf, buf_size);
> -if (ret != buf_size) {
> -error_report("Failed to read bitmap bits");
> -g_free(buf);
> -return -EIO;
> +cancel_incoming_locked(s);
> +return 0;
>  }
>  
>  bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, 
> nr_bytes,
> @@ -700,14 +755,16 @@ static int dirty_bitmap_load_header(QEMUFile *f, 
> DBMLoadState *s)
>  error_report("Unable to read node name string");
>  return -EINVAL;
>  }
> -s->bs = bdrv_lookup_bs(s->node_name, s->node_name, _err);
> -if (!s->bs) {
> -error_report_err(local_err);
> -return -EINVAL;
> +if (!s->cancelled) {
> +s->bs = bdrv_lookup_bs(s->node_name, s->node_name, _err);
> +if (!s->bs) {
> +error_report_err(local_err);
> +cancel_incoming_locked(s);
> +}
>  }
> -} else if (!s->bs && !nothing) {
> +} else if (!s->bs && !nothing && !s->cancelled) {
>  error_report("Error: block device name is not set");
> -return -EINVAL;
> +cancel_incoming_locked(s);
>  }
>  
>  if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
> @@ -715,24 +772,38 @@ static int dirty_bitmap_load_header(QEMUFile *f, 
> DBMLoadState *s)
>  error_report("Unable to read bitmap name string");
>  return -EINVAL;
>  }
> -s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
> -
> -/* bitmap may be NULL here, it wouldn't be an error if it is the
> - * first occurrence of the bitmap */
> -if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
> -error_report("Error: unknown dirty bitmap "
> - "'%s' for block device '%s'",
> - s->bitmap_name, s->node_name);
> -return -EINVAL;
> +if (!s->cancelled) {
> +s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
> +
> +/*
> + * bitmap may be NULL here, it wouldn't be an error if it is the
> + * first occurrence of the bitmap
> + */
> +if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
> +error_report("Error: unknown dirty bitmap "
> + "'%s' for block device '%s'",
> + s->bitmap_name, s->node_name);
> +cancel_incoming_locked(s);
> +}
>  }
> -} else if (!s->bitmap && !nothing) {
> +} else if (!s->bitmap && !nothing && !s->cancelled) {
>  error_report("Error: block device name is not set");
> -return -EINVAL;
> +cancel_incoming_locked(s);
>  }
>  
>  return 0;
>  }
>  
> +/*
> + * dirty_bitmap_load
> + *
> + * Load sequence of dirty bitmap chunks. Return error only on fatal io stream
> + * violations. On other errors just cancel bitmaps incoming migration and 
> return
> + * 0.
> + *
> + * Note, than when incoming bitmap migration is canceled, we still must read 
> all
> + * our chunks (and just ignore them), to not affect other migration objects.
> + */
>  static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
>  {
>  DBMLoadState *s = &((DBMState *)opaque)->load;
> @@ -741,12 +812,19 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, 
> int version_id)
>  trace_dirty_bitmap_load_enter();
>  
>  if (version_id != 1) {
> +qemu_mutex_lock(>lock);
> +cancel_incoming_locked(s);
> +qemu_mutex_unlock(>lock);
>  return -EINVAL;
>  }
>  
>  do {
> +qemu_mutex_lock(>lock);

Would QEMU_LOCK_GUARD(>lock)  work there?
It avoids the need to catch the unlock on each of the failure cases.

Dave

>  ret = dirty_bitmap_load_header(f, s);
>  if (ret < 0) {
> +cancel_incoming_locked(s);
> +qemu_mutex_unlock(>lock);
>  return ret;
>  }
>  
> @@ -763,8 +841,12 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, 
> int version_id)
>  }
>  
>  if (ret) {
> +cancel_incoming_locked(s);
> +qemu_mutex_unlock(>lock);
>  return ret;
>  }
> +
> +qemu_mutex_unlock(>lock);
>  } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS));
>  
>  trace_dirty_bitmap_load_success();
> -- 
> 2.21.0
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 15/21] migration/block-dirty-bitmap: relax error handling in incoming part

2020-07-27 Thread Dr. David Alan Gilbert
* Vladimir Sementsov-Ogievskiy (vsement...@virtuozzo.com) wrote:
> 24.07.2020 20:35, Eric Blake wrote:
> > On 7/24/20 3:43 AM, Vladimir Sementsov-Ogievskiy wrote:
> > > Bitmaps data is not critical, and we should not fail the migration (or
> > > use postcopy recovering) because of dirty-bitmaps migration failure.
> > > Instead we should just lose unfinished bitmaps.
> > > 
> > > Still we have to report io stream violation errors, as they affect the
> > > whole migration stream.
> > > 
> > > Signed-off-by: Vladimir Sementsov-Ogievskiy 
> > > ---
> > >   migration/block-dirty-bitmap.c | 152 +
> > >   1 file changed, 117 insertions(+), 35 deletions(-)
> > > 
> > 
> > > @@ -650,15 +695,32 @@ static int dirty_bitmap_load_bits(QEMUFile *f, 
> > > DBMLoadState *s)
> > >       if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
> > >           trace_dirty_bitmap_load_bits_zeroes();
> > > -        bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, 
> > > first_byte, nr_bytes,
> > > -                                    
> > >          false);
> > > +        if (!s->cancelled) {
> > > +            bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, 
> > > first_byte,
> > > +                                    
> > >              nr_bytes, false);
> > > +        }
> > >       } else {
> > >           size_t ret;
> > >           uint8_t *buf;
> > >           uint64_t buf_size = qemu_get_be64(f);
> > 
> > Pre-existing, but if I understand, we are reading a value from the 
> > migration stream...
> 
> Hmm, actually, this becomes worse after patch, as before patch we had the 
> check, that the size at least corresponds to the bitmap.. But we want to 
> relax things in cancelled mode (and we may not have any bitmap). Most correct 
> thing is to use read in a loop to just skip the data from stream if we are in 
> cancelled mode (something like nbd_drop()).
> 
> I can fix this with a follow-up patch.

If the size is bogus, it's probably not worth trying to skip anything
because it could be just a broken/misaligned stream.

Dave

> > 
> > > -        uint64_t needed_size =
> > > -            bdrv_dirty_bitmap_serialization_size(s->bitmap,
> > > -                                    
> > >              first_byte, nr_bytes);
> > > +        uint64_t needed_size;
> > > +
> > > +        buf = g_malloc(buf_size);
> > > +        ret = qemu_get_buffer(f, buf, buf_size);
> > 
> > ...and using it to malloc memory.  Is that a potential risk of a malicious 
> > stream causing us to allocate too much memory in relation to the guest's 
> > normal size?  If so, fixing that should be done separately.
> > 
> > I'm not a migration expert, but the patch looks reasonable to me.
> > 
> > Reviewed-by: Eric Blake 
> > 
> 
> 
> -- 
> Best regards,
> Vladimir
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 10/21] migration/block-dirty-bitmap: move mutex init to dirty_bitmap_mig_init

2020-07-27 Thread Dr. David Alan Gilbert
* Vladimir Sementsov-Ogievskiy (vsement...@virtuozzo.com) wrote:
> No reasons to keep two public init functions.
> 
> Signed-off-by: Vladimir Sementsov-Ogievskiy 
> Reviewed-by: Andrey Shinkevich 

Yes so I think that means the initialisation of that lock is a little
later in startup, but OK

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  migration/migration.h  | 1 -
>  migration/block-dirty-bitmap.c | 6 +-
>  migration/migration.c  | 2 --
>  3 files changed, 1 insertion(+), 8 deletions(-)
> 
> diff --git a/migration/migration.h b/migration/migration.h
> index f617960522..ab20c756f5 100644
> --- a/migration/migration.h
> +++ b/migration/migration.h
> @@ -335,7 +335,6 @@ void migrate_send_rp_recv_bitmap(MigrationIncomingState 
> *mis,
>  void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value);
>  
>  void dirty_bitmap_mig_before_vm_start(void);
> -void init_dirty_bitmap_incoming_migration(void);
>  void migrate_add_address(SocketAddress *address);
>  
>  int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque);
> diff --git a/migration/block-dirty-bitmap.c b/migration/block-dirty-bitmap.c
> index 01a536d7d3..4b67e4f4fb 100644
> --- a/migration/block-dirty-bitmap.c
> +++ b/migration/block-dirty-bitmap.c
> @@ -148,11 +148,6 @@ typedef struct LoadBitmapState {
>  static GSList *enabled_bitmaps;
>  QemuMutex finish_lock;
>  
> -void init_dirty_bitmap_incoming_migration(void)
> -{
> -qemu_mutex_init(_lock);
> -}
> -
>  static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
>  {
>  uint8_t flags = qemu_get_byte(f);
> @@ -801,6 +796,7 @@ static SaveVMHandlers savevm_dirty_bitmap_handlers = {
>  void dirty_bitmap_mig_init(void)
>  {
>  QSIMPLEQ_INIT(_bitmap_mig_state.dbms_list);
> +qemu_mutex_init(_lock);
>  
>  register_savevm_live("dirty-bitmap", 0, 1,
>   _dirty_bitmap_handlers,
> diff --git a/migration/migration.c b/migration/migration.c
> index 2ed9923227..1c61428988 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -165,8 +165,6 @@ void migration_object_init(void)
>  qemu_sem_init(_incoming->postcopy_pause_sem_dst, 0);
>  qemu_sem_init(_incoming->postcopy_pause_sem_fault, 0);
>  
> -    init_dirty_bitmap_incoming_migration();
> -
>  if (!migration_object_check(current_migration, )) {
>  error_report_err(err);
>  exit(1);
> -- 
> 2.21.0
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v2 0/3] migration: Add block-bitmap-mapping parameter

2020-07-22 Thread Dr. David Alan Gilbert
* Max Reitz (mre...@redhat.com) wrote:
> RFC v1: https://lists.nongnu.org/archive/html/qemu-block/2020-05/msg00912.html
> RFC v2: https://lists.nongnu.org/archive/html/qemu-block/2020-05/msg00915.html
> v1: https://lists.nongnu.org/archive/html/qemu-devel/2020-06/msg09792.html
> 
> Branch: https://github.com/XanClic/qemu.git migration-bitmap-mapping-v2
> Branch: https://git.xanclic.moe/XanClic/qemu.git migration-bitmap-mapping-v2
> 
> Based-on: <20200626130658.76498-1-vsement...@virtuozzo.com>
>   (“migration/block-dirty-bitmap: fix add_bitmaps_to_list”)
> 
> 
> Hi,
> 
> This new migration parameter allows mapping block node names and bitmap
> names to aliases for the purpose of block dirty bitmap migration.

One random thought is that you might find these block name aliases turn
out to be useful in other places, and an alias list may well turn out to
be generically useful.

Dave

> This way, management tools can use different node names on the source
> and destination and pass the mapping of how bitmaps are to be
> transferred to qemu (on the source, the destination, or even both with
> arbitrary aliases in the migration stream).
> 
> 
> v2:
> - Dropped what used to be patch 1 (the memleak fix), I see the exact
>   same fix has been sent concurrently and has been merged as
>   9728ebfb77f0159f4
> 
> - Patch 1:
>   - Changed documentation to clarify the default behavior
>   - s/5.1/5.2/
>   - Dropped dead assignment
>   - Fixed bitmaps_map memleak
>   - Assert that the bs_name given to add_bitmaps_to_list() must be the
> BDS’s node name if an alias_map is given
>   - On the source side, unmapped bitmaps are simply dropped without
> error
>   - On the destination side, unmapped aliases still result in errors
> (see patch 1 for a short explanation on my reasoning)
>   - Fixed a bug in qmp_query_migrate_parameters(): We have to clone
> s->parameters.block_bitmap_mapping, not
> params->block_bitmap_mapping, or the latter will just stay NULL (and
> so qmp_query_migrate_parameters() won’t return a result for the
> block-bitmap-mapping)
>   - Emit the mapping through HMP’s “info migrate_parameters”
>   - Return an error when trying to set the mapping through HMP (instead
> of just crashing because of an “assert(0)” signalling an unhandled
> migration parameter)
> 
> - Patch 3:
>   - Type alias for BlockBitmapMapping
>   - Check the result of “info migrate_parameters” whenever setting the
> block-bitmap-mapping (just to test the new formatting code)
>   - Catch the qemu.machine.AbnormalShutdown exception on the destination
> VM whenever the migration is expected to fail
> (necessary since commit ef5d474472426eda6abf81)
>   - Cases where we don’t set up a mapping for some bitmap on the source
> are now expected to succeed (without the bitmap being migrated)
> 
> 
> git-backport-diff against v1:
> 
> Key:
> [] : patches are identical
> [] : number of functional differences between upstream/downstream patch
> [down] : patch is downstream-only
> The flags [FC] indicate (F)unctional and (C)ontextual differences, 
> respectively
> 
> 001/3:[0117] [FC] 'migration: Add block-bitmap-mapping parameter'
> 002/3:[] [--] 'iotests.py: Add wait_for_runstate()'
> 003/3:[0202] [FC] 'iotests: Test node/bitmap aliases during migration'
> 
> 
> Max Reitz (3):
>   migration: Add block-bitmap-mapping parameter
>   iotests.py: Add wait_for_runstate()
>   iotests: Test node/bitmap aliases during migration
> 
>  qapi/migration.json|  92 +-
>  migration/migration.h  |   3 +
>  migration/block-dirty-bitmap.c | 373 
>  migration/migration.c  |  30 ++
>  monitor/hmp-cmds.c |  30 ++
>  tests/qemu-iotests/300 | 511 +
>  tests/qemu-iotests/300.out |   5 +
>  tests/qemu-iotests/group   |   1 +
>  tests/qemu-iotests/iotests.py  |   4 +
>  9 files changed, 994 insertions(+), 55 deletions(-)
>  create mode 100755 tests/qemu-iotests/300
>  create mode 100644 tests/qemu-iotests/300.out
> 
> -- 
> 2.26.2
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v2 1/3] migration: Add block-bitmap-mapping parameter

2020-07-22 Thread Dr. David Alan Gilbert
* Max Reitz (mre...@redhat.com) wrote:
> This migration parameter allows mapping block node names and bitmap
> names to aliases for the purpose of block dirty bitmap migration.
> 
> This way, management tools can use different node and bitmap names on
> the source and destination and pass the mapping of how bitmaps are to be
> transferred to qemu (on the source, the destination, or even both with
> arbitrary aliases in the migration stream).
> 
> Suggested-by: Vladimir Sementsov-Ogievskiy 
> Signed-off-by: Max Reitz 

For both HMP and migration stuff:
Acked-by: Dr. David Alan Gilbert 

> ---
> Vladimir noted in v1 that it would be better to ignore bitmaps whose
> names aren't mapped, or that are on nodes whose names aren't mapped.
> One of the reasons he gave was that bitmap migration is inherently a
> form of postcopy migration, and we should not break the target when it
> is already running because of a bitmap issue.
> 
> So in this version, I changed the behavior to just ignore bitmaps
> without a mapping on the source side.  On the destination, however, I
> kept it an error when an incoming bitmap or node alias is unknown.
> 
> This is in violation of Vladimir's (rightful) reasoning that such
> errors should never break the already running target, but I decided to
> do so anyway for a couple of reasons:
> 
> - Ignoring unmapped bitmaps on the source is trivial: We just have to
>   not put them into the migration stream.
>   On the destination, it isn't so easy: We (I think) would have to
>   modify the code to actively skip the bitmap in the stream.
>   (On the other hand, erroring out is always easy.)
> 
> - Incoming bitmaps with unknown names are already an error before this
>   series.  So this isn't introducing new breakage.
> 
> - I think it makes sense to drop all bitmaps you don't want to migrate
>   (or implicitly drop them by just not specifying them if you don't care
>   about them) on the source.  I can't imagine a scenario where it would
>   be really useful if the destination could silently drop unknown
>   bitmaps.  Unknown bitmaps should just be dropped on the source.
> 
> - Choosing to make it an error now doesn't prevent us from relaxing that
>   restriction in the future.
> ---
>  qapi/migration.json|  92 +++-
>  migration/migration.h  |   3 +
>  migration/block-dirty-bitmap.c | 373 -
>  migration/migration.c  |  30 +++
>  monitor/hmp-cmds.c |  30 +++
>  5 files changed, 473 insertions(+), 55 deletions(-)
> 
> diff --git a/qapi/migration.json b/qapi/migration.json
> index d5000558c6..1b0fbcef96 100644
> --- a/qapi/migration.json
> +++ b/qapi/migration.json
> @@ -507,6 +507,44 @@
>'data': [ 'none', 'zlib',
>  { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
>  
> +##
> +# @BitmapMigrationBitmapAlias:
> +#
> +# @name: The name of the bitmap.
> +#
> +# @alias: An alias name for migration (for example the bitmap name on
> +# the opposite site).
> +#
> +# Since: 5.1
> +##
> +{ 'struct': 'BitmapMigrationBitmapAlias',
> +  'data': {
> +  'name': 'str',
> +  'alias': 'str'
> +  } }
> +
> +##
> +# @BitmapMigrationNodeAlias:
> +#
> +# Maps a block node name and the bitmaps it has to aliases for dirty
> +# bitmap migration.
> +#
> +# @node-name: A block node name.
> +#
> +# @alias: An alias block node name for migration (for example the
> +# node name on the opposite site).
> +#
> +# @bitmaps: Mappings for the bitmaps on this node.
> +#
> +# Since: 5.1
> +##
> +{ 'struct': 'BitmapMigrationNodeAlias',
> +  'data': {
> +  'node-name': 'str',
> +  'alias': 'str',
> +  'bitmaps': [ 'BitmapMigrationBitmapAlias' ]
> +  } }
> +
>  ##
>  # @MigrationParameter:
>  #
> @@ -641,6 +679,21 @@
>  #  will consume more CPU.
>  #  Defaults to 1. (Since 5.0)
>  #
> +# @block-bitmap-mapping: Maps block nodes and bitmaps on them to
> +#  aliases for the purpose of dirty bitmap migration.  Such
> +#  aliases may for example be the corresponding names on the
> +#  opposite site.
> +#  The mapping must be one-to-one, and on the destination also
> +#  complete: On the source, bitmaps on nodes where either the
> +#  bitmap or the node is not mapped will be ignored.  In
> +#  contrast, on the destination, receiving a bitmap (by alias)
> +#  from a node (by alias) when either alias is not mapped will
> +#  result in an error.
> +#  By default (when this parameter has never been set), bitmap
> +#  names are mapped to themselves.  Nodes are m

Re: [PATCH 2/6] migration: introduce savevm, loadvm, delvm QMP commands

2020-07-03 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Fri, Jul 03, 2020 at 06:00:50PM +0100, Dr. David Alan Gilbert wrote:
> > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > On Fri, Jul 03, 2020 at 05:22:46PM +0100, Dr. David Alan Gilbert wrote:
> > > > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > > > On Fri, Jul 03, 2020 at 05:10:12PM +0100, Dr. David Alan Gilbert 
> > > > > wrote:
> > > > > > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > > > > > On Fri, Jul 03, 2020 at 04:49:33PM +0100, Dr. David Alan Gilbert 
> > > > > > > wrote:
> > > > > > > > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > > > > > > > On Thu, Jul 02, 2020 at 01:12:52PM -0500, Eric Blake wrote:
> > > > > > > > > > On 7/2/20 12:57 PM, Daniel P. Berrangé wrote:
> > > > > > > > > > > savevm, loadvm and delvm are some of the few commands 
> > > > > > > > > > > that have never
> > > > > > > > > > > been converted to use QMP. The primary reason for this 
> > > > > > > > > > > lack of
> > > > > > > > > > > conversion is that they block execution of the thread for 
> > > > > > > > > > > as long as
> > > > > > > > > > > they run.
> > > > > > > > > > > 
> > > > > > > > > > > Despite this downside, however, libvirt and applications 
> > > > > > > > > > > using libvirt
> > > > > > > > > > > has used these commands for as long as QMP has existed, 
> > > > > > > > > > > via the
> > > > > > > > > > > "human-monitor-command" passthrough command. IOW, while 
> > > > > > > > > > > it is clearly
> > > > > > > > > > > desirable to be able to fix the blocking problem, this is 
> > > > > > > > > > > not an
> > > > > > > > > > > immediate obstacle to real world usage.
> > > > > > > > > > > 
> > > > > > > > > > > Meanwhile there is a need for other features which 
> > > > > > > > > > > involve adding new
> > > > > > > > > > > parameters to the commands. This is possible with HMP 
> > > > > > > > > > > passthrough, but
> > > > > > > > > > > it provides no reliable way for apps to introspect 
> > > > > > > > > > > features, so using
> > > > > > > > > > > QAPI modelling is highly desirable.
> > > > > > > > > > > 
> > > > > > > > > > > This patch thus introduces trival savevm, loadvm, delvm 
> > > > > > > > > > > commands
> > > > > > > > > > 
> > > > > > > > > > trivial
> > > > > > > > > > 
> > > > > > > > > > > to QMP that are functionally identical to the HMP 
> > > > > > > > > > > counterpart, including
> > > > > > > > > > > the blocking problem.
> > > > > > > > > > 
> > > > > > > > > > Should we name them 'x-savevm', 'x-loadvm', 'x-delvm' to 
> > > > > > > > > > give ourselves room
> > > > > > > > > > to change them when we DO solve the blocking issue?  Or 
> > > > > > > > > > will the solution of
> > > > > > > > > > the blocking issue introduce new QMP commands, at which 
> > > > > > > > > > point we can add QMP
> > > > > > > > > > deprecation markers on these commands to eventually retire 
> > > > > > > > > > them?
> > > > > > > > > 
> > > > > > > > > I was in two minds about this, so I'm open to arguments 
> > > > > > > > > either way.
> > > > > > > > > 
> > > > > > > > > The primary goal is for libvirt to consume the APIs as soon 
> > > > > > > > > as possible

Re: [PATCH 2/6] migration: introduce savevm, loadvm, delvm QMP commands

2020-07-03 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Fri, Jul 03, 2020 at 05:22:46PM +0100, Dr. David Alan Gilbert wrote:
> > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > On Fri, Jul 03, 2020 at 05:10:12PM +0100, Dr. David Alan Gilbert wrote:
> > > > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > > > On Fri, Jul 03, 2020 at 04:49:33PM +0100, Dr. David Alan Gilbert 
> > > > > wrote:
> > > > > > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > > > > > On Thu, Jul 02, 2020 at 01:12:52PM -0500, Eric Blake wrote:
> > > > > > > > On 7/2/20 12:57 PM, Daniel P. Berrangé wrote:
> > > > > > > > > savevm, loadvm and delvm are some of the few commands that 
> > > > > > > > > have never
> > > > > > > > > been converted to use QMP. The primary reason for this lack of
> > > > > > > > > conversion is that they block execution of the thread for as 
> > > > > > > > > long as
> > > > > > > > > they run.
> > > > > > > > > 
> > > > > > > > > Despite this downside, however, libvirt and applications 
> > > > > > > > > using libvirt
> > > > > > > > > has used these commands for as long as QMP has existed, via 
> > > > > > > > > the
> > > > > > > > > "human-monitor-command" passthrough command. IOW, while it is 
> > > > > > > > > clearly
> > > > > > > > > desirable to be able to fix the blocking problem, this is not 
> > > > > > > > > an
> > > > > > > > > immediate obstacle to real world usage.
> > > > > > > > > 
> > > > > > > > > Meanwhile there is a need for other features which involve 
> > > > > > > > > adding new
> > > > > > > > > parameters to the commands. This is possible with HMP 
> > > > > > > > > passthrough, but
> > > > > > > > > it provides no reliable way for apps to introspect features, 
> > > > > > > > > so using
> > > > > > > > > QAPI modelling is highly desirable.
> > > > > > > > > 
> > > > > > > > > This patch thus introduces trival savevm, loadvm, delvm 
> > > > > > > > > commands
> > > > > > > > 
> > > > > > > > trivial
> > > > > > > > 
> > > > > > > > > to QMP that are functionally identical to the HMP 
> > > > > > > > > counterpart, including
> > > > > > > > > the blocking problem.
> > > > > > > > 
> > > > > > > > Should we name them 'x-savevm', 'x-loadvm', 'x-delvm' to give 
> > > > > > > > ourselves room
> > > > > > > > to change them when we DO solve the blocking issue?  Or will 
> > > > > > > > the solution of
> > > > > > > > the blocking issue introduce new QMP commands, at which point 
> > > > > > > > we can add QMP
> > > > > > > > deprecation markers on these commands to eventually retire them?
> > > > > > > 
> > > > > > > I was in two minds about this, so I'm open to arguments either 
> > > > > > > way.
> > > > > > > 
> > > > > > > The primary goal is for libvirt to consume the APIs as soon as 
> > > > > > > possible,
> > > > > > > and generally libvirt doesn't want todo this is they are declared 
> > > > > > > experimental
> > > > > > > via a "x-" prefix. So that pushes me away from "x-".
> > > > > > > 
> > > > > > > If we don't have an "x-" prefix and want to make changes, we can 
> > > > > > > add extra
> > > > > > > parameters to trigger new behaviour in backwards compatible 
> > > > > > > manner. Or we can
> > > > > > > simply deprecate these commands, deleting them 2 releases later, 
> > > > > > > while adding
> > > > > > > completely new commands.
> > > >

Re: [PATCH 2/6] migration: introduce savevm, loadvm, delvm QMP commands

2020-07-03 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Fri, Jul 03, 2020 at 05:10:12PM +0100, Dr. David Alan Gilbert wrote:
> > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > On Fri, Jul 03, 2020 at 04:49:33PM +0100, Dr. David Alan Gilbert wrote:
> > > > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > > > On Thu, Jul 02, 2020 at 01:12:52PM -0500, Eric Blake wrote:
> > > > > > On 7/2/20 12:57 PM, Daniel P. Berrangé wrote:
> > > > > > > savevm, loadvm and delvm are some of the few commands that have 
> > > > > > > never
> > > > > > > been converted to use QMP. The primary reason for this lack of
> > > > > > > conversion is that they block execution of the thread for as long 
> > > > > > > as
> > > > > > > they run.
> > > > > > > 
> > > > > > > Despite this downside, however, libvirt and applications using 
> > > > > > > libvirt
> > > > > > > has used these commands for as long as QMP has existed, via the
> > > > > > > "human-monitor-command" passthrough command. IOW, while it is 
> > > > > > > clearly
> > > > > > > desirable to be able to fix the blocking problem, this is not an
> > > > > > > immediate obstacle to real world usage.
> > > > > > > 
> > > > > > > Meanwhile there is a need for other features which involve adding 
> > > > > > > new
> > > > > > > parameters to the commands. This is possible with HMP 
> > > > > > > passthrough, but
> > > > > > > it provides no reliable way for apps to introspect features, so 
> > > > > > > using
> > > > > > > QAPI modelling is highly desirable.
> > > > > > > 
> > > > > > > This patch thus introduces trival savevm, loadvm, delvm commands
> > > > > > 
> > > > > > trivial
> > > > > > 
> > > > > > > to QMP that are functionally identical to the HMP counterpart, 
> > > > > > > including
> > > > > > > the blocking problem.
> > > > > > 
> > > > > > Should we name them 'x-savevm', 'x-loadvm', 'x-delvm' to give 
> > > > > > ourselves room
> > > > > > to change them when we DO solve the blocking issue?  Or will the 
> > > > > > solution of
> > > > > > the blocking issue introduce new QMP commands, at which point we 
> > > > > > can add QMP
> > > > > > deprecation markers on these commands to eventually retire them?
> > > > > 
> > > > > I was in two minds about this, so I'm open to arguments either way.
> > > > > 
> > > > > The primary goal is for libvirt to consume the APIs as soon as 
> > > > > possible,
> > > > > and generally libvirt doesn't want todo this is they are declared 
> > > > > experimental
> > > > > via a "x-" prefix. So that pushes me away from "x-".
> > > > > 
> > > > > If we don't have an "x-" prefix and want to make changes, we can add 
> > > > > extra
> > > > > parameters to trigger new behaviour in backwards compatible manner. 
> > > > > Or we can
> > > > > simply deprecate these commands, deleting them 2 releases later, 
> > > > > while adding
> > > > > completely new commands.
> > > > > 
> > > > > If we think the prposed design will definitely need incompatible 
> > > > > changes in
> > > > > a very short time frame though, that would push towards "x-".
> > > > > 
> > > > > So IMHO the right answer largely depends on whether there is a 
> > > > > credible
> > > > > strategy to implement the ideal non-blocking solution in a reasonable 
> > > > > amount
> > > > > of time. I can't justify spending much time on this myself, but I'm 
> > > > > willing
> > > > > to consider & try proposals for solving the blocking problem if 
> > > > > they're not
> > > > > too complex / invasive.
> > > > 
> > > > Remind me, what was the problem with just making a block: migration
> > > > channel, and then

Re: [PATCH 2/6] migration: introduce savevm, loadvm, delvm QMP commands

2020-07-03 Thread Dr. David Alan Gilbert
* Peter Krempa (pkre...@redhat.com) wrote:
> On Fri, Jul 03, 2020 at 17:10:12 +0100, Dr. David Alan Gilbert wrote:
> > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > On Fri, Jul 03, 2020 at 04:49:33PM +0100, Dr. David Alan Gilbert wrote:
> > > > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > > > On Thu, Jul 02, 2020 at 01:12:52PM -0500, Eric Blake wrote:
> > > > > > On 7/2/20 12:57 PM, Daniel P. Berrangé wrote:
> 
> [...]
> 
> > > > Remind me, what was the problem with just making a block: migration
> > > > channel, and then we can migrate to it?
> > > 
> > > migration only does vmstate, not disks. The current blockdev commands
> > > are all related to external snapshots, nothing for internal snapshots
> > > AFAIK. So we still need commands to load/save internal snapshots of
> > > the disk data in the qcow2 files.
> > > 
> > > So we could look at loadvm/savevm conceptually as a syntax sugar above
> > > a migration transport that targets disk images, and blockdev QMP command
> > > that can do internal snapshots. Neither of these exist though and feel
> > > like a significantly larger amount of work than using existing 
> > > functionality
> > > that is currently working.
> > 
> > I think that's what we should aim for; adding this wrapper isn't gaining
> > that much without moving a bit towards that; so I would stick with the
> > x- for now.
> 
> Relying on the HMP variants is IMO even worse. Error handling is
> terrible there. I'd vote even for a straight wrapper without any logic
> at this point. IMO it's just necessary to document that it's an
> intermediate solution which WILL be deprecated and removed as soon as a
> suitable replacement is in place.
> 
> Not doing anything is the argument we hear for multiple years now and
> savevm/delvm/loadvm are now the only 3 commands used via the HMP wrapper
> in libvirt.
> 
> Since deprecation is now a thing I think we can add a disposable
> inteface. In the end HMP will or will not need to remain anyways and the
> deprecation there is IMO less clear.

Only if we come up with a list of what we actually need to do to
properly fix it; I'm not suggesting we actually need to do the work, but
we should figure out what we need to do.

Dave

--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 2/6] migration: introduce savevm, loadvm, delvm QMP commands

2020-07-03 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Fri, Jul 03, 2020 at 04:49:33PM +0100, Dr. David Alan Gilbert wrote:
> > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > On Thu, Jul 02, 2020 at 01:12:52PM -0500, Eric Blake wrote:
> > > > On 7/2/20 12:57 PM, Daniel P. Berrangé wrote:
> > > > > savevm, loadvm and delvm are some of the few commands that have never
> > > > > been converted to use QMP. The primary reason for this lack of
> > > > > conversion is that they block execution of the thread for as long as
> > > > > they run.
> > > > > 
> > > > > Despite this downside, however, libvirt and applications using libvirt
> > > > > has used these commands for as long as QMP has existed, via the
> > > > > "human-monitor-command" passthrough command. IOW, while it is clearly
> > > > > desirable to be able to fix the blocking problem, this is not an
> > > > > immediate obstacle to real world usage.
> > > > > 
> > > > > Meanwhile there is a need for other features which involve adding new
> > > > > parameters to the commands. This is possible with HMP passthrough, but
> > > > > it provides no reliable way for apps to introspect features, so using
> > > > > QAPI modelling is highly desirable.
> > > > > 
> > > > > This patch thus introduces trival savevm, loadvm, delvm commands
> > > > 
> > > > trivial
> > > > 
> > > > > to QMP that are functionally identical to the HMP counterpart, 
> > > > > including
> > > > > the blocking problem.
> > > > 
> > > > Should we name them 'x-savevm', 'x-loadvm', 'x-delvm' to give ourselves 
> > > > room
> > > > to change them when we DO solve the blocking issue?  Or will the 
> > > > solution of
> > > > the blocking issue introduce new QMP commands, at which point we can 
> > > > add QMP
> > > > deprecation markers on these commands to eventually retire them?
> > > 
> > > I was in two minds about this, so I'm open to arguments either way.
> > > 
> > > The primary goal is for libvirt to consume the APIs as soon as possible,
> > > and generally libvirt doesn't want todo this is they are declared 
> > > experimental
> > > via a "x-" prefix. So that pushes me away from "x-".
> > > 
> > > If we don't have an "x-" prefix and want to make changes, we can add extra
> > > parameters to trigger new behaviour in backwards compatible manner. Or we 
> > > can
> > > simply deprecate these commands, deleting them 2 releases later, while 
> > > adding
> > > completely new commands.
> > > 
> > > If we think the prposed design will definitely need incompatible changes 
> > > in
> > > a very short time frame though, that would push towards "x-".
> > > 
> > > So IMHO the right answer largely depends on whether there is a credible
> > > strategy to implement the ideal non-blocking solution in a reasonable 
> > > amount
> > > of time. I can't justify spending much time on this myself, but I'm 
> > > willing
> > > to consider & try proposals for solving the blocking problem if they're 
> > > not
> > > too complex / invasive.
> > 
> > Remind me, what was the problem with just making a block: migration
> > channel, and then we can migrate to it?
> 
> migration only does vmstate, not disks. The current blockdev commands
> are all related to external snapshots, nothing for internal snapshots
> AFAIK. So we still need commands to load/save internal snapshots of
> the disk data in the qcow2 files.
> 
> So we could look at loadvm/savevm conceptually as a syntax sugar above
> a migration transport that targets disk images, and blockdev QMP command
> that can do internal snapshots. Neither of these exist though and feel
> like a significantly larger amount of work than using existing functionality
> that is currently working.

I think that's what we should aim for; adding this wrapper isn't gaining
that much without moving a bit towards that; so I would stick with the
x- for now.

Dave

> Regards,
> Daniel
> -- 
> |: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-https://fstop138.berrange.com :|
> |: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 2/6] migration: introduce savevm, loadvm, delvm QMP commands

2020-07-03 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Thu, Jul 02, 2020 at 01:12:52PM -0500, Eric Blake wrote:
> > On 7/2/20 12:57 PM, Daniel P. Berrangé wrote:
> > > savevm, loadvm and delvm are some of the few commands that have never
> > > been converted to use QMP. The primary reason for this lack of
> > > conversion is that they block execution of the thread for as long as
> > > they run.
> > > 
> > > Despite this downside, however, libvirt and applications using libvirt
> > > has used these commands for as long as QMP has existed, via the
> > > "human-monitor-command" passthrough command. IOW, while it is clearly
> > > desirable to be able to fix the blocking problem, this is not an
> > > immediate obstacle to real world usage.
> > > 
> > > Meanwhile there is a need for other features which involve adding new
> > > parameters to the commands. This is possible with HMP passthrough, but
> > > it provides no reliable way for apps to introspect features, so using
> > > QAPI modelling is highly desirable.
> > > 
> > > This patch thus introduces trival savevm, loadvm, delvm commands
> > 
> > trivial
> > 
> > > to QMP that are functionally identical to the HMP counterpart, including
> > > the blocking problem.
> > 
> > Should we name them 'x-savevm', 'x-loadvm', 'x-delvm' to give ourselves room
> > to change them when we DO solve the blocking issue?  Or will the solution of
> > the blocking issue introduce new QMP commands, at which point we can add QMP
> > deprecation markers on these commands to eventually retire them?
> 
> I was in two minds about this, so I'm open to arguments either way.
> 
> The primary goal is for libvirt to consume the APIs as soon as possible,
> and generally libvirt doesn't want todo this is they are declared experimental
> via a "x-" prefix. So that pushes me away from "x-".
> 
> If we don't have an "x-" prefix and want to make changes, we can add extra
> parameters to trigger new behaviour in backwards compatible manner. Or we can
> simply deprecate these commands, deleting them 2 releases later, while adding
> completely new commands.
> 
> If we think the prposed design will definitely need incompatible changes in
> a very short time frame though, that would push towards "x-".
> 
> So IMHO the right answer largely depends on whether there is a credible
> strategy to implement the ideal non-blocking solution in a reasonable amount
> of time. I can't justify spending much time on this myself, but I'm willing
> to consider & try proposals for solving the blocking problem if they're not
> too complex / invasive.

Remind me, what was the problem with just making a block: migration
channel, and then we can migrate to it?

Dave

> I just don't want to end up having a "x-savevm" command for another 10 years,
> waiting for a perfect solution that never arrives because people always have
> higher priority items, as apps are clearly able to accept the blocking problem
> if the alternative is no snapshots at all.
> 
> 
> > > +
> > > +##
> > > +# @savevm:
> > > +#
> > > +# Save a VM snapshot
> > > +#
> > > +# @tag: name of the snapshot to create. If it already
> > > +# exists it will be replaced.
> > > +#
> > > +# Note that execution of the VM will be paused during the time
> > > +# it takes to save the snapshot
> > > +#
> > > +# Returns: nothing
> > > +#
> > > +# Example:
> > > +#
> > > +# -> { "execute": "savevm",
> > > +#  "data": {
> > > +# "tag": "my-snap"
> > > +#  }
> > > +#}
> > > +# <- { "return": { } }
> > > +#
> > > +# Since: 5.2
> > 
> > I guess you are NOT trying to make 5.1 soft freeze next week?
> 
> Correct. It is unrealistic to consider this for soft freeze.
> 
> I'd really love to have a solution in 5.2 though, even if it doesn't
> solve all our problems. Something that can at least unblock apps that
> want to use OVMF with internal snapshots today.
> 
> Regards,
> Daniel
> -- 
> |: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-https://fstop138.berrange.com :|
> |: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 1/6] migration: improve error reporting of block driver state name

2020-07-02 Thread Dr. David Alan Gilbert
* Eric Blake (ebl...@redhat.com) wrote:
> On 7/2/20 12:57 PM, Daniel P. Berrangé wrote:
> > With blockdev, a BlockDriverState may not have an device name,
> 
> s/an/a/
> 
> > so using a node name is required as an alternative.
> > 
> > Signed-off-by: Daniel P. Berrangé 
> > ---
> >   migration/savevm.c | 12 ++--
> >   1 file changed, 6 insertions(+), 6 deletions(-)
> > 
> 
> Reviewed-by: Eric Blake 

Why don't you send this one to trivial.

Dave

> 
> -- 
> Eric Blake, Principal Software Engineer
> Red Hat, Inc.       +1-919-301-3226
> Virtualization:  qemu.org | libvirt.org
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 2/4] migration: Add block-bitmap-mapping parameter

2020-07-02 Thread Dr. David Alan Gilbert
* Max Reitz (mre...@redhat.com) wrote:
> On 30.06.20 12:51, Dr. David Alan Gilbert wrote:
> > * Max Reitz (mre...@redhat.com) wrote:
> >> This migration parameter allows mapping block node names and bitmap
> >> names to aliases for the purpose of block dirty bitmap migration.
> >>
> >> This way, management tools can use different node and bitmap names on
> >> the source and destination and pass the mapping of how bitmaps are to be
> >> transferred to qemu (on the source, the destination, or even both with
> >> arbitrary aliases in the migration stream).
> >>
> >> Suggested-by: Vladimir Sementsov-Ogievskiy 
> >> Signed-off-by: Max Reitz 
> >> ---
> >>  qapi/migration.json|  83 +++-
> >>  migration/migration.h  |   3 +
> >>  migration/block-dirty-bitmap.c | 372 -
> >>  migration/migration.c  |  29 +++
> >>  4 files changed, 432 insertions(+), 55 deletions(-)
> >>
> >> diff --git a/qapi/migration.json b/qapi/migration.json
> >> index d5000558c6..5aeae9bea8 100644
> >> --- a/qapi/migration.json
> >> +++ b/qapi/migration.json
> >> @@ -507,6 +507,44 @@
> >>'data': [ 'none', 'zlib',
> >>  { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
> >>  
> >> +##
> >> +# @BitmapMigrationBitmapAlias:
> >> +#
> >> +# @name: The name of the bitmap.
> >> +#
> >> +# @alias: An alias name for migration (for example the bitmap name on
> >> +# the opposite site).
> >> +#
> >> +# Since: 5.1
> >> +##
> >> +{ 'struct': 'BitmapMigrationBitmapAlias',
> >> +  'data': {
> >> +  'name': 'str',
> >> +  'alias': 'str'
> >> +  } }
> >> +
> >> +##
> >> +# @BitmapMigrationNodeAlias:
> >> +#
> >> +# Maps a block node name and the bitmaps it has to aliases for dirty
> >> +# bitmap migration.
> >> +#
> >> +# @node-name: A block node name.
> >> +#
> >> +# @alias: An alias block node name for migration (for example the
> >> +# node name on the opposite site).
> >> +#
> >> +# @bitmaps: Mappings for the bitmaps on this node.
> >> +#
> >> +# Since: 5.1
> >> +##
> >> +{ 'struct': 'BitmapMigrationNodeAlias',
> >> +  'data': {
> >> +  'node-name': 'str',
> >> +  'alias': 'str',
> >> +  'bitmaps': [ 'BitmapMigrationBitmapAlias' ]
> >> +  } }
> >> +
> >>  ##
> >>  # @MigrationParameter:
> >>  #
> >> @@ -641,6 +679,18 @@
> >>  #  will consume more CPU.
> >>  #  Defaults to 1. (Since 5.0)
> >>  #
> >> +# @block-bitmap-mapping: Maps block nodes and bitmaps on them to
> >> +#  aliases for the purpose of dirty bitmap migration.  Such
> >> +#  aliases may for example be the corresponding names on the
> >> +#  opposite site.
> >> +#  The mapping must be one-to-one and complete: On the source,
> >> +#  migrating a bitmap from a node when either is not mapped
> >> +#  will result in an error.  On the destination, similarly,
> >> +#  receiving a bitmap (by alias) from a node (by alias) when
> >> +#  either alias is not mapped will result in an error.
> >> +#  By default, all node names and bitmap names are mapped to
> >> +#  themselves. (Since 5.1)
> >> +#
> >>  # Since: 2.4
> >>  ##
> >>  { 'enum': 'MigrationParameter',
> >> @@ -655,7 +705,8 @@
> >> 'multifd-channels',
> >> 'xbzrle-cache-size', 'max-postcopy-bandwidth',
> >> 'max-cpu-throttle', 'multifd-compression',
> >> -   'multifd-zlib-level' ,'multifd-zstd-level' ] }
> >> +   'multifd-zlib-level' ,'multifd-zstd-level',
> >> +   'block-bitmap-mapping' ] }
> >>  
> >>  ##
> >>  # @MigrateSetParameters:
> >> @@ -781,6 +832,18 @@
> >>  #  will consume more CPU.
> >>  #  Defaults to 1. (Since 5.0)
> >>  #
> >> +# @block-bitmap-mapping: Maps block nodes and bitmaps on them to
> >> +#  aliases for the purpose of dirty bitmap migration.  Such
> >> +#  aliases may for example be the corresponding names on the
> >> +#  opposite site.
> >> +#  The mapping must be one-to-

Re: [PATCH 2/4] migration: Add block-bitmap-mapping parameter

2020-06-30 Thread Dr. David Alan Gilbert
->node_name, s->node_name, _err);
> +
> +if (alias_map) {
> +const AliasMapInnerNode *amin;
> +
> +amin = g_hash_table_lookup(alias_map, s->node_alias);
> +if (!amin) {
> +error_report("Error: Unknown node alias '%s'", 
> s->node_alias);
> +return -EINVAL;
> +}
> +
> +bitmap_alias_map = amin->subtree;
> +s->bs = bdrv_lookup_bs(NULL, amin->string, _err);
> +} else {
> +s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias, _err);
> +}
>  if (!s->bs) {
>  error_report_err(local_err);
>  return -EINVAL;
>  }
> -} else if (!s->bs && !nothing) {
> +} else if (s->bs) {
> +if (alias_map) {
> +const AliasMapInnerNode *amin;
> +
> +/* Must be present in the map, or s->bs would not be set */
> +amin = g_hash_table_lookup(alias_map, s->node_alias);
> +assert(amin != NULL);
> +
> +bitmap_alias_map = amin->subtree;
> +}
> +} else if (!nothing) {
>  error_report("Error: block device name is not set");
>  return -EINVAL;
>  }
>  
> +assert(!!alias_map == !!bitmap_alias_map);
> +
>  if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
> -if (!qemu_get_counted_string(f, s->bitmap_name)) {
> +const char *bitmap_name;
> +
> +if (!qemu_get_counted_string(f, s->bitmap_alias)) {
>  error_report("Unable to read bitmap name string");
>  return -EINVAL;
>  }
> +
> +if (bitmap_alias_map) {
> +bitmap_name = g_hash_table_lookup(bitmap_alias_map,
> +  s->bitmap_alias);
> +if (!bitmap_name) {
> +error_report("Error: Unknown bitmap alias '%s' on node '%s' "
> + "(alias '%s')", s->bitmap_alias, 
> s->bs->node_name,
> + s->node_alias);
> +return -EINVAL;
> +}
> +} else {
> +bitmap_name = s->bitmap_alias;
> +}
> +
> +g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name));
>  s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
>  
>  /* bitmap may be NULL here, it wouldn't be an error if it is the
> @@ -698,7 +954,7 @@ static int dirty_bitmap_load_header(QEMUFile *f, 
> DirtyBitmapLoadState *s)
>  if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
>  error_report("Error: unknown dirty bitmap "
>   "'%s' for block device '%s'",
> - s->bitmap_name, s->node_name);
> + s->bitmap_name, s->bs->node_name);
>  return -EINVAL;
>  }
>  } else if (!s->bitmap && !nothing) {
> @@ -711,6 +967,8 @@ static int dirty_bitmap_load_header(QEMUFile *f, 
> DirtyBitmapLoadState *s)
>  
>  static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
>  {
> +GHashTable *alias_map = NULL;
> +const MigrationParameters *mig_params = 
> _get_current()->parameters;
>  static DirtyBitmapLoadState s;
>  int ret = 0;
>  
> @@ -720,10 +978,15 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, 
> int version_id)
>  return -EINVAL;
>  }
>  
> +if (mig_params->has_block_bitmap_mapping) {
> +alias_map = construct_alias_map(mig_params->block_bitmap_mapping,
> +false, _abort);
> +}
> +
>  do {
> -ret = dirty_bitmap_load_header(f, );
> +ret = dirty_bitmap_load_header(f, , alias_map);
>  if (ret < 0) {
> -return ret;
> +goto fail;
>  }
>  
>  if (s.flags & DIRTY_BITMAP_MIG_FLAG_START) {
> @@ -739,12 +1002,17 @@ static int dirty_bitmap_load(QEMUFile *f, void 
> *opaque, int version_id)
>  }
>  
>      if (ret) {
> -return ret;
> +goto fail;
>  }
>  } while (!(s.flags & DIRTY_BITMAP_MIG_FLAG_EOS));
>  
>  trace_dirty_bitmap_load_success();
> -return 0;
> +ret = 0;
> +fail:
> +if (alias_map) {
> +g_hash_table_destroy(alias_map);
> +}
> +return ret;
>  }
>  
>  static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque)
> diff --git a/migration/migration.c b/migration/migration.c
> index 47c7da4e55..23fc13e527 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -35,6 +35,7 @@
>  #include "block/block.h"
>  #include "qapi/error.h"
>  #include "qapi/clone-visitor.h"
> +#include "qapi/qapi-visit-migration.h"
>  #include "qapi/qapi-visit-sockets.h"
>  #include "qapi/qapi-commands-migration.h"
>  #include "qapi/qapi-events-migration.h"
> @@ -825,6 +826,12 @@ MigrationParameters *qmp_query_migrate_parameters(Error 
> **errp)
>  params->has_announce_step = true;
>  params->announce_step = s->parameters.announce_step;
>  
> +if (s->parameters.has_block_bitmap_mapping) {
> +params->has_block_bitmap_mapping = true;
> +params->block_bitmap_mapping = 
> QAPI_CLONE(BitmapMigrationNodeAliasList,
> +  
> params->block_bitmap_mapping);
> +}
> +
>  return params;
>  }
>  
> @@ -1292,6 +1299,13 @@ static bool migrate_params_check(MigrationParameters 
> *params, Error **errp)
> "is invalid, it must be in the range of 1 to 1 ms");
> return false;
>  }
> +
> +if (params->has_block_bitmap_mapping &&
> +!check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, 
> errp)) {
> +error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: 
> ");
> +return false;
> +}
> +
>  return true;
>  }
>  
> @@ -1386,6 +1400,11 @@ static void 
> migrate_params_test_apply(MigrateSetParameters *params,
>  if (params->has_announce_step) {
>  dest->announce_step = params->announce_step;
>  }
> +
> +if (params->has_block_bitmap_mapping) {
> +dest->has_block_bitmap_mapping = true;
> +dest->block_bitmap_mapping = params->block_bitmap_mapping;
> +}
>  }
>  
>  static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
> @@ -1498,6 +1517,16 @@ static void migrate_params_apply(MigrateSetParameters 
> *params, Error **errp)
>  if (params->has_announce_step) {
>  s->parameters.announce_step = params->announce_step;
>  }
> +
> +if (params->has_block_bitmap_mapping) {
> +qapi_free_BitmapMigrationNodeAliasList(
> +s->parameters.block_bitmap_mapping);
> +
> +s->parameters.has_block_bitmap_mapping = true;
> +s->parameters.block_bitmap_mapping =
> +QAPI_CLONE(BitmapMigrationNodeAliasList,
> +   params->block_bitmap_mapping);
> +}
>  }
>  
>  void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
> -- 
> 2.26.2
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 1/4] migration: Prevent memleak by ...params_test_apply

2020-06-30 Thread Dr. David Alan Gilbert
* Max Reitz (mre...@redhat.com) wrote:
> The created structure is not really a proper QAPI object, so we cannot
> and will not free its members.  Strings therein should therefore not be
> duplicated, or we will leak them.
> 
> Signed-off-by: Max Reitz 
> ---
>  migration/migration.c | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/migration/migration.c b/migration/migration.c
> index 481a590f72..47c7da4e55 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -1336,12 +1336,12 @@ static void 
> migrate_params_test_apply(MigrateSetParameters *params,
>  
>  if (params->has_tls_creds) {
>  assert(params->tls_creds->type == QTYPE_QSTRING);
> -dest->tls_creds = g_strdup(params->tls_creds->u.s);
> +dest->tls_creds = params->tls_creds->u.s;
>  }
>  
>  if (params->has_tls_hostname) {
>  assert(params->tls_hostname->type == QTYPE_QSTRING);
> -dest->tls_hostname = g_strdup(params->tls_hostname->u.s);
> +dest->tls_hostname = params->tls_hostname->u.s;
>      }

Yeh I think I agree.

Reviewed-by: Dr. David Alan Gilbert 

>  
>  if (params->has_max_bandwidth) {
> -- 
> 2.26.2
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v2 00/16] Crazy shit around -global (pardon my french)

2020-06-29 Thread Dr. David Alan Gilbert
* Markus Armbruster (arm...@redhat.com) wrote:
> Cc: David for insurance against me spewing nonsense about migration.
> 
> John Snow  writes:
> 
> > On 6/25/20 12:45 AM, Markus Armbruster wrote:
> >> John Snow  writes:
> >> 
> >>> On 6/22/20 5:42 AM, Markus Armbruster wrote:
> >>>> There are three ways to configure backends:
> >>>>
> >>>> * -nic, -serial, -drive, ... (onboard devices)
> >>>>
> >>>> * Set the property with -device, or, if you feel masochistic, with
> >>>>   -set device (pluggable devices)
> >>>>
> >>>> * Set the property with -global (both)
> >>>>
> >>>> The trouble is -global is terrible.
> >>>>
> >>>> It gets applied in object_new(), which can't fail.  We treat failure
> >>>> to apply -global as fatal error, except when hot-plugging, where we
> >>>> treat it as warning *boggle*.  I'm not addressing that today.
> >>>>
> >>>> Some code falls apart when you use both -global and the other way.
> >>>>
> >>>> To make life more interesting, we gave -drive two roles: with
> >>>> interface type other than none, it's for configuring onboard devices,
> >>>> and with interface type none, it's for defining backends for use with
> >>>> -device and such.  Since we neglect to require interface type none for
> >>>> the latter, you can use one -drive in both roles.  This confuses the
> >>>> code about as much as you, dear reader, probably are by now.
> >>>>
> >>>> Because this still isn't interesting enough, there's yet another way
> >>>> to configure backends, just for floppies: set the floppy controller's
> >>>> property.  Goes back to the time when floppy wasn't a separate device,
> >>>> and involves some Bad Magic.  Now -global can interact with itself!
> >>>>
> >>>> Digging through all this took me an embarrassing amount of time.
> >>>> Hair, too.
> >>>>
> >>>> My patches reject some the silliest uses outright, and deprecate some
> >>>> not so silly ones that have replacements.
> >>>>
> >>>> Apply on top of my "[PATCH v2 00/58] qdev: Rework how we plug into the
> >>>> parent bus".
> >>>>
> >>>
> >>> Oof. Thank you for your work in fixing our darkest corners. I sincerely
> >>> appreciate it.
> >>>
> >>> The qdev tree ordering problems don't cause any issues for migration, do
> >>> they?
> >> 
> >> This series should only change device configuration, not device state or
> >> its encoding in the migration stream.
> >> 
> >> I'm not sure what you mean by "qdev tree ordering problems".  Ist it
> >> commit e8c9e65816 'qom: Make "info qom-tree" show children sorted'?
> >> 
> >>> (I see you already sent a PR, so whatever!)
> >> 
> >> A question that might avoid a later migration debugging session is
> >> *never* "whatever"!
> >> 
> >
> > I thought I had read that one of these patches changes the order in
> > which devices get instantiated, which I thought might change their QOM
> > paths. Which I thought *might* have some ramifications for migration,
> > but wasn't sure.
> 
> Device instantiation order changes should not break migration.

They shouldn't; although I only narrowly stopped a new device from
making a mistake that would have made it dependent.
Of course you do have to explicitly state PCI/USB slot IDs otherwise the
allocation of those is order dependent.

> The order in which devices appear in the migration stream should not
> matter.

Order in the stream is a separate issue; we have ways to enforce that;
for example you want the interrupt controller to arrive before a device
that will raise an interrupt.

Dave


Dave

> > If it's just showing the same path outputs *sorted*, then there's no
> > problem.
> >
> > Likely misread.
> >
> > --js
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 0/4] block: seriously improve savevm performance

2020-06-15 Thread Dr. David Alan Gilbert
* Denis V. Lunev (d...@openvz.org) wrote:
> On 6/15/20 3:17 PM, Dr. David Alan Gilbert wrote:
> > * Denis V. Lunev (d...@openvz.org) wrote:
> >> This series do standard basic things:
> >> - it creates intermediate buffer for all writes from QEMU migration code
> >>   to QCOW2 image,
> >> - this buffer is sent to disk asynchronously, allowing several writes to
> >>   run in parallel.
> >>
> >> In general, migration code is fantastically inefficent (by observation),
> >> buffers are not aligned and sent with arbitrary pieces, a lot of time
> >> less than 100 bytes at a chunk, which results in read-modify-write
> >> operations with non-cached operations. It should also be noted that all
> >> operations are performed into unallocated image blocks, which also suffer
> >> due to partial writes to such new clusters.
> > It surprises me a little that you're not benefiting from the buffer
> > internal to qemu-file.c
> >
> > Dave
> There are a lot of problems with this buffer:
> 
> Flushes to block driver state are performed in the abstract places,
> pushing
>   a) small IO
>   b) non-aligned IO both to
>        1) page size
>        2) cluster size
> It should also be noted that buffer in QEMU file is quite small and
> all IO operations with it are synchronous. IO, like ethernet, wants
> good queues.

Yeh, for ethernet we immediately get the kernels buffer so it's not too
bad; and I guess the async page writes are easier as well.

Dave

> The difference is on the table.
> 
> Den
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 0/4] block: seriously improve savevm performance

2020-06-15 Thread Dr. David Alan Gilbert
* Denis V. Lunev (d...@openvz.org) wrote:
> This series do standard basic things:
> - it creates intermediate buffer for all writes from QEMU migration code
>   to QCOW2 image,
> - this buffer is sent to disk asynchronously, allowing several writes to
>   run in parallel.
> 
> In general, migration code is fantastically inefficent (by observation),
> buffers are not aligned and sent with arbitrary pieces, a lot of time
> less than 100 bytes at a chunk, which results in read-modify-write
> operations with non-cached operations. It should also be noted that all
> operations are performed into unallocated image blocks, which also suffer
> due to partial writes to such new clusters.

It surprises me a little that you're not benefiting from the buffer
internal to qemu-file.c

Dave

> This patch series is an implementation of idea discussed in the RFC
> posted by Denis Plotnikov
> https://lists.gnu.org/archive/html/qemu-devel/2020-04/msg01925.html
> Results with this series over NVME are better than original code
> original rfcthis
> cached:  1.79s  2.38s   1.27s
> non-cached:  3.29s  1.31s   0.81s
> 
> Changes from v2:
> - code moved from QCOW2 level to generic block level
> - created bdrv_flush_vmstate helper to fix 022, 029 tests
> - added recursive for bs->file in bdrv_co_flush_vmstate (fix 267)
> - fixed blk_save_vmstate helper
> - fixed coroutine wait as Vladimir suggested with waiting fixes from me
> 
> Changes from v1:
> - patchew warning fixed
> - fixed validation that only 1 waiter is allowed in patch 1
> 
> Signed-off-by: Denis V. Lunev 
> CC: Kevin Wolf 
> CC: Max Reitz 
> CC: Stefan Hajnoczi 
> CC: Fam Zheng 
> CC: Juan Quintela 
> CC: "Dr. David Alan Gilbert" 
> CC: Vladimir Sementsov-Ogievskiy 
> CC: Denis Plotnikov 
> 
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 1/4] migration/savevm: respect qemu_fclose() error code in save_snapshot()

2020-06-15 Thread Dr. David Alan Gilbert
* Denis V. Lunev (d...@openvz.org) wrote:
> qemu_fclose() could return error, f.e. if bdrv_co_flush() will return
> the error.
> 
> This validation will become more important once we will start waiting of
> asynchronous IO operations, started from bdrv_write_vmstate(), which are
> coming soon.
> 
> Signed-off-by: Denis V. Lunev 
> CC: Kevin Wolf 
> CC: Max Reitz 
> CC: Stefan Hajnoczi 
> CC: Fam Zheng 
> CC: Juan Quintela 
> CC: "Dr. David Alan Gilbert" 
> CC: Vladimir Sementsov-Ogievskiy 
> CC: Denis Plotnikov 

We check the return value in very few other places; I think in the
migration case we do flushes and assume that if the flushes work we
were OK; then most of the closes happen on error paths or after points
we think we're done.

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  migration/savevm.c | 8 ++--
>  1 file changed, 6 insertions(+), 2 deletions(-)
> 
> diff --git a/migration/savevm.c b/migration/savevm.c
> index c00a6807d9..0ff5bb40ed 100644
> --- a/migration/savevm.c
> +++ b/migration/savevm.c
> @@ -2628,7 +2628,7 @@ int save_snapshot(const char *name, Error **errp)
>  {
>  BlockDriverState *bs, *bs1;
>  QEMUSnapshotInfo sn1, *sn = , old_sn1, *old_sn = _sn1;
> -int ret = -1;
> +int ret = -1, ret2;
>  QEMUFile *f;
>  int saved_vm_running;
>  uint64_t vm_state_size;
> @@ -2712,10 +2712,14 @@ int save_snapshot(const char *name, Error **errp)
>  }
>  ret = qemu_savevm_state(f, errp);
>  vm_state_size = qemu_ftell(f);
> -qemu_fclose(f);
> +ret2 = qemu_fclose(f);
>  if (ret < 0) {
>  goto the_end;
>  }
> +if (ret2 < 0) {
> +ret = ret2;
> +goto the_end;
> +}
>  
>  /* The bdrv_all_create_snapshot() call that follows acquires the 
> AioContext
>   * for itself.  BDRV_POLL_WHILE() does not support nested locking because
> -- 
> 2.17.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH] iotests: 194: wait migration completion on target too

2020-06-04 Thread Dr. David Alan Gilbert
* Vladimir Sementsov-Ogievskiy (vsement...@virtuozzo.com) wrote:
> It is possible, that shutdown on target occurs earlier than migration
> finish. In this case we crash in bdrv_release_dirty_bitmap_locked()
> on assertion "assert(!bdrv_dirty_bitmap_busy(bitmap));" as we do have
> busy bitmap, as bitmap migration is ongoing.
> 
> We'll fix bitmap migration to gracefully cancel on early shutdown soon.
> Now let's fix iotest 194 to wait migration completion before shutdown.
> 
> Note that in this test dest_vm.shutdown() is called implicitly, as vms
> used as context-providers, see __exit__() method of QEMUMachine class.
> 
> Actually, not waiting migration finish is a wrong thing, but the test
> started to crash after commit ae00aa239847682
> "iotests: 194: test also migration of dirty bitmap", which added dirty
> bitmaps here. So, Fixes: tag won't hurt.

Without knowing the iotests framework, the actual problem sounds right;
I just fixed a similar bug in the acceptance test migration code.
Every migration test seems to make the same mistake!

Dave

> Fixes: ae00aa2398476824f0eca80461da215e7cdc1c3b
> Reported-by: Thomas Huth 
> Signed-off-by: Vladimir Sementsov-Ogievskiy 
> ---
>  tests/qemu-iotests/194 | 10 ++
>  tests/qemu-iotests/194.out |  5 +
>  2 files changed, 15 insertions(+)
> 
> diff --git a/tests/qemu-iotests/194 b/tests/qemu-iotests/194
> index 3fad7c6c1a..6dc2bc94d7 100755
> --- a/tests/qemu-iotests/194
> +++ b/tests/qemu-iotests/194
> @@ -87,4 +87,14 @@ with iotests.FilePath('source.img') as source_img_path, \
>  iotests.log(dest_vm.qmp('nbd-server-stop'))
>  break
>  
> +iotests.log('Wait migration completion on target...')
> +migr_events = (('MIGRATION', {'data': {'status': 'completed'}}),
> +   ('MIGRATION', {'data': {'status': 'failed'}}))
> +event = dest_vm.events_wait(migr_events)
> +iotests.log(event, filters=[iotests.filter_qmp_event])
> +
> +iotests.log('Check bitmaps on source:')
>  iotests.log(source_vm.qmp('query-block')['return'][0]['dirty-bitmaps'])
> +
> +iotests.log('Check bitmaps on target:')
> +iotests.log(dest_vm.qmp('query-block')['return'][0]['dirty-bitmaps'])
> diff --git a/tests/qemu-iotests/194.out b/tests/qemu-iotests/194.out
> index dd60dcc14f..f70cf7610e 100644
> --- a/tests/qemu-iotests/194.out
> +++ b/tests/qemu-iotests/194.out
> @@ -21,4 +21,9 @@ Gracefully ending the `drive-mirror` job on source...
>  {"data": {"device": "mirror-job0", "len": 1073741824, "offset": 1073741824, 
> "speed": 0, "type": "mirror"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": 
> {"microseconds": "USECS", "seconds": "SECS"}}
>  Stopping the NBD server on destination...
>  {"return": {}}
> +Wait migration completion on target...
> +{"data": {"status": "completed"}, "event": "MIGRATION", "timestamp": 
> {"microseconds": "USECS", "seconds": "SECS"}}
> +Check bitmaps on source:
> +[{"busy": false, "count": 0, "granularity": 65536, "name": "bitmap0", 
> "persistent": false, "recording": true, "status": "active"}]
> +Check bitmaps on target:
>  [{"busy": false, "count": 0, "granularity": 65536, "name": "bitmap0", 
> "persistent": false, "recording": true, "status": "active"}]
> -- 
> 2.21.0
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 2/5] vhost: involve device backends in feature negotiation

2020-05-29 Thread Dr. David Alan Gilbert
g them in
> feature negotiation without breaking existing backends.
> 
> Would you like me to rephrase this comment in some way?
> 
> > > + *
> > > + * New feature bits added to the VIRTIO spec should usually be included 
> > > here
> > > + * so that existing vhost device backends that do not support them yet 
> > > continue
> > > + * to work.
> > 
> > 
> > It actually depends on the type of backend.
> > 
> > Kernel vhost-net does not validate GSO features since qemu can talk directly
> > to TAP and vhost doesn't report those features. But for vhost-user GSO
> > features must be validated by qemu since qemu can't see what is behind
> > vhost-user.
> 
> Maybe the comment should say "New transport/vring feature bits". I'm
> thinking about things like VIRTIO_F_RING_PACKED that are not
> device-specific but require backend support.
> 
> The GSO features you mentioned are device-specific. Devices that want to
> let the backend advertise device-specific features cannot use
> vhost_default_feature_bits[].
> 
> > > + */
> > > +static const int vhost_default_feature_bits[] = {
> > > +VIRTIO_F_IOMMU_PLATFORM,
> > > +VIRTIO_F_RING_PACKED,
> > > +VHOST_INVALID_FEATURE_BIT
> > > +};
> > > +
> > >   bool vhost_has_free_slot(void)
> > >   {
> > >   unsigned int slots_limit = ~0U;
> > > @@ -1468,6 +1485,11 @@ uint64_t vhost_get_features(struct vhost_dev 
> > > *hdev, const int *feature_bits,
> > >   return features;
> > >   }
> > > +uint64_t vhost_get_default_features(struct vhost_dev *hdev, uint64_t 
> > > features)
> > > +{
> > > +return vhost_get_features(hdev, vhost_default_feature_bits, 
> > > features);
> > > +}
> > 
> > 
> > There's probably no need for a new helper, we can do all these inside
> > vhost_get_features().
> 
> Would you prefer:
> 
>   extern const int vhost_default_feature_bits[];
> 
> And then callers do:
> 
>   vhost_get_features(hdev, vhost_default_feature_bits, features);
> 
> ?


--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 5/5] virtio: enable VIRTIO_F_RING_PACKED for all devices

2020-05-26 Thread Dr. David Alan Gilbert
* Stefan Hajnoczi (stefa...@redhat.com) wrote:
> The packed virtqueue layout was introduced in VIRTIO 1.1. It is a single
> ring instead of a split avail/used ring design. There are CPU cache
> advantages to this layout and it is also suited better to hardware
> implementation.
> 
> The vhost-net backend has already supported packed virtqueues for some
> time. Performance benchmarks show that virtio-blk performance on NVMe
> drives is also improved.
> 
> Go ahead and enable this feature for all VIRTIO devices. Keep it
> disabled for QEMU 5.0 and earlier machine types.
> 
> Signed-off-by: Stefan Hajnoczi 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  include/hw/virtio/virtio.h |  2 +-
>  hw/core/machine.c  | 18 +-
>  2 files changed, 18 insertions(+), 2 deletions(-)
> 
> diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h
> index b69d517496..fd5b4a2044 100644
> --- a/include/hw/virtio/virtio.h
> +++ b/include/hw/virtio/virtio.h
> @@ -292,7 +292,7 @@ typedef struct VirtIORNGConf VirtIORNGConf;
>  DEFINE_PROP_BIT64("iommu_platform", _state, _field, \
>VIRTIO_F_IOMMU_PLATFORM, false), \
>  DEFINE_PROP_BIT64("packed", _state, _field, \
> -  VIRTIO_F_RING_PACKED, false)
> +  VIRTIO_F_RING_PACKED, true)
>  
>  hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n);
>  bool virtio_queue_enabled(VirtIODevice *vdev, int n);
> diff --git a/hw/core/machine.c b/hw/core/machine.c
> index bb3a7b18b1..3598c3c825 100644
> --- a/hw/core/machine.c
> +++ b/hw/core/machine.c
> @@ -28,7 +28,23 @@
>  #include "hw/mem/nvdimm.h"
>  #include "migration/vmstate.h"
>  
> -GlobalProperty hw_compat_5_0[] = {};
> +GlobalProperty hw_compat_5_0[] = {
> +{ "vhost-user-blk", "packed", "off" },
> +{ "vhost-user-fs-device", "packed", "off" },
> +{ "vhost-vsock-device", "packed", "off" },
> +{ "virtio-9p-device", "packed", "off" },
> +{ "virtio-balloon-device", "packed", "off" },
> +{ "virtio-blk-device", "packed", "off" },
> +{ "virtio-crypto-device", "packed", "off" },
> +{ "virtio-gpu-device", "packed", "off" },
> +{ "virtio-input-device", "packed", "off" },
> +{ "virtio-iommu-device", "packed", "off" },
> +{ "virtio-net-device", "packed", "off" },
> +{ "virtio-pmem", "packed", "off" },
> +{ "virtio-rng-device", "packed", "off" },
> +{ "virtio-scsi-common", "packed", "off" },
> +{ "virtio-serial-device", "packed", "off" },
> +};
>  const size_t hw_compat_5_0_len = G_N_ELEMENTS(hw_compat_5_0);
>  
>  GlobalProperty hw_compat_4_2[] = {
> -- 
> 2.25.3
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v2 5/5] vhost: add device started check in migration set log

2020-05-19 Thread Dr. David Alan Gilbert
* Dima Stepanov (dimas...@yandex-team.ru) wrote:
> On Mon, May 18, 2020 at 10:53:59AM +0100, Dr. David Alan Gilbert wrote:
> > * Dima Stepanov (dimas...@yandex-team.ru) wrote:
> > > On Mon, May 18, 2020 at 10:50:39AM +0800, Jason Wang wrote:
> > > > 
> > > > On 2020/5/16 上午12:54, Dima Stepanov wrote:
> > > > >On Thu, May 14, 2020 at 03:34:24PM +0800, Jason Wang wrote:
> > > > >>On 2020/5/13 下午5:47, Dima Stepanov wrote:
> > > > >>>>> case CHR_EVENT_CLOSED:
> > > > >>>>> /* a close event may happen during a read/write, but vhost
> > > > >>>>>  * code assumes the vhost_dev remains setup, so delay the
> > > > >>>>>  * stop & clear to idle.
> > > > >>>>>  * FIXME: better handle failure in vhost code, remove bh
> > > > >>>>>  */
> > > > >>>>> if (s->watch) {
> > > > >>>>> AioContext *ctx = qemu_get_current_aio_context();
> > > > >>>>>
> > > > >>>>> g_source_remove(s->watch);
> > > > >>>>> s->watch = 0;
> > > > >>>>> qemu_chr_fe_set_handlers(>chr, NULL, NULL, NULL, 
> > > > >>>>> NULL,
> > > > >>>>>  NULL, NULL, false);
> > > > >>>>>
> > > > >>>>> aio_bh_schedule_oneshot(ctx, chr_closed_bh, opaque);
> > > > >>>>> }
> > > > >>>>> break;
> > > > >>>>>
> > > > >>>>>I think it's time we dropped the FIXME and moved the handling to 
> > > > >>>>>common
> > > > >>>>>code. Jason? Marc-André?
> > > > >>>>I agree. Just to confirm, do you prefer bh or doing changes like 
> > > > >>>>what is
> > > > >>>>done in this series? It looks to me bh can have more easier codes.
> > > > >>>Could it be a good idea just to make disconnect in the char device 
> > > > >>>but
> > > > >>>postphone clean up in the vhost-user-blk (or any other vhost-user
> > > > >>>device) itself? So we are moving the postphone logic and decision 
> > > > >>>from
> > > > >>>the char device to vhost-user device. One of the idea i have is as
> > > > >>>follows:
> > > > >>>   - Put ourself in the INITIALIZATION state
> > > > >>>   - Start these vhost-user "handshake" commands
> > > > >>>   - If we got a disconnect error, perform disconnect, but don't 
> > > > >>> clean up
> > > > >>> device (it will be clean up on the roll back). I can be done by
> > > > >>> checking the state in vhost_user_..._disconnect routine or smth 
> > > > >>> like it
> > > > >>
> > > > >>Any issue you saw just using the aio bh as Michael posted above.
> > > > >>
> > > > >>Then we don't need to deal with the silent vhost_dev_stop() and we 
> > > > >>will have
> > > > >>codes that is much more easier to understand.
> > > > >I've implemented this solution inside
> > > > >hw/block/vhost-user-blk.c:vhost_user_blk_event() in the similar way by
> > > > >using the s->connected field. Looks good and more correct fix ). I have
> > > > >two questions here before i'll rework the fixes:
> > > > >1. Is it okay to make the similar fix inside vhost_user_blk_event() or
> > > > >we are looking for more generic vhost-user solution? What do you think?
> > > > 
> > > > 
> > > > I think I agree with Michael, it's better to have a generic vhost-user
> > > > solution. But if it turns out to be not easy, we can start from fixing
> > > > vhost-user-blk.
> > > I also agree, but as i see it right now the connect/disconnect events
> > > are handled inside each vhost-user device implementation file. So it will
> > > need some global refactoring. So i suggest having this fix first and
> > > after it refactoring the code:
> > >  - more devices will be involved
> > >  - i see there 

Re: [PATCH v2 5/5] vhost: add device started check in migration set log

2020-05-18 Thread Dr. David Alan Gilbert
; > >   Core was generated by `x86_64-softmmu/qemu-system-x86_64 -nodefaults 
> > > -no-user-config -M q35,sata=false'.
> > >   Program terminated with signal SIGABRT, Aborted.
> > >   #0  0x7fb56e729428 in raise () from /lib/x86_64-linux-gnu/libc.so.6
> > >   [Current thread is 1 (Thread 0x7fb486ef5700 (LWP 527734))]
> > >
> > >   (gdb) bt
> > >   #0  0x7fb56e729428 in raise () from /lib/x86_64-linux-gnu/libc.so.6
> > >   #1  0x7fb56e72b02a in abort () from /lib/x86_64-linux-gnu/libc.so.6
> > >   #2  0x5648ea376ee6 in vhost_log_global_start
> > >   (listener=0x5648ece4eb08) at ./hw/virtio/vhost.c:857
> > >   #3  0x5648ea2dde7e in memory_global_dirty_log_start ()
> > >   at ./memory.c:2611
> > >   #4  0x5648ea2e68e7 in ram_init_bitmaps (rs=0x7fb4740008c0)
> > >   at ./migration/ram.c:2305
> > >   #5  0x5648ea2e698b in ram_init_all (rsp=0x5648eb1f0f20 )
> > >   at ./migration/ram.c:2323
> > >   #6  0x5648ea2e6cc5 in ram_save_setup (f=0x5648ec609e00,
> > >   opaque=0x5648eb1f0f20 )
> > >   at ./migration/ram.c:2436
> > >   #7  0x5648ea67b7d3 in qemu_savevm_state_setup (f=0x5648ec609e00) at
> > >   migration/savevm.c:1176
> > >   #8  0x5648ea674511 in migration_thread (opaque=0x5648ec031ff0) at
> > >   migration/migration.c:3416
> > >   #9  0x5648ea85d65d in qemu_thread_start (args=0x5648ec6057f0) at
> > >   util/qemu-thread-posix.c:519
> > >   #10 0x7fb56eac56ba in start_thread () from
> > >   /lib/x86_64-linux-gnu/libpthread.so.0
> > >   #11 0x7fb56e7fb41d in clone () from /lib/x86_64-linux-gnu/libc.so.6
> > >   (gdb) frame 2
> > >   #2  0x5648ea376ee6 in vhost_log_global_start
> > >  (listener=0x5648ece4eb08) at ./hw/virtio/vhost.c:857
> > >   857 abort();
> > >   (gdb) list
> > >   852 {
> > >   853 int r;
> > >   854
> > >   855 r = vhost_migration_log(listener, true);
> > >   856 if (r < 0) {
> > >   857 abort();
> > >   858 }
> > >   859 }
> > >   860
> > >   861 static void vhost_log_global_stop(MemoryListener *listener)
> > >Since bh postphone the clean up, we can't use the ->started field.
> > >Do we have any mechanism to get the device type/state in the common
> > >vhost_migration_log() routine? So for example for the vhost-user/disconnect
> > >device we will be able to return 0. Or should we implement it and introduce
> > >it in this patch set?
> > 
> > 
> > This requires more thought, I will reply in Feng's mail.
> Okay, let's continue discussion there.
> 
> No other comments mixed in below.
> 
> Thanks, Dima.
> 
> > 
> > Thanks
> > 
> > 
> > >
> > >Thanks, Dima.
> > >
> > >>Thank
> > >>
> > >>
> > >>>   - vhost-user command returns error back to the _start() routine
> > >>>   - Rollback in one place in the start() routine, by calling this
> > >>> postphoned clean up for the disconnect
> > >>>
> > 
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [RFC v4 6/6] hmp: add x-debug-virtio commands

2020-05-15 Thread Dr. David Alan Gilbert
* Laurent Vivier (lviv...@redhat.com) wrote:
> On 13/05/2020 12:51, Dr. David Alan Gilbert wrote:
> > * Laurent Vivier (lviv...@redhat.com) wrote:
> >> This patch implements HMP version of the virtio QMP commands
> >>
> >> Signed-off-by: Laurent Vivier 
> > 
> > Reviewed-by: Dr. David Alan Gilbert 
> > 
> > With a thought below
> > 
> >> ---
> >>  Makefile|   2 +-
> >>  Makefile.target |   7 +-
> >>  docs/system/monitor.rst |   2 +
> >>  hmp-commands-virtio.hx  | 160 +
> >>  hmp-commands.hx |  10 +++
> >>  hw/virtio/virtio.c  | 193 +++-
> >>  include/monitor/hmp.h   |   4 +
> >>  monitor/misc.c  |  17 
> >>  8 files changed, 391 insertions(+), 4 deletions(-)
> >>  create mode 100644 hmp-commands-virtio.hx
> >>
> ...
> >> diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
> >> index 66dc2cef1b39..c3d6b783417e 100644
> >> --- a/hw/virtio/virtio.c
> >> +++ b/hw/virtio/virtio.c
> ...
> >> @@ -4033,6 +4092,92 @@ VirtioStatus *qmp_x_debug_virtio_status(const char* 
> >> path, Error **errp)
> >>  return status;
> >>  }
> >>  
> >> +#define DUMP_FEATURES(type, field)
> >>  \
> >> +do {  
> >>  \
> >> +type##FeatureList *list = features->device->u.field.data; 
> >>  \
> >> +if (list) {   
> >>  \
> >> +monitor_printf(mon, "");  
> >>  \
> >> +while (list) {
> >>  \
> >> +monitor_printf(mon, "%s", 
> >> type##Feature_str(list->value)); \
> >> +list = list->next;
> >>  \
> >> +if (list != NULL) {   
> >>  \
> >> +monitor_printf(mon, ", ");
> >>  \
> >> +} 
> >>  \
> >> +} 
> >>  \
> >> +monitor_printf(mon, "\n");
> >>  \
> >> +} 
> >>  \
> >> +} while (0)
> > 
> > It feels like you should be able to have an array of Feature_str's
> > indexed by VIRTIO_DEVICE_FEATURE_KIND_ enum, so that when a new
> > VIRTIO_DEVICE_FEATURE_KIND is added you don't need to fix this up.
> 
> I don't understand what you mean here.

Instead of the switch below, I'm thinking you could have something like:

if (features->device->type < something_MAX) {
features_str = anarray[features->device->type];


monitor_printf(mon, "%s", features_str(list->value));

}

with 'anarray' somewhere more central, so we don't have to keep
these switch structures and macros spread around.

Dave

> >> +
> >> +static void hmp_virtio_dump_features(Monitor *mon,
> >> + VirtioStatusFeatures *features)
> >> +{
> >> +VirtioTransportFeatureList *transport_list = features->transport;
> >> +while (transport_list) {
> >> +monitor_printf(mon, "%s",
> >> +   VirtioTransportFeature_str(transport_list->value));
> >> +transport_list = transport_list->next;
> >> +if (transport_list != NULL) {
> >> +monitor_printf(mon, ", ");
> >> +}
> >> +}
> >> +monitor_printf(mon, "\n");
> >> +switch (features->device->type) {
> >> +case VIRTIO_DEVICE_FEATURES_KIND_VIRTIO_SERIAL:
> >> +DUMP_FEATURES(VirtioSerial, virtio_serial);
> >> +break;
> >> +case VIRTIO_DEVICE_FEATURES_KIND_VIRTIO_BLK:
> >> +DUMP_FEATURES(VirtioBlk, virtio_blk);
> >> +break;
> >> +case VIRTIO_DEVICE_FEATURES_KIND_VIRTIO_GPU:
> >> +DUMP_FEATURES(VirtioGpu, virtio_gpu);
> >> +

Re: [PATCH 0/5] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-05-14 Thread Dr. David Alan Gilbert
* Lukas Straub (lukasstra...@web.de) wrote:
> Terminology:
> instance = one (nbd) blockdev/one chardev/the single migrationstate
> connection = one TCP connection
> 
> Hello Everyone,
> Having read all the comments, here is proposal v2:
> Every instance registers itself with a unique name in the form 
> "blockdev:", "chardev:" and "migration" using 
> yank_register_instance which will do some sanity checks like checking if the 
> same name exists already. Then (multiple) yank functions can be registered as 
> needed with that single name. When the instance exits/is removed, it 
> unregisters all yank functions and unregisters it's name with 
> yank_unregister_instance which will check if all yank functions where 
> unregistered.
> Every instance that supports the yank feature will register itself and the 
> yank functions unconditionally (No extra 'yank' option per instance).
> The 'query-yank' oob qmp command lists the names of all registered instances.
> The 'yank' oob qmp command takes a list of names and for every name calls all 
> yank functions registered with that name. Before doing anything, it will 
> check that all names exist.
> 
> If the instance has multiple connections (say, migration with multifd), i 
> don't think it makes much sense to just shutdown one connection. Calling 
> 'yank' on a instance will always shutdown all connections of that instance.

Agreed.

> Yank functions are generic and in no way limited to connections. Say, if 
> migration is started to an 'exec:' address, migration could register a yank 
> function to kill that external command on yank (Won't be implemented yet 
> though).

One thing we need to be care of is that the yank functions stay suitable
for OOB calling.

Dave

> Regards,
> Lukas Straub


--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [RFC v2] migration: Add migrate-set-bitmap-node-mapping

2020-05-14 Thread Dr. David Alan Gilbert
* Max Reitz (mre...@redhat.com) wrote:
> On 14.05.20 10:42, Dr. David Alan Gilbert wrote:
> > * Max Reitz (mre...@redhat.com) wrote:
> > 
> > 
> > 
> >> +void qmp_migrate_set_bitmap_node_mapping(MigrationBlockNodeMappingList 
> >> *mapping,
> >> + Error **errp)
> >> +{
> >> +QDict *in_mapping = qdict_new();
> >> +QDict *out_mapping = qdict_new();
> >> +
> >> +for (; mapping; mapping = mapping->next) {
> >> +MigrationBlockNodeMapping *entry = mapping->value;
> >> +
> >> +if (qdict_haskey(out_mapping, entry->node_name)) {
> >> +error_setg(errp, "Cannot map node name '%s' twice",
> >> +   entry->node_name);
> >> +goto fail;
> >> +}
> > 
> > I'm not too clear exactly which case this is protecting against;
> > I think that's protecting against mapping
> > 
> >   'src1'->'dst1' and 'src1'->'dst2'
> > which is a good check.s (or maybe it's checking against dst2 twice?)
> 
> This one is against mapping src1 twice.  Both checks together check that
> it’s a one-to-one bijective mapping.
> 
> The technical reason why it needs to be one-to-one is because we base
> two QDicts off of it, so the inverse mapping needs to work.
> 
> > What about cases where there is no mapping - e.g. imagine
> > that we have b1/b2 on the source and b2/b3 on the dest; now
> > if we add just a mapping:
> > 
> >   b1->b2
> > 
> > then we end up with:
> > 
> >   b1 -> b2
> >   b2 -> b2  (non-mapped)
> > b3
> > 
> > so we have a clash there - are we protected against that?
> 
> Oh, no, we aren’t.  That wasn’t intentional.  However, I’m not sure how
> we’d protect against it.  We can’t check it in
> qmp_migrate_set_bitmap_node_mapping(), because we don’t know yet which
> nodes will exist at the time of migration, and which of those will have
> bitmaps.
> 
> So we’d need to check it as part of the migration process (by looking up
> any unmapped entries that default to the identity mapping in the
> respective reverse mapping, to see whether anything maps to the same name).

Yeh a once through check of all the nodes at the start of the migration
would probably fix it.

> OTOH, Vladimir proposed adding a parameter to
> migrate-set-bitmap-node-mapping that would make migration fail if any
> bitmaps should be migrated off of unmapped nodes, or if any incoming
> alias is unmapped (instead of defaulting to the identity mapping).  If
> we just make that the only behavior, then we wouldn’t have a problem
> with that at all, because all unmapped nodes would always throw an error.

Yeh that would force you to put a full mapping table in place.

> (And on the third hand, I wonder whether we should actually allow
> migrating bitmaps from multiple nodes to a single one, but I suppose
> that would require two separate commands, one for incoming and one for
> outgoing...)

Wouldn't that get very messy?

Dave

> Max
> 



--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [RFC v2] migration: Add migrate-set-bitmap-node-mapping

2020-05-14 Thread Dr. David Alan Gilbert
* Max Reitz (mre...@redhat.com) wrote:



> +void qmp_migrate_set_bitmap_node_mapping(MigrationBlockNodeMappingList 
> *mapping,
> + Error **errp)
> +{
> +QDict *in_mapping = qdict_new();
> +QDict *out_mapping = qdict_new();
> +
> +for (; mapping; mapping = mapping->next) {
> +MigrationBlockNodeMapping *entry = mapping->value;
> +
> +if (qdict_haskey(out_mapping, entry->node_name)) {
> +error_setg(errp, "Cannot map node name '%s' twice",
> +   entry->node_name);
> +goto fail;
> +}

I'm not too clear exactly which case this is protecting against;
I think that's protecting against mapping

  'src1'->'dst1' and 'src1'->'dst2'
which is a good check.s (or maybe it's checking against dst2 twice?)

What about cases where there is no mapping - e.g. imagine
that we have b1/b2 on the source and b2/b3 on the dest; now
if we add just a mapping:

  b1->b2

then we end up with:

  b1 -> b2
  b2 -> b2  (non-mapped)
b3

so we have a clash there - are we protected against that?

Dave

> +if (qdict_haskey(in_mapping, entry->alias)) {
> +error_setg(errp, "Cannot use alias '%s' twice",
> +   entry->alias);
> +goto fail;
> +}
> +
> +qdict_put_str(in_mapping, entry->alias, entry->node_name);
> +qdict_put_str(out_mapping, entry->node_name, entry->alias);
> +}
> +
> +qobject_unref(dirty_bitmap_mig_state.node_in_mapping);
> +qobject_unref(dirty_bitmap_mig_state.node_out_mapping);
> +
> +dirty_bitmap_mig_state.node_in_mapping = in_mapping;
> +dirty_bitmap_mig_state.node_out_mapping = out_mapping;
> +
> +return;
> +
> +fail:
> +qobject_unref(in_mapping);
> +qobject_unref(out_mapping);
> +}
> +
>  static SaveVMHandlers savevm_dirty_bitmap_handlers = {
>  .save_setup = dirty_bitmap_save_setup,
>  .save_live_complete_postcopy = dirty_bitmap_save_complete,
> -- 
> 2.26.2
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 0/5] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-05-13 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Wed, May 13, 2020 at 01:56:24PM +0100, Dr. David Alan Gilbert wrote:
> > * Kevin Wolf (kw...@redhat.com) wrote:
> > > I guess it would be nice to have a single namespace for everything in
> > > QEMU, but the reality is that we have a few separate ones. As long as we
> > > consistently add a prefix that identifies the namespace in question, I
> > > think that would work.
> > 
> > > This means that if we're using node-name to identify the NBD connection,
> > > the namespace should be 'block' rather than 'nbd'.
> > > 
> > > One more thing to consider is, what if a single object has multiple
> > > connections? In the case of node-names, we have a limited set of allowed
> > > characters, so we can use one of the remaining characters as a separator
> > > and then suffix a counter. In other places, the identifier isn't
> > > restricted, so suffixing doesn't work. Maybe prefixing does, but it
> > > would have to be there from the beginning then.
> > 
> > Yeh I worry about whether on nbd if you can have multiple nbd
> > connections to one block device.
> 
> The kernel NBD driver now supports multiple parallel connections.
> QEMU hasn't implemented this in its NBD code yet, but I certainly
> see that being in scope for future.

It's not parallel for performance that worries me, it's more about
separateq connections for separate uses - e.g. if we're serving the same
read-only disk to multiple separate things.

Dave
 
> 
> Regards,
> Daniel
> -- 
> |: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-https://fstop138.berrange.com :|
> |: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 0/5] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-05-13 Thread Dr. David Alan Gilbert
* Kevin Wolf (kw...@redhat.com) wrote:
> Am 13.05.2020 um 12:53 hat Dr. David Alan Gilbert geschrieben:
> > * Kevin Wolf (kw...@redhat.com) wrote:
> > > Am 12.05.2020 um 11:43 hat Daniel P. Berrangé geschrieben:
> > > > On Tue, May 12, 2020 at 11:32:06AM +0200, Lukas Straub wrote:
> > > > > On Mon, 11 May 2020 16:46:45 +0100
> > > > > "Dr. David Alan Gilbert"  wrote:
> > > > > 
> > > > > > * Daniel P. Berrangé (berra...@redhat.com) wrote: 
> > > > > > > ...
> > > > > > > That way if QEMU does get stuck, you can start by tearing down the
> > > > > > > least distruptive channel. eg try tearing down the migration 
> > > > > > > connection
> > > > > > > first (which shouldn't negatively impact the guest), and only if 
> > > > > > > that
> > > > > > > doesn't work then, move on to tear down the NBD connection (which 
> > > > > > > risks
> > > > > > > data loss)  
> > > > > > 
> > > > > > I wonder if a different way would be to make all network connections
> > > > > > register with yank, but then make yank take a list of connections to
> > > > > > shutdown(2).
> > > > > 
> > > > > Good Idea. We could name the connections (/yank callbacks) in the
> > > > > form "nbd:", "chardev:" and "migration"
> > > > > (and add "netdev:...", etc. in the future). Then make yank take a
> > > > > list of connection names as you suggest and silently ignore 
> > > > > connections
> > > > > that don't exist. And maybe even add a 'query-yank' oob command 
> > > > > returning
> > > > > a list of registered connections so the management application can do
> > > > > pattern matching if it wants.
> > > 
> > > I'm generally not a big fan of silently ignoring things. Is there a
> > > specific requirement to do it in this case, or can management
> > > applications be expected to know which connections exist?
> > > 
> > > > Yes, that would make the yank command much more flexible in how it can
> > > > be used.
> > > > 
> > > > As an alternative to using formatted strings like this, it could be
> > > > modelled more explicitly in QAPI
> > > > 
> > > >   { 'struct':  'YankChannels',
> > > > 'data': { 'chardev': [ 'string' ],
> > > >   'nbd': ['string'],
> > > >   'migration': bool } }
> > > > 
> > > > In this example, 'chardev' would accept a list of chardev IDs which
> > > > have it enabled, 'nbd' would accept a list of block node IDs which
> > > > have it enabled, and migration is a singleton on/off.
> > > 
> > > Of course, it also means that the yank code needs to know about every
> > > single object that supports the operation, whereas if you only have
> > > strings, the objects could keep registering their connection with a
> > > generic function like yank_register_function() in this version.
> > > 
> > > I'm not sure if the additional complexity is worth the benefits.
> > 
> > I tend to agree; although we do have to ensure we either use an existing
> > naming scheme (e.g. QOM object names?) or make sure we've got a well
> > defined list of prefixes.
> 
> Not everything that has a network connection is a QOM object (in fact,
> neither migration nor chardev nor nbd are QOM objects).

Hmm, migrationstate is a qdev object.

> I guess it would be nice to have a single namespace for everything in
> QEMU, but the reality is that we have a few separate ones. As long as we
> consistently add a prefix that identifies the namespace in question, I
> think that would work.

> This means that if we're using node-name to identify the NBD connection,
> the namespace should be 'block' rather than 'nbd'.
> 
> One more thing to consider is, what if a single object has multiple
> connections? In the case of node-names, we have a limited set of allowed
> characters, so we can use one of the remaining characters as a separator
> and then suffix a counter. In other places, the identifier isn't
> restricted, so suffixing doesn't work. Maybe prefixing does, but it
> would have to be there from the beginning then.

Yeh I worry about whether on nbd if you can have multiple nbd
connections to one block device.

> And another thing: Do we really want to document this as limited to
> network connections? Another common cause of hangs is when you have
> image files on an NFS mount and the connection goes away. Of course, in
> the end this is still networking, but inside of QEMU it looks like
> accessing any other file. I'm not sure that we'll allow yanking access
> to image files anytime soon, but it might not hurt to keep it at the
> back of our mind as a potential option we might want the design to
> allow.

Yep.

Dave

> Kevin
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 0/5] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-05-13 Thread Dr. David Alan Gilbert
* Paolo Bonzini (pbonz...@redhat.com) wrote:
> On 13/05/20 13:26, Daniel P. Berrangé wrote:
> > Are you referring to the in-kernel NFS client hangs here ?  AFAIK, it is
> > impossible to do anything to get out of those hangs from userspace, because
> > the thread is stuck in an uninterruptable sleep in kernel space.
> > 
> > If using the in-QEMU NFS client, then there is a network connection that
> > can be yanked just like the NBD client.
> 
> But it's a bad idea to yank it (and also the NBD client) because you're
> not sure which wites have made it to the server (and to the medium) and
> which haven't.

No, that's OK - if you look at the COLO case, and some other cases,
you've got a dead storage device but your redundant pair might be OK;
so it's OK to yank it.
Other similar storage cases are trying to migrate a VM that has one dead
disk, even if you know and accept it's dead and unresponding, you often
can't kill it off if the device is hung.

> Effectively, the in-QEMU NFS client and NBD client are always operating
> in "soft" mode, but we should always treat that as a bug (which cannot
> be fixed) and not a feature for read-write images.
Dave

> 
> Paolo
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 0/5] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-05-13 Thread Dr. David Alan Gilbert
* Kevin Wolf (kw...@redhat.com) wrote:
> Am 12.05.2020 um 11:43 hat Daniel P. Berrangé geschrieben:
> > On Tue, May 12, 2020 at 11:32:06AM +0200, Lukas Straub wrote:
> > > On Mon, 11 May 2020 16:46:45 +0100
> > > "Dr. David Alan Gilbert"  wrote:
> > > 
> > > > * Daniel P. Berrangé (berra...@redhat.com) wrote: 
> > > > > ...
> > > > > That way if QEMU does get stuck, you can start by tearing down the
> > > > > least distruptive channel. eg try tearing down the migration 
> > > > > connection
> > > > > first (which shouldn't negatively impact the guest), and only if that
> > > > > doesn't work then, move on to tear down the NBD connection (which 
> > > > > risks
> > > > > data loss)  
> > > > 
> > > > I wonder if a different way would be to make all network connections
> > > > register with yank, but then make yank take a list of connections to
> > > > shutdown(2).
> > > 
> > > Good Idea. We could name the connections (/yank callbacks) in the
> > > form "nbd:", "chardev:" and "migration"
> > > (and add "netdev:...", etc. in the future). Then make yank take a
> > > list of connection names as you suggest and silently ignore connections
> > > that don't exist. And maybe even add a 'query-yank' oob command returning
> > > a list of registered connections so the management application can do
> > > pattern matching if it wants.
> 
> I'm generally not a big fan of silently ignoring things. Is there a
> specific requirement to do it in this case, or can management
> applications be expected to know which connections exist?
> 
> > Yes, that would make the yank command much more flexible in how it can
> > be used.
> > 
> > As an alternative to using formatted strings like this, it could be
> > modelled more explicitly in QAPI
> > 
> >   { 'struct':  'YankChannels',
> > 'data': { 'chardev': [ 'string' ],
> >   'nbd': ['string'],
> >   'migration': bool } }
> > 
> > In this example, 'chardev' would accept a list of chardev IDs which
> > have it enabled, 'nbd' would accept a list of block node IDs which
> > have it enabled, and migration is a singleton on/off.
> 
> Of course, it also means that the yank code needs to know about every
> single object that supports the operation, whereas if you only have
> strings, the objects could keep registering their connection with a
> generic function like yank_register_function() in this version.
> 
> I'm not sure if the additional complexity is worth the benefits.

I tend to agree; although we do have to ensure we either use an existing
naming scheme (e.g. QOM object names?) or make sure we've got a well
defined list of prefixes.

Dave

> 
> > The benefit of this modelling is that you can introspect QEMU
> > to discover what classes of channels support being yanked by
> > this QEMU build, as well as what instances are configured to
> > be yanked. ie you can distinguish between a QEMU that doesn't
> > support yanking network devices, from a QEMU that does support
> > yanking network devices, but doesn't have it enabled for any
> > network device instances.
> 
> This is true, though I think Lukas' suggestion with query-yank should be
> as good in practice (you can't check before creating the NBD device
> then, but would you actually want to do this?).
> 
> And if all else fails, we can still add a few more feature flags to the
> schema...
> 
> Kevin
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [RFC v4 6/6] hmp: add x-debug-virtio commands

2020-05-13 Thread Dr. David Alan Gilbert
* Laurent Vivier (lviv...@redhat.com) wrote:
> This patch implements HMP version of the virtio QMP commands
> 
> Signed-off-by: Laurent Vivier 

Reviewed-by: Dr. David Alan Gilbert 

With a thought below

> ---
>  Makefile|   2 +-
>  Makefile.target |   7 +-
>  docs/system/monitor.rst |   2 +
>  hmp-commands-virtio.hx  | 160 +
>  hmp-commands.hx |  10 +++
>  hw/virtio/virtio.c  | 193 +++-
>  include/monitor/hmp.h   |   4 +
>  monitor/misc.c  |  17 
>  8 files changed, 391 insertions(+), 4 deletions(-)
>  create mode 100644 hmp-commands-virtio.hx
> 
> diff --git a/Makefile b/Makefile
> index 34275f57c9cb..feb300ebb2d4 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -1099,7 +1099,7 @@ $(MANUAL_BUILDDIR)/interop/index.html: $(call 
> manual-deps,interop)
>  $(MANUAL_BUILDDIR)/specs/index.html: $(call manual-deps,specs)
>   $(call build-manual,specs,html)
>  
> -$(MANUAL_BUILDDIR)/system/index.html: $(call manual-deps,system) 
> $(SRC_PATH)/hmp-commands.hx $(SRC_PATH)/hmp-commands-info.hx 
> $(SRC_PATH)/qemu-options.hx
> +$(MANUAL_BUILDDIR)/system/index.html: $(call manual-deps,system) 
> $(SRC_PATH)/hmp-commands.hx $(SRC_PATH)/hmp-commands-info.hx 
> $(SRC_PATH)/qemu-options.hx $(SRC_PATH)/hmp-commands-virtio.hx
>   $(call build-manual,system,html)
>  
>  $(MANUAL_BUILDDIR)/tools/index.html: $(call manual-deps,tools) 
> $(SRC_PATH)/qemu-img-cmds.hx $(SRC_PATH)/docs/qemu-option-trace.rst.inc
> diff --git a/Makefile.target b/Makefile.target
> index 8ed1eba95b9c..66d3ff9bc350 100644
> --- a/Makefile.target
> +++ b/Makefile.target
> @@ -171,7 +171,7 @@ else
>  obj-y += hw/$(TARGET_BASE_ARCH)/
>  endif
>  
> -generated-files-y += hmp-commands.h hmp-commands-info.h
> +generated-files-y += hmp-commands.h hmp-commands-info.h hmp-commands-virtio.h
>  generated-files-y += config-devices.h
>  
>  endif # CONFIG_SOFTMMU
> @@ -220,10 +220,13 @@ hmp-commands.h: $(SRC_PATH)/hmp-commands.hx 
> $(SRC_PATH)/scripts/hxtool
>  hmp-commands-info.h: $(SRC_PATH)/hmp-commands-info.hx 
> $(SRC_PATH)/scripts/hxtool
>   $(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > 
> $@,"GEN","$(TARGET_DIR)$@")
>  
> +hmp-commands-virtio.h: $(SRC_PATH)/hmp-commands-virtio.hx 
> $(SRC_PATH)/scripts/hxtool
> + $(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > 
> $@,"GEN","$(TARGET_DIR)$@")
> +
>  clean: clean-target
>   rm -f *.a *~ $(PROGS)
>   rm -f $(shell find . -name '*.[od]')
> - rm -f hmp-commands.h gdbstub-xml.c
> + rm -f hmp-commands.h hmp-commands-virtio.h gdbstub-xml.c
>   rm -f trace/generated-helpers.c trace/generated-helpers.c-timestamp
>  ifdef CONFIG_TRACE_SYSTEMTAP
>   rm -f *.stp
> diff --git a/docs/system/monitor.rst b/docs/system/monitor.rst
> index 0bcd5da21644..985c3f51ffe7 100644
> --- a/docs/system/monitor.rst
> +++ b/docs/system/monitor.rst
> @@ -21,6 +21,8 @@ The following commands are available:
>  
>  .. hxtool-doc:: hmp-commands.hx
>  
> +.. hxtool-doc:: hmp-commands-virtio.hx
> +
>  .. hxtool-doc:: hmp-commands-info.hx
>  
>  Integer expressions
> diff --git a/hmp-commands-virtio.hx b/hmp-commands-virtio.hx
> new file mode 100644
> index ..14cb14bfcc70
> --- /dev/null
> +++ b/hmp-commands-virtio.hx
> @@ -0,0 +1,160 @@
> +HXCOMM Use DEFHEADING() to define headings in both help text and rST.
> +HXCOMM Text between SRST and ERST is copied to the rST version and
> +HXCOMM discarded from C version.
> +HXCOMM DEF(command, args, callback, arg_string, help) is used to construct
> +HXCOMM monitor info commands
> +HXCOMM HXCOMM can be used for comments, discarded from both rST and C.
> +HXCOMM
> +HXCOMM In this file, generally SRST fragments should have two extra
> +HXCOMM spaces of indent, so that the documentation list item for 
> "x-debug-virtio cmd"
> +HXCOMM appears inside the documentation list item for the top level
> +HXCOMM "x-debug-virtio" documentation entry. The exception is the first SRST
> +HXCOMM fragment that defines that top level entry.
> +
> +SRST
> +``x-debug-virtio`` *subcommand*
> +  Show various information about virtio.
> +
> +  Example:
> +
> +  List all sub-commands::
> +
> +(qemu) x-debug-virtio
> +x-debug-virtio query  -- List all available virtio devices
> +x-debug-virtio status path -- Display status of a given virtio device
> +x-debug-virtio queue-status path queue -- Display status of a given 
> virtio queue
> +x-debug-virtio queue-element path queue [index] -- Display element

Re: [PATCH 0/5] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-05-12 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Tue, May 12, 2020 at 11:32:06AM +0200, Lukas Straub wrote:
> > On Mon, 11 May 2020 16:46:45 +0100
> > "Dr. David Alan Gilbert"  wrote:
> > 
> > > * Daniel P. Berrangé (berra...@redhat.com) wrote: 
> > > > ...
> > > > That way if QEMU does get stuck, you can start by tearing down the
> > > > least distruptive channel. eg try tearing down the migration connection
> > > > first (which shouldn't negatively impact the guest), and only if that
> > > > doesn't work then, move on to tear down the NBD connection (which risks
> > > > data loss)  
> > > 
> > > I wonder if a different way would be to make all network connections
> > > register with yank, but then make yank take a list of connections to
> > > shutdown(2).
> > 
> > Good Idea. We could name the connections (/yank callbacks) in the
> > form "nbd:", "chardev:" and "migration"
> > (and add "netdev:...", etc. in the future). Then make yank take a
> > list of connection names as you suggest and silently ignore connections
> > that don't exist. And maybe even add a 'query-yank' oob command returning
> > a list of registered connections so the management application can do
> > pattern matching if it wants.
> 
> Yes, that would make the yank command much more flexible in how it can
> be used.
> 
> As an alternative to using formatted strings like this, it could be
> modelled more explicitly in QAPI
> 
>   { 'struct':  'YankChannels',
> 'data': { 'chardev': [ 'string' ],
>   'nbd': ['string'],
> 'migration': bool } }
> 
> In this example, 'chardev' would accept a list of chardev IDs which
> have it enabled, 'nbd' would accept a list of block node IDs which
> have it enabled, and migration is a singleton on/off.

Do we already have a QOM object name for each of these things?
Is that nbd/blockdevice unique - i.e. can you have multiple nbd clients
on the same node?

> The benefit of this modelling is that you can introspect QEMU
> to discover what classes of channels support being yanked by
> this QEMU build, as well as what instances are configured to
> be yanked. ie you can distinguish between a QEMU that doesn't
> support yanking network devices, from a QEMU that does support
> yanking network devices, but doesn't have it enabled for any
> network device instances.

What do we need to make it introspectable like that?

Dave

> Regards,
> Daniel
> -- 
> |: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-https://fstop138.berrange.com :|
> |: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 0/5] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-05-12 Thread Dr. David Alan Gilbert
* Lukas Straub (lukasstra...@web.de) wrote:
> On Mon, 11 May 2020 12:49:47 +0100
> Daniel P. Berrangé  wrote:
> 
> > On Mon, May 11, 2020 at 01:14:34PM +0200, Lukas Straub wrote:
> > > Hello Everyone,
> > > In many cases, if qemu has a network connection (qmp, migration, chardev, 
> > > etc.)
> > > to some other server and that server dies or hangs, qemu hangs too.  
> > 
> > If qemu as a whole hangs due to a stalled network connection, that is a
> > bug in QEMU that we should be fixing IMHO. QEMU should be doing non-blocking
> > I/O in general, such that if the network connection or remote server stalls,
> > we simply stop sending I/O - we shouldn't ever hang the QEMU process or main
> > loop.
> > 
> > There are places in QEMU code which are not well behaved in this respect,
> > but many are, and others are getting fixed where found to be important.
> > 
> > Arguably any place in QEMU code which can result in a hang of QEMU in the
> > event of a stalled network should be considered a security flaw, because
> > the network is untrusted in general.
> 
> The fact that out-of-band qmp commands exist at all shows that we have to 
> make tradeoffs of developer time vs. doing things right. Sure, the migration 
> code can be rewritten to use non-blocking i/o and finegrained locks. But as a 
> hobbyist I don't have time to fix this.

If it was just an hobbyist vs fulltime thing then I'd say that was a bad
way to make the decision; however the reality is that even those who are
paid to watch this code don't have the feeling we can make it fail
quickly/non-blocking - and for COLO you need to be absolutely sure you
nail every case, so you'd some how have to audit the whole lot, and keep
watching it.

(and thank you for taking your time to do this!)

Dave


> > > These patches introduce the new 'yank' out-of-band qmp command to recover 
> > > from
> > > these kinds of hangs. The different subsystems register callbacks which 
> > > get
> > > executed with the yank command. For example the callback can shutdown() a
> > > socket. This is intended for the colo use-case, but it can be used for 
> > > other
> > > things too of course.  
> > 
> > IIUC, invoking the "yank" command unconditionally kills every single
> > network connection in QEMU that has registered with the "yank" subsystem.
> > IMHO this is way too big of a hammer, even if we accept there are bugs in
> > QEMU not handling stalled networking well.
> > 
> > eg if a chardev hangs QEMU, and we tear down everything, killing the NBD
> > connection used for the guest disk, we needlessly break I/O.
> 
> Yeah, these patches are intended to solve the problems with the colo use-case 
> where all external connections (migration, chardevs, nbd) are just for 
> replication. In other use-cases you'd enable the yank feature only on the 
> non-essential connections.
> 
> > eg doing this in the chardev backend is not desirable, because the bugs
> > with hanging QEMU are typically caused by the way the frontend device
> > uses the chardev blocking I/O calls, instead of non-blocking I/O calls.
> > 
> > 
> > Regards,
> > Daniel
> 


--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 0/5] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-05-12 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Mon, May 11, 2020 at 08:12:18PM +0200, Lukas Straub wrote:
> > On Mon, 11 May 2020 12:49:47 +0100
> > Daniel P. Berrangé  wrote:
> > 
> > > On Mon, May 11, 2020 at 01:14:34PM +0200, Lukas Straub wrote:
> > > > Hello Everyone,
> > > > In many cases, if qemu has a network connection (qmp, migration, 
> > > > chardev, etc.)
> > > > to some other server and that server dies or hangs, qemu hangs too.  
> > > 
> > > If qemu as a whole hangs due to a stalled network connection, that is a
> > > bug in QEMU that we should be fixing IMHO. QEMU should be doing 
> > > non-blocking
> > > I/O in general, such that if the network connection or remote server 
> > > stalls,
> > > we simply stop sending I/O - we shouldn't ever hang the QEMU process or 
> > > main
> > > loop.
> > > 
> > > There are places in QEMU code which are not well behaved in this respect,
> > > but many are, and others are getting fixed where found to be important.
> > > 
> > > Arguably any place in QEMU code which can result in a hang of QEMU in the
> > > event of a stalled network should be considered a security flaw, because
> > > the network is untrusted in general.
> > 
> > The fact that out-of-band qmp commands exist at all shows that we have to 
> > make tradeoffs of developer time vs. doing things right. Sure, the 
> > migration code can be rewritten to use non-blocking i/o and finegrained 
> > locks. But as a hobbyist I don't have time to fix this.
> > 
> > > > These patches introduce the new 'yank' out-of-band qmp command to 
> > > > recover from
> > > > these kinds of hangs. The different subsystems register callbacks which 
> > > > get
> > > > executed with the yank command. For example the callback can shutdown() 
> > > > a
> > > > socket. This is intended for the colo use-case, but it can be used for 
> > > > other
> > > > things too of course.  
> > > 
> > > IIUC, invoking the "yank" command unconditionally kills every single
> > > network connection in QEMU that has registered with the "yank" subsystem.
> > > IMHO this is way too big of a hammer, even if we accept there are bugs in
> > > QEMU not handling stalled networking well.
> > > 
> > > eg if a chardev hangs QEMU, and we tear down everything, killing the NBD
> > > connection used for the guest disk, we needlessly break I/O.
> > 
> > Yeah, these patches are intended to solve the problems with the colo
> > use-case where all external connections (migration, chardevs, nbd)
> > are just for replication. In other use-cases you'd enable the yank
> > feature only on the non-essential connections.
> 
> That is a pretty inflexible design for other use cases though,
> as "non-essential" is not a black & white list in general. There
> are varying levels of importance to the different channels. We
> can afford to loose migration without any user visible effects.
> If that doesn't solve it, a serial device chardev, or VNC connection
> can be dropped at the inconvenience of loosing interactive console
> which is end user visible impact, so may only be want to be yanked
> if the migration yank didn't fix it. 

In the case of COLO that's not the case though - here we explicitly want
to kill the migration to be able to ensure that we can recover - and
we're under time pressure to get the other member of the pair running
again. 

Dave

> Regards,
> Daniel
> -- 
> |: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-https://fstop138.berrange.com :|
> |: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 3/5] block/nbd.c: Add yank feature

2020-05-11 Thread Dr. David Alan Gilbert
* Lukas Straub (lukasstra...@web.de) wrote:
> On Mon, 11 May 2020 17:19:09 +0100
> "Dr. David Alan Gilbert"  wrote:
> 
> > * Lukas Straub (lukasstra...@web.de) wrote:
> > > Add yank option, pass it to the socket-channel and register a yank
> > > function which sets s->state = NBD_CLIENT_QUIT. This is the same
> > > behaviour as if an error occured.
> > > 
> > > Signed-off-by: Lukas Straub   
> > 
> > > +static void nbd_yank(void *opaque)
> > > +{
> > > +BlockDriverState *bs = opaque;
> > > +BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
> > > +
> > > +atomic_set(>state, NBD_CLIENT_QUIT);  
> > 
> > I think I was expecting a shutdown on the socket here - why doesn't it
> > have one?
> 
> For nbd, we register two yank functions: This one and we enable the yank 
> feature on the qio channel (see function nbd_establish_connection below).

Oh I see; yeh that still surprises me a little; I'd expected one yank
per item.

Dave

> Regards,
> Lukas Straub
> 
> > Dave
> > 
> > > +}
> > > +
> > >  static void nbd_client_close(BlockDriverState *bs)
> > >  {
> > >  BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
> > > @@ -1407,14 +1421,17 @@ static void nbd_client_close(BlockDriverState *bs)
> > >  nbd_teardown_connection(bs);
> > >  }
> > >  
> > > -static QIOChannelSocket *nbd_establish_connection(SocketAddress *saddr,
> > > +static QIOChannelSocket *nbd_establish_connection(BlockDriverState *bs,
> > > +  SocketAddress *saddr,
> > >Error **errp)
> > >  {
> > > +BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
> > >  QIOChannelSocket *sioc;
> > >  Error *local_err = NULL;
> > >  
> > >  sioc = qio_channel_socket_new();
> > >  qio_channel_set_name(QIO_CHANNEL(sioc), "nbd-client");
> > > +qio_channel_set_yank(QIO_CHANNEL(sioc), s->yank);
> > >  
> > >  qio_channel_socket_connect_sync(sioc, saddr, _err);
> > >  if (local_err) {
> > > @@ -1438,7 +1455,7 @@ static int nbd_client_connect(BlockDriverState *bs, 
> > > Error **errp)
> > >   * establish TCP connection, return error if it fails
> > >   * TODO: Configurable retry-until-timeout behaviour.
> > >   */
> > > -QIOChannelSocket *sioc = nbd_establish_connection(s->saddr, errp);
> > > +QIOChannelSocket *sioc = nbd_establish_connection(bs, s->saddr, 
> > > errp);
> > >  
> > >  if (!sioc) {
> > >  return -ECONNREFUSED;
> > > @@ -1829,6 +1846,12 @@ static QemuOptsList nbd_runtime_opts = {
> > >  "future requests before a successful reconnect will "
> > >  "immediately fail. Default 0",
> > >  },
> > > +{
> > > +.name = "yank",
> > > +.type = QEMU_OPT_BOOL,
> > > +.help = "Forcibly close the connection and don't attempt to "
> > > +"reconnect when the 'yank' qmp command is executed.",
> > > +},
> > >  { /* end of list */ }
> > >  },
> > >  };
> > > @@ -1888,6 +1911,8 @@ static int nbd_process_options(BlockDriverState 
> > > *bs, QDict *options,
> > >  
> > >  s->reconnect_delay = qemu_opt_get_number(opts, "reconnect-delay", 0);
> > >  
> > > +s->yank = qemu_opt_get_bool(opts, "yank", false);
> > > +
> > >  ret = 0;
> > >  
> > >   error:
> > > @@ -1921,6 +1946,10 @@ static int nbd_open(BlockDriverState *bs, QDict 
> > > *options, int flags,
> > >  /* successfully connected */
> > >  s->state = NBD_CLIENT_CONNECTED;
> > >  
> > > +if (s->yank) {
> > > +yank_register_function(nbd_yank, bs);
> > > +}
> > > +
> > >  s->connection_co = qemu_coroutine_create(nbd_connection_entry, s);
> > >  bdrv_inc_in_flight(bs);
> > >  aio_co_schedule(bdrv_get_aio_context(bs), s->connection_co);
> > > @@ -1972,6 +2001,11 @@ static void nbd_close(BlockDriverState *bs)
> > >  BDRVNBDState *s = bs->opaque;
> > >  
> > >  nbd_client_close(bs);
> > > +

Re: [PATCH 3/5] block/nbd.c: Add yank feature

2020-05-11 Thread Dr. David Alan Gilbert
* Lukas Straub (lukasstra...@web.de) wrote:
> Add yank option, pass it to the socket-channel and register a yank
> function which sets s->state = NBD_CLIENT_QUIT. This is the same
> behaviour as if an error occured.
> 
> Signed-off-by: Lukas Straub 

> +static void nbd_yank(void *opaque)
> +{
> +BlockDriverState *bs = opaque;
> +BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
> +
> +atomic_set(>state, NBD_CLIENT_QUIT);

I think I was expecting a shutdown on the socket here - why doesn't it
have one?

Dave

> +}
> +
>  static void nbd_client_close(BlockDriverState *bs)
>  {
>  BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
> @@ -1407,14 +1421,17 @@ static void nbd_client_close(BlockDriverState *bs)
>  nbd_teardown_connection(bs);
>  }
>  
> -static QIOChannelSocket *nbd_establish_connection(SocketAddress *saddr,
> +static QIOChannelSocket *nbd_establish_connection(BlockDriverState *bs,
> +  SocketAddress *saddr,
>Error **errp)
>  {
> +BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
>  QIOChannelSocket *sioc;
>  Error *local_err = NULL;
>  
>  sioc = qio_channel_socket_new();
>  qio_channel_set_name(QIO_CHANNEL(sioc), "nbd-client");
> +qio_channel_set_yank(QIO_CHANNEL(sioc), s->yank);
>  
>  qio_channel_socket_connect_sync(sioc, saddr, _err);
>  if (local_err) {
> @@ -1438,7 +1455,7 @@ static int nbd_client_connect(BlockDriverState *bs, 
> Error **errp)
>   * establish TCP connection, return error if it fails
>   * TODO: Configurable retry-until-timeout behaviour.
>   */
> -QIOChannelSocket *sioc = nbd_establish_connection(s->saddr, errp);
> +QIOChannelSocket *sioc = nbd_establish_connection(bs, s->saddr, errp);
>  
>  if (!sioc) {
>  return -ECONNREFUSED;
> @@ -1829,6 +1846,12 @@ static QemuOptsList nbd_runtime_opts = {
>  "future requests before a successful reconnect will "
>  "immediately fail. Default 0",
>  },
> +{
> +.name = "yank",
> +.type = QEMU_OPT_BOOL,
> +.help = "Forcibly close the connection and don't attempt to "
> +"reconnect when the 'yank' qmp command is executed.",
> +},
>  { /* end of list */ }
>  },
>  };
> @@ -1888,6 +1911,8 @@ static int nbd_process_options(BlockDriverState *bs, 
> QDict *options,
>  
>  s->reconnect_delay = qemu_opt_get_number(opts, "reconnect-delay", 0);
>  
> +s->yank = qemu_opt_get_bool(opts, "yank", false);
> +
>  ret = 0;
>  
>   error:
> @@ -1921,6 +1946,10 @@ static int nbd_open(BlockDriverState *bs, QDict 
> *options, int flags,
>  /* successfully connected */
>  s->state = NBD_CLIENT_CONNECTED;
>  
> +if (s->yank) {
> +yank_register_function(nbd_yank, bs);
> +}
> +
>  s->connection_co = qemu_coroutine_create(nbd_connection_entry, s);
>  bdrv_inc_in_flight(bs);
>  aio_co_schedule(bdrv_get_aio_context(bs), s->connection_co);
> @@ -1972,6 +2001,11 @@ static void nbd_close(BlockDriverState *bs)
>  BDRVNBDState *s = bs->opaque;
>  
>  nbd_client_close(bs);
> +
> +if (s->yank) {
> +yank_unregister_function(nbd_yank, bs);
> +}
> +
>  nbd_clear_bdrvstate(s);
>  }
>  
> diff --git a/qapi/block-core.json b/qapi/block-core.json
> index 943df1926a..1c1578160e 100644
> --- a/qapi/block-core.json
> +++ b/qapi/block-core.json
> @@ -3862,6 +3862,8 @@
>  #   reconnect. After that time, any delayed requests and all
>  #   future requests before a successful reconnect will
>  #   immediately fail. Default 0 (Since 4.2)
> +# @yank: Forcibly close the connection and don't attempt to reconnect when
> +#the 'yank' qmp command is executed. (Since: 5.1)
>  #
>  # Since: 2.9
>  ##
> @@ -3870,7 +3872,8 @@
>  '*export': 'str',
>  '*tls-creds': 'str',
>  '*x-dirty-bitmap': 'str',
> -'*reconnect-delay': 'uint32' } }
> +'*reconnect-delay': 'uint32',
> + 'yank': 'bool' } }
>  
>  ##
>  # @BlockdevOptionsRaw:
> -- 
> 2.20.1
> 


--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 0/5] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-05-11 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Mon, May 11, 2020 at 01:07:18PM +0100, Dr. David Alan Gilbert wrote:
> > * Daniel P. Berrangé (berra...@redhat.com) wrote:
> > > On Mon, May 11, 2020 at 01:14:34PM +0200, Lukas Straub wrote:
> > > > Hello Everyone,
> > > > In many cases, if qemu has a network connection (qmp, migration, 
> > > > chardev, etc.)
> > > > to some other server and that server dies or hangs, qemu hangs too.
> > > 
> > > If qemu as a whole hangs due to a stalled network connection, that is a
> > > bug in QEMU that we should be fixing IMHO. QEMU should be doing 
> > > non-blocking
> > > I/O in general, such that if the network connection or remote server 
> > > stalls,
> > > we simply stop sending I/O - we shouldn't ever hang the QEMU process or 
> > > main
> > > loop.
> > > 
> > > There are places in QEMU code which are not well behaved in this respect,
> > > but many are, and others are getting fixed where found to be important.
> > > 
> > > Arguably any place in QEMU code which can result in a hang of QEMU in the
> > > event of a stalled network should be considered a security flaw, because
> > > the network is untrusted in general.
> > 
> > That's not really true of the 'management network' - people trust that
> > and I don't see a lot of the qemu code getting fixed safely for all of
> > them.
> 
> It depends on the user / app / deployment scenario. In OpenStack alot of
> work was done to beef up security between services on the mgmt network,
> with TLS encryption as standard to reduce attack vectors.
> 
> > > > These patches introduce the new 'yank' out-of-band qmp command to 
> > > > recover from
> > > > these kinds of hangs. The different subsystems register callbacks which 
> > > > get
> > > > executed with the yank command. For example the callback can shutdown() 
> > > > a
> > > > socket. This is intended for the colo use-case, but it can be used for 
> > > > other
> > > > things too of course.
> > > 
> > > IIUC, invoking the "yank" command unconditionally kills every single
> > > network connection in QEMU that has registered with the "yank" subsystem.
> > > IMHO this is way too big of a hammer, even if we accept there are bugs in
> > > QEMU not handling stalled networking well.
> > 
> > But isn't this hammer conditional - I see that it's a migration
> > capabiltiy for the migration socket, and a flag in nbd - so it only
> > yanks things you've told it to.
> 
> IIUC, you have to set these flags upfront when you launch QEMU, or
> hotplug the device using the feature. When something gets stuck,
> and you issue the "yank" command, then everything that has the flag
> enabled gets torn down. So in practice it looks like the flag will
> get enabled for everything at QEMU startup, and yanking down tear
> down everything.

For COLO I really expect it for the migration stream, the disk mirroring
stream and probably the network comparison/forwarding streams.

> > > eg if a chardev hangs QEMU, and we tear down everything, killing the NBD
> > > connection used for the guest disk, we needlessly break I/O.
> > > 
> > > eg doing this in the chardev backend is not desirable, because the bugs
> > > with hanging QEMU are typically caused by the way the frontend device
> > > uses the chardev blocking I/O calls, instead of non-blocking I/O calls.
> > > 
> > 
> > Having a way to get out of any of these problems from a single point is
> > quite nice.  To be useful in COLO you need to know for sure you can get
> > out of any network screwup.
> > 
> > We already use shutdown(2) in migrate_cancel and migrate-pause for
> > basically the same reason; I don't think we've got anything similar for
> > NBD, and we probably should have (I think I asked for it fairly
> > recently).
> 
> Yes, the migrate_cancel is an example of a more fine grained way to
> recover. I was thinking that we need an equivalent fine control knob
> for NBD too.

I feel it might be nice not to have to create so many separate knobs.

> That way if QEMU does get stuck, you can start by tearing down the
> least distruptive channel. eg try tearing down the migration connection
> first (which shouldn't negatively impact the guest), and only if that
> doesn't work then, move on to tear down the NBD connection (which risks
> data loss)

I wonder if a different way would be to make all network connections
register with yank, but then make yank take a list of connections to
shutdown(2).

Dave

> Regards,
> Daniel
> -- 
> |: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-https://fstop138.berrange.com :|
> |: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 0/5] Introduce 'yank' oob qmp command to recover from hanging qemu

2020-05-11 Thread Dr. David Alan Gilbert
* Daniel P. Berrangé (berra...@redhat.com) wrote:
> On Mon, May 11, 2020 at 01:14:34PM +0200, Lukas Straub wrote:
> > Hello Everyone,
> > In many cases, if qemu has a network connection (qmp, migration, chardev, 
> > etc.)
> > to some other server and that server dies or hangs, qemu hangs too.
> 
> If qemu as a whole hangs due to a stalled network connection, that is a
> bug in QEMU that we should be fixing IMHO. QEMU should be doing non-blocking
> I/O in general, such that if the network connection or remote server stalls,
> we simply stop sending I/O - we shouldn't ever hang the QEMU process or main
> loop.
> 
> There are places in QEMU code which are not well behaved in this respect,
> but many are, and others are getting fixed where found to be important.
> 
> Arguably any place in QEMU code which can result in a hang of QEMU in the
> event of a stalled network should be considered a security flaw, because
> the network is untrusted in general.

That's not really true of the 'management network' - people trust that
and I don't see a lot of the qemu code getting fixed safely for all of
them.

> > These patches introduce the new 'yank' out-of-band qmp command to recover 
> > from
> > these kinds of hangs. The different subsystems register callbacks which get
> > executed with the yank command. For example the callback can shutdown() a
> > socket. This is intended for the colo use-case, but it can be used for other
> > things too of course.
> 
> IIUC, invoking the "yank" command unconditionally kills every single
> network connection in QEMU that has registered with the "yank" subsystem.
> IMHO this is way too big of a hammer, even if we accept there are bugs in
> QEMU not handling stalled networking well.

But isn't this hammer conditional - I see that it's a migration
capabiltiy for the migration socket, and a flag in nbd - so it only
yanks things you've told it to.

> eg if a chardev hangs QEMU, and we tear down everything, killing the NBD
> connection used for the guest disk, we needlessly break I/O.
> 
> eg doing this in the chardev backend is not desirable, because the bugs
> with hanging QEMU are typically caused by the way the frontend device
> uses the chardev blocking I/O calls, instead of non-blocking I/O calls.
> 

Having a way to get out of any of these problems from a single point is
quite nice.  To be useful in COLO you need to know for sure you can get
out of any network screwup.

We already use shutdown(2) in migrate_cancel and migrate-pause for
basically the same reason; I don't think we've got anything similar for
NBD, and we probably should have (I think I asked for it fairly
recently).

Dave



> Regards,
> Daniel
> -- 
> |: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-https://fstop138.berrange.com :|
> |: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [RFC PATCH 03/17] hw/misc/temp-sensor: Add 'info temp' HMP command

2020-04-21 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (f4...@amsat.org) wrote:
> Add a command to display current devices temperature in the monitor:
> 
>   (qemu) info temp
>   Temperatures (in C):
>   videocore  25.00
>   bcm2835-thermal-0  25.00
> 
> Signed-off-by: Philippe Mathieu-Daudé 

How do I set the temperature?

> ---
>  include/monitor/hmp.h |  1 +
>  hw/misc/temp-sensor.c | 29 +
>  hmp-commands-info.hx  | 11 +++
>  3 files changed, 41 insertions(+)
> 
> diff --git a/include/monitor/hmp.h b/include/monitor/hmp.h
> index e33ca5a911..f023230bd1 100644
> --- a/include/monitor/hmp.h
> +++ b/include/monitor/hmp.h
> @@ -129,5 +129,6 @@ void hmp_hotpluggable_cpus(Monitor *mon, const QDict 
> *qdict);
>  void hmp_info_vm_generation_id(Monitor *mon, const QDict *qdict);
>  void hmp_info_memory_size_summary(Monitor *mon, const QDict *qdict);
>  void hmp_info_sev(Monitor *mon, const QDict *qdict);
> +void hmp_info_temp(Monitor *mon, const QDict *qdict);
>  
>  #endif
> diff --git a/hw/misc/temp-sensor.c b/hw/misc/temp-sensor.c
> index 27750c533d..5f591bd9c3 100644
> --- a/hw/misc/temp-sensor.c
> +++ b/hw/misc/temp-sensor.c
> @@ -12,6 +12,8 @@
>  #include "hw/misc/temp-sensor.h"
>  #include "qapi/qapi-commands-misc.h"
>  #include "qapi/error.h"
> +#include "monitor/monitor.h"
> +#include "monitor/hmp.h"
>  
>  static int query_temperature_sensors_foreach(Object *obj, void *opaque)
>  {
> @@ -59,6 +61,33 @@ TemperatureSensorList *qmp_query_temperature_sensors(Error 
> **errp)
>  return list;
>  }
>  
> +void hmp_info_temp(Monitor *mon, const QDict *qdict)
> +{
> +TemperatureSensorList *list, *sensor;
> +Error *err = NULL;
> +
> +list = qmp_query_temperature_sensors();
> +if (!list) {
> +monitor_printf(mon, "No temperature sensors\n");
> +return;
> +}
> +if (err) {
> +monitor_printf(mon, "Error while getting temperatures: %s\n",
> +   error_get_pretty(err));
> +error_free(err);

Maybe use hmp_handle_error

> +goto out;
> +}
> +
> +monitor_printf(mon, "Temperatures (in C):\n");
> +for (sensor = list; sensor; sensor = sensor->next) {
> +monitor_printf(mon, "%-33s %6.2f\n", sensor->value->name,
> +   sensor->value->temperature);

See my question on the earlier patch; I'm curious here whether we want
device name, and then subname within that device.

Dave

> +}
> +
> +out:
> +qapi_free_TemperatureSensorList(list);
> +}
> +
>  static TypeInfo tempsensor_interface_type_info = {
>  .name = TYPE_TEMPSENSOR_INTERFACE,
>  .parent = TYPE_INTERFACE,
> diff --git a/hmp-commands-info.hx b/hmp-commands-info.hx
> index ca5198438d..77f1c43ce3 100644
> --- a/hmp-commands-info.hx
> +++ b/hmp-commands-info.hx
> @@ -880,4 +880,15 @@ SRST
>  Show SEV information.
>  ERST
>  
> +{
> +.name   = "temp",
> +.args_type  = "",
> +.params = "",
> +.help   = "show device temperatures",
> +.cmd= hmp_info_temp,
> +},
>  
> +SRST
> +  ``info temp``
> +Show device temperatures.
> +ERST
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [RFC PATCH 02/17] hw/misc/temp-sensor: Add 'query-temperature-sensors' QMP command

2020-04-21 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (f4...@amsat.org) wrote:
> Add a command to query current temperature from all sensors able
> to report it:
> 
>   { "execute": "query-temperature-sensors" }
>   {
>   "return": [
>   {
>   "temperature": 25,
>   "name": "videocore"
>   },
>   {
>   "temperature": 25,
>   "name": "bcm2835-thermal-0"
>   }
>   ]
>   }
> 
> Signed-off-by: Philippe Mathieu-Daudé 
> ---
>  qapi/misc.json| 24 ++
>  hw/misc/temp-sensor.c | 48 +++
>  2 files changed, 72 insertions(+)
> 
> diff --git a/qapi/misc.json b/qapi/misc.json
> index 99b90ac80b..51881931e2 100644
> --- a/qapi/misc.json
> +++ b/qapi/misc.json
> @@ -1550,3 +1550,27 @@
>  ##
>  { 'command': 'query-vm-generation-id', 'returns': 'GuidInfo' }
>  
> +##
> +# @TemperatureSensor:
> +#
> +# Temperature sensor information.
> +#
> +# @name: the name of the sensor
> +#
> +# @temperature: the current temperature of the sensor (in C)
> +#
> +# Since: 5.1
> +##
> +{ 'struct': 'TemperatureSensor',
> +  'data': { 'name': 'str',
> +'temperature': 'number' } }
> +
> +##
> +# @query-temperature-sensors:
> +#
> +# Return a list of TemperatureSensor for devices that support
> +# the TYPE_TEMPSENSOR_INTERFACE.
> +#
> +# Since: 5.1
> +##
> +{ 'command': 'query-temperature-sensors', 'returns': ['TemperatureSensor']}
> diff --git a/hw/misc/temp-sensor.c b/hw/misc/temp-sensor.c
> index b7c1eb2d87..27750c533d 100644
> --- a/hw/misc/temp-sensor.c
> +++ b/hw/misc/temp-sensor.c
> @@ -10,6 +10,54 @@
>  
>  #include "qemu/osdep.h"
>  #include "hw/misc/temp-sensor.h"
> +#include "qapi/qapi-commands-misc.h"
> +#include "qapi/error.h"
> +
> +static int query_temperature_sensors_foreach(Object *obj, void *opaque)
> +{
> +TemperatureSensorList **list = opaque;
> +TempSensor *sensor;
> +TempSensorClass *k;
> +
> +if (!object_dynamic_cast(obj, TYPE_TEMPSENSOR_INTERFACE)) {
> +return 0;
> +}
> +
> +k = TEMPSENSOR_INTERFACE_GET_CLASS(obj);
> +if (!k->get_temperature) {
> +return 0;
> +}
> +
> +sensor = TEMPSENSOR_INTERFACE(obj);
> +for (size_t i = 0; i < k->sensor_count; i++) {
> +TemperatureSensorList *info = g_malloc0(sizeof(*info));
> +TemperatureSensor *value = g_malloc0(sizeof(*value));
> +
> +if (k->get_name) {
> +value->name = g_strdup(k->get_name(sensor, i));
> +} else {
> +value->name = g_strdup_printf("%s-%zu",
> +  object_get_typename(obj), i);
> +}

How do the names work if you've got multiple of the same device; e.g.
is the get_name() method supposed to return the device path as well as
the sensor name?  if I have an emulated PCI device with two sensors on
and I then instantiate two of the PCI devices, do I get a name with the
PCI path in?

Dave

> +value->temperature = k->get_temperature(sensor, i);
> +
> +info->value = value;
> +info->next = *list;
> +*list = info;
> +}
> +
> +    return 0;
> +}
> +
> +TemperatureSensorList *qmp_query_temperature_sensors(Error **errp)
> +{
> +TemperatureSensorList *list = NULL;
> +
> +object_child_foreach_recursive(object_get_root(),
> +   query_temperature_sensors_foreach,
> +   );
> +return list;
> +}
>  
>  static TypeInfo tempsensor_interface_type_info = {
>  .name = TYPE_TEMPSENSOR_INTERFACE,
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 5/7] qapi: add filter-node-name to block-stream

2020-04-21 Thread Dr. David Alan Gilbert
RT, false, false, false, false,
> - );
> + BLOCKDEV_ON_ERROR_REPORT, false, NULL, false, false,
> + false, false, );
>  
>  hmp_handle_error(mon, );

That's moved now into block/monitor/block-hmp-cmds.c
Feel free to add the extra parameter to the HMP side as well!

Dave

>  }
> diff --git a/qapi/block-core.json b/qapi/block-core.json
> index 3c54717..169f8ea 100644
> --- a/qapi/block-core.json
> +++ b/qapi/block-core.json
> @@ -2552,6 +2552,11 @@
>  #'stop' and 'enospc' can only be used if the block device
>  #supports io-status (see BlockInfo).  Since 1.3.
>  #
> +# @filter-node-name: the node name that should be assigned to the
> +#filter driver that the stream job inserts into the graph
> +#above @device. If this option is not given, a node name 
> is
> +#autogenerated. (Since: 5.0)
> +#
>  # @auto-finalize: When false, this job will wait in a PENDING state after it 
> has
>  # finished its work, waiting for @block-job-finalize before
>  #         making any block graph changes.
> @@ -2581,6 +2586,7 @@
>'data': { '*job-id': 'str', 'device': 'str', '*base': 'str',
>  '*base-node': 'str', '*backing-file': 'str', '*speed': 'int',
>  '*on-error': 'BlockdevOnError',
> +'*filter-node-name': 'str',
>  '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } }
>  
>  ##
> -- 
> 1.8.3.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v2 0/5] fix migration with bitmaps and mirror

2020-04-03 Thread Dr. David Alan Gilbert
* Vladimir Sementsov-Ogievskiy (vsement...@virtuozzo.com) wrote:
> 03.04.2020 14:29, Vladimir Sementsov-Ogievskiy wrote:
> > 03.04.2020 14:23, Peter Krempa wrote:
> > > On Fri, Apr 03, 2020 at 14:02:47 +0300, Vladimir Sementsov-Ogievskiy 
> > > wrote:
> > > > 19.12.2019 13:36, Peter Krempa wrote:
> > > > > On Thu, Dec 19, 2019 at 11:51:01 +0300, Vladimir Sementsov-Ogievskiy 
> > > > > wrote:
> > > > > > Hi all!
> > > > > > 
> > > > > > It's a continuation for
> > > > > > "bitmap migration bug with -drive while block mirror runs"
> > > > > > <315cff78-dcdb-a3ce-2742-da3cc9f0c...@redhat.com>
> > > > > > https://lists.gnu.org/archive/html/qemu-devel/2019-09/msg07241.html
> > > > > > 
> > > > > > The problem is that bitmaps migrated to node with same node-name or
> > > > > > blk-parent name. And currently only the latter actually work in 
> > > > > > libvirt.
> > > > > > And with mirror-top filter it doesn't work, because
> > > > > > bdrv_get_device_or_node_name don't go through filters.
> > > > > 
> > > > > I want to point out that since libvirt-5.10 we use -blockdev to
> > > > > configure the backend of storage devices with qemu-4.2 and later. This
> > > > > means unfortunately that the BlockBackend of the drive does not have a
> > > > > name any more and thus the above will not work even if you make the
> > > > > lookup code to see through filters.
> > > > 
> > > > Not that this series doesn't make things worse, as it loops through 
> > > > named
> > > > block backends when trying to use their name for migration. So with 
> > > > these
> > > > patches applied, qemu will just work in more possible scenarios.
> > > 
> > > Okay, if that's so it's fair enough in this case.
> > > 
> > > I'm just very firmly against baking in the assumption that
> > > node names mean the same thing accross migration, because that will
> > > create a precedent situation and more stuff may be baked in on top of
> > > this in the future. It seems that it has already happened though and
> > > it's wrong. And the worst part is that it's never mentioned that this
> > > might occur. But again, don't do that and preferrably remove the
> > > matching of node names for bitmaps altogether until we can control it
> > > arbitrarily.
> > > 
> > > We've also seen this already before with the backend name of memory
> > > devices being baked in to the migration stream which creates an unwanted
> > > dependancy.
> > > 
> > 
> > Hmm. Actually, matching by node-name never worked. May be just drop it now, 
> > and allow only matching by blk-name?
> > 
> > And then (in 5.1) implement special qmp commands for precise mapping.
> > 
> 
> Hmm, it may break someones setup... Bad idea. Probably we can forbid 
> auto-generated node-names.

If we want to remove it I guess we have to go through a proper
deprecation; but that's OK.

The thing to keep in mind is that when people say 'the commandline
should match' on source/destination - that's just not true;
so we have to define what actually needs to stay the same for bitmap
migration to work.

Dave

> -- 
> Best regards,
> Vladimir
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH] Compress lines for immediate return

2020-04-01 Thread Dr. David Alan Gilbert
* Simran Singhal (singhalsimr...@gmail.com) wrote:
> Compress two lines into a single line if immediate return statement is found.
> 
> It also remove variables progress, val, data, ret and sock
> as they are no longer needed.
> 
> Remove space between function "mixer_load" and '(' to fix the
> checkpatch.pl error:-
> ERROR: space prohibited between function name and open parenthesis '('
> 
> Signed-off-by: Simran Singhal 
> ---
>  block/file-posix.c  | 3 +--
>  block/nfs.c | 3 +--
>  block/nvme.c| 4 +---
>  block/vhdx.c| 3 +--
>  hw/audio/ac97.c | 4 +---
>  hw/audio/adlib.c| 5 +
>  hw/display/cirrus_vga.c | 4 +---
>  migration/ram.c | 4 +---
>  ui/gtk.c| 3 +--
>  util/qemu-sockets.c | 5 +
>  10 files changed, 10 insertions(+), 28 deletions(-)
> 
> diff --git a/block/file-posix.c b/block/file-posix.c
> index 7e19bbff5f..dc01f0d4d3 100644
> --- a/block/file-posix.c
> +++ b/block/file-posix.c
> @@ -1627,8 +1627,7 @@ static int handle_aiocb_write_zeroes_unmap(void *opaque)
>  
>  /* If we couldn't manage to unmap while guaranteed that the area reads as
>   * all-zero afterwards, just write zeroes without unmapping */
> -ret = handle_aiocb_write_zeroes(aiocb);
> -return ret;
> +return handle_aiocb_write_zeroes(aiocb);
>  }
>  
>  #ifndef HAVE_COPY_FILE_RANGE
> diff --git a/block/nfs.c b/block/nfs.c
> index cc2413d5ab..100f15bd1f 100644
> --- a/block/nfs.c
> +++ b/block/nfs.c
> @@ -623,8 +623,7 @@ static int nfs_file_open(BlockDriverState *bs, QDict 
> *options, int flags,
>  }
>  
>  bs->total_sectors = ret;
> -ret = 0;
> -return ret;
> +return 0;
>  }
>  
>  static QemuOptsList nfs_create_opts = {
> diff --git a/block/nvme.c b/block/nvme.c
> index 7b7c0cc5d6..eb2f54dd9d 100644
> --- a/block/nvme.c
> +++ b/block/nvme.c
> @@ -575,11 +575,9 @@ static bool nvme_poll_cb(void *opaque)
>  {
>  EventNotifier *e = opaque;
>  BDRVNVMeState *s = container_of(e, BDRVNVMeState, irq_notifier);
> -bool progress = false;
>  
>  trace_nvme_poll_cb(s);
> -progress = nvme_poll_queues(s);
> -return progress;
> +return nvme_poll_queues(s);
>  }
>  
>  static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
> diff --git a/block/vhdx.c b/block/vhdx.c
> index 33e57cd656..2c0e7ee44d 100644
> --- a/block/vhdx.c
> +++ b/block/vhdx.c
> @@ -411,8 +411,7 @@ int vhdx_update_headers(BlockDriverState *bs, 
> BDRVVHDXState *s,
>  if (ret < 0) {
>  return ret;
>  }
> -ret = vhdx_update_header(bs, s, generate_data_write_guid, log_guid);
> -return ret;
> +return vhdx_update_header(bs, s, generate_data_write_guid, log_guid);
>  }
>  
>  /* opens the specified header block from the VHDX file header section */
> diff --git a/hw/audio/ac97.c b/hw/audio/ac97.c
> index 1ec87feec0..8a9b9924c4 100644
> --- a/hw/audio/ac97.c
> +++ b/hw/audio/ac97.c
> @@ -573,11 +573,9 @@ static uint32_t nam_readb (void *opaque, uint32_t addr)
>  static uint32_t nam_readw (void *opaque, uint32_t addr)
>  {
>  AC97LinkState *s = opaque;
> -uint32_t val = ~0U;
>  uint32_t index = addr;
>  s->cas = 0;
> -val = mixer_load (s, index);
> -return val;
> +return mixer_load(s, index);
>  }
>  
>  static uint32_t nam_readl (void *opaque, uint32_t addr)
> diff --git a/hw/audio/adlib.c b/hw/audio/adlib.c
> index d6c1fb0586..7c3b67dcfb 100644
> --- a/hw/audio/adlib.c
> +++ b/hw/audio/adlib.c
> @@ -120,13 +120,10 @@ static void adlib_write(void *opaque, uint32_t nport, 
> uint32_t val)
>  static uint32_t adlib_read(void *opaque, uint32_t nport)
>  {
>  AdlibState *s = opaque;
> -uint8_t data;
>  int a = nport & 3;
>  
>  adlib_kill_timers (s);
> -data = OPLRead (s->opl, a);
> -
> -return data;
> +return OPLRead (s->opl, a);
>  }
>  
>  static void timer_handler (void *opaque, int c, double interval_Sec)
> diff --git a/hw/display/cirrus_vga.c b/hw/display/cirrus_vga.c
> index 0d391e1300..1f29731ffe 100644
> --- a/hw/display/cirrus_vga.c
> +++ b/hw/display/cirrus_vga.c
> @@ -2411,12 +2411,10 @@ static uint64_t cirrus_linear_bitblt_read(void 
> *opaque,
>unsigned size)
>  {
>  CirrusVGAState *s = opaque;
> -uint32_t ret;
>  
>  /* XXX handle bitblt */
>  (void)s;
> -ret = 0xff;
> -return ret;
> +return 0xff;
>  }
>  
>  static void cirrus_linear_bitblt_write(void *opaque,
> diff --git a/migration/ram.c b/

Re: [PATCH 5/6] migration/ram: fix use after free of local_err

2020-03-24 Thread Dr. David Alan Gilbert
* Vladimir Sementsov-Ogievskiy (vsement...@virtuozzo.com) wrote:
> local_err is used again in migration_bitmap_sync_precopy() after
> precopy_notify(), so we must zero it. Otherwise try to set
> non-NULL local_err will crash.
> 
> Signed-off-by: Vladimir Sementsov-Ogievskiy 
> ---
>  migration/ram.c | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/migration/ram.c b/migration/ram.c
> index c12cfdbe26..04f13feb2e 100644
> --- a/migration/ram.c
> +++ b/migration/ram.c
> @@ -980,6 +980,7 @@ static void migration_bitmap_sync_precopy(RAMState *rs)
>   */
>  if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, _err)) {
>  error_report_err(local_err);
> +    local_err = NULL;

Reviewed-by: Dr. David Alan Gilbert 

and queued.


>  }
>  
>  migration_bitmap_sync(rs);
> -- 
> 2.21.0
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 4/6] migration/colo: fix use after free of local_err

2020-03-24 Thread Dr. David Alan Gilbert
* Vladimir Sementsov-Ogievskiy (vsement...@virtuozzo.com) wrote:
> local_err is used again in secondary_vm_do_failover() after
> replication_stop_all(), so we must zero it. Otherwise try to set
> non-NULL local_err will crash.
> 
> Signed-off-by: Vladimir Sementsov-Ogievskiy 

Reviewed-by: Dr. David Alan Gilbert 

I'll queue this

> ---
>  migration/colo.c | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/migration/colo.c b/migration/colo.c
> index 44942c4e23..a54ac84f41 100644
> --- a/migration/colo.c
> +++ b/migration/colo.c
> @@ -93,6 +93,7 @@ static void secondary_vm_do_failover(void)
>  replication_stop_all(true, _err);
>  if (local_err) {
>  error_report_err(local_err);
> +local_err = NULL;
>  }
>  
>  /* Notify all filters of all NIC to do checkpoint */
> -- 
> 2.21.0
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 2/2] lockable: replaced locks with lock guard macros where appropriate

2020-03-20 Thread Dr. David Alan Gilbert
* Paolo Bonzini (pbonz...@redhat.com) wrote:
> On 20/03/20 13:56, Dr. David Alan Gilbert wrote:
> >> According to https://wiki.qemu.org/ToDo/LockGuards cases that are trivial 
> >> (no
> >> conditional logic) shouldn't be replaced.
> > OK
> 
> I don't think that has to be either-or.  Trivial lock/unlock sequences
> are not the first ones that should be converted, but there's an
> advantage in having a single patch that converts all possible uses of a
> lock.  Trivial sequences certainly do not belong in a bigger patch like
> this, as they would make the patch even bigger.
> 
> > So for what you've already got there,
> > 
> > For migration:
> > Acked-by: Dr. David Alan Gilbert 
> > 
> 
> Can you just extract that and queue it yourself (for 5.1 probably)?


I can, although it would be easier if Daniel did that; there's no rush
given it's for 5.1

> Paolo
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 2/2] lockable: replaced locks with lock guard macros where appropriate

2020-03-20 Thread Dr. David Alan Gilbert
* Daniel Brodsky (dnbrd...@gmail.com) wrote:
> On Fri, Mar 20, 2020 at 5:34 AM Dr. David Alan Gilbert
>  wrote:
> >
> > * dnbrd...@gmail.com (dnbrd...@gmail.com) wrote:
> > > From: Daniel Brodsky 
> > >
> > > - ran regexp "qemu_mutex_lock\(.*\).*\n.*if" to find targets
> > > - replaced result with QEMU_LOCK_GUARD if all unlocks at function end
> > > - replaced result with WITH_QEMU_LOCK_GUARD if unlock not at end
> > >
> > > Signed-off-by: Daniel Brodsky 
> > > ---
> > >  block/iscsi.c | 11 +++---
> > >  block/nfs.c   | 51 ---
> > >  cpus-common.c | 13 +--
> > >  hw/display/qxl.c  | 43 +---
> > >  hw/vfio/platform.c|  5 ++---
> > >  migration/migration.c |  3 +--
> > >  migration/multifd.c   |  8 +++
> > >  migration/ram.c   |  3 +--
> > >  monitor/misc.c|  4 +---
> > >  ui/spice-display.c| 14 ++--
> > >  util/log.c|  4 ++--
> > >  util/qemu-timer.c | 17 +++
> > >  util/rcu.c|  8 +++
> > >  util/thread-pool.c|  3 +--
> > >  util/vfio-helpers.c   |  4 ++--
> > >  15 files changed, 84 insertions(+), 107 deletions(-)
> > >
> >
> > 
> >
> > > diff --git a/migration/migration.c b/migration/migration.c
> > > index c1d88ace7f..2f0bd6d8b4 100644
> > > --- a/migration/migration.c
> > > +++ b/migration/migration.c
> > > @@ -1652,11 +1652,10 @@ static void migrate_fd_cleanup_bh(void *opaque)
> > >
> > >  void migrate_set_error(MigrationState *s, const Error *error)
> > >  {
> > > -qemu_mutex_lock(>error_mutex);
> > > +QEMU_LOCK_GUARD(>error_mutex);
> > >  if (!s->error) {
> > >  s->error = error_copy(error);
> > >  }
> > > -qemu_mutex_unlock(>error_mutex);
> > >  }
> >
> > There are some other places in migration.c that would really benefit;
> > for example, migrate_send_rp_message, if you use a LOCK_QUARD
> > there, then you can remove the 'int ret', and the goto error.
> > In postcopy_pause, the locks on qemu_file_lock would work well using the
> > WITH_QEMU_LOCK_GUARD.
> 
> I did a basic pass through for targets and that one didn't come up. I can add
> more replacements later, but there are ~300 mutex locks that might be worth
> replacing and going through them manually in one go is too tedious.

Sure; the send_rp_message case is quite a nice example of where the
guard makes the code simpler.

> > >  void migrate_fd_error(MigrationState *s, const Error *error)
> > > diff --git a/migration/multifd.c b/migration/multifd.c
> > > index cb6a4a3ab8..9123c111a3 100644
> > > --- a/migration/multifd.c
> > > +++ b/migration/multifd.c
> > > @@ -894,11 +894,11 @@ void multifd_recv_sync_main(void)
> > >  for (i = 0; i < migrate_multifd_channels(); i++) {
> > >  MultiFDRecvParams *p = _recv_state->params[i];
> > >
> > > -qemu_mutex_lock(>mutex);
> > > -if (multifd_recv_state->packet_num < p->packet_num) {
> > > -multifd_recv_state->packet_num = p->packet_num;
> > > +WITH_QEMU_LOCK_GUARD(>mutex) {
> > > +if (multifd_recv_state->packet_num < p->packet_num) {
> > > +multifd_recv_state->packet_num = p->packet_num;
> > > +}
> > >  }
> > > -qemu_mutex_unlock(>mutex);
> > >  trace_multifd_recv_sync_main_signal(p->id);
> > >  qemu_sem_post(>sem_sync);
> > >  }
> >
> > > diff --git a/migration/ram.c b/migration/ram.c
> > > index c12cfdbe26..87a670cfbf 100644
> > > --- a/migration/ram.c
> > > +++ b/migration/ram.c
> > > @@ -1368,7 +1368,7 @@ static RAMBlock *unqueue_page(RAMState *rs, 
> > > ram_addr_t *offset)
> > >  return NULL;
> > >  }
> > >
> > > -qemu_mutex_lock(>src_page_req_mutex);
> > > +    QEMU_LOCK_GUARD(>src_page_req_mutex);
> > >  if (!QSIMPLEQ_EMPTY(>src_page_requests)) {
> > >  struct RAMSrcPageRequest *entry =
> > >  QSIMPLEQ_FIRST(>src_page_requests);
> > > @@ -1385,7 +1385,6 @@ static RAMBlock *unqueue_page(RAMState *rs, 
> > > ram_addr_t *offset)
> > >  

Re: [PATCH v3 2/2] lockable: replaced locks with lock guard macros where appropriate

2020-03-20 Thread Dr. David Alan Gilbert
k);
>  
>  /* Write RCU-protected pointers before reading p_rcu_reader->ctr.
>   * Pairs with smp_mb_placeholder() in rcu_read_lock().
>   */
>  smp_mb_global();
>  
> -qemu_mutex_lock(_registry_lock);
> +QEMU_LOCK_GUARD(_registry_lock);
>  if (!QLIST_EMPTY()) {
>  /* In either case, the atomic_mb_set below blocks stores that free
>   * old RCU-protected pointers.
> @@ -169,9 +170,6 @@ void synchronize_rcu(void)
>  
>  wait_for_readers();
>  }
> -
> -qemu_mutex_unlock(_registry_lock);
> -qemu_mutex_unlock(_sync_lock);
>  }
>  
>  
> diff --git a/util/thread-pool.c b/util/thread-pool.c
> index 4ed9b89ab2..d763cea505 100644
> --- a/util/thread-pool.c
> +++ b/util/thread-pool.c
> @@ -210,7 +210,7 @@ static void thread_pool_cancel(BlockAIOCB *acb)
>  
>  trace_thread_pool_cancel(elem, elem->common.opaque);
>  
> -qemu_mutex_lock(>lock);
> +QEMU_LOCK_GUARD(>lock);
>  if (elem->state == THREAD_QUEUED &&
>  /* No thread has yet started working on elem. we can try to "steal"
>   * the item from the worker if we can get a signal from the
> @@ -225,7 +225,6 @@ static void thread_pool_cancel(BlockAIOCB *acb)
>  elem->ret = -ECANCELED;
>  }
>  
> -qemu_mutex_unlock(>lock);
>  }
>  
>  static AioContext *thread_pool_get_aio_context(BlockAIOCB *acb)
> diff --git a/util/vfio-helpers.c b/util/vfio-helpers.c
> index ddd9a96e76..b310b23003 100644
> --- a/util/vfio-helpers.c
> +++ b/util/vfio-helpers.c
> @@ -21,6 +21,7 @@
>  #include "standard-headers/linux/pci_regs.h"
>  #include "qemu/event_notifier.h"
>  #include "qemu/vfio-helpers.h"
> +#include "qemu/lockable.h"
>  #include "trace.h"
>  
>  #define QEMU_VFIO_DEBUG 0
> @@ -667,14 +668,13 @@ int qemu_vfio_dma_reset_temporary(QEMUVFIOState *s)
>  .size = QEMU_VFIO_IOVA_MAX - s->high_water_mark,
>  };
>  trace_qemu_vfio_dma_reset_temporary(s);
> -qemu_mutex_lock(>lock);
> +QEMU_LOCK_GUARD(>lock);
>  if (ioctl(s->container, VFIO_IOMMU_UNMAP_DMA, )) {
>  error_report("VFIO_UNMAP_DMA failed: %s", strerror(errno));
>  qemu_mutex_unlock(>lock);
>  return -errno;
>  }
>  s->high_water_mark = QEMU_VFIO_IOVA_MAX;
> -qemu_mutex_unlock(>lock);
>  return 0;
>  }
>  
> -- 
> 2.25.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v5 00/11] HMP monitor handlers refactoring

2020-03-09 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> This patch series is bunch of cleanups to the hmp monitor code.
> It mostly moves the blockdev related hmp handlers to its own file,
> and does some minor refactoring.
> 
> No functional changes expected.

Queued for HMP, with the commit message fix up in 05.

Dave

> Changes from V1:
>* move the handlers to block/monitor/block-hmp-cmds.c
>* tiny cleanup for the commit messages
> 
> Changes from V2:
>* Moved all the function prototypes to new header (blockdev-hmp-cmds.h)
>* Set the license of blockdev-hmp-cmds.c to GPLv2+
>* Moved hmp_snapshot_* functions to blockdev-hmp-cmds.c
>* Moved hmp_drive_add_node to blockdev-hmp-cmds.c
>  (this change needed some new exports, thus in separate new patch)
>* Moved hmp_qemu_io and hmp_eject to blockdev-hmp-cmds.c
>* Added 'error:' prefix to vreport, and updated the iotests
>  This is invasive change, but really feels like the right one
>* Added minor refactoring patch that drops an unused #include
> 
> Changes from V3:
>* Dropped the error prefix patches for now due to fact that it seems
>  that libvirt doesn't need that after all. Oh well...
>  I'll send them in a separate series.
> 
>* Hopefully correctly merged the copyright info the new files
>  Both files are GPLv2 now (due to code from hmp.h/hmp-cmds.c)
> 
>* Addressed review feedback
>* Renamed the added header to block-hmp-cmds.h
> 
>* Got rid of checkpatch.pl warnings in the moved code
>  (cosmetic code changes only)
> 
>* I kept the reviewed-by tags, since the changes I did are minor.
>  I hope that this is right thing to do.
> 
> Changes from V4:
>* Rebase with recent changes
>* Fixed review feedback
> 
> Best regards,
>   Maxim Levitsky
> 
> Maxim Levitsky (11):
>   usb/dev-storage: remove unused include
>   monitor/hmp: inline add_init_drive
>   monitor/hmp: rename device-hotplug.c to block/monitor/block-hmp-cmds.c
>   monitor/hmp: move hmp_drive_del and hmp_commit to block-hmp-cmds.c
>   monitor/hmp: move hmp_drive_mirror and hmp_drive_backup to
> block-hmp-cmds.c Moved code was added after 2012-01-13, thus under
> GPLv2+
>   monitor/hmp: move hmp_block_job* to block-hmp-cmds.c
>   monitor/hmp: move hmp_snapshot_* to block-hmp-cmds.c
>   monitor/hmp: move hmp_nbd_server* to block-hmp-cmds.c
>   monitor/hmp: move remaining hmp_block* functions to block-hmp-cmds.c
>   monitor/hmp: move hmp_info_block* to block-hmp-cmds.c
>   monitor/hmp: Move hmp_drive_add_node to block-hmp-cmds.c
> 
>  MAINTAINERS|1 +
>  Makefile.objs  |2 +-
>  block/Makefile.objs|1 +
>  block/monitor/Makefile.objs|1 +
>  block/monitor/block-hmp-cmds.c | 1015 
>  blockdev.c |  137 +
>  device-hotplug.c   |   91 ---
>  hw/usb/dev-storage.c   |1 -
>  include/block/block-hmp-cmds.h |   54 ++
>  include/block/block_int.h  |5 +-
>  include/monitor/hmp.h  |   24 -
>  include/sysemu/blockdev.h  |4 -
>  include/sysemu/sysemu.h|3 -
>  monitor/hmp-cmds.c |  782 
>  monitor/misc.c |1 +
>  15 files changed, 1085 insertions(+), 1037 deletions(-)
>  create mode 100644 block/monitor/Makefile.objs
>  create mode 100644 block/monitor/block-hmp-cmds.c
>  delete mode 100644 device-hotplug.c
>  create mode 100644 include/block/block-hmp-cmds.h
> 
> -- 
> 2.17.2
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v5 05/11] monitor/hmp: move hmp_drive_mirror and hmp_drive_backup to block-hmp-cmds.c Moved code was added after 2012-01-13, thus under GPLv2+

2020-03-09 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> 
> I see that I have the same issue of long subject line here.
> Its because I forgot the space after first line, when adding this.
> If I need to resend another version of this patchset I'll fix this,
> but otherwise maybe that can be fixed when applying this to one of 
> maintainer's
> trees.
> 
> Sorry for noise.

I can just fix the commit message.

Dave

> Best regards,
>   Maxim Levitsky
> 
> On Sun, 2020-03-08 at 11:24 +0200, Maxim Levitsky wrote:
> > Signed-off-by: Maxim Levitsky 
> > Reviewed-by: Dr. David Alan Gilbert 
> > ---
> >  block/monitor/block-hmp-cmds.c | 60 ++
> >  include/block/block-hmp-cmds.h | 12 +--
> >  include/monitor/hmp.h  |  2 --
> >  monitor/hmp-cmds.c | 58 
> >  4 files changed, 69 insertions(+), 63 deletions(-)
> > 
> > diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> > index ad727a6b08..d6dd5d97f7 100644
> > --- a/block/monitor/block-hmp-cmds.c
> > +++ b/block/monitor/block-hmp-cmds.c
> > @@ -37,10 +37,12 @@
> >  #include "qapi/qapi-commands-block.h"
> >  #include "qapi/qmp/qdict.h"
> >  #include "qapi/error.h"
> > +#include "qapi/qmp/qerror.h"
> >  #include "qemu/config-file.h"
> >  #include "qemu/option.h"
> >  #include "sysemu/sysemu.h"
> >  #include "monitor/monitor.h"
> > +#include "monitor/hmp.h"
> >  #include "block/block_int.h"
> >  #include "block/block-hmp-cmds.h"
> >  
> > @@ -187,3 +189,61 @@ void hmp_commit(Monitor *mon, const QDict *qdict)
> >  error_report("'commit' error for '%s': %s", device, 
> > strerror(-ret));
> >  }
> >  }
> > +
> > +void hmp_drive_mirror(Monitor *mon, const QDict *qdict)
> > +{
> > +const char *filename = qdict_get_str(qdict, "target");
> > +const char *format = qdict_get_try_str(qdict, "format");
> > +bool reuse = qdict_get_try_bool(qdict, "reuse", false);
> > +bool full = qdict_get_try_bool(qdict, "full", false);
> > +Error *err = NULL;
> > +DriveMirror mirror = {
> > +.device = (char *)qdict_get_str(qdict, "device"),
> > +.target = (char *)filename,
> > +.has_format = !!format,
> > +.format = (char *)format,
> > +.sync = full ? MIRROR_SYNC_MODE_FULL : MIRROR_SYNC_MODE_TOP,
> > +.has_mode = true,
> > +.mode = reuse ? NEW_IMAGE_MODE_EXISTING : 
> > NEW_IMAGE_MODE_ABSOLUTE_PATHS,
> > +.unmap = true,
> > +};
> > +
> > +if (!filename) {
> > +error_setg(, QERR_MISSING_PARAMETER, "target");
> > +hmp_handle_error(mon, err);
> > +return;
> > +}
> > +qmp_drive_mirror(, );
> > +hmp_handle_error(mon, err);
> > +}
> > +
> > +void hmp_drive_backup(Monitor *mon, const QDict *qdict)
> > +{
> > +const char *device = qdict_get_str(qdict, "device");
> > +const char *filename = qdict_get_str(qdict, "target");
> > +const char *format = qdict_get_try_str(qdict, "format");
> > +bool reuse = qdict_get_try_bool(qdict, "reuse", false);
> > +bool full = qdict_get_try_bool(qdict, "full", false);
> > +bool compress = qdict_get_try_bool(qdict, "compress", false);
> > +Error *err = NULL;
> > +DriveBackup backup = {
> > +.device = (char *)device,
> > +.target = (char *)filename,
> > +.has_format = !!format,
> > +.format = (char *)format,
> > +.sync = full ? MIRROR_SYNC_MODE_FULL : MIRROR_SYNC_MODE_TOP,
> > +.has_mode = true,
> > +.mode = reuse ? NEW_IMAGE_MODE_EXISTING : 
> > NEW_IMAGE_MODE_ABSOLUTE_PATHS,
> > +.has_compress = !!compress,
> > +.compress = compress,
> > +};
> > +
> > +if (!filename) {
> > +error_setg(, QERR_MISSING_PARAMETER, "target");
> > +hmp_handle_error(mon, err);
> > +return;
> > +}
> > +
> > +qmp_drive_backup(, );
> > +hmp_handle_error(mon, err);
> > +}
> > diff --git a/include/block/block-hmp-cmds.h b/include/block/block-hmp-cmds.h
> > index 30b0f56415..a64b737b3a 100644
> > --- a/include/block/block-hmp-cmds.h
> > +++ b/include/blo

Re: [PATCH v1] block/nvme: introduce PMR support from NVMe 1.4 spec

2020-02-21 Thread Dr. David Alan Gilbert
* Stefan Hajnoczi (stefa...@gmail.com) wrote:
> On Tue, Feb 18, 2020 at 03:48:11PM -0700, Andrzej Jakowski wrote:
> > This patch introduces support for PMR that has been defined as part of NVMe 
> > 1.4
> > spec. User can now specify a pmr_file which will be mmap'ed into qemu 
> > address
> > space and subsequently in PCI BAR 2. Guest OS can perform mmio read and 
> > writes
> > to the PMR region that will stay persistent accross system reboot.
> > 
> > Signed-off-by: Andrzej Jakowski 
> > ---
> >  hw/block/nvme.c   | 145 ++-
> >  hw/block/nvme.h   |   5 ++
> >  hw/block/trace-events |   5 ++
> >  include/block/nvme.h  | 172 ++
> >  4 files changed, 326 insertions(+), 1 deletion(-)
> 
> NVDIMM folks, please take a look.  There seems to be commonality here.
> 
> Can this use -object memory-backend-file instead of manually opening and
> mapping a file?
> 
> Also CCing David Gilbert because there is some similarity with the
> vhost-user-fs's DAX Window feature where QEMU mmaps regions of files
> into a BAR.

I guess the biggest difference here is that the read can have the side
effect; in my world I don't have to set read/write/endian ops - I just
map a chunk of memory and use memory_region_add_subregion, so all the
read/writes are native read/writes - I assume that would be a lot
faster - I guess it depends if NVME_PMRCAP_PMRWBM is something constant
you know early on; if you know that you don't need to do side effects in
the read you could do the same trick and avoid the IO ops altogether.


Isn't there also a requirement that BARs are powers of two? Wouldn't
you need to ensure the PMR file is a power of 2 in size?

Dave

> > @@ -1303,6 +1327,38 @@ static const MemoryRegionOps nvme_cmb_ops = {
> >  },
> >  };
> >  
> > +static void nvme_pmr_write(void *opaque, hwaddr addr, uint64_t data,
> > +unsigned size)
> > +{
> > +NvmeCtrl *n = (NvmeCtrl *)opaque;
> > +stn_le_p(>pmrbuf[addr], size, data);
> > +}
> > +
> > +static uint64_t nvme_pmr_read(void *opaque, hwaddr addr, unsigned size)
> > +{
> > +NvmeCtrl *n = (NvmeCtrl *)opaque;
> > +if (!NVME_PMRCAP_PMRWBM(n->bar.pmrcap)) {
> > +int ret;
> > +ret = msync(n->pmrbuf, n->f_pmr_size, MS_SYNC);
> > +    if (!ret) {
> > +NVME_GUEST_ERR(nvme_ub_mmiowr_pmrread_barrier,
> > +   "error while persisting data");
> > +}
> > +}
> 
> Why is msync(2) done on memory loads instead of stores?


--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v1] block/nvme: introduce PMR support from NVMe 1.4 spec

2020-02-21 Thread Dr. David Alan Gilbert
* Stefan Hajnoczi (stefa...@gmail.com) wrote:
> On Fri, Feb 21, 2020 at 3:36 PM Andrzej Jakowski
>  wrote:
> > On 2/21/20 6:45 AM, Stefan Hajnoczi wrote:
> > > Why is msync(2) done on memory loads instead of stores?
> >
> > This is my interpretation of NVMe spec wording with regards to PMRWBM field
> > which says:
> >
> > "The completion of a memory read from any Persistent
> > Memory Region address ensures that all prior writes to the
> > Persistent Memory Region have completed and are
> > persistent."
> 
> Thanks, I haven't read the PMR section of the spec :).
> 
> A synchronous operation is bad for virtualization performance.  While
> the sync may be a cheap operation in hardware, it can be arbitrarily
> expensive with msync(2).  The vCPU will be stuck until msync(2)
> completes on the host.
> 
> It's also a strange design choice since performance will suffer when
> an unrelated read has to wait for writes to complete.  This is
> especially problematic for multi-threaded applications or multi-core
> systems where I guess this case is hit frequently.  Maybe it's so
> cheap in hardware that it doesn't matter?  But then why didn't NVDIMM
> use this mechanism?
> 
> If anyone knows the answer I'd be interested in learning.  But this
> isn't a criticism of the patch - of course it needs to implement the
> hardware spec and we can't change it.

Is this coming from the underlying PCIe spec ?
In PCIe Base 4.0 Rev 0.3 Feb19-2014, section 2.4.1 Transaction ordering,
there's a Table 2-39 and entry B2a in that table is:


  'A Read Request must not pass a Posted Request unless B2b applies.'

and a posted request is defined as a 'Memory Write Request or a Message
Request'.

???

Dave

> Stefan
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH] migration/block: rename BLOCK_SIZE macro

2020-02-18 Thread Dr. David Alan Gilbert
* Stefan Hajnoczi (stefa...@redhat.com) wrote:
> Both  and  define BLOCK_SIZE macros.  Avoiding
> using that name in block/migration.c.
> 
> I noticed this when including  (Linux io_uring) from
> "block/aio.h" and compilation failed.  Although patches adding that
> include haven't been sent yet, it makes sense to rename the macro now in
> case someone else stumbles on it in the meantime.
> 
> Signed-off-by: Stefan Hajnoczi 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  migration/block.c | 39 ---
>  1 file changed, 20 insertions(+), 19 deletions(-)
> 
> diff --git a/migration/block.c b/migration/block.c
> index c90288ed29..737b6499f9 100644
> --- a/migration/block.c
> +++ b/migration/block.c
> @@ -27,8 +27,8 @@
>  #include "migration/vmstate.h"
>  #include "sysemu/block-backend.h"
>  
> -#define BLOCK_SIZE   (1 << 20)
> -#define BDRV_SECTORS_PER_DIRTY_CHUNK (BLOCK_SIZE >> BDRV_SECTOR_BITS)
> +#define BLK_MIG_BLOCK_SIZE   (1 << 20)
> +#define BDRV_SECTORS_PER_DIRTY_CHUNK (BLK_MIG_BLOCK_SIZE >> BDRV_SECTOR_BITS)
>  
>  #define BLK_MIG_FLAG_DEVICE_BLOCK   0x01
>  #define BLK_MIG_FLAG_EOS0x02
> @@ -133,7 +133,7 @@ static void blk_send(QEMUFile *f, BlkMigBlock * blk)
>  uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK;
>  
>  if (block_mig_state.zero_blocks &&
> -buffer_is_zero(blk->buf, BLOCK_SIZE)) {
> +buffer_is_zero(blk->buf, BLK_MIG_BLOCK_SIZE)) {
>  flags |= BLK_MIG_FLAG_ZERO_BLOCK;
>  }
>  
> @@ -154,7 +154,7 @@ static void blk_send(QEMUFile *f, BlkMigBlock * blk)
>  return;
>  }
>  
> -qemu_put_buffer(f, blk->buf, BLOCK_SIZE);
> +qemu_put_buffer(f, blk->buf, BLK_MIG_BLOCK_SIZE);
>  }
>  
>  int blk_mig_active(void)
> @@ -309,7 +309,7 @@ static int mig_save_device_bulk(QEMUFile *f, 
> BlkMigDevState *bmds)
>  }
>  
>  blk = g_new(BlkMigBlock, 1);
> -blk->buf = g_malloc(BLOCK_SIZE);
> +blk->buf = g_malloc(BLK_MIG_BLOCK_SIZE);
>  blk->bmds = bmds;
>  blk->sector = cur_sector;
>  blk->nr_sectors = nr_sectors;
> @@ -350,7 +350,8 @@ static int set_dirty_tracking(void)
>  
>  QSIMPLEQ_FOREACH(bmds, _mig_state.bmds_list, entry) {
>  bmds->dirty_bitmap = bdrv_create_dirty_bitmap(blk_bs(bmds->blk),
> -  BLOCK_SIZE, NULL, 
> NULL);
> +  BLK_MIG_BLOCK_SIZE,
> +  NULL, NULL);
>  if (!bmds->dirty_bitmap) {
>  ret = -errno;
>  goto fail;
> @@ -548,7 +549,7 @@ static int mig_save_device_dirty(QEMUFile *f, 
> BlkMigDevState *bmds,
>  bdrv_dirty_bitmap_unlock(bmds->dirty_bitmap);
>  
>  blk = g_new(BlkMigBlock, 1);
> -blk->buf = g_malloc(BLOCK_SIZE);
> +blk->buf = g_malloc(BLK_MIG_BLOCK_SIZE);
>  blk->bmds = bmds;
>  blk->sector = sector;
>  blk->nr_sectors = nr_sectors;
> @@ -770,7 +771,7 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
>  
>  /* control the rate of transfer */
>  blk_mig_lock();
> -while (block_mig_state.read_done * BLOCK_SIZE <
> +while (block_mig_state.read_done * BLK_MIG_BLOCK_SIZE <
> qemu_file_get_rate_limit(f) &&
> block_mig_state.submitted < MAX_PARALLEL_IO &&
> (block_mig_state.submitted + block_mig_state.read_done) <
> @@ -874,13 +875,13 @@ static void block_save_pending(QEMUFile *f, void 
> *opaque, uint64_t max_size,
>  qemu_mutex_unlock_iothread();
>  
>  blk_mig_lock();
> -pending += block_mig_state.submitted * BLOCK_SIZE +
> -   block_mig_state.read_done * BLOCK_SIZE;
> +pending += block_mig_state.submitted * BLK_MIG_BLOCK_SIZE +
> +   block_mig_state.read_done * BLK_MIG_BLOCK_SIZE;
>  blk_mig_unlock();
>  
>  /* Report at least one block pending during bulk phase */
>  if (pending <= max_size && !block_mig_state.bulk_completed) {
> -pending = max_size + BLOCK_SIZE;
> +pending = max_size + BLK_MIG_BLOCK_SIZE;
>  }
>  
>  DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
> @@ -901,7 +902,7 @@ static int block_load(QEMUFile *f, void *opaque, int 
> version_id)
>  int nr_sectors;
>  int ret;
>  BlockDriverInfo bdi;
> -int cluster_size = BLOCK_SIZE;
> +int cluster_size 

Re: [PATCH RESEND 13/13] contrib/rdmacm-mux: Remove superfluous semicolon

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: a5d2f6f8773
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
> Cc: Shamir Rabinovitch 
> ---
>  contrib/rdmacm-mux/main.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/contrib/rdmacm-mux/main.c b/contrib/rdmacm-mux/main.c
> index de53048f06..bd82abbad3 100644
> --- a/contrib/rdmacm-mux/main.c
> +++ b/contrib/rdmacm-mux/main.c
> @@ -490,7 +490,7 @@ static int read_and_process(int fd)
>  
>  static int accept_all(void)
>  {
> -int fd, rc = 0;;
> +int fd, rc = 0;
>  
>  pthread_rwlock_wrlock();
>  
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 12/13] tests/qtest/libqos/qgraph: Remove superfluous semicolons

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: fc281c80202
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
> Cc: Emanuele Giuseppe Esposito 
> ---
>  tests/qtest/libqos/qgraph.c | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/tests/qtest/libqos/qgraph.c b/tests/qtest/libqos/qgraph.c
> index 7a7ae2a19e..ca01de0743 100644
> --- a/tests/qtest/libqos/qgraph.c
> +++ b/tests/qtest/libqos/qgraph.c
> @@ -474,7 +474,7 @@ QOSEdgeType qos_graph_edge_get_type(QOSGraphEdge *edge)
>  if (!edge) {
>  return -1;
>  }
> -return edge->type;;
> +return edge->type;
>  }
>  
>  char *qos_graph_edge_get_dest(QOSGraphEdge *edge)
> @@ -590,7 +590,7 @@ void qos_add_test(const char *name, const char *interface,
>QOSTestFunc test_func, QOSGraphTestOptions *opts)
>  {
>  QOSGraphNode *node;
> -char *test_name = g_strdup_printf("%s-tests/%s", interface, name);;
> +char *test_name = g_strdup_printf("%s-tests/%s", interface, name);
>  QOSGraphTestOptions def_opts = { };
>  
>  if (!opts) {
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 11/13] target/i386/whpx: Remove superfluous semicolon

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: 812d49f2a3e
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
> Cc: Justin Terry (VM) 
> ---
>  target/i386/whpx-all.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/target/i386/whpx-all.c b/target/i386/whpx-all.c
> index 3ed2aa1892..35601b8176 100644
> --- a/target/i386/whpx-all.c
> +++ b/target/i386/whpx-all.c
> @@ -511,7 +511,7 @@ static void whpx_get_registers(CPUState *cpu)
>  /* WHvX64RegisterPat - Skipped */
>  
>  assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
> -env->sysenter_cs = vcxt.values[idx++].Reg64;;
> +env->sysenter_cs = vcxt.values[idx++].Reg64;
>  assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
>  env->sysenter_eip = vcxt.values[idx++].Reg64;
>  assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 07/13] hw/scsi/esp: Remove superfluous semicolon

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: 74d71ea16bc
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
> Cc: Dr. David Alan Gilbert 
> Cc: Mark Cave-Ayland 
> Cc: Laurent Vivier 
> Cc: Paolo Bonzini 
> ---
>  hw/scsi/esp.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/hw/scsi/esp.c b/hw/scsi/esp.c
> index f8fc30cccb..405f8b7cbc 100644
> --- a/hw/scsi/esp.c
> +++ b/hw/scsi/esp.c
> @@ -293,7 +293,7 @@ static void handle_satn_stop(ESPState *s)
>  s->dma_cb = handle_satn_stop;
>  return;
>  }
> -s->pdma_cb = satn_stop_pdma_cb;;
> +s->pdma_cb = satn_stop_pdma_cb;
>  s->cmdlen = get_cmd(s, s->cmdbuf, sizeof(s->cmdbuf));
>  if (s->cmdlen) {
>  trace_esp_handle_satn_stop(s->cmdlen);
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 10/13] ui/input-barrier: Remove superfluous semicolon

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: 6105683da35
> Signed-off-by: Philippe Mathieu-Daudé 
> ---
> Cc: Laurent Vivier 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  ui/input-barrier.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/ui/input-barrier.c b/ui/input-barrier.c
> index fe35049b83..527c75e130 100644
> --- a/ui/input-barrier.c
> +++ b/ui/input-barrier.c
> @@ -455,7 +455,7 @@ static gboolean writecmd(InputBarrier *ib, struct 
> barrierMsg *msg)
>  break;
>  default:
>  write_cmd(p, barrierCmdEUnknown, avail);
> -break;;
> +break;
>  }
>  
>  len = MAX_HELLO_LENGTH - avail - sizeof(int);
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 04/13] block/io_uring: Remove superfluous semicolon

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: 6663a0a3376
> Signed-off-by: Philippe Mathieu-Daudé 
> ---
> Cc: Stefano Garzarella 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block/io_uring.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/block/io_uring.c b/block/io_uring.c
> index 56892fd1ab..a3142ca989 100644
> --- a/block/io_uring.c
> +++ b/block/io_uring.c
> @@ -187,7 +187,7 @@ static void luring_process_completions(LuringState *s)
>  ret = 0;
>  }
>  } else {
> -ret = -ENOSPC;;
> +ret = -ENOSPC;
>          }
>      }
>  end:
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 09/13] migration/multifd: Remove superfluous semicolon

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: d32ca5ad798
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  migration/multifd.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/migration/multifd.c b/migration/multifd.c
> index b3e8ae9bcc..cfaba1369e 100644
> --- a/migration/multifd.c
> +++ b/migration/multifd.c
> @@ -305,7 +305,7 @@ static int multifd_send_pages(QEMUFile *f)
>  + p->packet_len;
>  qemu_file_update_transfer(f, transferred);
>  ram_counters.multifd_bytes += transferred;
> -ram_counters.transferred += transferred;;
> +ram_counters.transferred += transferred;
>  qemu_mutex_unlock(>mutex);
>  qemu_sem_post(>sem);
>  
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 08/13] hw/vfio/display: Remove superfluous semicolon

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: 8b818e059bf
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
> Cc: Gerd Hoffmann 
> ---
>  hw/vfio/display.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/hw/vfio/display.c b/hw/vfio/display.c
> index a5a608c5b2..f4977c66e1 100644
> --- a/hw/vfio/display.c
> +++ b/hw/vfio/display.c
> @@ -287,7 +287,7 @@ static void vfio_display_dmabuf_update(void *opaque)
>  VFIOPCIDevice *vdev = opaque;
>  VFIODisplay *dpy = vdev->dpy;
>  VFIODMABuf *primary, *cursor;
> -bool free_bufs = false, new_cursor = false;;
> +bool free_bufs = false, new_cursor = false;
>  
>  primary = vfio_display_get_dmabuf(vdev, DRM_PLANE_TYPE_PRIMARY);
>  if (primary == NULL) {
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 03/13] block: Remove superfluous semicolons

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: 132ada80c4a
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block.c | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/block.c b/block.c
> index 9c810534d6..9db0b973fe 100644
> --- a/block.c
> +++ b/block.c
> @@ -2435,13 +2435,13 @@ BdrvChild *bdrv_root_attach_child(BlockDriverState 
> *child_bs,
>  if (bdrv_get_aio_context(child_bs) != ctx) {
>  ret = bdrv_try_set_aio_context(child_bs, ctx, _err);
>  if (ret < 0 && child_role->can_set_aio_ctx) {
> -GSList *ignore = g_slist_prepend(NULL, child);;
> +GSList *ignore = g_slist_prepend(NULL, child);
>  ctx = bdrv_get_aio_context(child_bs);
>  if (child_role->can_set_aio_ctx(child, ctx, , NULL)) {
>  error_free(local_err);
>  ret = 0;
>  g_slist_free(ignore);
> -ignore = g_slist_prepend(NULL, child);;
> +ignore = g_slist_prepend(NULL, child);
>  child_role->set_aio_ctx(child, ctx, );
>  }
>  g_slist_free(ignore);
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 06/13] hw/m68k/next-cube: Remove superfluous semicolon

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: 956a78118bf
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  hw/m68k/next-cube.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/hw/m68k/next-cube.c b/hw/m68k/next-cube.c
> index e5343348d0..350c6fec78 100644
> --- a/hw/m68k/next-cube.c
> +++ b/hw/m68k/next-cube.c
> @@ -734,7 +734,7 @@ void next_irq(void *opaque, int number, int level)
>  switch (number) {
>  /* level 3 - floppy, kbd/mouse, power, ether rx/tx, scsi, clock */
>  case NEXT_FD_I:
> -shift = 7;;
> +shift = 7;
>  break;
>  case NEXT_KBD_I:
>      shift = 3;
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 01/13] scripts/checkpatch.pl: Detect superfluous semicolon in C code

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Display error when a commit contains superfluous semicolon:
> 
>   $ git show 6663a0a3376 | scripts/checkpatch.pl -q -
>   ERROR: superfluous trailing semicolon
>   #276: FILE: block/io_uring.c:186:
>   +ret = -ENOSPC;;
>   total: 1 errors, 1 warnings, 485 lines checked
> 
> Reported-by: Luc Michel 
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
> Cc: Paolo Bonzini 
> ---
>  scripts/checkpatch.pl | 5 +
>  1 file changed, 5 insertions(+)
> 
> diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
> index ce43a306f8..11512a8a09 100755
> --- a/scripts/checkpatch.pl
> +++ b/scripts/checkpatch.pl
> @@ -1830,6 +1830,11 @@ sub process {
>   ERROR("suspicious ; after while (0)\n" . $herecurr);
>   }
>  
> +# Check superfluous trailing ';'
> + if ($line =~ /;;$/) {
> + ERROR("superfluous trailing semicolon\n" . $herecurr);
> + }
> +
>  # Check relative indent for conditionals and blocks.
>   if ($line =~ /\b(?:(?:if|while|for)\s*\(|do\b)/ && $line !~ 
> /^.\s*#/ && $line !~ /\}\s*while\s*/) {
>   my ($s, $c) = ($stat, $cond);
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 05/13] hw/arm/xlnx-versal: Remove superfluous semicolon

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: 6f16da53ffe
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  hw/arm/xlnx-versal-virt.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/hw/arm/xlnx-versal-virt.c b/hw/arm/xlnx-versal-virt.c
> index 462493c467..0d2e3bdda1 100644
> --- a/hw/arm/xlnx-versal-virt.c
> +++ b/hw/arm/xlnx-versal-virt.c
> @@ -350,7 +350,7 @@ static void create_virtio_regions(VersalVirt *s)
>  int i;
>  
>  for (i = 0; i < NUM_VIRTIO_TRANSPORT; i++) {
> -char *name = g_strdup_printf("virtio%d", i);;
> +char *name = g_strdup_printf("virtio%d", i);
>  hwaddr base = MM_TOP_RSVD + i * virtio_mmio_size;
>  int irq = VERSAL_RSVD_IRQ_FIRST + i;
>  MemoryRegion *mr;
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH RESEND 02/13] audio/alsaaudio: Remove superfluous semicolons

2020-02-18 Thread Dr. David Alan Gilbert
* Philippe Mathieu-Daudé (phi...@redhat.com) wrote:
> Fixes: 286a5d201e4
> Signed-off-by: Philippe Mathieu-Daudé 

Reviewed-by: Dr. David Alan Gilbert 

> ---
> Cc: "Kővágó, Zoltán" 
> ---
>  audio/alsaaudio.c | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/audio/alsaaudio.c b/audio/alsaaudio.c
> index a23a5a0b60..a8e62542f9 100644
> --- a/audio/alsaaudio.c
> +++ b/audio/alsaaudio.c
> @@ -819,7 +819,7 @@ static size_t alsa_read(HWVoiceIn *hw, void *buf, size_t 
> len)
>  switch (nread) {
>  case 0:
>  trace_alsa_read_zero(len);
> -return pos;;
> +return pos;
>  
>  case -EPIPE:
>  if (alsa_recover(alsa->handle)) {
> @@ -835,7 +835,7 @@ static size_t alsa_read(HWVoiceIn *hw, void *buf, size_t 
> len)
>  default:
>  alsa_logerr(nread, "Failed to read %zu frames to %p\n",
>  len, dst);
> -    return pos;;
> +return pos;
>  }
>  }
>  
> -- 
> 2.21.1
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH 3/3] iotests: Test external snapshot with VM state

2020-02-10 Thread Dr. David Alan Gilbert
* Kevin Wolf (kw...@redhat.com) wrote:
> Am 02.01.2020 um 14:25 hat Dr. David Alan Gilbert geschrieben:
> > * Kevin Wolf (kw...@redhat.com) wrote:
> > > Am 19.12.2019 um 15:26 hat Max Reitz geschrieben:
> > > > On 17.12.19 15:59, Kevin Wolf wrote:
> > > > > This tests creating an external snapshot with VM state (which results 
> > > > > in
> > > > > an active overlay over an inactive backing file, which is also the 
> > > > > root
> > > > > node of an inactive BlockBackend), re-activating the images and
> > > > > performing some operations to test that the re-activation worked as
> > > > > intended.
> > > > > 
> > > > > Signed-off-by: Kevin Wolf 
> > > > 
> > > > [...]
> > > > 
> > > > > diff --git a/tests/qemu-iotests/280.out b/tests/qemu-iotests/280.out
> > > > > new file mode 100644
> > > > > index 00..5d382faaa8
> > > > > --- /dev/null
> > > > > +++ b/tests/qemu-iotests/280.out
> > > > > @@ -0,0 +1,50 @@
> > > > > +Formatting 'TEST_DIR/PID-base', fmt=qcow2 size=67108864 
> > > > > cluster_size=65536 lazy_refcounts=off refcount_bits=16
> > > > > +
> > > > > +=== Launch VM ===
> > > > > +Enabling migration QMP events on VM...
> > > > > +{"return": {}}
> > > > > +
> > > > > +=== Migrate to file ===
> > > > > +{"execute": "migrate", "arguments": {"uri": "exec:cat > /dev/null"}}
> > > > > +{"return": {}}
> > > > > +{"data": {"status": "setup"}, "event": "MIGRATION", "timestamp": 
> > > > > {"microseconds": "USECS", "seconds": "SECS"}}
> > > > > +{"data": {"status": "active"}, "event": "MIGRATION", "timestamp": 
> > > > > {"microseconds": "USECS", "seconds": "SECS"}}
> > > > > +{"data": {"status": "completed"}, "event": "MIGRATION", "timestamp": 
> > > > > {"microseconds": "USECS", "seconds": "SECS"}}
> > > > > +
> > > > > +VM is now stopped:
> > > > > +completed
> > > > > +{"execute": "query-status", "arguments": {}}
> > > > > +{"return": {"running": false, "singlestep": false, "status": 
> > > > > "postmigrate"}}
> > > > 
> > > > Hmmm, I get a finish-migrate status here (on tmpfs)...
> > > 
> > > Dave, is it intentional that the "completed" migration event is emitted
> > > while we are still in finish-migration rather than postmigrate?
> > 
> > Yes it looks like it;  it's that the migration state machine hits
> > COMPLETED that then _causes_ the runstate transitition to POSTMIGRATE.
> > 
> > static void migration_iteration_finish(MigrationState *s)
> > {
> > /* If we enabled cpu throttling for auto-converge, turn it off. */
> > cpu_throttle_stop();
> > 
> > qemu_mutex_lock_iothread();
> > switch (s->state) {
> > case MIGRATION_STATUS_COMPLETED:
> > migration_calculate_complete(s);
> > runstate_set(RUN_STATE_POSTMIGRATE);
> > break;
> > 
> > then there are a bunch of error cases where if it landed in
> > FAILED/CANCELLED etc then we either restart the VM or also go to
> > POSTMIGRATE.
> 
> Yes, I read the code. My question was more if there is a reason why we
> want things to look like this in the external interface.
> 
> I just thought that it was confusing that migration is already called
> completed when it will still change the runstate. But I guess the
> opposite could be confusing as well (if we're in postmigrate, why should
> the migration status still change?)
> 
> > > I guess we could change wait_migration() in qemu-iotests to wait for the
> > > postmigrate state rather than the "completed" event, but maybe it would
> > > be better to change the migration code to avoid similar races in other
> > > QMP clients.
> > 
> > Given that the migration state machine is driving the runstate state
> > machine I think it currently makes sens

Re: [PATCH v4 00/11] RFC: [for 5.0]: HMP monitor handlers refactoring

2020-02-07 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> On Mon, 2020-02-03 at 19:57 +0000, Dr. David Alan Gilbert wrote:
> > * Maxim Levitsky (mlevi...@redhat.com) wrote:
> > > This patch series is bunch of cleanups to the hmp monitor code.
> > > It mostly moves the blockdev related hmp handlers to its own file,
> > > and does some minor refactoring.
> > > 
> > > No functional changes expected.
> > 
> > You've still got the title marked as RFC - are you actually ready for
> > this log?
> 
> I forgot to update this to be honest, I don't consider this an RFC,
> especially since I dropped for now the patches that might cause
> issues. This is now just a nice refactoring.

OK, so if we can get some block people to say they're happy, then
I'd be happy to take this through HMP or they can take it through block.

Dave

> Best regards,
>   Maxim Levitsky
> 
> > 
> > Dave
> > 
> > > 
> > > Changes from V1:
> > >* move the handlers to block/monitor/block-hmp-cmds.c
> > >* tiny cleanup for the commit messages
> > > 
> > > Changes from V2:
> > >* Moved all the function prototypes to new header (blockdev-hmp-cmds.h)
> > >* Set the license of blockdev-hmp-cmds.c to GPLv2+
> > >* Moved hmp_snapshot_* functions to blockdev-hmp-cmds.c
> > >* Moved hmp_drive_add_node to blockdev-hmp-cmds.c
> > >  (this change needed some new exports, thus in separate new patch)
> > >* Moved hmp_qemu_io and hmp_eject to blockdev-hmp-cmds.c
> > >* Added 'error:' prefix to vreport, and updated the iotests
> > >  This is invasive change, but really feels like the right one
> > >* Added minor refactoring patch that drops an unused #include
> > > 
> > > Changes from V3:
> > >* Dropped the error prefix patches for now due to fact that it seems
> > >  that libvirt doesn't need that after all. Oh well...
> > >  I'll send them in a separate series.
> > > 
> > >* Hopefully correctly merged the copyright info the new files
> > >  Both files are GPLv2 now (due to code from hmp.h/hmp-cmds.c)
> > > 
> > >* Addressed review feedback
> > >* Renamed the added header to block-hmp-cmds.h
> > > 
> > >* Got rid of checkpatch.pl warnings in the moved code
> > >  (cosmetic code changes only)
> > > 
> > >* I kept the reviewed-by tags, since the changes I did are minor.
> > >  I hope that this is right thing to do.
> > > 
> > > Best regards,
> > >   Maxim Levitsky
> > > 
> > > Maxim Levitsky (11):
> > >   usb/dev-storage: remove unused include
> > >   monitor/hmp: uninline add_init_drive
> > >   monitor/hmp: rename device-hotplug.c to block/monitor/block-hmp-cmds.c
> > >   monitor/hmp: move hmp_drive_del and hmp_commit to block-hmp-cmds.c
> > >   monitor/hmp: move hmp_drive_mirror and hmp_drive_backup to
> > > block-hmp-cmds.c Moved code was added after 2012-01-13, thus under
> > > GPLv2+
> > >   monitor/hmp: move hmp_block_job* to block-hmp-cmds.c
> > >   monitor/hmp: move hmp_snapshot_* to block-hmp-cmds.c
> > > hmp_snapshot_blkdev is from GPLv2 version of the hmp-cmds.c thus
> > > have to change the licence to GPLv2
> > >   monitor/hmp: move hmp_nbd_server* to block-hmp-cmds.c
> > >   monitor/hmp: move remaining hmp_block* functions to block-hmp-cmds.c
> > >   monitor/hmp: move hmp_info_block* to block-hmp-cmds.c
> > >   monitor/hmp: Move hmp_drive_add_node to block-hmp-cmds.c
> > > 
> > >  MAINTAINERS|1 +
> > >  Makefile.objs  |2 +-
> > >  block/Makefile.objs|1 +
> > >  block/monitor/Makefile.objs|1 +
> > >  block/monitor/block-hmp-cmds.c | 1002 
> > >  blockdev.c |  137 +
> > >  device-hotplug.c   |   91 ---
> > >  hw/usb/dev-storage.c   |1 -
> > >  include/block/block-hmp-cmds.h |   54 ++
> > >  include/block/block_int.h      |    5 +-
> > >  include/monitor/hmp.h  |   24 -
> > >  include/sysemu/blockdev.h  |4 -
> > >  include/sysemu/sysemu.h|3 -
> > >  monitor/hmp-cmds.c |  769 
> > >  monitor/misc.c |1 +
> > >  15 files changed, 1072 insertions(+), 1024 deletions(-)
> > >  create mode 100644 block/monitor/Makefile.objs
> > >  create mode 100644 block/monitor/block-hmp-cmds.c
> > >  delete mode 100644 device-hotplug.c
> > >  create mode 100644 include/block/block-hmp-cmds.h
> > > 
> > > -- 
> > > 2.17.2
> > > 
> > 
> > --
> > Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK
> 
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 00/13] RFC: [for 5.0]: HMP monitor handlers cleanups

2020-02-06 Thread Dr. David Alan Gilbert
* John Snow (js...@redhat.com) wrote:
> 
> 
> On 1/28/20 11:47 AM, Dr. David Alan Gilbert wrote:
> > * John Snow (js...@redhat.com) wrote:
> >>
> >>
> >> On 1/27/20 3:43 PM, Peter Krempa wrote:
> >>> On Mon, Jan 27, 2020 at 14:39:02 -0500, John Snow wrote:
> >>>>
> >>>>
> >>>> On 1/27/20 5:36 AM, Maxim Levitsky wrote:
> >>>>> This patch series is bunch of cleanups
> >>>>> to the hmp monitor code.
> >>>>>
> >>>>> This series only touched blockdev related hmp handlers.
> >>>>>
> >>>>> No functional changes expected other that
> >>>>> light error message changes by the last patch.
> >>>>>
> >>>>> This was inspired by this bugzilla:
> >>>>> https://bugzilla.redhat.com/show_bug.cgi?id=1719169
> >>>>>
> >>>>> Basically some users still parse hmp error messages,
> >>>>> and they would like to have them prefixed with 'Error:'
> >>>>>
> >>>>
> >>>> HMP isn't meant to be parsed. It's explicitly *not* API or ABI. I do
> >>>> like consistency in my UIs (it's useful for human eyes, too), but I'd
> >>>> like to know more about the request.
> >>>
> >>> That's true as long as there's an stable replacement ... see below.
> >>>
> >>
> >> Thanks for the context!
> >>
> >>>>
> >>>> Is this request coming from libvirt? Can we wean them off of this
> >>>> interface? What do they need as a replacement?
> >>>
> >>> There are 5 commands that libvirt still has HMP interfaces for:
> >>>
> >>> drive_add
> >>> drive_del
> >>>
> >>> savevm
> >>> loadvm
> >>> delvm
> >>>
> >>> From upstream point of view there's no value in adding the 'error'
> >>> prefix to drive_add/drive_del as libvirt now uses blockdev-add/del QMP
> >>> command instead which have implicit error propagation.
> >>>
> >>
> >> As thought.
> >>
> >>> There are no replacements for the internal snapshot commands, but they
> >>> reported the 'error' prefix for some time even before this series.
> >>>
> >>> Said that, please don't break savevm/loadvm/delvm until a QMP
> >>> replacement is added.
> >>>
> >>
> >> Yes, noted. I wonder where userfaultfd write support is these days...
> > 
> > How would that help you there?
> > 
> 
> Left at the traffic lights, but there was a thought that we'd be able to
> get transactionable save-memory support in QMP if we could use
> userfaultfd to do just-in-time copies of memory as needed, until the job
> is complete.
> 
> This way we could support it properly in QMP and we'd have a replacement
> for the HMP version which -- from memory -- is not appropriate for the
> QMP channel.
> 
> Maybe I imagined this restriction.

The restriction is there; but it's not related to the memory saving;
savevm mostly uses the core migration code (which would normally run in
a separate thread) but uses it itself in it's own loop in the main
thread with a bunch of bdrv code glued around it to do the snapshots
there. 
It shouldn't be that hard to convert it to be much more like a normal
migration - except it needs some hook then to do the snapshotting stuff
at the end.

Dave

> --js
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v4 00/11] RFC: [for 5.0]: HMP monitor handlers refactoring

2020-02-03 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> This patch series is bunch of cleanups to the hmp monitor code.
> It mostly moves the blockdev related hmp handlers to its own file,
> and does some minor refactoring.
> 
> No functional changes expected.

You've still got the title marked as RFC - are you actually ready for
this log?

Dave

> 
> Changes from V1:
>* move the handlers to block/monitor/block-hmp-cmds.c
>* tiny cleanup for the commit messages
> 
> Changes from V2:
>* Moved all the function prototypes to new header (blockdev-hmp-cmds.h)
>* Set the license of blockdev-hmp-cmds.c to GPLv2+
>* Moved hmp_snapshot_* functions to blockdev-hmp-cmds.c
>* Moved hmp_drive_add_node to blockdev-hmp-cmds.c
>  (this change needed some new exports, thus in separate new patch)
>* Moved hmp_qemu_io and hmp_eject to blockdev-hmp-cmds.c
>* Added 'error:' prefix to vreport, and updated the iotests
>  This is invasive change, but really feels like the right one
>* Added minor refactoring patch that drops an unused #include
> 
> Changes from V3:
>* Dropped the error prefix patches for now due to fact that it seems
>  that libvirt doesn't need that after all. Oh well...
>  I'll send them in a separate series.
> 
>* Hopefully correctly merged the copyright info the new files
>  Both files are GPLv2 now (due to code from hmp.h/hmp-cmds.c)
> 
>* Addressed review feedback
>* Renamed the added header to block-hmp-cmds.h
> 
>* Got rid of checkpatch.pl warnings in the moved code
>  (cosmetic code changes only)
> 
>* I kept the reviewed-by tags, since the changes I did are minor.
>  I hope that this is right thing to do.
> 
> Best regards,
>   Maxim Levitsky
> 
> Maxim Levitsky (11):
>   usb/dev-storage: remove unused include
>   monitor/hmp: uninline add_init_drive
>   monitor/hmp: rename device-hotplug.c to block/monitor/block-hmp-cmds.c
>   monitor/hmp: move hmp_drive_del and hmp_commit to block-hmp-cmds.c
>   monitor/hmp: move hmp_drive_mirror and hmp_drive_backup to
> block-hmp-cmds.c Moved code was added after 2012-01-13, thus under
> GPLv2+
>   monitor/hmp: move hmp_block_job* to block-hmp-cmds.c
>   monitor/hmp: move hmp_snapshot_* to block-hmp-cmds.c
> hmp_snapshot_blkdev is from GPLv2 version of the hmp-cmds.c thus
> have to change the licence to GPLv2
>   monitor/hmp: move hmp_nbd_server* to block-hmp-cmds.c
>   monitor/hmp: move remaining hmp_block* functions to block-hmp-cmds.c
>   monitor/hmp: move hmp_info_block* to block-hmp-cmds.c
>   monitor/hmp: Move hmp_drive_add_node to block-hmp-cmds.c
> 
>  MAINTAINERS|1 +
>  Makefile.objs  |2 +-
>  block/Makefile.objs|1 +
>  block/monitor/Makefile.objs|1 +
>  block/monitor/block-hmp-cmds.c | 1002 
>  blockdev.c |  137 +
>  device-hotplug.c   |   91 ---
>  hw/usb/dev-storage.c   |1 -
>  include/block/block-hmp-cmds.h |   54 ++
>  include/block/block_int.h  |5 +-
>  include/monitor/hmp.h  |   24 -
>  include/sysemu/blockdev.h  |4 -
>  include/sysemu/sysemu.h|3 -
>  monitor/hmp-cmds.c |  769 
>  monitor/misc.c |1 +
>  15 files changed, 1072 insertions(+), 1024 deletions(-)
>  create mode 100644 block/monitor/Makefile.objs
>  create mode 100644 block/monitor/block-hmp-cmds.c
>  delete mode 100644 device-hotplug.c
>  create mode 100644 include/block/block-hmp-cmds.h
> 
> -- 
> 2.17.2
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 10/13] monitor/hmp: move hmp_info_block* to block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> Signed-off-by: Maxim Levitsky 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block/monitor/block-hmp-cmds.c | 388 +
>  include/block/block-hmp-commands.h |   4 +
>  include/monitor/hmp.h  |   4 -
>  monitor/hmp-cmds.c | 388 -
>  4 files changed, 392 insertions(+), 392 deletions(-)
> 
> diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> index 60d63bfe18..a4b1604aee 100644
> --- a/block/monitor/block-hmp-cmds.c
> +++ b/block/monitor/block-hmp-cmds.c
> @@ -23,11 +23,13 @@
>  #include "qemu/config-file.h"
>  #include "qemu/option.h"
>  #include "qemu/sockets.h"
> +#include "qemu/cutils.h"
>  #include "sysemu/sysemu.h"
>  #include "monitor/monitor.h"
>  #include "block/nbd.h"
>  #include "block/block_int.h"
>  #include "block/block-hmp-commands.h"
> +#include "block/qapi.h"
>  #include "monitor/hmp.h"
>  #include "qemu-io.h"
>  
> @@ -553,3 +555,389 @@ fail:
>  blk_unref(local_blk);
>  hmp_handle_error(mon, err);
>  }
> +
> +static void print_block_info(Monitor *mon, BlockInfo *info,
> + BlockDeviceInfo *inserted, bool verbose)
> +{
> +ImageInfo *image_info;
> +
> +assert(!info || !info->has_inserted || info->inserted == inserted);
> +
> +if (info && *info->device) {
> +monitor_printf(mon, "%s", info->device);
> +if (inserted && inserted->has_node_name) {
> +monitor_printf(mon, " (%s)", inserted->node_name);
> +}
> +} else {
> +assert(info || inserted);
> +monitor_printf(mon, "%s",
> +   inserted && inserted->has_node_name ? 
> inserted->node_name
> +   : info && info->has_qdev ? info->qdev
> +   : "");
> +}
> +
> +if (inserted) {
> +monitor_printf(mon, ": %s (%s%s%s)\n",
> +   inserted->file,
> +   inserted->drv,
> +   inserted->ro ? ", read-only" : "",
> +   inserted->encrypted ? ", encrypted" : "");
> +} else {
> +monitor_printf(mon, ": [not inserted]\n");
> +}
> +
> +if (info) {
> +if (info->has_qdev) {
> +monitor_printf(mon, "Attached to:  %s\n", info->qdev);
> +}
> +if (info->has_io_status && info->io_status != 
> BLOCK_DEVICE_IO_STATUS_OK) {
> +monitor_printf(mon, "I/O status:   %s\n",
> +   BlockDeviceIoStatus_str(info->io_status));
> +}
> +
> +if (info->removable) {
> +monitor_printf(mon, "Removable device: %slocked, tray %s\n",
> +   info->locked ? "" : "not ",
> +   info->tray_open ? "open" : "closed");
> +}
> +}
> +
> +
> +if (!inserted) {
> +return;
> +}
> +
> +monitor_printf(mon, "Cache mode:   %s%s%s\n",
> +   inserted->cache->writeback ? "writeback" : "writethrough",
> +   inserted->cache->direct ? ", direct" : "",
> +   inserted->cache->no_flush ? ", ignore flushes" : "");
> +
> +if (inserted->has_backing_file) {
> +monitor_printf(mon,
> +   "Backing file: %s "
> +   "(chain depth: %" PRId64 ")\n",
> +   inserted->backing_file,
> +   inserted->backing_file_depth);
> +}
> +
> +if (inserted->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF) {
> +monitor_printf(mon, "Detect zeroes:%s\n",
> +BlockdevDetectZeroesOptions_str(inserted->detect_zeroes));
> +}
> +
> +if (inserted->bps  || inserted->bps_rd  || inserted->bps_wr  ||
> +inserted->iops || inserted->iops_rd || inserted->iops_wr)
> +{
> +monitor_printf(mon, "I/O throttling:   bps=%" PRId64
> +" bps_rd=%" PRId64  " bps_wr=%" PRId64
> +" 

Re: [PATCH 9/9] monitor/hmp: Prefer to use hmp_handle_error for error reporting in block hmp commands

2020-01-28 Thread Dr. David Alan Gilbert
rror and what's something else, decoration can help.
> Yes, also this way it is consistent

Yes I also like it; I wouldn't worry too much about things parsing error
messages for the exact error message; if anything is doing that then the
corresponding case needs to have big red flags around it.

Dave

> > 
> > Perhaps you can give some examples where the proposed decoration helps.
> It helps to tag most monitor messages with error prefix which was the root 
> cause of
> me starting to work on this refactoring.
> You suggested this, and I kind of like that idea.
> 
> > 
> > > > That leaves the ones that are still reported with monitor_printf().
> > > > Converting those to error_report() looks far more tractable to me.
> > > 
> > > Yep, in fact I grepped the tree for monitor_printf and there are not
> > > that much instances of this used for error reporting, so it might
> > > be possible to have 'error' prefix on all monitor errors that way
> > > and not only for the block layer.
> > 
> > I figure "all" would be more useful than "just for the block layer".
> Yep, the cover letter is outdated, now this patch series touch way
> more that the block layer.
> 
> Best regards,
>   Maxim Levitsky
> 
> 
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




Re: [PATCH v3 09/13] monitor/hmp: move remaining hmp_block* functions to block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> Signed-off-by: Maxim Levitsky 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block/monitor/block-hmp-cmds.c | 138 +
>  include/block/block-hmp-commands.h |   9 ++
>  include/monitor/hmp.h  |   6 --
>  monitor/hmp-cmds.c | 137 
>  4 files changed, 147 insertions(+), 143 deletions(-)
> 
> diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> index df0178d0f9..60d63bfe18 100644
> --- a/block/monitor/block-hmp-cmds.c
> +++ b/block/monitor/block-hmp-cmds.c
> @@ -29,6 +29,7 @@
>  #include "block/block_int.h"
>  #include "block/block-hmp-commands.h"
>  #include "monitor/hmp.h"
> +#include "qemu-io.h"
>  
>  void hmp_drive_add(Monitor *mon, const QDict *qdict)
>  {
> @@ -415,3 +416,140 @@ void hmp_nbd_server_stop(Monitor *mon, const QDict 
> *qdict)
>  qmp_nbd_server_stop();
>  hmp_handle_error(mon, err);
>  }
> +
> +void hmp_block_resize(Monitor *mon, const QDict *qdict)
> +{
> +const char *device = qdict_get_str(qdict, "device");
> +int64_t size = qdict_get_int(qdict, "size");
> +Error *err = NULL;
> +
> +qmp_block_resize(true, device, false, NULL, size, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_block_stream(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +const char *base = qdict_get_try_str(qdict, "base");
> +int64_t speed = qdict_get_try_int(qdict, "speed", 0);
> +
> +qmp_block_stream(true, device, device, base != NULL, base, false, NULL,
> + false, NULL, qdict_haskey(qdict, "speed"), speed, true,
> + BLOCKDEV_ON_ERROR_REPORT, false, false, false, false,
> + );
> +
> +hmp_handle_error(mon, error);
> +}
> +
> +void hmp_block_passwd(Monitor *mon, const QDict *qdict)
> +{
> +const char *device = qdict_get_str(qdict, "device");
> +const char *password = qdict_get_str(qdict, "password");
> +Error *err = NULL;
> +
> +qmp_block_passwd(true, device, false, NULL, password, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_block_set_io_throttle(Monitor *mon, const QDict *qdict)
> +{
> +Error *err = NULL;
> +char *device = (char *) qdict_get_str(qdict, "device");
> +BlockIOThrottle throttle = {
> +.bps = qdict_get_int(qdict, "bps"),
> +.bps_rd = qdict_get_int(qdict, "bps_rd"),
> +.bps_wr = qdict_get_int(qdict, "bps_wr"),
> +.iops = qdict_get_int(qdict, "iops"),
> +.iops_rd = qdict_get_int(qdict, "iops_rd"),
> +.iops_wr = qdict_get_int(qdict, "iops_wr"),
> +};
> +
> +/* qmp_block_set_io_throttle has separate parameters for the
> + * (deprecated) block device name and the qdev ID but the HMP
> + * version has only one, so we must decide which one to pass. */
> +if (blk_by_name(device)) {
> +throttle.has_device = true;
> +throttle.device = device;
> +} else {
> +throttle.has_id = true;
> +throttle.id = device;
> +}
> +
> +qmp_block_set_io_throttle(, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_eject(Monitor *mon, const QDict *qdict)
> +{
> +bool force = qdict_get_try_bool(qdict, "force", false);
> +const char *device = qdict_get_str(qdict, "device");
> +Error *err = NULL;
> +
> +qmp_eject(true, device, false, NULL, true, force, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_qemu_io(Monitor *mon, const QDict *qdict)
> +{
> +BlockBackend *blk;
> +BlockBackend *local_blk = NULL;
> +bool qdev = qdict_get_try_bool(qdict, "qdev", false);
> +const char* device = qdict_get_str(qdict, "device");
> +const char* command = qdict_get_str(qdict, "command");
> +Error *err = NULL;
> +int ret;
> +
> +if (qdev) {
> +blk = blk_by_qdev_id(device, );
> +if (!blk) {
> +goto fail;
> +}
> +} else {
> +blk = blk_by_name(device);
> +if (!blk) {
> +BlockDriverState *bs = bdrv_lookup_bs(NULL, device, );
> +if (bs) {
> +blk = local_blk = blk_new(bdrv_get_aio_context(bs),
> +  0, BLK_PERM_ALL);
> +ret = blk_insert_bs(blk, 

Re: [PATCH v3 06/13] monitor/hmp: move hmp_block_job* to block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> Signed-off-by: Maxim Levitsky 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block/monitor/block-hmp-cmds.c | 52 ++
>  include/block/block-hmp-commands.h |  6 
>  include/monitor/hmp.h  |  5 ---
>  monitor/hmp-cmds.c | 52 --
>  4 files changed, 58 insertions(+), 57 deletions(-)
> 
> diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> index ae3890aaab..ed3c350143 100644
> --- a/block/monitor/block-hmp-cmds.c
> +++ b/block/monitor/block-hmp-cmds.c
> @@ -228,3 +228,55 @@ void hmp_drive_backup(Monitor *mon, const QDict *qdict)
>  qmp_drive_backup(, );
>  hmp_handle_error(mon, err);
>  }
> +
> +void hmp_block_job_set_speed(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +int64_t value = qdict_get_int(qdict, "speed");
> +
> +qmp_block_job_set_speed(device, value, );
> +
> +hmp_handle_error(mon, error);
> +}
> +
> +void hmp_block_job_cancel(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +bool force = qdict_get_try_bool(qdict, "force", false);
> +
> +qmp_block_job_cancel(device, true, force, );
> +
> +hmp_handle_error(mon, error);
> +}
> +
> +void hmp_block_job_pause(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +
> +qmp_block_job_pause(device, );
> +
> +hmp_handle_error(mon, error);
> +}
> +
> +void hmp_block_job_resume(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +
> +qmp_block_job_resume(device, );
> +
> +hmp_handle_error(mon, error);
> +}
> +
> +void hmp_block_job_complete(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +
> +qmp_block_job_complete(device, );
> +
> +hmp_handle_error(mon, error);
> +}
> diff --git a/include/block/block-hmp-commands.h 
> b/include/block/block-hmp-commands.h
> index fcaf753118..ea6578a5f6 100644
> --- a/include/block/block-hmp-commands.h
> +++ b/include/block/block-hmp-commands.h
> @@ -11,4 +11,10 @@ void hmp_drive_del(Monitor *mon, const QDict *qdict);
>  void hmp_drive_mirror(Monitor *mon, const QDict *qdict);
>  void hmp_drive_backup(Monitor *mon, const QDict *qdict);
>  
> +void hmp_block_job_set_speed(Monitor *mon, const QDict *qdict);
> +void hmp_block_job_cancel(Monitor *mon, const QDict *qdict);
> +void hmp_block_job_pause(Monitor *mon, const QDict *qdict);
> +void hmp_block_job_resume(Monitor *mon, const QDict *qdict);
> +void hmp_block_job_complete(Monitor *mon, const QDict *qdict);
> +
>  #endif
> diff --git a/include/monitor/hmp.h b/include/monitor/hmp.h
> index c1b363ee57..592ce0ccfe 100644
> --- a/include/monitor/hmp.h
> +++ b/include/monitor/hmp.h
> @@ -87,11 +87,6 @@ void hmp_eject(Monitor *mon, const QDict *qdict);
>  void hmp_change(Monitor *mon, const QDict *qdict);
>  void hmp_block_set_io_throttle(Monitor *mon, const QDict *qdict);
>  void hmp_block_stream(Monitor *mon, const QDict *qdict);
> -void hmp_block_job_set_speed(Monitor *mon, const QDict *qdict);
> -void hmp_block_job_cancel(Monitor *mon, const QDict *qdict);
> -void hmp_block_job_pause(Monitor *mon, const QDict *qdict);
> -void hmp_block_job_resume(Monitor *mon, const QDict *qdict);
> -void hmp_block_job_complete(Monitor *mon, const QDict *qdict);
>  void hmp_migrate(Monitor *mon, const QDict *qdict);
>  void hmp_device_add(Monitor *mon, const QDict *qdict);
>  void hmp_device_del(Monitor *mon, const QDict *qdict);
> diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
> index a70bcb1d16..996ce96430 100644
> --- a/monitor/hmp-cmds.c
> +++ b/monitor/hmp-cmds.c
> @@ -1975,58 +1975,6 @@ void hmp_block_stream(Monitor *mon, const QDict *qdict)
>  hmp_handle_error(mon, error);
>  }
>  
> -void hmp_block_job_set_speed(Monitor *mon, const QDict *qdict)
> -{
> -Error *error = NULL;
> -const char *device = qdict_get_str(qdict, "device");
> -int64_t value = qdict_get_int(qdict, "speed");
> -
> -qmp_block_job_set_speed(device, value, );
> -
> -hmp_handle_error(mon, error);
> -}
> -
> -void hmp_block_job_cancel(Monitor *mon, const QDict *qdict)
> -{
> -Error *error = NULL;
> -const char *devic

Re: [PATCH v3 11/13] monitor: Move hmp_drive_add_node to block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> Signed-off-by: Maxim Levitsky 

Looks OK to me, I'm not clear on the name for 'bdrv_set_monitor_owned'
I'd want a block person to OK that, but:


Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block/monitor/block-hmp-cmds.c | 30 
>  blockdev.c | 42 +++---
>  include/block/block_int.h  |  5 ++--
>  3 files changed, 41 insertions(+), 36 deletions(-)
> 
> diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> index a4b1604aee..7bbe4e3814 100644
> --- a/block/monitor/block-hmp-cmds.c
> +++ b/block/monitor/block-hmp-cmds.c
> @@ -33,6 +33,36 @@
>  #include "monitor/hmp.h"
>  #include "qemu-io.h"
>  
> +static void hmp_drive_add_node(Monitor *mon, const char *optstr)
> +{
> +QemuOpts *opts;
> +QDict *qdict;
> +Error *local_err = NULL;
> +
> +opts = qemu_opts_parse_noisily(_drive_opts, optstr, false);
> +if (!opts) {
> +return;
> +}
> +
> +qdict = qemu_opts_to_qdict(opts, NULL);
> +
> +if (!qdict_get_try_str(qdict, "node-name")) {
> +qobject_unref(qdict);
> +error_report("'node-name' needs to be specified");
> +goto out;
> +}
> +
> +BlockDriverState *bs = bds_tree_init(qdict, _err);
> +if (!bs) {
> +error_report_err(local_err);
> +goto out;
> +}
> +
> +bdrv_set_monitor_owned(bs);
> +out:
> +qemu_opts_del(opts);
> +}
> +
>  void hmp_drive_add(Monitor *mon, const QDict *qdict)
>  {
>  Error *err = NULL;
> diff --git a/blockdev.c b/blockdev.c
> index df43e0aaef..63805f34b5 100644
> --- a/blockdev.c
> +++ b/blockdev.c
> @@ -64,7 +64,7 @@
>  #include "qemu/main-loop.h"
>  #include "qemu/throttle-options.h"
>  
> -static QTAILQ_HEAD(, BlockDriverState) monitor_bdrv_states =
> +QTAILQ_HEAD(, BlockDriverState) monitor_bdrv_states =
>  QTAILQ_HEAD_INITIALIZER(monitor_bdrv_states);
>  
>  static int do_open_tray(const char *blk_name, const char *qdev_id,
> @@ -75,6 +75,11 @@ static void blockdev_insert_medium(bool has_device, const 
> char *device,
> bool has_id, const char *id,
> const char *node_name, Error **errp);
>  
> +void bdrv_set_monitor_owned(BlockDriverState *bs)
> +{
> +QTAILQ_INSERT_TAIL(_bdrv_states, bs, monitor_list);
> +}
> +
>  static const char *const if_name[IF_COUNT] = {
>  [IF_NONE] = "none",
>  [IF_IDE] = "ide",
> @@ -652,7 +657,7 @@ err_no_opts:
>  }
>  
>  /* Takes the ownership of bs_opts */
> -static BlockDriverState *bds_tree_init(QDict *bs_opts, Error **errp)
> +BlockDriverState *bds_tree_init(QDict *bs_opts, Error **errp)
>  {
>  int bdrv_flags = 0;
>  
> @@ -4201,37 +4206,6 @@ out:
>  aio_context_release(aio_context);
>  }
>  
> -void hmp_drive_add_node(Monitor *mon, const char *optstr)
> -{
> -QemuOpts *opts;
> -QDict *qdict;
> -Error *local_err = NULL;
> -
> -opts = qemu_opts_parse_noisily(_drive_opts, optstr, false);
> -if (!opts) {
> -return;
> -}
> -
> -qdict = qemu_opts_to_qdict(opts, NULL);
> -
> -if (!qdict_get_try_str(qdict, "node-name")) {
> -qobject_unref(qdict);
> -error_report("'node-name' needs to be specified");
> -goto out;
> -}
> -
> -BlockDriverState *bs = bds_tree_init(qdict, _err);
> -if (!bs) {
> -error_report_err(local_err);
> -goto out;
> -}
> -
> -QTAILQ_INSERT_TAIL(_bdrv_states, bs, monitor_list);
> -
> -out:
> -qemu_opts_del(opts);
> -}
> -
>  void qmp_blockdev_add(BlockdevOptions *options, Error **errp)
>  {
>  BlockDriverState *bs;
> @@ -4261,7 +4235,7 @@ void qmp_blockdev_add(BlockdevOptions *options, Error 
> **errp)
>  goto fail;
>  }
>  
> -QTAILQ_INSERT_TAIL(_bdrv_states, bs, monitor_list);
> +bdrv_set_monitor_owned(bs);
>  
>  fail:
>  visit_free(v);
> diff --git a/include/block/block_int.h b/include/block/block_int.h
> index dd033d0b37..10df257a61 100644
> --- a/include/block/block_int.h
> +++ b/include/block/block_int.h
> @@ -1217,8 +1217,6 @@ BlockJob *backup_job_create(const char *job_id, 
> BlockDriverState *bs,
>  BlockCompletionFunc *cb, void *opaque,
>  JobTxn *txn, Error **errp);
>  
> -void hmp_drive_add_node(Monitor *mon, const char *optstr);
> -
>  BdrvChild *bdrv_root_attach_child(BlockDrive

Re: [PATCH v3 06/13] monitor/hmp: move hmp_block_job* to block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> Signed-off-by: Maxim Levitsky 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block/monitor/block-hmp-cmds.c | 52 ++
>  include/block/block-hmp-commands.h |  6 
>  include/monitor/hmp.h  |  5 ---
>  monitor/hmp-cmds.c | 52 --
>  4 files changed, 58 insertions(+), 57 deletions(-)
> 
> diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> index ae3890aaab..ed3c350143 100644
> --- a/block/monitor/block-hmp-cmds.c
> +++ b/block/monitor/block-hmp-cmds.c
> @@ -228,3 +228,55 @@ void hmp_drive_backup(Monitor *mon, const QDict *qdict)
>  qmp_drive_backup(, );
>  hmp_handle_error(mon, err);
>  }
> +
> +void hmp_block_job_set_speed(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +int64_t value = qdict_get_int(qdict, "speed");
> +
> +qmp_block_job_set_speed(device, value, );
> +
> +hmp_handle_error(mon, error);
> +}
> +
> +void hmp_block_job_cancel(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +bool force = qdict_get_try_bool(qdict, "force", false);
> +
> +qmp_block_job_cancel(device, true, force, );
> +
> +hmp_handle_error(mon, error);
> +}
> +
> +void hmp_block_job_pause(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +
> +qmp_block_job_pause(device, );
> +
> +hmp_handle_error(mon, error);
> +}
> +
> +void hmp_block_job_resume(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +
> +qmp_block_job_resume(device, );
> +
> +hmp_handle_error(mon, error);
> +}
> +
> +void hmp_block_job_complete(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +
> +qmp_block_job_complete(device, );
> +
> +hmp_handle_error(mon, error);
> +}
> diff --git a/include/block/block-hmp-commands.h 
> b/include/block/block-hmp-commands.h
> index fcaf753118..ea6578a5f6 100644
> --- a/include/block/block-hmp-commands.h
> +++ b/include/block/block-hmp-commands.h
> @@ -11,4 +11,10 @@ void hmp_drive_del(Monitor *mon, const QDict *qdict);
>  void hmp_drive_mirror(Monitor *mon, const QDict *qdict);
>  void hmp_drive_backup(Monitor *mon, const QDict *qdict);
>  
> +void hmp_block_job_set_speed(Monitor *mon, const QDict *qdict);
> +void hmp_block_job_cancel(Monitor *mon, const QDict *qdict);
> +void hmp_block_job_pause(Monitor *mon, const QDict *qdict);
> +void hmp_block_job_resume(Monitor *mon, const QDict *qdict);
> +void hmp_block_job_complete(Monitor *mon, const QDict *qdict);
> +
>  #endif
> diff --git a/include/monitor/hmp.h b/include/monitor/hmp.h
> index c1b363ee57..592ce0ccfe 100644
> --- a/include/monitor/hmp.h
> +++ b/include/monitor/hmp.h
> @@ -87,11 +87,6 @@ void hmp_eject(Monitor *mon, const QDict *qdict);
>  void hmp_change(Monitor *mon, const QDict *qdict);
>  void hmp_block_set_io_throttle(Monitor *mon, const QDict *qdict);
>  void hmp_block_stream(Monitor *mon, const QDict *qdict);
> -void hmp_block_job_set_speed(Monitor *mon, const QDict *qdict);
> -void hmp_block_job_cancel(Monitor *mon, const QDict *qdict);
> -void hmp_block_job_pause(Monitor *mon, const QDict *qdict);
> -void hmp_block_job_resume(Monitor *mon, const QDict *qdict);
> -void hmp_block_job_complete(Monitor *mon, const QDict *qdict);
>  void hmp_migrate(Monitor *mon, const QDict *qdict);
>  void hmp_device_add(Monitor *mon, const QDict *qdict);
>  void hmp_device_del(Monitor *mon, const QDict *qdict);
> diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
> index a70bcb1d16..996ce96430 100644
> --- a/monitor/hmp-cmds.c
> +++ b/monitor/hmp-cmds.c
> @@ -1975,58 +1975,6 @@ void hmp_block_stream(Monitor *mon, const QDict *qdict)
>  hmp_handle_error(mon, error);
>  }
>  
> -void hmp_block_job_set_speed(Monitor *mon, const QDict *qdict)
> -{
> -Error *error = NULL;
> -const char *device = qdict_get_str(qdict, "device");
> -int64_t value = qdict_get_int(qdict, "speed");
> -
> -qmp_block_job_set_speed(device, value, );
> -
> -hmp_handle_error(mon, error);
> -}
> -
> -void hmp_block_job_cancel(Monitor *mon, const QDict *qdict)
> -{
> -Error *error = NULL;
> -const char *devic

Re: [PATCH v3 08/13] monitor/hmp: move hmp_nbd_server* to block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> Signed-off-by: Maxim Levitsky 

Yes, I think that's OK; I can imagine nbd might want to move on it's own
somewhere since it's not really core block code; copying in Eric.


Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block/monitor/block-hmp-cmds.c | 88 ++
>  include/block/block-hmp-commands.h |  5 ++
>  include/monitor/hmp.h  |  4 --
>  monitor/hmp-cmds.c | 87 -
>  4 files changed, 93 insertions(+), 91 deletions(-)
> 
> diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> index 9aa94ea6e0..df0178d0f9 100644
> --- a/block/monitor/block-hmp-cmds.c
> +++ b/block/monitor/block-hmp-cmds.c
> @@ -22,8 +22,10 @@
>  #include "qapi/qmp/qerror.h"
>  #include "qemu/config-file.h"
>  #include "qemu/option.h"
> +#include "qemu/sockets.h"
>  #include "sysemu/sysemu.h"
>  #include "monitor/monitor.h"
> +#include "block/nbd.h"
>  #include "block/block_int.h"
>  #include "block/block-hmp-commands.h"
>  #include "monitor/hmp.h"
> @@ -327,3 +329,89 @@ void hmp_snapshot_delete_blkdev_internal(Monitor *mon, 
> const QDict *qdict)
> true, name, );
>  hmp_handle_error(mon, err);
>  }
> +
> +void hmp_nbd_server_start(Monitor *mon, const QDict *qdict)
> +{
> +const char *uri = qdict_get_str(qdict, "uri");
> +bool writable = qdict_get_try_bool(qdict, "writable", false);
> +bool all = qdict_get_try_bool(qdict, "all", false);
> +Error *local_err = NULL;
> +BlockInfoList *block_list, *info;
> +SocketAddress *addr;
> +
> +if (writable && !all) {
> +error_setg(_err, "-w only valid together with -a");
> +goto exit;
> +}
> +
> +/* First check if the address is valid and start the server.  */
> +addr = socket_parse(uri, _err);
> +if (local_err != NULL) {
> +goto exit;
> +}
> +
> +nbd_server_start(addr, NULL, NULL, _err);
> +qapi_free_SocketAddress(addr);
> +if (local_err != NULL) {
> +goto exit;
> +}
> +
> +if (!all) {
> +return;
> +}
> +
> +/* Then try adding all block devices.  If one fails, close all and
> + * exit.
> + */
> +block_list = qmp_query_block(NULL);
> +
> +for (info = block_list; info; info = info->next) {
> +if (!info->value->has_inserted) {
> +continue;
> +}
> +
> +qmp_nbd_server_add(info->value->device, false, NULL,
> +   true, writable, false, NULL, _err);
> +
> +if (local_err != NULL) {
> +qmp_nbd_server_stop(NULL);
> +break;
> +}
> +}
> +
> +qapi_free_BlockInfoList(block_list);
> +
> +exit:
> +hmp_handle_error(mon, local_err);
> +}
> +
> +void hmp_nbd_server_add(Monitor *mon, const QDict *qdict)
> +{
> +const char *device = qdict_get_str(qdict, "device");
> +const char *name = qdict_get_try_str(qdict, "name");
> +bool writable = qdict_get_try_bool(qdict, "writable", false);
> +Error *local_err = NULL;
> +
> +qmp_nbd_server_add(device, !!name, name, true, writable,
> +   false, NULL, _err);
> +hmp_handle_error(mon, local_err);
> +}
> +
> +void hmp_nbd_server_remove(Monitor *mon, const QDict *qdict)
> +{
> +const char *name = qdict_get_str(qdict, "name");
> +bool force = qdict_get_try_bool(qdict, "force", false);
> +Error *err = NULL;
> +
> +/* Rely on NBD_SERVER_REMOVE_MODE_SAFE being the default */
> +qmp_nbd_server_remove(name, force, NBD_SERVER_REMOVE_MODE_HARD, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_nbd_server_stop(Monitor *mon, const QDict *qdict)
> +{
> +Error *err = NULL;
> +
> +qmp_nbd_server_stop();
> +hmp_handle_error(mon, err);
> +}
> diff --git a/include/block/block-hmp-commands.h 
> b/include/block/block-hmp-commands.h
> index 3fc2daf3a9..721b9a1978 100644
> --- a/include/block/block-hmp-commands.h
> +++ b/include/block/block-hmp-commands.h
> @@ -21,4 +21,9 @@ void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict);
>  void hmp_snapshot_blkdev_internal(Monitor *mon, const QDict *qdict);
>  void hmp_snapshot_delete_blkdev_internal(Monitor *mon, const QDict *qdict);
>  
> +void hmp_nbd_server_start(Monitor *mon, const QDict *qdict);
> +void hmp_nbd_server_add(Monitor *mon, const QDi

Re: [PATCH v3 07/13] monitor/hmp: move hmp_snapshot_* to block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> Signed-off-by: Maxim Levitsky 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block/monitor/block-hmp-cmds.c | 47 ++
>  include/block/block-hmp-commands.h |  4 +++
>  include/monitor/hmp.h  |  3 --
>  monitor/hmp-cmds.c | 47 --
>  4 files changed, 51 insertions(+), 50 deletions(-)
> 
> diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> index ed3c350143..9aa94ea6e0 100644
> --- a/block/monitor/block-hmp-cmds.c
> +++ b/block/monitor/block-hmp-cmds.c
> @@ -280,3 +280,50 @@ void hmp_block_job_complete(Monitor *mon, const QDict 
> *qdict)
>  
>  hmp_handle_error(mon, error);
>  }
> +
> +void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict)
> +{
> +const char *device = qdict_get_str(qdict, "device");
> +const char *filename = qdict_get_try_str(qdict, "snapshot-file");
> +const char *format = qdict_get_try_str(qdict, "format");
> +bool reuse = qdict_get_try_bool(qdict, "reuse", false);
> +enum NewImageMode mode;
> +Error *err = NULL;
> +
> +if (!filename) {
> +/* In the future, if 'snapshot-file' is not specified, the snapshot
> +   will be taken internally. Today it's actually required. */
> +error_setg(, QERR_MISSING_PARAMETER, "snapshot-file");
> +hmp_handle_error(mon, err);
> +return;
> +}
> +
> +mode = reuse ? NEW_IMAGE_MODE_EXISTING : NEW_IMAGE_MODE_ABSOLUTE_PATHS;
> +qmp_blockdev_snapshot_sync(true, device, false, NULL,
> +   filename, false, NULL,
> +   !!format, format,
> +   true, mode, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_snapshot_blkdev_internal(Monitor *mon, const QDict *qdict)
> +{
> +const char *device = qdict_get_str(qdict, "device");
> +const char *name = qdict_get_str(qdict, "name");
> +Error *err = NULL;
> +
> +qmp_blockdev_snapshot_internal_sync(device, name, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_snapshot_delete_blkdev_internal(Monitor *mon, const QDict *qdict)
> +{
> +const char *device = qdict_get_str(qdict, "device");
> +const char *name = qdict_get_str(qdict, "name");
> +const char *id = qdict_get_try_str(qdict, "id");
> +Error *err = NULL;
> +
> +qmp_blockdev_snapshot_delete_internal_sync(device, !!id, id,
> +   true, name, );
> +hmp_handle_error(mon, err);
> +}
> diff --git a/include/block/block-hmp-commands.h 
> b/include/block/block-hmp-commands.h
> index ea6578a5f6..3fc2daf3a9 100644
> --- a/include/block/block-hmp-commands.h
> +++ b/include/block/block-hmp-commands.h
> @@ -17,4 +17,8 @@ void hmp_block_job_pause(Monitor *mon, const QDict *qdict);
>  void hmp_block_job_resume(Monitor *mon, const QDict *qdict);
>  void hmp_block_job_complete(Monitor *mon, const QDict *qdict);
>  
> +void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict);
> +void hmp_snapshot_blkdev_internal(Monitor *mon, const QDict *qdict);
> +void hmp_snapshot_delete_blkdev_internal(Monitor *mon, const QDict *qdict);
> +
>  #endif
> diff --git a/include/monitor/hmp.h b/include/monitor/hmp.h
> index 592ce0ccfe..6d34e29bb6 100644
> --- a/include/monitor/hmp.h
> +++ b/include/monitor/hmp.h
> @@ -61,9 +61,6 @@ void hmp_set_link(Monitor *mon, const QDict *qdict);
>  void hmp_block_passwd(Monitor *mon, const QDict *qdict);
>  void hmp_balloon(Monitor *mon, const QDict *qdict);
>  void hmp_block_resize(Monitor *mon, const QDict *qdict);
> -void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict);
> -void hmp_snapshot_blkdev_internal(Monitor *mon, const QDict *qdict);
> -void hmp_snapshot_delete_blkdev_internal(Monitor *mon, const QDict *qdict);
>  void hmp_loadvm(Monitor *mon, const QDict *qdict);
>  void hmp_savevm(Monitor *mon, const QDict *qdict);
>  void hmp_delvm(Monitor *mon, const QDict *qdict);
> diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
> index 996ce96430..46b46b6dd7 100644
> --- a/monitor/hmp-cmds.c
> +++ b/monitor/hmp-cmds.c
> @@ -1337,53 +1337,6 @@ void hmp_block_resize(Monitor *mon, const QDict *qdict)
>  hmp_handle_error(mon, err);
>  }
>  
> -void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict)
> -{
> -const char *device = qdict_get_str(qdict, "device");
> -const char *filename = qdict_get_try_str(qdict, "snapshot-file");
> -const char *format = qdict_get_try_str(qdict

Re: [PATCH v3 09/13] monitor/hmp: move remaining hmp_block* functions to block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> Signed-off-by: Maxim Levitsky 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block/monitor/block-hmp-cmds.c | 138 +
>  include/block/block-hmp-commands.h |   9 ++
>  include/monitor/hmp.h  |   6 --
>  monitor/hmp-cmds.c | 137 
>  4 files changed, 147 insertions(+), 143 deletions(-)
> 
> diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> index df0178d0f9..60d63bfe18 100644
> --- a/block/monitor/block-hmp-cmds.c
> +++ b/block/monitor/block-hmp-cmds.c
> @@ -29,6 +29,7 @@
>  #include "block/block_int.h"
>  #include "block/block-hmp-commands.h"
>  #include "monitor/hmp.h"
> +#include "qemu-io.h"
>  
>  void hmp_drive_add(Monitor *mon, const QDict *qdict)
>  {
> @@ -415,3 +416,140 @@ void hmp_nbd_server_stop(Monitor *mon, const QDict 
> *qdict)
>  qmp_nbd_server_stop();
>  hmp_handle_error(mon, err);
>  }
> +
> +void hmp_block_resize(Monitor *mon, const QDict *qdict)
> +{
> +const char *device = qdict_get_str(qdict, "device");
> +int64_t size = qdict_get_int(qdict, "size");
> +Error *err = NULL;
> +
> +qmp_block_resize(true, device, false, NULL, size, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_block_stream(Monitor *mon, const QDict *qdict)
> +{
> +Error *error = NULL;
> +const char *device = qdict_get_str(qdict, "device");
> +const char *base = qdict_get_try_str(qdict, "base");
> +int64_t speed = qdict_get_try_int(qdict, "speed", 0);
> +
> +qmp_block_stream(true, device, device, base != NULL, base, false, NULL,
> + false, NULL, qdict_haskey(qdict, "speed"), speed, true,
> + BLOCKDEV_ON_ERROR_REPORT, false, false, false, false,
> + );
> +
> +hmp_handle_error(mon, error);
> +}
> +
> +void hmp_block_passwd(Monitor *mon, const QDict *qdict)
> +{
> +const char *device = qdict_get_str(qdict, "device");
> +const char *password = qdict_get_str(qdict, "password");
> +Error *err = NULL;
> +
> +qmp_block_passwd(true, device, false, NULL, password, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_block_set_io_throttle(Monitor *mon, const QDict *qdict)
> +{
> +Error *err = NULL;
> +char *device = (char *) qdict_get_str(qdict, "device");
> +BlockIOThrottle throttle = {
> +.bps = qdict_get_int(qdict, "bps"),
> +.bps_rd = qdict_get_int(qdict, "bps_rd"),
> +.bps_wr = qdict_get_int(qdict, "bps_wr"),
> +.iops = qdict_get_int(qdict, "iops"),
> +.iops_rd = qdict_get_int(qdict, "iops_rd"),
> +.iops_wr = qdict_get_int(qdict, "iops_wr"),
> +};
> +
> +/* qmp_block_set_io_throttle has separate parameters for the
> + * (deprecated) block device name and the qdev ID but the HMP
> + * version has only one, so we must decide which one to pass. */
> +if (blk_by_name(device)) {
> +throttle.has_device = true;
> +throttle.device = device;
> +} else {
> +throttle.has_id = true;
> +throttle.id = device;
> +}
> +
> +qmp_block_set_io_throttle(, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_eject(Monitor *mon, const QDict *qdict)
> +{
> +bool force = qdict_get_try_bool(qdict, "force", false);
> +const char *device = qdict_get_str(qdict, "device");
> +Error *err = NULL;
> +
> +qmp_eject(true, device, false, NULL, true, force, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_qemu_io(Monitor *mon, const QDict *qdict)
> +{
> +BlockBackend *blk;
> +BlockBackend *local_blk = NULL;
> +bool qdev = qdict_get_try_bool(qdict, "qdev", false);
> +const char* device = qdict_get_str(qdict, "device");
> +const char* command = qdict_get_str(qdict, "command");
> +Error *err = NULL;
> +int ret;
> +
> +if (qdev) {
> +blk = blk_by_qdev_id(device, );
> +if (!blk) {
> +goto fail;
> +}
> +} else {
> +blk = blk_by_name(device);
> +if (!blk) {
> +BlockDriverState *bs = bdrv_lookup_bs(NULL, device, );
> +if (bs) {
> +blk = local_blk = blk_new(bdrv_get_aio_context(bs),
> +  0, BLK_PERM_ALL);
> +ret = blk_insert_bs(blk, 

Re: [PATCH v3 05/13] monitor/hmp: move hmp_drive_mirror and hmp_drive_backup to block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> Signed-off-by: Maxim Levitsky 

Reviewed-by: Dr. David Alan Gilbert 

> ---
>  block/monitor/block-hmp-cmds.c | 64 ++
>  include/block/block-hmp-commands.h |  3 ++
>  include/monitor/hmp.h  |  2 -
>  monitor/hmp-cmds.c | 58 ---
>  4 files changed, 67 insertions(+), 60 deletions(-)
> 
> diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> index 9614c67e77..ae3890aaab 100644
> --- a/block/monitor/block-hmp-cmds.c
> +++ b/block/monitor/block-hmp-cmds.c
> @@ -2,6 +2,10 @@
>   * Blockdev HMP commands
>   *
>   * Copyright (c) 2004 Fabrice Bellard
> + * Copyright IBM, Corp. 2011
> + *
> + * Authors:
> + *  Anthony Liguori   
>   *
>   * This work is licensed under the terms of the GNU GPL, version 2.
>   * or (at your option) any later version.
> @@ -15,12 +19,14 @@
>  #include "qapi/qapi-commands-block.h"
>  #include "qapi/qmp/qdict.h"
>  #include "qapi/error.h"
> +#include "qapi/qmp/qerror.h"
>  #include "qemu/config-file.h"
>  #include "qemu/option.h"
>  #include "sysemu/sysemu.h"
>  #include "monitor/monitor.h"
>  #include "block/block_int.h"
>  #include "block/block-hmp-commands.h"
> +#include "monitor/hmp.h"
>  
>  void hmp_drive_add(Monitor *mon, const QDict *qdict)
>  {
> @@ -164,3 +170,61 @@ void hmp_commit(Monitor *mon, const QDict *qdict)
>  error_report("'commit' error for '%s': %s", device, strerror(-ret));
>  }
>  }
> +
> +void hmp_drive_mirror(Monitor *mon, const QDict *qdict)
> +{
> +const char *filename = qdict_get_str(qdict, "target");
> +const char *format = qdict_get_try_str(qdict, "format");
> +bool reuse = qdict_get_try_bool(qdict, "reuse", false);
> +bool full = qdict_get_try_bool(qdict, "full", false);
> +Error *err = NULL;
> +DriveMirror mirror = {
> +.device = (char *)qdict_get_str(qdict, "device"),
> +.target = (char *)filename,
> +.has_format = !!format,
> +.format = (char *)format,
> +.sync = full ? MIRROR_SYNC_MODE_FULL : MIRROR_SYNC_MODE_TOP,
> +.has_mode = true,
> +.mode = reuse ? NEW_IMAGE_MODE_EXISTING : 
> NEW_IMAGE_MODE_ABSOLUTE_PATHS,
> +.unmap = true,
> +};
> +
> +if (!filename) {
> +error_setg(, QERR_MISSING_PARAMETER, "target");
> +hmp_handle_error(mon, err);
> +return;
> +}
> +qmp_drive_mirror(, );
> +hmp_handle_error(mon, err);
> +}
> +
> +void hmp_drive_backup(Monitor *mon, const QDict *qdict)
> +{
> +const char *device = qdict_get_str(qdict, "device");
> +const char *filename = qdict_get_str(qdict, "target");
> +const char *format = qdict_get_try_str(qdict, "format");
> +bool reuse = qdict_get_try_bool(qdict, "reuse", false);
> +bool full = qdict_get_try_bool(qdict, "full", false);
> +bool compress = qdict_get_try_bool(qdict, "compress", false);
> +Error *err = NULL;
> +DriveBackup backup = {
> +.device = (char *)device,
> +.target = (char *)filename,
> +.has_format = !!format,
> +.format = (char *)format,
> +.sync = full ? MIRROR_SYNC_MODE_FULL : MIRROR_SYNC_MODE_TOP,
> +.has_mode = true,
> +.mode = reuse ? NEW_IMAGE_MODE_EXISTING : 
> NEW_IMAGE_MODE_ABSOLUTE_PATHS,
> +.has_compress = !!compress,
> +.compress = compress,
> +};
> +
> +if (!filename) {
> +error_setg(, QERR_MISSING_PARAMETER, "target");
> +hmp_handle_error(mon, err);
> +return;
> +}
> +
> +qmp_drive_backup(, );
> +hmp_handle_error(mon, err);
> +}
> diff --git a/include/block/block-hmp-commands.h 
> b/include/block/block-hmp-commands.h
> index c5e394c0fc..fcaf753118 100644
> --- a/include/block/block-hmp-commands.h
> +++ b/include/block/block-hmp-commands.h
> @@ -8,4 +8,7 @@ void hmp_drive_add(Monitor *mon, const QDict *qdict);
>  void hmp_commit(Monitor *mon, const QDict *qdict);
>  void hmp_drive_del(Monitor *mon, const QDict *qdict);
>  
> +void hmp_drive_mirror(Monitor *mon, const QDict *qdict);
> +void hmp_drive_backup(Monitor *mon, const QDict *qdict);
> +
>  #endif
> diff --git a/include/monitor/hmp.h b/include/monitor/hmp.h
> index 3d329853b2..c1b363ee57 100644
> --- a/include/monitor/hmp.h
> +++ b/include/monitor/hmp.h
> @@ -64,8 +64,6 @@ void hmp_block_

Re: [PATCH v3 04/13] monitor/hmp: move hmp_drive_del and hmp_commit to block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
* Maxim Levitsky (mlevi...@redhat.com) wrote:
> Signed-off-by: Maxim Levitsky 

Reviewed-by: Dr. David Alan Gilbert 

(It's easier to compare if you keep the function order the same)

> ---
>  block/monitor/block-hmp-cmds.c | 97 +-
>  blockdev.c | 95 -
>  include/block/block-hmp-commands.h |  3 +
>  include/sysemu/blockdev.h  |  4 --
>  4 files changed, 99 insertions(+), 100 deletions(-)
> 
> diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
> index c65aaa86ea..9614c67e77 100644
> --- a/block/monitor/block-hmp-cmds.c
> +++ b/block/monitor/block-hmp-cmds.c
> @@ -12,6 +12,7 @@
>  #include "hw/boards.h"
>  #include "sysemu/block-backend.h"
>  #include "sysemu/blockdev.h"
> +#include "qapi/qapi-commands-block.h"
>  #include "qapi/qmp/qdict.h"
>  #include "qapi/error.h"
>  #include "qemu/config-file.h"
> @@ -21,7 +22,6 @@
>  #include "block/block_int.h"
>  #include "block/block-hmp-commands.h"
>  
> -
>  void hmp_drive_add(Monitor *mon, const QDict *qdict)
>  {
>  Error *err = NULL;
> @@ -69,3 +69,98 @@ err:
>  blk_unref(blk);
>  }
>  }
> +
> +void hmp_drive_del(Monitor *mon, const QDict *qdict)
> +{
> +const char *id = qdict_get_str(qdict, "id");
> +BlockBackend *blk;
> +BlockDriverState *bs;
> +AioContext *aio_context;
> +Error *local_err = NULL;
> +
> +bs = bdrv_find_node(id);
> +if (bs) {
> +qmp_blockdev_del(id, _err);
> +if (local_err) {
> +error_report_err(local_err);
> +}
> +return;
> +}
> +
> +blk = blk_by_name(id);
> +if (!blk) {
> +error_report("Device '%s' not found", id);
> +return;
> +}
> +
> +if (!blk_legacy_dinfo(blk)) {
> +error_report("Deleting device added with blockdev-add"
> + " is not supported");
> +return;
> +}
> +
> +aio_context = blk_get_aio_context(blk);
> +aio_context_acquire(aio_context);
> +
> +bs = blk_bs(blk);
> +if (bs) {
> +if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, _err)) {
> +error_report_err(local_err);
> +aio_context_release(aio_context);
> +return;
> +}
> +
> +blk_remove_bs(blk);
> +}
> +
> +/* Make the BlockBackend and the attached BlockDriverState anonymous */
> +monitor_remove_blk(blk);
> +
> +/* If this BlockBackend has a device attached to it, its refcount will be
> + * decremented when the device is removed; otherwise we have to do so 
> here.
> + */
> +if (blk_get_attached_dev(blk)) {
> +/* Further I/O must not pause the guest */
> +blk_set_on_error(blk, BLOCKDEV_ON_ERROR_REPORT,
> + BLOCKDEV_ON_ERROR_REPORT);
> +} else {
> +blk_unref(blk);
> +}
> +
> +aio_context_release(aio_context);
> +}
> +
> +void hmp_commit(Monitor *mon, const QDict *qdict)
> +{
> +const char *device = qdict_get_str(qdict, "device");
> +BlockBackend *blk;
> +int ret;
> +
> +if (!strcmp(device, "all")) {
> +ret = blk_commit_all();
> +} else {
> +BlockDriverState *bs;
> +AioContext *aio_context;
> +
> +blk = blk_by_name(device);
> +if (!blk) {
> +error_report("Device '%s' not found", device);
> +return;
> +}
> +if (!blk_is_available(blk)) {
> +error_report("Device '%s' has no medium", device);
> +return;
> +}
> +
> +bs = blk_bs(blk);
> +aio_context = bdrv_get_aio_context(bs);
> +aio_context_acquire(aio_context);
> +
> +ret = bdrv_commit(bs);
> +
> +aio_context_release(aio_context);
> +}
> +if (ret < 0) {
> +error_report("'commit' error for '%s': %s", device, strerror(-ret));
> +}
> +}
> diff --git a/blockdev.c b/blockdev.c
> index 8e029e9c01..df43e0aaef 100644
> --- a/blockdev.c
> +++ b/blockdev.c
> @@ -1074,41 +1074,6 @@ static BlockBackend *qmp_get_blk(const char *blk_name, 
> const char *qdev_id,
>  return blk;
>  }
>  
> -void hmp_commit(Monitor *mon, const QDict *qdict)
> -{
> -const char *device = qdict_get_str(qdict, "device");
> -BlockBackend *blk;
> -int ret;
> -
> -if (!strcmp(device, "all")) {
>

Re: [PATCH v3 03/13] monitor/hmp: rename device-hotplug.c to block/monitor/block-hmp-cmds.c

2020-01-28 Thread Dr. David Alan Gilbert
w file mode 100644
> index 00..4f9033a8a6
> --- /dev/null
> +++ b/include/block/block-hmp-commands.h
> @@ -0,0 +1,8 @@
> +#ifndef BLOCK_HMP_COMMANDS_H
> +#define BLOCK_HMP_COMMANDS_H
> +
> +/* HMP commands related to the block layer*/

Should this file get a copyright header as well?

> +
> +void hmp_drive_add(Monitor *mon, const QDict *qdict);
> +
> +#endif
> diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
> index 80c57fdc4e..c48635666d 100644
> --- a/include/sysemu/sysemu.h
> +++ b/include/sysemu/sysemu.h
> @@ -68,9 +68,6 @@ extern int nb_option_roms;
>  extern const char *prom_envs[MAX_PROM_ENVS];
>  extern unsigned int nb_prom_envs;
>  
> -/* generic hotplug */
> -void hmp_drive_add(Monitor *mon, const QDict *qdict);
> -
>  /* pcie aer error injection */
>  void hmp_pcie_aer_inject_error(Monitor *mon, const QDict *qdict);
>  
> diff --git a/monitor/misc.c b/monitor/misc.c
> index de1ca4d114..0466c00830 100644
> --- a/monitor/misc.c
> +++ b/monitor/misc.c
> @@ -79,6 +79,7 @@
>  #include "sysemu/cpus.h"
>  #include "qemu/cutils.h"
>  #include "tcg/tcg.h"
> +#include "block/block-hmp-commands.h"
>  
>  #if defined(TARGET_S390X)
>  #include "hw/s390x/storage-keys.h"
> -- 
> 2.17.2
> 
--
Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK




  1   2   3   4   >