Re: [Qemu-block] [PATCH 5/5] nbd/server: refactor nbd_trip: split out nbd_handle_request

[Eric Blake]

On 03/08/2018 12:46 PM, Vladimir Sementsov-Ogievskiy wrote:

Split out request handling logic.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
  nbd/server.c | 129 +++
  1 file changed, 67 insertions(+), 62 deletions(-)




+
+switch (request->type) {
+case NBD_CMD_READ:
+return nbd_do_cmd_read(client, request, data, errp);
+
+case NBD_CMD_WRITE:
+flags = 0;
+if (request->flags & NBD_CMD_FLAG_FUA) {
+flags |= BDRV_REQ_FUA;
+}
+ret = blk_pwrite(exp->blk, request->from + exp->dev_offset,
+ data, request->len, flags);
+
+return nbd_send_generic_reply(client, request->handle, ret,
+  "writing to file failed", errp);
+case NBD_CMD_WRITE_ZEROES:


Inconsistent spacing between return and the next case label.

But switching whitespace is trivial, so

Reviewed-by: Eric Blake 

--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 4/5] nbd/server: refactor nbd_trip: cmd_read and generic reply

[Eric Blake]

On 03/08/2018 12:46 PM, Vladimir Sementsov-Ogievskiy wrote:

nbd_trip has difficult logic of sending reply: it tries to use one
code path for all replies. It is ok for simple replies, but is not
comfortable for structured replies. Also, two types of error (and
corresponding message in local_err) - fatal (leading to disconnect)
and not-fatal (just to be sent to the client) are difficult to follow.

To make things a bit clearer, the following is done:
  - split CMD_READ logic to separate function. It is the most difficult
command for now, and it is definitely cramped inside nbd_trip. Also,
it is difficult to follow CMD_READ logic, shared between
"case NBD_CMD_READ" and "if"s under "reply:" label.


Yay - I already admitted when adding sparse read replies that splitting 
the logic was weird.



  - create separate helper function nbd_send_generic_reply() and use it
both in new nbd_do_cmd_read and for other command in nbd_trip instead


s/command/commands/


of common code-path under "reply:" label in nbd_trip. The helper
supports error message, so logic with local_err in nbd_trip is


s/supports/supports an/


simplified.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
  nbd/server.c | 164 ---
  1 file changed, 88 insertions(+), 76 deletions(-)


Gains a few lines, but I think the end result is more legible.



diff --git a/nbd/server.c b/nbd/server.c
index 97b45a21fa..a2f5f73d52 100644
--- a/nbd/server.c
+++ b/nbd/server.c
@@ -1520,6 +1520,70 @@ static int nbd_co_receive_request(NBDRequestData *req, 
NBDRequest *request,
  return 0;
  }
  
+/* Send simple reply without a payload or structured error


s/payload or/payload, or a /


+ * @error_msg is ignored if @ret >= 0 */


Maybe mention the return value (0 if connection is still live, -errno on 
failure to communicate to client)



+static coroutine_fn int nbd_send_generic_reply(NBDClient *client,
+   uint64_t handle,
+   int ret,
+   const char *error_msg,
+   Error **errp)
+{
+if (client->structured_reply && ret < 0) {
+return nbd_co_send_structured_error(client, handle, -ret, error_msg,
+errp);
+} else {
+return nbd_co_send_simple_reply(client, handle, ret < 0 ? -ret : 0,
+NULL, 0, errp);
+}
+}
+
+/* Handle NBD_CMD_READ request.
+ * Return -errno if sending fails. Other errors are reported directly to the
+ * client as an error reply. */
+static coroutine_fn int nbd_do_cmd_read(NBDClient *client, NBDRequest *request,
+uint8_t *data, Error **errp)
+{



+
  /* Owns a reference to the NBDClient passed as opaque.  */
  static coroutine_fn void nbd_trip(void *opaque)
  {
@@ -1529,7 +1593,6 @@ static coroutine_fn void nbd_trip(void *opaque)
  NBDRequest request = { 0 };/* GCC thinks it can be used uninitialized 
*/
  int ret;
  int flags;
-int reply_data_len = 0;
  Error *local_err = NULL;
  char *msg = NULL;
  
@@ -1556,39 +1619,21 @@ static coroutine_fn void nbd_trip(void *opaque)

  }
  
  if (ret < 0) {

-goto reply;
+/* It's not a -EIO, so, accordingly to nbd_co_receive_request()


It wasn't -EIO, so according to nbd_co_receive_request()


+ * semantics, we should return the error to the client. */
+Error *export_err = local_err;
+
+local_err = NULL;
+ret = nbd_send_generic_reply(client, request.handle, -EINVAL,
+ error_get_pretty(export_err), &local_err);
+error_free(export_err);
+
+goto replied;
  }
  
  switch (request.type) {

  case NBD_CMD_READ:



-reply_data_len = request.len;
+ret = nbd_do_cmd_read(client, &request, req->data, &local_err);
  
  break;


We could also collapse the empty line before break.


  case NBD_CMD_WRITE:
@@ -1598,9 +1643,8 @@ static coroutine_fn void nbd_trip(void *opaque)
  }
  ret = blk_pwrite(exp->blk, request.from + exp->dev_offset,
   req->data, request.len, flags);
-if (ret < 0) {
-error_setg_errno(&local_err, -ret, "writing to file failed");
-}
+ret = nbd_send_generic_reply(client, request.handle, ret,
+ "writing to file failed", &local_err);


I like how this works out.

  
  break;

  default:
-error_setg(&local_err, "invalid request type (%" PRIu32 ") received",
-   request.type);
-ret = -EINVAL;
-}
-
-reply:
-if (local_err) {
-/* If we get here, local_err was not a fatal error, and should be sent
- * to the client. */
-assert(ret < 0);
-  

[Qemu-block] [PATCH 4/5] block/blkreplay: Remove protocol-related fields

[Fabiano Rosas]

The blkreplay driver is not a protocol so it should implement bdrv_open
instead of bdrv_file_open and not provide a protocol_name.

Attempts to invoke this driver using protocol syntax
(i.e. blkreplay:) will now fail gracefully:

  $ qemu-img info blkreplay:foo
  qemu-img: Could not open 'blkreplay:foo': Unknown protocol 'blkreplay'

Signed-off-by: Fabiano Rosas 
---
 block/blkreplay.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/block/blkreplay.c b/block/blkreplay.c
index 61e44a1949..fe5a9b4a98 100755
--- a/block/blkreplay.c
+++ b/block/blkreplay.c
@@ -129,10 +129,9 @@ static int coroutine_fn 
blkreplay_co_flush(BlockDriverState *bs)
 
 static BlockDriver bdrv_blkreplay = {
 .format_name= "blkreplay",
-.protocol_name  = "blkreplay",
 .instance_size  = 0,
 
-.bdrv_file_open = blkreplay_open,
+.bdrv_open  = blkreplay_open,
 .bdrv_close = blkreplay_close,
 .bdrv_child_perm= bdrv_filter_default_perms,
 .bdrv_getlength = blkreplay_getlength,
-- 
2.13.6

[Qemu-block] [PATCH 3/5] block/throttle: Remove protocol-related fields

[Fabiano Rosas]

The throttle driver is not a protocol so it should implement bdrv_open
instead of bdrv_file_open and not provide a protocol_name.

Attempts to invoke this driver using protocol syntax
(i.e. throttle:) will now fail gracefully:

  $ qemu-img info throttle:foo
  qemu-img: Could not open 'throttle:foo': Unknown protocol 'throttle'

Signed-off-by: Fabiano Rosas 
---
 block/throttle.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/block/throttle.c b/block/throttle.c
index 5f4d43d0fc..95ed06acd8 100644
--- a/block/throttle.c
+++ b/block/throttle.c
@@ -215,10 +215,9 @@ static void coroutine_fn 
throttle_co_drain_end(BlockDriverState *bs)
 
 static BlockDriver bdrv_throttle = {
 .format_name=   "throttle",
-.protocol_name  =   "throttle",
 .instance_size  =   sizeof(ThrottleGroupMember),
 
-.bdrv_file_open =   throttle_open,
+.bdrv_open  =   throttle_open,
 .bdrv_close =   throttle_close,
 .bdrv_co_flush  =   throttle_co_flush,
 
-- 
2.13.6

[Qemu-block] [PATCH 1/5] block/replication: Remove protocol_name field

[Fabiano Rosas]

The replication driver is only selected explicitly (via
driver=replication,mode=primary,...) so it is not a protocol driver.

This patch removes the protocol_name field from the brdv_replication
structure so that attempts to invoke this driver using protocol
syntax (i.e. replication:) will fail gracefully:

  $ qemu-img info replication:foo
  qemu-img: Could not open 'replication:': Unknown protocol 'replication'

Buglink: https://bugs.launchpad.net/qemu/+bug/1726733
Signed-off-by: Fabiano Rosas 
---
 block/replication.c | 1 -
 replication.h   | 1 -
 2 files changed, 2 deletions(-)

diff --git a/block/replication.c b/block/replication.c
index f98ef094b9..6c0c7186d9 100644
--- a/block/replication.c
+++ b/block/replication.c
@@ -703,7 +703,6 @@ static void replication_stop(ReplicationState *rs, bool 
failover, Error **errp)
 
 BlockDriver bdrv_replication = {
 .format_name= "replication",
-.protocol_name  = "replication",
 .instance_size  = sizeof(BDRVReplicationState),
 
 .bdrv_open  = replication_open,
diff --git a/replication.h b/replication.h
index 8faefe005f..4c8354de23 100644
--- a/replication.h
+++ b/replication.h
@@ -67,7 +67,6 @@ typedef struct ReplicationState ReplicationState;
  *
  * BlockDriver bdrv_replication = {
  * .format_name= "replication",
- * .protocol_name  = "replication",
  * .instance_size  = sizeof(BDRVReplicationState),
  *
  * .bdrv_open  = replication_open,
-- 
2.13.6

[Qemu-block] [PATCH 0/5] block: Ensure non-protocol drivers can only be selected explicitly

[Fabiano Rosas]

Block drivers can be selected by either protocol syntax:

  :[:options]

or explicitly:

  driver=[,option=...]

For the protocol syntax to work, drivers should set the protocol_name
field of the BlockDriver structure and provide bdrv_file_open and
bdrv_parse_filename implementations.

Conversely, block drivers that do not support the protocol syntax
should instead implement bdrv_open and not have a protocol_name field.

Some drivers do not currently adhere to this and errors arise when
trying to select them using the protocol syntax. For instance:

  $ qemu-img info replication:foo
  qemu-img: block.c:2401: bdrv_open_inherit: \
  Assertion `!!(flags & BDRV_O_PROTOCOL) == !!drv->bdrv_file_open' failed.
  Aborted (core dumped)

This patch-set ensures that the following drivers are meeting the
above criteria:
- blkreplay
- quorum
- replication
- throttle

Aside from that, documentation was added to make the above more
explicit.


Fabiano Rosas (5):
  block/replication: Remove protocol_name field
  block/quorum: Remove protocol-related fields
  block/throttle: Remove protocol-related fields
  block/blkreplay: Remove protocol-related fields
  include/block/block_int: Document protocol related functions

 block/blkreplay.c | 3 +--
 block/quorum.c| 3 +--
 block/replication.c   | 1 -
 block/throttle.c  | 3 +--
 include/block/block_int.h | 6 ++
 replication.h | 1 -
 6 files changed, 9 insertions(+), 8 deletions(-)

--
2.13.6

[Qemu-block] [PATCH 2/5] block/quorum: Remove protocol-related fields

[Fabiano Rosas]

The quorum driver is not a protocol so it should implement bdrv_open
instead of bdrv_file_open and not provide a protocol_name.

Attempts to invoke this driver using protocol syntax
(i.e. quorum:) will now fail gracefully:

  $ qemu-img info quorum:foo
  qemu-img: Could not open 'quorum:foo': Unknown protocol 'quorum'

Signed-off-by: Fabiano Rosas 
---
 block/quorum.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/block/quorum.c b/block/quorum.c
index 14333c18aa..cfe484a945 100644
--- a/block/quorum.c
+++ b/block/quorum.c
@@ -1098,11 +1098,10 @@ static void quorum_refresh_filename(BlockDriverState 
*bs, QDict *options)
 
 static BlockDriver bdrv_quorum = {
 .format_name= "quorum",
-.protocol_name  = "quorum",
 
 .instance_size  = sizeof(BDRVQuorumState),
 
-.bdrv_file_open = quorum_open,
+.bdrv_open  = quorum_open,
 .bdrv_close = quorum_close,
 .bdrv_refresh_filename  = quorum_refresh_filename,
 
-- 
2.13.6

[Qemu-block] [PATCH 5/5] include/block/block_int: Document protocol related functions

[Fabiano Rosas]

Clarify that for protocols the brdv_file_open function is used instead
of bdrv_open and that protocol_name is expected to be set.

Signed-off-by: Fabiano Rosas 
---
 include/block/block_int.h | 6 ++
 1 file changed, 6 insertions(+)

diff --git a/include/block/block_int.h b/include/block/block_int.h
index 64a5700f2b..d5e864c2dc 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -126,6 +126,8 @@ struct BlockDriver {
 
 int (*bdrv_open)(BlockDriverState *bs, QDict *options, int flags,
  Error **errp);
+
+/* Protocol drivers should implement this instead of bdrv_open */
 int (*bdrv_file_open)(BlockDriverState *bs, QDict *options, int flags,
   Error **errp);
 void (*bdrv_close)(BlockDriverState *bs);
@@ -247,6 +249,10 @@ struct BlockDriver {
  */
 int coroutine_fn (*bdrv_co_flush_to_os)(BlockDriverState *bs);
 
+/*
+ * Drivers that set this field should also provide a
+ * bdrv_file_open implementation
+ */
 const char *protocol_name;
 int (*bdrv_truncate)(BlockDriverState *bs, int64_t offset,
  PreallocMode prealloc, Error **errp);
-- 
2.13.6

Re: [Qemu-block] [PATCH 4/7] qed: Support .bdrv_co_create

[Eric Blake]

On 03/09/2018 03:46 PM, Kevin Wolf wrote:

This adds the .bdrv_co_create driver callback to qed, which
enables image creation over QMP.

Signed-off-by: Kevin Wolf 
---
  qapi/block-core.json |  25 ++-
  block/qed.c  | 204 ++-
  2 files changed, 162 insertions(+), 67 deletions(-)


Similar question as to qcow (who still creates qed images these days? 
and if no one seriously does it outside of our testsuite, would it be 
better to not allow QMP creation of qed images?).  On the other hand, 
qed is newer than qcow so it doesn't have quite the legacy of poor 
usage, so it may also mean that qed gets a longer deprecation cycle than 
qcow.


--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 3/7] qcow: Support .bdrv_co_create

[Eric Blake]

On 03/09/2018 03:46 PM, Kevin Wolf wrote:

This adds the .bdrv_co_create driver callback to qcow, which
enables image creation over QMP.

Signed-off-by: Kevin Wolf 
---
  qapi/block-core.json |  21 +-
  block/qcow.c | 196 ++-
  2 files changed, 150 insertions(+), 67 deletions(-)


Pre-review question: do we REALLY want to support creation of new qcow 
images from QMP?  Or are we at the point where we want to declare qcow a 
read-only format where we only support it to the extent that you can 
convert an existing qcow file into a better supported format like qcow2?


--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 3/5] nbd/server: fix: check client->closing before reply sending

[Eric Blake]

On 03/08/2018 12:46 PM, Vladimir Sementsov-Ogievskiy wrote:

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---

It's like an RFC. I'm not sure, but this place looks like a bug. Shouldn't
we chack client-closing even before nbd_client_receive_next_request() call?

  nbd/server.c | 8 
  1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/nbd/server.c b/nbd/server.c
index e0de431e10..97b45a21fa 100644
--- a/nbd/server.c
+++ b/nbd/server.c
@@ -1547,10 +1547,6 @@ static coroutine_fn void nbd_trip(void *opaque)


More context:

ret = nbd_co_receive_request(req, &request, &local_err);
client->recv_coroutine = NULL;
nbd_client_receive_next_request(client);
if (ret == -EIO) {


  goto disconnect;
  }
  


Calling nbd_client_receive_next_request() checks whether recv_coroutine 
is NULL (it is, because we just set it that way) and whether we are up 
to our maximum in parallel request handling; so it likely queues another 
coroutine to call nbd_trip() again.  However, when the next nbd_trip() 
is invoked, the first thing it does (after a trace call) is check 
client->closing, at which point it is a nop.


Your argument is that if ret was -EIO, we goto disconnect (which sets 
client->closing if it was not already set), and thus the just-scheduled 
next nbd_trip() will be a nop.  If ret was anything else, we used to try 
to reply to the client no matter what, which generally works; although 
if client->closing is already set, the attempt to reply is instead 
likely to fail and result in a later attempt to goto disconnect - but at 
that point disconnect is a nop since client->closing is already set. 
Whereas your patch skips the reply to the client if client->closing (and 
can't reach the disconnect code, but that doesn't matter as the 
disconnect attempt did nothing).  Swapping the check for client->closing 
to be earlier than the reply to the client thus looks safe.


Your RFC question is whether we can just check client->closing before 
checking ret, and skip nbd_client_receive_next_request() in that case 
(in other words, why even bother to queue up a coroutine that will do 
nothing, if we already know the client is going away).  And my answer is 
yes, I think that makes more sense.  So that would be:


diff --git c/nbd/server.c w/nbd/server.c
index 5f292064af0..b230ecb4fb8 100644
--- c/nbd/server.c
+++ w/nbd/server.c
@@ -1543,14 +1543,6 @@ static coroutine_fn void nbd_trip(void *opaque)
 req = nbd_request_get(client);
 ret = nbd_co_receive_request(req, &request, &local_err);
 client->recv_coroutine = NULL;
-nbd_client_receive_next_request(client);
-if (ret == -EIO) {
-goto disconnect;
-}
-
-if (ret < 0) {
-goto reply;
-}

 if (client->closing) {
 /*
@@ -1560,6 +1552,15 @@ static coroutine_fn void nbd_trip(void *opaque)
 goto done;
 }

+nbd_client_receive_next_request(client);
+if (ret == -EIO) {
+goto disconnect;
+}
+
+if (ret < 0) {
+goto reply;
+}
+
 switch (request.type) {
 case NBD_CMD_READ:
 /* XXX: NBD Protocol only documents use of FUA with WRITE */


Unless this revised form fails testing or gets any other review 
comments, I will go ahead and amend your commit in this manner.


--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 4/6] luks: Turn invalid assertion into check

[Kevin Wolf]

Am 09.03.2018 um 21:19 hat Eric Blake geschrieben:
> On 03/09/2018 11:27 AM, Kevin Wolf wrote:
> > The .bdrv_getlength implementation of the crypto block driver asserted
> > that the payload offset isn't after EOF. This is an invalid assertion to
> > make as the image file could be corrupted. Instead, check it and return
> > -EIO if the file is too small for the payload offset.
> 
> Good catch.  Probably not a CVE (unless someone can argue some way that
> causing a crash on an attempt to load a maliciously corrupted file can be
> used as a denial of service across a privilege boundary), but definitely
> needs fixing.
> 
> > 
> > Zero length images are fine, so trigger -EIO only on offset > len, not
> > on offset >= len as the assertion did before.
> > 
> > Signed-off-by: Kevin Wolf 
> > ---
> >   block/crypto.c | 5 -
> >   1 file changed, 4 insertions(+), 1 deletion(-)
> > 
> > diff --git a/block/crypto.c b/block/crypto.c
> > index 2035f9ab13..4908d8627f 100644
> > --- a/block/crypto.c
> > +++ b/block/crypto.c
> > @@ -518,7 +518,10 @@ static int64_t block_crypto_getlength(BlockDriverState 
> > *bs)
> >   uint64_t offset = qcrypto_block_get_payload_offset(crypto->block);
> >   assert(offset < INT64_MAX);
> 
> Umm, if the file can be corrupted, what's to prevent someone from sticking
> in a negative size that fails this assertion?

In qcrypto_block_luks_open():

block->sector_size = QCRYPTO_BLOCK_LUKS_SECTOR_SIZE;
block->payload_offset = luks->header.payload_offset *
block->sector_size

The sector size is 512LL, and luks->header.payload_offset is 32 bit.

But I just saw that block_crypto_truncate() has another wrong assertion.
Maybe I should fix that and write a test case for it. Not sure if I'll
add it to this series or as a follow-up during the freeze.

Kevin

[Qemu-block] [PATCH 7/7] vpc: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to vpc, which
enables image creation over QMP.

Signed-off-by: Kevin Wolf 
---
 qapi/block-core.json |  33 ++-
 block/vpc.c  | 152 ++-
 2 files changed, 147 insertions(+), 38 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 3a65909c47..ca645a0067 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3734,6 +3734,37 @@
 '*block-state-zero':'bool' } }
 
 ##
+# @BlockdevVpcSubformat:
+#
+# @dynamic: Growing image file
+# @fixed:   Preallocated fixed-size imge file
+#
+# Since: 2.12
+##
+{ 'enum': 'BlockdevVpcSubformat',
+  'data': [ 'dynamic', 'fixed' ] }
+
+##
+# @BlockdevCreateOptionsVpc:
+#
+# Driver specific image creation options for vpc (VHD).
+#
+# @file Node to create the image format on
+# @size Size of the virtual disk in bytes
+# @subformatvhdx subformat (default: dynamic)
+# @force-size   Force use of the exact byte size instead of rounding to the
+#   next size that can be represented in CHS geometry
+#   (default: false)
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsVpc',
+  'data': { 'file': 'BlockdevRef',
+'size': 'size',
+'*subformat':   'BlockdevVpcSubformat',
+'*force-size':  'bool' } }
+
+##
 # @BlockdevCreateNotSupported:
 #
 # This is used for all drivers that don't support creating images.
@@ -3790,7 +3821,7 @@
   'vdi':'BlockdevCreateOptionsVdi',
   'vhdx':   'BlockdevCreateOptionsVhdx',
   'vmdk':   'BlockdevCreateNotSupported',
-  'vpc':'BlockdevCreateNotSupported',
+  'vpc':'BlockdevCreateOptionsVpc',
   'vvfat':  'BlockdevCreateNotSupported',
   'vxhs':   'BlockdevCreateNotSupported'
   } }
diff --git a/block/vpc.c b/block/vpc.c
index b2e2b9ebd4..8824211713 100644
--- a/block/vpc.c
+++ b/block/vpc.c
@@ -32,6 +32,9 @@
 #include "migration/blocker.h"
 #include "qemu/bswap.h"
 #include "qemu/uuid.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qobject-input-visitor.h"
+#include "qapi/qapi-visit-block-core.h"
 
 /**/
 
@@ -166,6 +169,8 @@ static QemuOptsList vpc_runtime_opts = {
 }
 };
 
+static QemuOptsList vpc_create_opts;
+
 static uint32_t vpc_checksum(uint8_t* buf, size_t size)
 {
 uint32_t res = 0;
@@ -897,12 +902,15 @@ static int create_fixed_disk(BlockBackend *blk, uint8_t 
*buf,
 return ret;
 }
 
-static int coroutine_fn vpc_co_create_opts(const char *filename, QemuOpts 
*opts,
-   Error **errp)
+static int coroutine_fn vpc_co_create(BlockdevCreateOptions *opts,
+  Error **errp)
 {
+BlockdevCreateOptionsVpc *vpc_opts;
+BlockBackend *blk = NULL;
+BlockDriverState *bs = NULL;
+
 uint8_t buf[1024];
 VHDFooter *footer = (VHDFooter *) buf;
-char *disk_type_param;
 int i;
 uint16_t cyls = 0;
 uint8_t heads = 0;
@@ -911,45 +919,38 @@ static int coroutine_fn vpc_co_create_opts(const char 
*filename, QemuOpts *opts,
 int64_t total_size;
 int disk_type;
 int ret = -EIO;
-bool force_size;
-Error *local_err = NULL;
-BlockBackend *blk = NULL;
 
-/* Read out options */
-total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
-  BDRV_SECTOR_SIZE);
-disk_type_param = qemu_opt_get_del(opts, BLOCK_OPT_SUBFMT);
-if (disk_type_param) {
-if (!strcmp(disk_type_param, "dynamic")) {
-disk_type = VHD_DYNAMIC;
-} else if (!strcmp(disk_type_param, "fixed")) {
-disk_type = VHD_FIXED;
-} else {
-error_setg(errp, "Invalid disk type, %s", disk_type_param);
-ret = -EINVAL;
-goto out;
-}
-} else {
+assert(opts->driver == BLOCKDEV_DRIVER_VPC);
+vpc_opts = &opts->u.vpc;
+
+/* Validate options and set default values */
+total_size = vpc_opts->size;
+
+if (!vpc_opts->has_subformat) {
+vpc_opts->subformat = BLOCKDEV_VPC_SUBFORMAT_DYNAMIC;
+}
+switch (vpc_opts->subformat) {
+case BLOCKDEV_VPC_SUBFORMAT_DYNAMIC:
 disk_type = VHD_DYNAMIC;
+break;
+case BLOCKDEV_VPC_SUBFORMAT_FIXED:
+disk_type = VHD_FIXED;
+break;
+default:
+g_assert_not_reached();
 }
 
-force_size = qemu_opt_get_bool_del(opts, VPC_OPT_FORCE_SIZE, false);
-
-ret = bdrv_create_file(filename, opts, &local_err);
-if (ret < 0) {
-error_propagate(errp, local_err);
-goto out;
+/* Create BlockBackend to write to the image */
+bs = bdrv_open_blockdev_ref(vpc_opts->file, errp);
+if (bs == NULL) {
+return -EIO;
 }
 
-blk = blk_new_open(filename, NUL

[Qemu-block] [PATCH 6/7] vhdx: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to vhdx, which
enables image creation over QMP.

Signed-off-by: Kevin Wolf 
---
 qapi/block-core.json |  37 ++-
 block/vhdx.c | 174 ++-
 2 files changed, 167 insertions(+), 44 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 2eba0eef7e..3a65909c47 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3699,6 +3699,41 @@
 '*static':  'bool' } }
 
 ##
+# @BlockdevVhdxSubformat:
+#
+# @dynamic: Growing image file
+# @fixed:   Preallocated fixed-size imge file
+#
+# Since: 2.12
+##
+{ 'enum': 'BlockdevVhdxSubformat',
+  'data': [ 'dynamic', 'fixed' ] }
+
+##
+# @BlockdevCreateOptionsVhdx:
+#
+# Driver specific image creation options for vhdx.
+#
+# @file Node to create the image format on
+# @size Size of the virtual disk in bytes
+# @log-size Log size in bytes (default: 1 MB)
+# @block-size   Block size in bytes (default: 1 MB)
+# @subformatvhdx subformat (default: dynamic)
+# @block-state-zero Force use of payload blocks of type 'ZERO'. Non-standard,
+#   but default.  Do not set to 'off' when using 'qemu-img
+#   convert' with subformat=dynamic.
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsVhdx',
+  'data': { 'file': 'BlockdevRef',
+'size': 'size',
+'*log-size':'size',
+'*block-size':  'size',
+'*subformat':   'BlockdevVhdxSubformat',
+'*block-state-zero':'bool' } }
+
+##
 # @BlockdevCreateNotSupported:
 #
 # This is used for all drivers that don't support creating images.
@@ -3753,7 +3788,7 @@
   'ssh':'BlockdevCreateOptionsSsh',
   'throttle':   'BlockdevCreateNotSupported',
   'vdi':'BlockdevCreateOptionsVdi',
-  'vhdx':   'BlockdevCreateNotSupported',
+  'vhdx':   'BlockdevCreateOptionsVhdx',
   'vmdk':   'BlockdevCreateNotSupported',
   'vpc':'BlockdevCreateNotSupported',
   'vvfat':  'BlockdevCreateNotSupported',
diff --git a/block/vhdx.c b/block/vhdx.c
index d82350d07c..0ce972381f 100644
--- a/block/vhdx.c
+++ b/block/vhdx.c
@@ -26,6 +26,9 @@
 #include "block/vhdx.h"
 #include "migration/blocker.h"
 #include "qemu/uuid.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qobject-input-visitor.h"
+#include "qapi/qapi-visit-block-core.h"
 
 /* Options for VHDX creation */
 
@@ -39,6 +42,8 @@ typedef enum VHDXImageType {
 VHDX_TYPE_DIFFERENCING,   /* Currently unsupported */
 } VHDXImageType;
 
+static QemuOptsList vhdx_create_opts;
+
 /* Several metadata and region table data entries are identified by
  * guids in  a MS-specific GUID format. */
 
@@ -1792,54 +1797,63 @@ exit:
  *. ~ --- ~  ~  ~ ---.
  *   1MB
  */
-static int coroutine_fn vhdx_co_create_opts(const char *filename, QemuOpts 
*opts,
-Error **errp)
+static int coroutine_fn vhdx_co_create(BlockdevCreateOptions *opts,
+   Error **errp)
 {
+BlockdevCreateOptionsVhdx *vhdx_opts;
+BlockBackend *blk = NULL;
+BlockDriverState *bs = NULL;
+
 int ret = 0;
-uint64_t image_size = (uint64_t) 2 * GiB;
-uint32_t log_size   = 1 * MiB;
-uint32_t block_size = 0;
+uint64_t image_size;
+uint32_t log_size;
+uint32_t block_size;
 uint64_t signature;
 uint64_t metadata_offset;
 bool use_zero_blocks = false;
 
 gunichar2 *creator = NULL;
 glong creator_items;
-BlockBackend *blk;
-char *type = NULL;
 VHDXImageType image_type;
-Error *local_err = NULL;
 
-image_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
-  BDRV_SECTOR_SIZE);
-log_size = qemu_opt_get_size_del(opts, VHDX_BLOCK_OPT_LOG_SIZE, 0);
-block_size = qemu_opt_get_size_del(opts, VHDX_BLOCK_OPT_BLOCK_SIZE, 0);
-type = qemu_opt_get_del(opts, BLOCK_OPT_SUBFMT);
-use_zero_blocks = qemu_opt_get_bool_del(opts, VHDX_BLOCK_OPT_ZERO, true);
+assert(opts->driver == BLOCKDEV_DRIVER_VHDX);
+vhdx_opts = &opts->u.vhdx;
+
+/* Validate options and set default values */
+
+image_size = vhdx_opts->size;
+block_size = vhdx_opts->block_size;
+
+if (!vhdx_opts->has_log_size) {
+log_size = DEFAULT_LOG_SIZE;
+} else {
+log_size = vhdx_opts->log_size;
+}
+
+if (!vhdx_opts->has_block_state_zero) {
+use_zero_blocks = true;
+} else {
+use_zero_blocks = vhdx_opts->block_state_zero;
+}
 
 if (image_size > VHDX_MAX_IMAGE_SIZE) {
 error_setg_errno(errp, EINVAL, "Image size too large; max of 64TB");
-ret = -EINVAL;
-goto exit;
+return -EINVAL;
 }
 
-if (type == NULL) {
-typ

[Qemu-block] [PATCH 0/7] block: .bdrv_co_create for format drivers

[Kevin Wolf]

This series adds a .bdrv_co_create implementation to almost all format
drivers that support creating images where its still missing. The only
exception is VMDK because its support for extents will make the QAPI
design a bit more complicated.

The other format driver not covered in this series are qcow2 (already
merged) and luks (already posted in a separate series).

Kevin Wolf (7):
  parallels: Support .bdrv_co_create
  qemu-iotests: Enable write tests for parallels
  qcow: Support .bdrv_co_create
  qed: Support .bdrv_co_create
  vdi: Support .bdrv_co_create
  vhdx: Support .bdrv_co_create
  vpc: Support .bdrv_co_create

 qapi/block-core.json | 155 +--
 block/parallels.c| 199 ++---
 block/qcow.c | 196 ++---
 block/qed.c  | 204 ---
 block/vdi.c  | 169 +--
 block/vhdx.c | 174 ++--
 block/vpc.c  | 152 ++-
 tests/qemu-iotests/181   |   2 +-
 tests/qemu-iotests/check |   1 -
 9 files changed, 943 insertions(+), 309 deletions(-)

-- 
2.13.6

[Qemu-block] [PATCH 4/7] qed: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to qed, which
enables image creation over QMP.

Signed-off-by: Kevin Wolf 
---
 qapi/block-core.json |  25 ++-
 block/qed.c  | 204 ++-
 2 files changed, 162 insertions(+), 67 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index c81677c434..1e2edbc063 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3559,6 +3559,29 @@
 '*refcount-bits':   'int' } }
 
 ##
+# @BlockdevCreateOptionsQed:
+#
+# Driver specific image creation options for qed.
+#
+# @file Node to create the image format on
+# @size Size of the virtual disk in bytes
+# @backing-file File name of the backing file if a backing file
+#   should be used
+# @backing-fmt  Name of the block driver to use for the backing file
+# @cluster-size Cluster size in bytes (default: 65536)
+# @table-size   L1/L2 table size (in clusters)
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsQed',
+  'data': { 'file': 'BlockdevRef',
+'size': 'size',
+'*backing-file':'str',
+'*backing-fmt': 'BlockdevDriver',
+'*cluster-size':'size',
+'*table-size':  'int' } }
+
+##
 # @BlockdevCreateOptionsRbd:
 #
 # Driver specific image creation options for rbd/Ceph.
@@ -3702,7 +3725,7 @@
   'parallels':  'BlockdevCreateOptionsParallels',
   'qcow':   'BlockdevCreateOptionsQcow',
   'qcow2':  'BlockdevCreateOptionsQcow2',
-  'qed':'BlockdevCreateNotSupported',
+  'qed':'BlockdevCreateOptionsQed',
   'quorum': 'BlockdevCreateNotSupported',
   'raw':'BlockdevCreateNotSupported',
   'rbd':'BlockdevCreateOptionsRbd',
diff --git a/block/qed.c b/block/qed.c
index 5e6a6bfaa0..46a84beeed 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -20,6 +20,11 @@
 #include "trace.h"
 #include "qed.h"
 #include "sysemu/block-backend.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qobject-input-visitor.h"
+#include "qapi/qapi-visit-block-core.h"
+
+static QemuOptsList qed_create_opts;
 
 static int bdrv_qed_probe(const uint8_t *buf, int buf_size,
   const char *filename)
@@ -594,57 +599,95 @@ static void bdrv_qed_close(BlockDriverState *bs)
 qemu_vfree(s->l1_table);
 }
 
-static int qed_create(const char *filename, uint32_t cluster_size,
-  uint64_t image_size, uint32_t table_size,
-  const char *backing_file, const char *backing_fmt,
-  QemuOpts *opts, Error **errp)
+static int coroutine_fn bdrv_qed_co_create(BlockdevCreateOptions *opts,
+   Error **errp)
 {
-QEDHeader header = {
-.magic = QED_MAGIC,
-.cluster_size = cluster_size,
-.table_size = table_size,
-.header_size = 1,
-.features = 0,
-.compat_features = 0,
-.l1_table_offset = cluster_size,
-.image_size = image_size,
-};
+BlockdevCreateOptionsQed *qed_opts;
+BlockBackend *blk = NULL;
+BlockDriverState *bs = NULL;
+
+QEDHeader header;
 QEDHeader le_header;
 uint8_t *l1_table = NULL;
-size_t l1_size = header.cluster_size * header.table_size;
-Error *local_err = NULL;
+size_t l1_size;
 int ret = 0;
-BlockBackend *blk;
 
-ret = bdrv_create_file(filename, opts, &local_err);
-if (ret < 0) {
-error_propagate(errp, local_err);
-return ret;
+assert(opts->driver == BLOCKDEV_DRIVER_QED);
+qed_opts = &opts->u.qed;
+
+/* Validate options and set default values */
+if (!qed_opts->has_cluster_size) {
+qed_opts->cluster_size = QED_DEFAULT_CLUSTER_SIZE;
+}
+if (!qed_opts->has_table_size) {
+qed_opts->table_size = QED_DEFAULT_TABLE_SIZE;
 }
 
-blk = blk_new_open(filename, NULL, NULL,
-   BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_PROTOCOL,
-   &local_err);
-if (blk == NULL) {
-error_propagate(errp, local_err);
+if (!qed_is_cluster_size_valid(qed_opts->cluster_size)) {
+error_setg(errp, "QED cluster size must be within range [%u, %u] "
+ "and power of 2",
+   QED_MIN_CLUSTER_SIZE, QED_MAX_CLUSTER_SIZE);
+return -EINVAL;
+}
+if (!qed_is_table_size_valid(qed_opts->table_size)) {
+error_setg(errp, "QED table size must be within range [%u, %u] "
+ "and power of 2",
+   QED_MIN_TABLE_SIZE, QED_MAX_TABLE_SIZE);
+return -EINVAL;
+}
+if (!qed_is_image_size_valid(qed_opts->size, qed_opts->cluster_size,
+ qed_opts->table_size))
+{
+error_setg(errp, "QED image size must be a non-zero multiple of "
+ "cluster si

[Qemu-block] [PATCH 3/7] qcow: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to qcow, which
enables image creation over QMP.

Signed-off-by: Kevin Wolf 
---
 qapi/block-core.json |  21 +-
 block/qcow.c | 196 ++-
 2 files changed, 150 insertions(+), 67 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index d38058eeab..c81677c434 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3497,6 +3497,25 @@
 '*cluster-size':'size' } }
 
 ##
+# @BlockdevCreateOptionsQcow:
+#
+# Driver specific image creation options for qcow.
+#
+# @file Node to create the image format on
+# @size Size of the virtual disk in bytes
+# @backing-file File name of the backing file if a backing file
+#   should be used
+# @encrypt  Encryption options if the image should be encrypted
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsQcow',
+  'data': { 'file': 'BlockdevRef',
+'size': 'size',
+'*backing-file':'str',
+'*encrypt': 'QCryptoBlockCreateOptions' } }
+
+##
 # @BlockdevQcow2Version:
 #
 # @v2:  The original QCOW2 format as introduced in qemu 0.10 (version 2)
@@ -3681,8 +3700,8 @@
   'null-co':'BlockdevCreateNotSupported',
   'nvme':   'BlockdevCreateNotSupported',
   'parallels':  'BlockdevCreateOptionsParallels',
+  'qcow':   'BlockdevCreateOptionsQcow',
   'qcow2':  'BlockdevCreateOptionsQcow2',
-  'qcow':   'BlockdevCreateNotSupported',
   'qed':'BlockdevCreateNotSupported',
   'quorum': 'BlockdevCreateNotSupported',
   'raw':'BlockdevCreateNotSupported',
diff --git a/block/qcow.c b/block/qcow.c
index 47a18d9a3a..2e3770ca63 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -33,6 +33,8 @@
 #include 
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qstring.h"
+#include "qapi/qobject-input-visitor.h"
+#include "qapi/qapi-visit-block-core.h"
 #include "crypto/block.h"
 #include "migration/blocker.h"
 #include "block/crypto.h"
@@ -86,6 +88,8 @@ typedef struct BDRVQcowState {
 Error *migration_blocker;
 } BDRVQcowState;
 
+static QemuOptsList qcow_create_opts;
+
 static int decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset);
 
 static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename)
@@ -810,62 +814,50 @@ static void qcow_close(BlockDriverState *bs)
 error_free(s->migration_blocker);
 }
 
-static int coroutine_fn qcow_co_create_opts(const char *filename, QemuOpts 
*opts,
-Error **errp)
+static int coroutine_fn qcow_co_create(BlockdevCreateOptions *opts,
+   Error **errp)
 {
+BlockdevCreateOptionsQcow *qcow_opts;
 int header_size, backing_filename_len, l1_size, shift, i;
 QCowHeader header;
 uint8_t *tmp;
 int64_t total_size = 0;
-char *backing_file = NULL;
-Error *local_err = NULL;
 int ret;
+BlockDriverState *bs;
 BlockBackend *qcow_blk;
-char *encryptfmt = NULL;
-QDict *options;
-QDict *encryptopts = NULL;
-QCryptoBlockCreateOptions *crypto_opts = NULL;
 QCryptoBlock *crypto = NULL;
 
-/* Read out options */
-total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
-  BDRV_SECTOR_SIZE);
+assert(opts->driver == BLOCKDEV_DRIVER_QCOW);
+qcow_opts = &opts->u.qcow;
+
+/* Sanity checks */
+total_size = qcow_opts->size;
 if (total_size == 0) {
 error_setg(errp, "Image size is too small, cannot be zero length");
-ret = -EINVAL;
-goto cleanup;
+return -EINVAL;
 }
 
-backing_file = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FILE);
-encryptfmt = qemu_opt_get_del(opts, BLOCK_OPT_ENCRYPT_FORMAT);
-if (encryptfmt) {
-if (qemu_opt_get(opts, BLOCK_OPT_ENCRYPT)) {
-error_setg(errp, "Options " BLOCK_OPT_ENCRYPT " and "
-   BLOCK_OPT_ENCRYPT_FORMAT " are mutually exclusive");
-ret = -EINVAL;
-goto cleanup;
-}
-} else if (qemu_opt_get_bool_del(opts, BLOCK_OPT_ENCRYPT, false)) {
-encryptfmt = g_strdup("aes");
+if (qcow_opts->has_encrypt &&
+qcow_opts->encrypt->format != Q_CRYPTO_BLOCK_FORMAT_QCOW)
+{
+error_setg(errp, "Unsupported encryption format");
+return -EINVAL;
 }
 
-ret = bdrv_create_file(filename, opts, &local_err);
-if (ret < 0) {
-error_propagate(errp, local_err);
-goto cleanup;
+/* Create BlockBackend to write to the image */
+bs = bdrv_open_blockdev_ref(qcow_opts->file, errp);
+if (bs == NULL) {
+return -EIO;
 }
 
-qcow_blk = blk_new_open(filename, NULL, NULL,
-BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_PROTOCOL,
-&loc

[Qemu-block] [PATCH 1/7] parallels: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to parallels, which
enables image creation over QMP.

Signed-off-by: Kevin Wolf 
---
 qapi/block-core.json |  18 -
 block/parallels.c| 199 ++-
 2 files changed, 168 insertions(+), 49 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 07039bfe9c..d38058eeab 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3481,6 +3481,22 @@
 'size': 'size' } }
 
 ##
+# @BlockdevCreateOptionsParallels:
+#
+# Driver specific image creation options for parallels.
+#
+# @file Node to create the image format on
+# @size Size of the virtual disk in bytes
+# @cluster-size Cluster size in bytes (default: 1 MB)
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsParallels',
+  'data': { 'file': 'BlockdevRef',
+'size': 'size',
+'*cluster-size':'size' } }
+
+##
 # @BlockdevQcow2Version:
 #
 # @v2:  The original QCOW2 format as introduced in qemu 0.10 (version 2)
@@ -3664,7 +3680,7 @@
   'null-aio':   'BlockdevCreateNotSupported',
   'null-co':'BlockdevCreateNotSupported',
   'nvme':   'BlockdevCreateNotSupported',
-  'parallels':  'BlockdevCreateNotSupported',
+  'parallels':  'BlockdevCreateOptionsParallels',
   'qcow2':  'BlockdevCreateOptionsQcow2',
   'qcow':   'BlockdevCreateNotSupported',
   'qed':'BlockdevCreateNotSupported',
diff --git a/block/parallels.c b/block/parallels.c
index c13cb619e6..2da5e56a9d 100644
--- a/block/parallels.c
+++ b/block/parallels.c
@@ -34,6 +34,9 @@
 #include "sysemu/block-backend.h"
 #include "qemu/module.h"
 #include "qemu/option.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qobject-input-visitor.h"
+#include "qapi/qapi-visit-block-core.h"
 #include "qemu/bswap.h"
 #include "qemu/bitmap.h"
 #include "migration/blocker.h"
@@ -79,6 +82,25 @@ static QemuOptsList parallels_runtime_opts = {
 },
 };
 
+static QemuOptsList parallels_create_opts = {
+.name = "parallels-create-opts",
+.head = QTAILQ_HEAD_INITIALIZER(parallels_create_opts.head),
+.desc = {
+{
+.name = BLOCK_OPT_SIZE,
+.type = QEMU_OPT_SIZE,
+.help = "Virtual disk size",
+},
+{
+.name = BLOCK_OPT_CLUSTER_SIZE,
+.type = QEMU_OPT_SIZE,
+.help = "Parallels image cluster size",
+.def_value_str = stringify(DEFAULT_CLUSTER_SIZE),
+},
+{ /* end of list */ }
+}
+};
+
 
 static int64_t bat2sect(BDRVParallelsState *s, uint32_t idx)
 {
@@ -480,46 +502,62 @@ out:
 }
 
 
-static int coroutine_fn parallels_co_create_opts(const char *filename,
- QemuOpts *opts,
- Error **errp)
+static int coroutine_fn parallels_co_create(BlockdevCreateOptions* opts,
+Error **errp)
 {
+BlockdevCreateOptionsParallels *parallels_opts;
+BlockDriverState *bs;
+BlockBackend *blk;
 int64_t total_size, cl_size;
-uint8_t tmp[BDRV_SECTOR_SIZE];
-Error *local_err = NULL;
-BlockBackend *file;
 uint32_t bat_entries, bat_sectors;
 ParallelsHeader header;
+uint8_t tmp[BDRV_SECTOR_SIZE];
 int ret;
 
-total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
-  BDRV_SECTOR_SIZE);
-cl_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE,
-  DEFAULT_CLUSTER_SIZE), BDRV_SECTOR_SIZE);
+assert(opts->driver == BLOCKDEV_DRIVER_PARALLELS);
+parallels_opts = &opts->u.parallels;
+
+/* Sanity checks */
+total_size = parallels_opts->size;
+
+if (parallels_opts->has_cluster_size) {
+cl_size = parallels_opts->cluster_size;
+} else {
+cl_size = DEFAULT_CLUSTER_SIZE;
+}
+
 if (total_size >= MAX_PARALLELS_IMAGE_FACTOR * cl_size) {
-error_propagate(errp, local_err);
+error_setg(errp, "Image size is too large for this cluster size");
 return -E2BIG;
 }
 
-ret = bdrv_create_file(filename, opts, &local_err);
-if (ret < 0) {
-error_propagate(errp, local_err);
-return ret;
+if (!QEMU_IS_ALIGNED(total_size, BDRV_SECTOR_SIZE)) {
+error_setg(errp, "Image size must be a multiple of 512 bytes");
+return -EINVAL;
 }
 
-file = blk_new_open(filename, NULL, NULL,
-BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_PROTOCOL,
-&local_err);
-if (file == NULL) {
-error_propagate(errp, local_err);
+if (!QEMU_IS_ALIGNED(cl_size, BDRV_SECTOR_SIZE)) {
+error_setg(errp, "Cluster size must be a multiple of 512 bytes");
+return -EINVAL;
+}
+
+/* Create BlockBackend to write to the image */
+bs

[Qemu-block] [PATCH 5/7] vdi: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to vdi, which
enables image creation over QMP.

Signed-off-by: Kevin Wolf 
---
 qapi/block-core.json |  21 ++-
 block/vdi.c  | 169 ++-
 2 files changed, 148 insertions(+), 42 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 1e2edbc063..2eba0eef7e 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3680,6 +3680,25 @@
 'size': 'size' } }
 
 ##
+# @BlockdevCreateOptionsVdi:
+#
+# Driver specific image creation options for vdi.
+#
+# @file Node to create the image format on
+# @size Size of the virtual disk in bytes
+# @cluster-size Cluster size in bytes (default: 1 MB)
+# @static   Whether to create a static (preallocated) image
+#   (default: false)
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsVdi',
+  'data': { 'file': 'BlockdevRef',
+'size': 'size',
+'*cluster-size':'size',
+'*static':  'bool' } }
+
+##
 # @BlockdevCreateNotSupported:
 #
 # This is used for all drivers that don't support creating images.
@@ -3733,7 +3752,7 @@
   'sheepdog':   'BlockdevCreateOptionsSheepdog',
   'ssh':'BlockdevCreateOptionsSsh',
   'throttle':   'BlockdevCreateNotSupported',
-  'vdi':'BlockdevCreateNotSupported',
+  'vdi':'BlockdevCreateOptionsVdi',
   'vhdx':   'BlockdevCreateNotSupported',
   'vmdk':   'BlockdevCreateNotSupported',
   'vpc':'BlockdevCreateNotSupported',
diff --git a/block/vdi.c b/block/vdi.c
index 2b5ddd0666..c60ddc58c0 100644
--- a/block/vdi.c
+++ b/block/vdi.c
@@ -60,6 +60,9 @@
 #include "qemu/coroutine.h"
 #include "qemu/cutils.h"
 #include "qemu/uuid.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qobject-input-visitor.h"
+#include "qapi/qapi-visit-block-core.h"
 
 /* Code configuration options. */
 
@@ -182,6 +185,8 @@ typedef struct {
 Error *migration_blocker;
 } BDRVVdiState;
 
+static QemuOptsList vdi_create_opts;
+
 static void vdi_header_to_cpu(VdiHeader *header)
 {
 le32_to_cpus(&header->signature);
@@ -716,67 +721,72 @@ nonallocating_write:
 return ret;
 }
 
-static int coroutine_fn vdi_co_create_opts(const char *filename, QemuOpts 
*opts,
-   Error **errp)
+static int coroutine_fn vdi_co_create(BlockdevCreateOptions *opts,
+  Error **errp)
 {
+BlockdevCreateOptionsVdi *vdi_opts;
+BlockBackend *blk = NULL;
+BlockDriverState *bs = NULL;
+
 int ret = 0;
-uint64_t bytes = 0;
 uint32_t blocks;
-size_t block_size = DEFAULT_CLUSTER_SIZE;
 uint32_t image_type = VDI_TYPE_DYNAMIC;
 VdiHeader header;
 size_t i;
 size_t bmap_size;
 int64_t offset = 0;
-Error *local_err = NULL;
-BlockBackend *blk = NULL;
 uint32_t *bmap = NULL;
 
 logout("\n");
 
-/* Read out options. */
-bytes = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
- BDRV_SECTOR_SIZE);
-#if defined(CONFIG_VDI_BLOCK_SIZE)
-/* TODO: Additional checks (SECTOR_SIZE * 2^n, ...). */
-block_size = qemu_opt_get_size_del(opts,
-   BLOCK_OPT_CLUSTER_SIZE,
-   DEFAULT_CLUSTER_SIZE);
-#endif
-#if defined(CONFIG_VDI_STATIC_IMAGE)
-if (qemu_opt_get_bool_del(opts, BLOCK_OPT_STATIC, false)) {
-image_type = VDI_TYPE_STATIC;
+assert(opts->driver == BLOCKDEV_DRIVER_VDI);
+vdi_opts = &opts->u.vdi;
+
+/* Validate options and set default values */
+if (!vdi_opts->has_cluster_size) {
+vdi_opts->cluster_size = DEFAULT_CLUSTER_SIZE;
 }
-#endif
 
-if (bytes > VDI_DISK_SIZE_MAX) {
-ret = -ENOTSUP;
+if (vdi_opts->size > VDI_DISK_SIZE_MAX) {
 error_setg(errp, "Unsupported VDI image size (size is 0x%" PRIx64
   ", max supported is 0x%" PRIx64 ")",
-  bytes, VDI_DISK_SIZE_MAX);
-goto exit;
+  vdi_opts->size, VDI_DISK_SIZE_MAX);
+return -ENOTSUP;
 }
 
-ret = bdrv_create_file(filename, opts, &local_err);
-if (ret < 0) {
-error_propagate(errp, local_err);
-goto exit;
+#if !defined(CONFIG_VDI_BLOCK_SIZE)
+if (vdi_opts->has_cluster_size) {
+error_setg(errp, "Non-default cluster size not supported");
+return -ENOTSUP;
+}
+#endif
+#if !defined(CONFIG_VDI_STATIC_IMAGE)
+if (vdi_opts->has_static) {
+error_setg(errp, "Static images not supported");
+return -ENOTSUP;
+}
+#else
+if (vdi_opts->q_static) {
+image_type = VDI_TYPE_STATIC;
+}
+#endif
+
+/* Create BlockBackend to write to the image */
+bs = bdrv_open_blockdev_ref(vdi_opts->file, errp);
+if (bs == NULL) {
+

[Qemu-block] [PATCH 2/7] qemu-iotests: Enable write tests for parallels

[Kevin Wolf]

Originally we added parallels as a read-only format to qemu-iotests
where we did just some tests with a binary image. Since then, write and
image creation support has been added to the driver, so we can now
enable it in _supported_fmt generic.

The driver doesn't support migration yet, though, so we need to add it
to the list of exceptions in 181.

Signed-off-by: Kevin Wolf 
---
 tests/qemu-iotests/181   | 2 +-
 tests/qemu-iotests/check | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/qemu-iotests/181 b/tests/qemu-iotests/181
index 0c91e8f9de..5e767c6195 100755
--- a/tests/qemu-iotests/181
+++ b/tests/qemu-iotests/181
@@ -44,7 +44,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt generic
 # Formats that do not support live migration
-_unsupported_fmt qcow vdi vhdx vmdk vpc vvfat
+_unsupported_fmt qcow vdi vhdx vmdk vpc vvfat parallels
 _supported_proto generic
 _supported_os Linux
 
diff --git a/tests/qemu-iotests/check b/tests/qemu-iotests/check
index e6b6ff7a04..469142cd58 100755
--- a/tests/qemu-iotests/check
+++ b/tests/qemu-iotests/check
@@ -284,7 +284,6 @@ testlist options
 
 -parallels)
 IMGFMT=parallels
-IMGFMT_GENERIC=false
 xpand=false
 ;;
 
-- 
2.13.6

Re: [Qemu-block] [PATCH 6/6] qemu-iotests: Test luks QMP image creation

[Eric Blake]

On 03/09/2018 11:27 AM, Kevin Wolf wrote:

Signed-off-by: Kevin Wolf 
---
  tests/qemu-iotests/208   | 211 +++
  tests/qemu-iotests/208.out   | 136 
  tests/qemu-iotests/common.rc |   2 +-
  tests/qemu-iotests/group |   1 +
  4 files changed, 349 insertions(+), 1 deletion(-)
  create mode 100755 tests/qemu-iotests/208
  create mode 100644 tests/qemu-iotests/208.out


I've seen another patch using 208 - someone gets to renumber ;)


+# creator
+owner=kw...@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1   # failure is the default!


Whatever happened to Jeff's efforts to reduce pointless boilerplate?



+function do_run_qemu()
+{
+echo Testing: "$@"
+$QEMU -nographic -qmp stdio -serial none "$@"


Is -nodefaults better than -serial none?

At any rate, my comments are trivial, whether or not you do something 
about them, so


Reviewed-by: Eric Blake 

--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 5/6] luks: Catch integer overflow for huge sizes

[Eric Blake]

On 03/09/2018 11:27 AM, Kevin Wolf wrote:

When you request an image size close to UINT64_MAX, the addition of the
crypto header may cause an integer overflow. Catch it instead of
silently truncating the image size.

Signed-off-by: Kevin Wolf 
---
  block/crypto.c | 5 +
  1 file changed, 5 insertions(+)

diff --git a/block/crypto.c b/block/crypto.c
index 4908d8627f..1b46519c53 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -102,6 +102,11 @@ static ssize_t block_crypto_init_func(QCryptoBlock *block,
  {
  struct BlockCryptoCreateData *data = opaque;
  
+if (headerlen > UINT64_MAX - data->size) {


INT64_MAX, please.  We are further bounded by having to fit within off_t 
(signed) rather than uint64_t.



+error_setg(errp, "The requested file size is too large");
+return -EFBIG;
+}
+
  /* User provided size should reflect amount of space made
   * available to the guest, so we must take account of that
   * which will be used by the crypto header



--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 4/6] luks: Turn invalid assertion into check

[Eric Blake]

On 03/09/2018 11:27 AM, Kevin Wolf wrote:

The .bdrv_getlength implementation of the crypto block driver asserted
that the payload offset isn't after EOF. This is an invalid assertion to
make as the image file could be corrupted. Instead, check it and return
-EIO if the file is too small for the payload offset.


Good catch.  Probably not a CVE (unless someone can argue some way that 
causing a crash on an attempt to load a maliciously corrupted file can 
be used as a denial of service across a privilege boundary), but 
definitely needs fixing.




Zero length images are fine, so trigger -EIO only on offset > len, not
on offset >= len as the assertion did before.

Signed-off-by: Kevin Wolf 
---
  block/crypto.c | 5 -
  1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/block/crypto.c b/block/crypto.c
index 2035f9ab13..4908d8627f 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -518,7 +518,10 @@ static int64_t block_crypto_getlength(BlockDriverState *bs)
  
  uint64_t offset = qcrypto_block_get_payload_offset(crypto->block);

  assert(offset < INT64_MAX);


Umm, if the file can be corrupted, what's to prevent someone from 
sticking in a negative size that fails this assertion?



-assert(offset < len);
+
+if (offset > len) {
+   return -EIO;
+}
  
  len -= offset;
  



--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 3/6] luks: Support .bdrv_co_create

[Eric Blake]

On 03/09/2018 11:27 AM, Kevin Wolf wrote:

This adds the .bdrv_co_create driver callback to luks, which enables
image creation over QMP.

Signed-off-by: Kevin Wolf 
---
  qapi/block-core.json | 17 -
  block/crypto.c   | 34 ++
  2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 524d51567a..07039bfe9c 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3452,6 +3452,21 @@
  '*preallocation':   'PreallocMode' } }
  
  ##

+# @BlockdevCreateOptionsLUKS:
+#
+# Driver specific image creation options for LUKS.
+#
+# @file Node to create the image format on
+# @size Size of the virtual disk in bytes
+#
+# Since: 2.12


Well, as long as we make it by Tuesday :)

Reviewed-by: Eric Blake 

--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH v3 0/7] block: Handle null backing link

[Eric Blake]

On 02/24/2018 09:40 AM, Max Reitz wrote:

Currently, we try to rewrite every occurrence of "backing": null into
"backing": "" in qmp_blockdev_add().  However, that breaks using the
same "backing": null construction in json:{} file names (which do not go
through qmp_blockdev_add()).  Currently, these then just behave as if
the option has not been specified.

Since there is actually only one place where we evaluate the @backing
option to find out whether not to use a backing file, we can instead
just check for null there.  It doesn't matter that this changes the
runtime state of the option from "" to null, because nobody really does
anything with that runtime state anyway (except put it into qemu again,
but qemu doesn't care whether it's "" or null).

And in the future, it's much better if we get it to be null in that
runtime state sooner than later -- see patch 7.


Note that it was Markus (who's away having a good time, I hope) who
proposed qobject_to(), so I guess he won't object too much to seeing the
concept having landed in his tree once he returns.
(Although he hasn't reviewed the previous iteration of this series,
  which included it already.)


v3:
- Added patch 1 so we can use a common macro in patch 2 (instead of
   invoking _Static_assert() directly), but still keep the explanatory
   message



This series mostly touches QAPI, so I'm probably going to include it in 
my pending qapi pull request in time for soft freeze.


--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 2/6] luks: Create block_crypto_co_create_generic()

[Eric Blake]

On 03/09/2018 11:27 AM, Kevin Wolf wrote:

Everything that refers to the protocol layer or QemuOpts is moved out of
block_crypto_create_generic(), so that the remaining function is
suitable to be called by a .bdrv_co_create implementation.

LUKS is the only driver that actually implements the old interface, and
we don't intend to use it in any new drivers, so put the moved out code
directly into a LUKS function rather than creating a generic
intermediate one.

Signed-off-by: Kevin Wolf 
---
  block/crypto.c | 95 +-
  1 file changed, 61 insertions(+), 34 deletions(-)



Reviewed-by: Eric Blake 

--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [Qemu-devel] [PATCH 3/5] migration/block: rename MAX_INFLIGHT_IO to MAX_IO_BUFFERS

[Peter Lieven]

Am 09.03.2018 um 15:58 schrieb Dr. David Alan Gilbert:
> * Peter Lieven (p...@kamp.de) wrote:
>> this actually limits (as the original commit mesage suggests) the
>> number of I/O buffers that can be allocated and not the number
>> of parallel (inflight) I/O requests.
>>
>> Signed-off-by: Peter Lieven 
> I've queued 1-3 (which have Juan's R-b).

Thank you. It would be good if we find a proper solution for the rest as this 
is also important.

Peter

Re: [Qemu-block] [PATCH v2 2/5] nbd/server: fix sparse read

[Eric Blake]

On 03/08/2018 12:46 PM, Vladimir Sementsov-Ogievskiy wrote:

In case of io error in nbd_co_send_sparse_read we should not
"goto reply:", as it is fatal error and common behavior is
disconnect in this case. We should not try to send client an


s/send/send the/


error reply, representing channel-io error on previous try to
send a reply.


s/representing .../since we already hit a channel-io error on our 
previous attempt to send a reply/




Fix this by handle block-status error in nbd_co_send_sparse_read,


s/handle/handling/


so nbd_co_send_sparse_read fails only on io error. Then just skip
common "reply:" code path in nbd_trip.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---

v2: code movement splitted to 01
 remove error_report
 fix indent
 fix commit message subject line
 add comment to nbd_co_send_sparse_read (be free to adjust it)


Not a problem, I know English is not your native tongue, so I don't mind 
touching it up, and I already admire your efforts at programming in a 
second language (a skill I lack).




  nbd/server.c | 17 ++---
  1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/nbd/server.c b/nbd/server.c
index c406b0656d..e0de431e10 100644
--- a/nbd/server.c
+++ b/nbd/server.c
@@ -1365,6 +1365,10 @@ static int coroutine_fn 
nbd_co_send_structured_error(NBDClient *client,
  return nbd_co_send_iov(client, iov, 1 + !!iov[1].iov_len, errp);
  }
  
+/* Do sparse read and send structured reply to the client.


s/Do/Do a/


+ * Returns -errno if sending fails. bdrv_block_status_above() fail is not


s/fail/failure/


+ * considered as an error, but reported to the client instead (as a structured
+ * error). */


The comment tail */ usually gets its own line.

Here's what I used:

/* Do a sparse read and send the structured reply to the client.
 * Returns -errno if sending fails. bdrv_block_status_above() failure is
 * reported to the client, at which point this function succeeds.
 */


  static int coroutine_fn nbd_co_send_sparse_read(NBDClient *client,
  uint64_t handle,
  uint64_t offset,
@@ -1385,8 +1389,13 @@ static int coroutine_fn 
nbd_co_send_sparse_read(NBDClient *client,
  bool final;
  
  if (status < 0) {

-error_setg_errno(errp, -status, "unable to check for holes");
-return status;
+char *msg = g_strdup_printf("unable to check for holes: %s",
+strerror(-status));


I had to double-check block/io.c, but it looks like 
bdrv_block-status_above() does indeed document a return of negative 
errno on failure.  (My suspicion is that the documentation might be 
wrong, because I did not audit error returns when doing my byte-based 
work; but a quick audit of a couple of random drivers shows that at 
least iscsi.c and qed.c appear to comply.)



+
+ret = nbd_co_send_structured_error(client, handle, -status, msg,
+   errp);
+g_free(msg);
+return ret;


I wonder if a separate patch should clean this up to let 
nbd_co_send_structured_error() do message formatting (making it varargs) 
- but doesn't affect this patch.


With grammar tweaks,
Reviewed-by: Eric Blake 

Re: [Qemu-block] [PATCH 1/6] luks: Separate image file creation from formatting

[Eric Blake]

On 03/09/2018 11:27 AM, Kevin Wolf wrote:

The crypto driver used to create the image file in a callback from the
crypto subsystem. If we want to implement .bdrv_co_create, this needs to
go away because that callback will get a reference to an already
existing block node.

Move the image file creation to block_crypto_create_generic().

Signed-off-by: Kevin Wolf 
---
  block/crypto.c | 37 +
  1 file changed, 17 insertions(+), 20 deletions(-)



Reviewed-by: Eric Blake 

--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 0/9] nbd block status base:allocation

[Vladimir Sementsov-Ogievskiy]

09.03.2018 22:08, Eric Blake wrote:

On 02/15/2018 07:51 AM, Vladimir Sementsov-Ogievskiy wrote:

Hi all.

Here is minimal realization of base:allocation context of NBD
block-status extension, which allows to get block status through
NBD.

Vladimir Sementsov-Ogievskiy (9):
   nbd/server: add nbd_opt_invalid helper
   nbd: change indenting in nbd.h
   nbd: BLOCK_STATUS for standard get_block_status function: server part
   block/nbd-client: save first fatal error in nbd_iter_error
   nbd/client: fix error messages in nbd_handle_reply_err
   nbd: BLOCK_STATUS for standard get_block_status function: client part
   iotests.py: tiny refactor: move system imports up
   iotests: add file_path helper
   iotests: new test 206 for NBD BLOCK_STATUS


I'd really like to send a PULL request for NBD on Monday, in order to 
make the 2.12 softfreeze deadline (this is a new feature, so if we 
miss Tuesday, we have to wait until 2.13 or whatever the next release 
is called).  Where do you stand on rebasing this, and what help can I 
offer? (I know you have factored out some of the patches in another 
thread that I'm in the middle of reviewing as well; you can submit the 
later patches even before the earlier ones land, and use a 'Based-on:' 
tag in the cover letter to make it obvious the dependencies between 
series).




I'm now at start of "Re: [PATCH 6/9] nbd: BLOCK_STATUS for standard 
get_block_status function: client part", and I think rebasing on 
byte-based is too much for me for that moment (10:20 pm =). I'll do my 
best on Monday morning as early as I can.


--
Best regards,
Vladimir

Re: [Qemu-block] [Qemu-devel] [PATCH 0/5] nbd server fixing and refactoring before BLOCK_STATUS

[Eric Blake]

On 03/09/2018 10:41 AM, Eric Blake wrote:

On 03/08/2018 12:46 PM, Vladimir Sementsov-Ogievskiy wrote:

01 and 02 are splitted and updated "[PATCH] nbd/server: fix space read",
others are new.

Vladimir Sementsov-Ogievskiy (5):
   nbd/server: move nbd_co_send_structured_error up
   nbd/server: fix sparse read
   nbd/server: fix: check client->closing before reply sending
   nbd/server: refactor nbd_trip: cmd_read and generic reply
   nbd/server: refactor nbd_trip: split out nbd_handle_request


I had a tough time applying this one:

Applying: nbd/server: move nbd_co_send_structured_error up
Applying: nbd/server: fix: check client->closing before reply sending
Applying: nbd/server: refactor nbd_trip: cmd_read and generic reply
error: sha1 information is lacking or useless (nbd/server.c).
error: could not build fake ancestor
Patch failed at 0004 nbd/server: refactor nbd_trip: cmd_read and generic 
reply


Aha - I see my problem - the patches were applied out of order because 
only patch 2 had a 'v2' in the subject line.  If I tell git to apply 
them one at a time, instead of trying to apply them on 'maildir/*' where 
the sorting botches the ordering, things work better.


I think I've resolved the conflicts correctly, but if I get through 
reviewing this series and posting it to my NBD queue, you may want to 
double-check things.


No need to repost this series, I'm doing a lot better now that I see my 
mistake.


--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 0/9] nbd block status base:allocation

[Eric Blake]

On 02/15/2018 07:51 AM, Vladimir Sementsov-Ogievskiy wrote:

Hi all.

Here is minimal realization of base:allocation context of NBD
block-status extension, which allows to get block status through
NBD.

Vladimir Sementsov-Ogievskiy (9):
   nbd/server: add nbd_opt_invalid helper
   nbd: change indenting in nbd.h
   nbd: BLOCK_STATUS for standard get_block_status function: server part
   block/nbd-client: save first fatal error in nbd_iter_error
   nbd/client: fix error messages in nbd_handle_reply_err
   nbd: BLOCK_STATUS for standard get_block_status function: client part
   iotests.py: tiny refactor: move system imports up
   iotests: add file_path helper
   iotests: new test 206 for NBD BLOCK_STATUS


I'd really like to send a PULL request for NBD on Monday, in order to 
make the 2.12 softfreeze deadline (this is a new feature, so if we miss 
Tuesday, we have to wait until 2.13 or whatever the next release is 
called).  Where do you stand on rebasing this, and what help can I 
offer? (I know you have factored out some of the patches in another 
thread that I'm in the middle of reviewing as well; you can submit the 
later patches even before the earlier ones land, and use a 'Based-on:' 
tag in the cover letter to make it obvious the dependencies between series).


--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH 7/9] iotests.py: tiny refactor: move system imports up

[Vladimir Sementsov-Ogievskiy]

16.02.2018 23:44, Eric Blake wrote:

On 02/15/2018 07:51 AM, Vladimir Sementsov-Ogievskiy wrote:

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
  tests/qemu-iotests/iotests.py | 5 +++--
  1 file changed, 3 insertions(+), 2 deletions(-)


What breaks if they aren't moved?  But stylistically, this looks 
reasonable; and can be merged independently of NBD stuff if someone 
else wants.


This movement also make it a bit nearer to PEP8, as it dislike "E402 
module level import not at top of file", because of "sys.path.append" 
before imports.. So, I'd just saved as many imports as I could)




Reviewed-by: Eric Blake 




--
Best regards,
Vladimir

Re: [Qemu-block] [PATCH 6/9] nbd: BLOCK_STATUS for standard get_block_status function: client part

[Vladimir Sementsov-Ogievskiy]

16.02.2018 23:40, Eric Blake wrote:

On 02/15/2018 07:51 AM, Vladimir Sementsov-Ogievskiy wrote:

Minimal realization: only one extent in server answer is supported.
Flag NBD_CMD_FLAG_REQ_ONE is used to force this behavior.


[...]


+    memcpy(extent, payload, sizeof(*extent));
+    be32_to_cpus(&extent->length);
+    be32_to_cpus(&extent->flags);


Instead of doing a memcpy() and then in-place bit-swizzling, you could 
do the swapping as part of assignment, for one less function call (and 
make the code a bit easier to extend, if we later drop our REQ_ONE 
limitation on only having one extent, because you'll advance payload 
as needed):


extent->length = payload_advance32(&payload);
extent->flags = payload_advance32(&payload);


Aha, yes. The funny thing is that these are my own helpers.



We should probably validate that the length field is a multiple of 
min_block (if a server tells us that all operations must be 512-byte 
aligned, then reports an extent that is smaller than 512 bytes, we 
have no way to ask for the status of the second half of the sector). 
Probably also something that needs to be explicitly stated in the NBD 
spec. [1]



+
+    if (extent->length > orig_length) {
+    error_setg(errp, "Protocol error: server sent chunk 
exceeding requested"

+ " region");
+    return -EINVAL;


That matches the current spec wording, but I'm not sure I agree with 
it - what's wrong with a server providing a final extent that extends 
beyond the request, if the information was already available for free 
(the classic example: if the server never replies with HOLE or ZERO, 
then the entire file has the same status, so all requests could 
trivially be replied to by taking the starting offset to the end of 
the file as the returned length, rather than just clamping at the 
requested length).


Maybe. But our already released clients not prepared to such change =(

But, on the other hand, this gives us possibility to understand, the the 
whole target (for backup/mirror) is zero in one request, skipping 
target-zeroing loop by 4gb chunks.. What about adding such possibility 
with an additionally negotiated option or something like this? (and 
don't we now have same possibility with something like INFO?)





+    }
+
+    return 0;
+}
+
  /* nbd_parse_error_payload
   * on success @errp contains message describing nbd error reply
   */


--
Best regards,
Vladimir

Re: [Qemu-block] [PULL 0/7] Block patches

[Peter Maydell]

On 9 March 2018 at 13:19, Stefan Hajnoczi  wrote:
> The following changes since commit 0ab4537f08e09b13788db67efd760592fb7db769:
>
>   Merge remote-tracking branch 
> 'remotes/stefanberger/tags/pull-tpm-2018-03-07-1' into staging (2018-03-08 
> 12:56:39 +)
>
> are available in the Git repository at:
>
>   git://github.com/stefanha/qemu.git tags/block-pull-request
>
> for you to fetch changes up to 4486e89c219c0d1b9bd8dfa0b1dd5b0d51ff2268:
>
>   vl: introduce vm_shutdown() (2018-03-08 17:38:51 +)
>
> 
>
> 

Applied, thanks.

-- PMM

[Qemu-block] [PATCH 6/6] qemu-iotests: Test luks QMP image creation

[Kevin Wolf]

Signed-off-by: Kevin Wolf 
---
 tests/qemu-iotests/208   | 211 +++
 tests/qemu-iotests/208.out   | 136 
 tests/qemu-iotests/common.rc |   2 +-
 tests/qemu-iotests/group |   1 +
 4 files changed, 349 insertions(+), 1 deletion(-)
 create mode 100755 tests/qemu-iotests/208
 create mode 100644 tests/qemu-iotests/208.out

diff --git a/tests/qemu-iotests/208 b/tests/qemu-iotests/208
new file mode 100755
index 00..ab4650d0ae
--- /dev/null
+++ b/tests/qemu-iotests/208
@@ -0,0 +1,211 @@
+#!/bin/bash
+#
+# Test luks and file image creation
+#
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+#
+
+# creator
+owner=kw...@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1   # failure is the default!
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt luks
+_supported_proto file
+_supported_os Linux
+
+function do_run_qemu()
+{
+echo Testing: "$@"
+$QEMU -nographic -qmp stdio -serial none "$@"
+echo
+}
+
+function run_qemu()
+{
+do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp \
+  | _filter_qemu | _filter_imgfmt \
+  | _filter_actual_image_size
+}
+
+echo
+echo "=== Successful image creation (defaults) ==="
+echo
+
+size=$((128 * 1024 * 1024))
+
+run_qemu -object secret,id=keysec0,data="foo" <&1 | \
+$QEMU_IMG info $QEMU_IMG_EXTRA_ARGS "$@" "$TEST_IMG" 2>&1 | \
 sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
 -e "s#$TEST_DIR#TEST_DIR#g" \
 -e "s#$IMGFMT#IMGFMT#g" \
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index c401791fcd..574227e761 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -204,3 +204,4 @@
 205 rw auto quick
 206 rw auto
 207 rw auto
+208 rw auto
-- 
2.13.6

[Qemu-block] [PATCH 0/6] luks: Implement .bdrv_co_create

[Kevin Wolf]

This series implements the .bdrv_co_create callback for luks, adds an
image creation test for it and contains some bonus fixes for bugs that
the test triggered.

Kevin Wolf (6):
  luks: Separate image file creation from formatting
  luks: Create block_crypto_co_create_generic()
  luks: Support .bdrv_co_create
  luks: Turn invalid assertion into check
  luks: Catch integer overflow for huge sizes
  qemu-iotests: Test luks QMP image creation

 qapi/block-core.json |  17 +++-
 block/crypto.c   | 150 +-
 tests/qemu-iotests/208   | 211 +++
 tests/qemu-iotests/208.out   | 136 
 tests/qemu-iotests/common.rc |   2 +-
 tests/qemu-iotests/group |   1 +
 6 files changed, 473 insertions(+), 44 deletions(-)
 create mode 100755 tests/qemu-iotests/208
 create mode 100644 tests/qemu-iotests/208.out

-- 
2.13.6

[Qemu-block] [PATCH 4/6] luks: Turn invalid assertion into check

[Kevin Wolf]

The .bdrv_getlength implementation of the crypto block driver asserted
that the payload offset isn't after EOF. This is an invalid assertion to
make as the image file could be corrupted. Instead, check it and return
-EIO if the file is too small for the payload offset.

Zero length images are fine, so trigger -EIO only on offset > len, not
on offset >= len as the assertion did before.

Signed-off-by: Kevin Wolf 
---
 block/crypto.c | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/block/crypto.c b/block/crypto.c
index 2035f9ab13..4908d8627f 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -518,7 +518,10 @@ static int64_t block_crypto_getlength(BlockDriverState *bs)
 
 uint64_t offset = qcrypto_block_get_payload_offset(crypto->block);
 assert(offset < INT64_MAX);
-assert(offset < len);
+
+if (offset > len) {
+   return -EIO;
+}
 
 len -= offset;
 
-- 
2.13.6

[Qemu-block] [PATCH 3/6] luks: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to luks, which enables
image creation over QMP.

Signed-off-by: Kevin Wolf 
---
 qapi/block-core.json | 17 -
 block/crypto.c   | 34 ++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 524d51567a..07039bfe9c 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3452,6 +3452,21 @@
 '*preallocation':   'PreallocMode' } }
 
 ##
+# @BlockdevCreateOptionsLUKS:
+#
+# Driver specific image creation options for LUKS.
+#
+# @file Node to create the image format on
+# @size Size of the virtual disk in bytes
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsLUKS',
+  'data': { 'file': 'BlockdevRef',
+'qcrypto':  'QCryptoBlockCreateOptionsLUKS',
+'size': 'size' } }
+
+##
 # @BlockdevCreateOptionsNfs:
 #
 # Driver specific image creation options for NFS.
@@ -3643,7 +3658,7 @@
   'http':   'BlockdevCreateNotSupported',
   'https':  'BlockdevCreateNotSupported',
   'iscsi':  'BlockdevCreateNotSupported',
-  'luks':   'BlockdevCreateNotSupported',
+  'luks':   'BlockdevCreateOptionsLUKS',
   'nbd':'BlockdevCreateNotSupported',
   'nfs':'BlockdevCreateOptionsNfs',
   'null-aio':   'BlockdevCreateNotSupported',
diff --git a/block/crypto.c b/block/crypto.c
index b0a4cb3388..2035f9ab13 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -543,6 +543,39 @@ static int block_crypto_open_luks(BlockDriverState *bs,
  bs, options, flags, errp);
 }
 
+static int coroutine_fn
+block_crypto_co_create_luks(BlockdevCreateOptions *create_options, Error 
**errp)
+{
+BlockdevCreateOptionsLUKS *luks_opts;
+BlockDriverState *bs = NULL;
+QCryptoBlockCreateOptions create_opts;
+int ret;
+
+assert(create_options->driver == BLOCKDEV_DRIVER_LUKS);
+luks_opts = &create_options->u.luks;
+
+bs = bdrv_open_blockdev_ref(luks_opts->file, errp);
+if (bs == NULL) {
+return -EIO;
+}
+
+create_opts = (QCryptoBlockCreateOptions) {
+.format = Q_CRYPTO_BLOCK_FORMAT_LUKS,
+.u.luks = *luks_opts->qcrypto,
+};
+
+ret = block_crypto_co_create_generic(bs, luks_opts->size, &create_opts,
+ errp);
+if (ret < 0) {
+goto fail;
+}
+
+ret = 0;
+fail:
+bdrv_unref(bs);
+return ret;
+}
+
 static int coroutine_fn block_crypto_co_create_opts_luks(const char *filename,
  QemuOpts *opts,
  Error **errp)
@@ -647,6 +680,7 @@ BlockDriver bdrv_crypto_luks = {
 .bdrv_open  = block_crypto_open_luks,
 .bdrv_close = block_crypto_close,
 .bdrv_child_perm= bdrv_format_default_perms,
+.bdrv_co_create = block_crypto_co_create_luks,
 .bdrv_co_create_opts = block_crypto_co_create_opts_luks,
 .bdrv_truncate  = block_crypto_truncate,
 .create_opts= &block_crypto_create_opts_luks,
-- 
2.13.6

[Qemu-block] [PATCH 2/6] luks: Create block_crypto_co_create_generic()

[Kevin Wolf]

Everything that refers to the protocol layer or QemuOpts is moved out of
block_crypto_create_generic(), so that the remaining function is
suitable to be called by a .bdrv_co_create implementation.

LUKS is the only driver that actually implements the old interface, and
we don't intend to use it in any new drivers, so put the moved out code
directly into a LUKS function rather than creating a generic
intermediate one.

Signed-off-by: Kevin Wolf 
---
 block/crypto.c | 95 +-
 1 file changed, 61 insertions(+), 34 deletions(-)

diff --git a/block/crypto.c b/block/crypto.c
index 77871640cc..b0a4cb3388 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -306,43 +306,29 @@ static int block_crypto_open_generic(QCryptoBlockFormat 
format,
 }
 
 
-static int block_crypto_create_generic(QCryptoBlockFormat format,
-   const char *filename,
-   QemuOpts *opts,
-   Error **errp)
+static int block_crypto_co_create_generic(BlockDriverState *bs,
+  int64_t size,
+  QCryptoBlockCreateOptions *opts,
+  Error **errp)
 {
-int ret = -EINVAL;
-QCryptoBlockCreateOptions *create_opts = NULL;
+int ret;
+BlockBackend *blk;
 QCryptoBlock *crypto = NULL;
-struct BlockCryptoCreateData data = {
-.size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
- BDRV_SECTOR_SIZE),
-};
-QDict *cryptoopts;
-
-/* Parse options */
-cryptoopts = qemu_opts_to_qdict(opts, NULL);
+struct BlockCryptoCreateData data;
 
-create_opts = block_crypto_create_opts_init(format, cryptoopts, errp);
-if (!create_opts) {
-return -1;
-}
+blk = blk_new(BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
 
-/* Create protocol layer */
-ret = bdrv_create_file(filename, opts, errp);
+ret = blk_insert_bs(blk, bs, errp);
 if (ret < 0) {
-return ret;
+goto cleanup;
 }
 
-data.blk = blk_new_open(filename, NULL, NULL,
-BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_PROTOCOL,
-errp);
-if (!data.blk) {
-return -EINVAL;
-}
+data = (struct BlockCryptoCreateData) {
+.blk = blk,
+.size = size,
+};
 
-/* Create format layer */
-crypto = qcrypto_block_create(create_opts, NULL,
+crypto = qcrypto_block_create(opts, NULL,
   block_crypto_init_func,
   block_crypto_write_func,
   &data,
@@ -355,10 +341,8 @@ static int block_crypto_create_generic(QCryptoBlockFormat 
format,
 
 ret = 0;
  cleanup:
-QDECREF(cryptoopts);
 qcrypto_block_free(crypto);
-blk_unref(data.blk);
-qapi_free_QCryptoBlockCreateOptions(create_opts);
+blk_unref(blk);
 return ret;
 }
 
@@ -563,8 +547,51 @@ static int coroutine_fn 
block_crypto_co_create_opts_luks(const char *filename,
  QemuOpts *opts,
  Error **errp)
 {
-return block_crypto_create_generic(Q_CRYPTO_BLOCK_FORMAT_LUKS,
-   filename, opts, errp);
+QCryptoBlockCreateOptions *create_opts = NULL;
+BlockDriverState *bs = NULL;
+QDict *cryptoopts;
+int64_t size;
+int ret;
+
+/* Parse options */
+size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+
+cryptoopts = qemu_opts_to_qdict_filtered(opts, NULL,
+ &block_crypto_create_opts_luks,
+ true);
+
+create_opts = block_crypto_create_opts_init(Q_CRYPTO_BLOCK_FORMAT_LUKS,
+cryptoopts, errp);
+if (!create_opts) {
+ret = -EINVAL;
+goto fail;
+}
+
+/* Create protocol layer */
+ret = bdrv_create_file(filename, opts, errp);
+if (ret < 0) {
+return ret;
+}
+
+bs = bdrv_open(filename, NULL, NULL,
+   BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_PROTOCOL, errp);
+if (!bs) {
+ret = -EINVAL;
+goto fail;
+}
+
+/* Create format layer */
+ret = block_crypto_co_create_generic(bs, size, create_opts, errp);
+if (ret < 0) {
+goto fail;
+}
+
+ret = 0;
+fail:
+bdrv_unref(bs);
+qapi_free_QCryptoBlockCreateOptions(create_opts);
+QDECREF(cryptoopts);
+return ret;
 }
 
 static int block_crypto_get_info_luks(BlockDriverState *bs,
-- 
2.13.6

[Qemu-block] [PATCH v6 2/2] qapi: add block latency histogram interface

[Vladimir Sementsov-Ogievskiy]

Set (and clear) histogram through new command
block-latency-histogram-set and show new statistics in
query-blockstats results.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 qapi/block-core.json | 111 ++-
 block/qapi.c |  41 +++
 blockdev.c   |  43 
 3 files changed, 194 insertions(+), 1 deletion(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 00475f08d4..efe8fe92ff 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -451,6 +451,106 @@
'status': 'DirtyBitmapStatus'} }
 
 ##
+# @XBlockLatencyHistogramInfo:
+#
+# Block latency histogram.
+#
+# @boundaries: list of interval boundary values in nanoseconds, all greater
+#  than zero and in ascending order.
+#  For example, the list [10, 50, 100] produces the following
+#  histogram intervals: [0, 10), [10, 50), [50, 100), [100, +inf).
+#
+# @bins: list of io request counts corresponding to histogram intervals.
+#len(@bins) = len(@boundaries) + 1
+#For the example above, @bins may be something like [3, 1, 5, 2],
+#and corresponding histogram looks like:
+#
+#5|   *
+#4|   *
+#3| * *
+#2| * **
+#1| ****
+# +--
+# 10   50   100
+#
+# Since: 2.12
+##
+{ 'struct': 'XBlockLatencyHistogramInfo',
+  'data': {'boundaries': ['uint64'], 'bins': ['uint64'] } }
+
+##
+# @x-block-latency-histogram-set:
+#
+# Manage read, write and flush latency histograms for the device.
+#
+# If only @device parameter is specified, remove all present latency histograms
+# for the device. Otherwise, add/reset some of (or all) latency histograms.
+#
+# @device: device name to set latency histogram for.
+#
+# @boundaries: list of interval boundary values (see description in
+#  XBlockLatencyHistogramInfo definition). If specified, all
+#  latency histograms are removed, and empty ones created for all
+#  io types with intervals corresponding to @boundaries (except for
+#  io types, for which specific boundaries are set through the
+#  following parameters).
+#
+# @boundaries-read: list of interval boundary values for read latency
+#   histogram. If specified, old read latency histogram is
+#   removed, and empty one created with interavals
+#   corresponding to @boundaries-read. The parameter has higher
+#   priority then @boundaries.
+#
+# @boundaries-write: list of interaval boundary values for write latency
+#histogram.
+#
+# @boundaries-flush: list of interaval boundary values for flush latency
+#histogram.
+#
+# Returns: error if device is not found or @boundaries* arrays are invalid.
+#
+# Since: 2.12
+#
+# Example: set new histograms for all io types with intervals
+# [0, 10), [10, 50), [50, 100), [100, +inf):
+#
+# -> { "execute": "block-latency-histogram-set",
+#  "arguments": { "device": "drive0",
+# "boundaries": [10, 50, 100] } }
+# <- { "return": {} }
+#
+# Example: set new histogram only for write, other histograms will remain
+# not changed (or not created):
+#
+# -> { "execute": "block-latency-histogram-set",
+#  "arguments": { "device": "drive0",
+# "boundaries-write": [10, 50, 100] } }
+# <- { "return": {} }
+#
+# Example: set new histograms with the following intervals:
+#   read, flush: [0, 10), [10, 50), [50, 100), [100, +inf)
+#   write: [0, 1000), [1000, 5000), [5000, +inf)
+#
+# -> { "execute": "block-latency-histogram-set",
+#  "arguments": { "device": "drive0",
+# "boundaries": [10, 50, 100],
+# "boundaries-write": [1000, 5000] } }
+# <- { "return": {} }
+#
+# Example: remove all latency histograms:
+#
+# -> { "execute": "block-latency-histogram-set",
+#  "arguments": { "device": "drive0" } }
+# <- { "return": {} }
+##
+{ 'command': 'x-block-latency-histogram-set',
+  'data': {'device': 'str',
+   '*boundaries': ['uint64'],
+   '*boundaries-read': ['uint64'],
+   '*boundaries-write': ['uint64'],
+   '*boundaries-flush': ['uint64'] } }
+
+##
 # @BlockInfo:
 #
 # Block device information.  This structure describes a virtual device and
@@ -730,6 +830,12 @@
 # @timed_stats: Statistics specific to the set of previously defined
 #   intervals of time (Since 2.5)
 #
+# @x_rd_latency_histogram: @BlockLatencyHistogramInfo. (Since 2.12)
+#
+# @x_wr_latency_histogram: @BlockLatencyHistogramInfo. (Since 2.12)
+#
+# @x_flush_latency_histogram: @BlockLatencyHistogramInfo. (Since 2.12)
+#
 # Since: 0.14.0
 ##
 { 'struct': 'BlockDeviceStats',
@@ -742,7 +848,10 @@
'failed_flush_operations': 'int', 'invalid_rd_operations': 'int',
'invalid_

[Qemu-block] [PATCH v6 0/2] block latency histogram

[Vladimir Sementsov-Ogievskiy]

v6:

Use correct header qapi/qapi-builtin-types.h, to fix build again.
Sorry for spam =(


v5:

Revert to v3 and just add qapi-types.h header.


v4:

Move block_latency_histogram_set from block/accounting.c to
blockdev.c, as it uses qapi type uint64List and this fact breaks
build.


v3:

- semantics, naming and wording changed a lot
- x prefixes added to new qapi names
- bug fixed about calculation of new_size (new_nbins now)
- drop g_renew
- in _clear() set nbinst to zero too

v2:

01: add block_latency_histogram_clear()
02: fix spelling (sorry =()
some rewordings
remove histogram if latency parameter unspecified

Vladimir Sementsov-Ogievskiy (2):
  block/accounting: introduce latency histogram
  qapi: add block latency histogram interface

 qapi/block-core.json   | 111 -
 include/block/accounting.h |  35 ++
 block/accounting.c |  91 +
 block/qapi.c   |  41 +
 blockdev.c |  43 ++
 5 files changed, 320 insertions(+), 1 deletion(-)

-- 
2.11.1

[Qemu-block] [PATCH 1/6] luks: Separate image file creation from formatting

[Kevin Wolf]

The crypto driver used to create the image file in a callback from the
crypto subsystem. If we want to implement .bdrv_co_create, this needs to
go away because that callback will get a reference to an already
existing block node.

Move the image file creation to block_crypto_create_generic().

Signed-off-by: Kevin Wolf 
---
 block/crypto.c | 37 +
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/block/crypto.c b/block/crypto.c
index e6095e7807..77871640cc 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -71,8 +71,6 @@ static ssize_t block_crypto_read_func(QCryptoBlock *block,
 
 
 struct BlockCryptoCreateData {
-const char *filename;
-QemuOpts *opts;
 BlockBackend *blk;
 uint64_t size;
 };
@@ -103,27 +101,13 @@ static ssize_t block_crypto_init_func(QCryptoBlock *block,
   Error **errp)
 {
 struct BlockCryptoCreateData *data = opaque;
-int ret;
 
 /* User provided size should reflect amount of space made
  * available to the guest, so we must take account of that
  * which will be used by the crypto header
  */
-data->size += headerlen;
-
-qemu_opt_set_number(data->opts, BLOCK_OPT_SIZE, data->size, &error_abort);
-ret = bdrv_create_file(data->filename, data->opts, errp);
-if (ret < 0) {
-return -1;
-}
-
-data->blk = blk_new_open(data->filename, NULL, NULL,
- BDRV_O_RDWR | BDRV_O_PROTOCOL, errp);
-if (!data->blk) {
-return -1;
-}
-
-return 0;
+return blk_truncate(data->blk, data->size + headerlen, PREALLOC_MODE_OFF,
+errp);
 }
 
 
@@ -333,11 +317,10 @@ static int block_crypto_create_generic(QCryptoBlockFormat 
format,
 struct BlockCryptoCreateData data = {
 .size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
  BDRV_SECTOR_SIZE),
-.opts = opts,
-.filename = filename,
 };
 QDict *cryptoopts;
 
+/* Parse options */
 cryptoopts = qemu_opts_to_qdict(opts, NULL);
 
 create_opts = block_crypto_create_opts_init(format, cryptoopts, errp);
@@ -345,6 +328,20 @@ static int block_crypto_create_generic(QCryptoBlockFormat 
format,
 return -1;
 }
 
+/* Create protocol layer */
+ret = bdrv_create_file(filename, opts, errp);
+if (ret < 0) {
+return ret;
+}
+
+data.blk = blk_new_open(filename, NULL, NULL,
+BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_PROTOCOL,
+errp);
+if (!data.blk) {
+return -EINVAL;
+}
+
+/* Create format layer */
 crypto = qcrypto_block_create(create_opts, NULL,
   block_crypto_init_func,
   block_crypto_write_func,
-- 
2.13.6

[Qemu-block] [PATCH 5/6] luks: Catch integer overflow for huge sizes

[Kevin Wolf]

When you request an image size close to UINT64_MAX, the addition of the
crypto header may cause an integer overflow. Catch it instead of
silently truncating the image size.

Signed-off-by: Kevin Wolf 
---
 block/crypto.c | 5 +
 1 file changed, 5 insertions(+)

diff --git a/block/crypto.c b/block/crypto.c
index 4908d8627f..1b46519c53 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -102,6 +102,11 @@ static ssize_t block_crypto_init_func(QCryptoBlock *block,
 {
 struct BlockCryptoCreateData *data = opaque;
 
+if (headerlen > UINT64_MAX - data->size) {
+error_setg(errp, "The requested file size is too large");
+return -EFBIG;
+}
+
 /* User provided size should reflect amount of space made
  * available to the guest, so we must take account of that
  * which will be used by the crypto header
-- 
2.13.6

Re: [Qemu-block] [PATCH 0/5] nbd server fixing and refactoring before BLOCK_STATUS

[Vladimir Sementsov-Ogievskiy]

09.03.2018 19:41, Eric Blake wrote:

On 03/08/2018 12:46 PM, Vladimir Sementsov-Ogievskiy wrote:

01 and 02 are splitted and updated "[PATCH] nbd/server: fix space read",
others are new.

Vladimir Sementsov-Ogievskiy (5):
   nbd/server: move nbd_co_send_structured_error up
   nbd/server: fix sparse read
   nbd/server: fix: check client->closing before reply sending
   nbd/server: refactor nbd_trip: cmd_read and generic reply
   nbd/server: refactor nbd_trip: split out nbd_handle_request


I had a tough time applying this one:

Applying: nbd/server: move nbd_co_send_structured_error up
Applying: nbd/server: fix: check client->closing before reply sending
Applying: nbd/server: refactor nbd_trip: cmd_read and generic reply
error: sha1 information is lacking or useless (nbd/server.c).
error: could not build fake ancestor
Patch failed at 0004 nbd/server: refactor nbd_trip: cmd_read and 
generic reply

The copy of the patch that failed is found in: .git/rebase-apply/patch
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".

I think I've resolved the conflicts correctly, but if I get through 
reviewing this series and posting it to my NBD queue, you may want to 
double-check things.




Oh, sorry for this. I should have to rebase it(

--
Best regards,
Vladimir

Re: [Qemu-block] [PATCH v4 0/2] block latency histogram

[Vladimir Sementsov-Ogievskiy]

09.03.2018 19:37, Eric Blake wrote:

On 03/09/2018 10:31 AM, Vladimir Sementsov-Ogievskiy wrote:

09.03.2018 18:40, Eric Blake wrote:

On 03/09/2018 09:33 AM, Vladimir Sementsov-Ogievskiy wrote:

v4:

Move block_latency_histogram_set from block/accounting.c to
blockdev.c, as it uses qapi type uint64List and this fact breaks
build.


Was the cross file motion necessary, or could you just fix the 
#includes according to the recent qapi header refactoring?




Looks like it's not as simple..

/tmp/qemu-test/src/include/block/accounting.h:30:10: fatal error: 
qapi-types.h: No such file or directory

  #include "qapi-types.h


what to do now? Looks like qapi-types are not generated for some 
builds. So, this version (v4) should take place.


top-level qapi-types.h was replaced by lots of qapi/qapi*.h modular 
headers.  Unfortunately, incremental builds don't delete the stale 
qapi-types.h that was left in tree from the point prior to the qapi 
header refactoring.


To get uint64List, use qapi/qapi-builtin-types.h.  To get any type 
defined in a qapi/MODULE.json file, use qapi/qapi-MODULE-types.h.




thanks, now it works, after git clean..

--
Best regards,
Vladimir

Re: [Qemu-block] [PATCH 1/5] nbd/server: move nbd_co_send_structured_error up

[Eric Blake]

On 03/08/2018 12:46 PM, Vladimir Sementsov-Ogievskiy wrote:

To be reused in nbd_co_send_sparse_read() in the following patch.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
  nbd/server.c | 48 
  1 file changed, 24 insertions(+), 24 deletions(-)


Reviewed-by: Eric Blake 

--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

Re: [Qemu-block] [PATCH v4 0/2] block latency histogram

[Eric Blake]

On 03/09/2018 10:31 AM, Vladimir Sementsov-Ogievskiy wrote:

09.03.2018 18:40, Eric Blake wrote:

On 03/09/2018 09:33 AM, Vladimir Sementsov-Ogievskiy wrote:

v4:

Move block_latency_histogram_set from block/accounting.c to
blockdev.c, as it uses qapi type uint64List and this fact breaks
build.


Was the cross file motion necessary, or could you just fix the 
#includes according to the recent qapi header refactoring?




Looks like it's not as simple..

/tmp/qemu-test/src/include/block/accounting.h:30:10: fatal error: 
qapi-types.h: No such file or directory

  #include "qapi-types.h


what to do now? Looks like qapi-types are not generated for some builds. 
So, this version (v4) should take place.


top-level qapi-types.h was replaced by lots of qapi/qapi*.h modular 
headers.  Unfortunately, incremental builds don't delete the stale 
qapi-types.h that was left in tree from the point prior to the qapi 
header refactoring.


To get uint64List, use qapi/qapi-builtin-types.h.  To get any type 
defined in a qapi/MODULE.json file, use qapi/qapi-MODULE-types.h.


--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

[Qemu-block] [PULL 54/56] iotests: Skip test for ENOMEM error

[Kevin Wolf]

From: Fam Zheng 

The AFL image is to exercise the code validating image size, which
doesn't work on 32 bit or when out of memory (there is a large
allocation before the interesting point). So check that and skip the
test, instead of faking the result.

Signed-off-by: Fam Zheng 
Message-id: 20180301011413.11531-1-f...@redhat.com
Reviewed-by: Eric Blake 
Signed-off-by: Max Reitz 
---
 tests/qemu-iotests/059 | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059
index 40f89eae18..530bbbe6ce 100755
--- a/tests/qemu-iotests/059
+++ b/tests/qemu-iotests/059
@@ -152,9 +152,8 @@ done
 echo
 echo "=== Testing afl image with a very large capacity ==="
 _use_sample_img afl9.vmdk.bz2
-# The sed makes this test pass on machines with little RAM
-# (and also with 32 bit builds)
-_img_info | sed -e 's/Cannot allocate memory/Invalid argument/'
+_img_info | grep -q 'Cannot allocate memory' && _notrun "Insufficent memory, 
skipped test"
+_img_info
 _cleanup_test_img
 
 # success, all done
-- 
2.13.6

[Qemu-block] [PULL 51/56] qemu-iotests: Test ssh image creation over QMP

[Kevin Wolf]

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 tests/qemu-iotests/207 | 261 +
 tests/qemu-iotests/207.out |  75 +
 tests/qemu-iotests/group   |   1 +
 3 files changed, 337 insertions(+)
 create mode 100755 tests/qemu-iotests/207
 create mode 100644 tests/qemu-iotests/207.out

diff --git a/tests/qemu-iotests/207 b/tests/qemu-iotests/207
new file mode 100755
index 00..f5c77852d1
--- /dev/null
+++ b/tests/qemu-iotests/207
@@ -0,0 +1,261 @@
+#!/bin/bash
+#
+# Test ssh image creation
+#
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+#
+
+# creator
+owner=kw...@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1   # failure is the default!
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt raw
+_supported_proto ssh
+_supported_os Linux
+
+function do_run_qemu()
+{
+echo Testing: "$@"
+$QEMU -nographic -qmp stdio -serial none "$@"
+echo
+}
+
+function run_qemu()
+{
+do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp \
+  | _filter_qemu | _filter_imgfmt \
+  | _filter_actual_image_size
+}
+
+echo
+echo "=== Successful image creation (defaults) ==="
+echo
+
+run_qemu 

[Qemu-block] [PATCH v6 1/2] block/accounting: introduce latency histogram

[Vladimir Sementsov-Ogievskiy]

Introduce latency histogram statics for block devices.
For each accounted operation type latency region [0, +inf) is
divided into subregions by several points. Then, calculate
hits for each subregion.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 include/block/accounting.h | 35 ++
 block/accounting.c | 91 ++
 2 files changed, 126 insertions(+)

diff --git a/include/block/accounting.h b/include/block/accounting.h
index b833d26d6c..d1f67b10dd 100644
--- a/include/block/accounting.h
+++ b/include/block/accounting.h
@@ -27,6 +27,7 @@
 
 #include "qemu/timed-average.h"
 #include "qemu/thread.h"
+#include "qapi/qapi-builtin-types.h"
 
 typedef struct BlockAcctTimedStats BlockAcctTimedStats;
 typedef struct BlockAcctStats BlockAcctStats;
@@ -45,6 +46,36 @@ struct BlockAcctTimedStats {
 QSLIST_ENTRY(BlockAcctTimedStats) entries;
 };
 
+typedef struct BlockLatencyHistogram {
+/* The following histogram is represented like this:
+ *
+ * 5|   *
+ * 4|   *
+ * 3| * *
+ * 2| * **
+ * 1| ****
+ *  +--
+ *  10   50   100
+ *
+ * BlockLatencyHistogram histogram = {
+ * .nbins = 4,
+ * .boundaries = {10, 50, 100},
+ * .bins = {3, 1, 5, 2},
+ * };
+ *
+ * @boundaries array define histogram intervals as follows:
+ * [0, boundaries[0]), [boundaries[0], boundaries[1]), ...
+ * [boundaries[nbins-2], +inf)
+ *
+ * So, for example above, histogram intervals are:
+ * [0, 10), [10, 50), [50, 100), [100, +inf)
+ */
+int nbins;
+uint64_t *boundaries; /* @nbins-1 numbers here
+ (all boundaries, except 0 and +inf) */
+uint64_t *bins;
+} BlockLatencyHistogram;
+
 struct BlockAcctStats {
 QemuMutex lock;
 uint64_t nr_bytes[BLOCK_MAX_IOTYPE];
@@ -57,6 +88,7 @@ struct BlockAcctStats {
 QSLIST_HEAD(, BlockAcctTimedStats) intervals;
 bool account_invalid;
 bool account_failed;
+BlockLatencyHistogram latency_histogram[BLOCK_MAX_IOTYPE];
 };
 
 typedef struct BlockAcctCookie {
@@ -82,5 +114,8 @@ void block_acct_merge_done(BlockAcctStats *stats, enum 
BlockAcctType type,
 int64_t block_acct_idle_time_ns(BlockAcctStats *stats);
 double block_acct_queue_depth(BlockAcctTimedStats *stats,
   enum BlockAcctType type);
+int block_latency_histogram_set(BlockAcctStats *stats, enum BlockAcctType type,
+uint64List *boundaries);
+void block_latency_histograms_clear(BlockAcctStats *stats);
 
 #endif
diff --git a/block/accounting.c b/block/accounting.c
index 87ef5bbfaa..70a3d9a426 100644
--- a/block/accounting.c
+++ b/block/accounting.c
@@ -94,6 +94,94 @@ void block_acct_start(BlockAcctStats *stats, BlockAcctCookie 
*cookie,
 cookie->type = type;
 }
 
+/* block_latency_histogram_compare_func:
+ * Compare @key with interval [@it[0], @it[1]).
+ * Return: -1 if @key < @it[0]
+ *  0 if @key in [@it[0], @it[1])
+ * +1 if @key >= @it[1]
+ */
+static int block_latency_histogram_compare_func(const void *key, const void 
*it)
+{
+uint64_t k = *(uint64_t *)key;
+uint64_t a = ((uint64_t *)it)[0];
+uint64_t b = ((uint64_t *)it)[1];
+
+return k < a ? -1 : (k < b ? 0 : 1);
+}
+
+static void block_latency_histogram_account(BlockLatencyHistogram *hist,
+int64_t latency_ns)
+{
+uint64_t *pos;
+
+if (hist->bins == NULL) {
+/* histogram disabled */
+return;
+}
+
+
+if (latency_ns < hist->boundaries[0]) {
+hist->bins[0]++;
+return;
+}
+
+if (latency_ns >= hist->boundaries[hist->nbins - 2]) {
+hist->bins[hist->nbins - 1]++;
+return;
+}
+
+pos = bsearch(&latency_ns, hist->boundaries, hist->nbins - 2,
+  sizeof(hist->boundaries[0]),
+  block_latency_histogram_compare_func);
+assert(pos != NULL);
+
+hist->bins[pos - hist->boundaries + 1]++;
+}
+
+int block_latency_histogram_set(BlockAcctStats *stats, enum BlockAcctType type,
+uint64List *boundaries)
+{
+BlockLatencyHistogram *hist = &stats->latency_histogram[type];
+uint64List *entry;
+uint64_t *ptr;
+uint64_t prev = 0;
+int new_nbins = 1;
+
+for (entry = boundaries; entry; entry = entry->next) {
+if (entry->value <= prev) {
+return -EINVAL;
+}
+new_nbins++;
+prev = entry->value;
+}
+
+hist->nbins = new_nbins;
+g_free(hist->boundaries);
+hist->boundaries = g_new(uint64_t, hist->nbins - 1);
+for (entry = boundaries, ptr = hist->boundaries; entry;
+ entry = entry->next, ptr++)
+{
+*ptr = entry->value;
+}
+
+g_free(hist->bins);
+hist->bins = g_new0(uint64_t, hist->nbins);
+
+return 0;
+}
+
+void block_latency_histogram

[Qemu-block] [PULL 47/56] ssh: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to ssh, which enables
image creation over QMP.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 qapi/block-core.json | 16 +-
 block/ssh.c  | 83 ++--
 2 files changed, 63 insertions(+), 36 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 4814bb7db7..524d51567a 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3593,6 +3593,20 @@
 '*object-size': 'size' } }
 
 ##
+# @BlockdevCreateOptionsSsh:
+#
+# Driver specific image creation options for SSH.
+#
+# @location Where to store the new image file
+# @size Size of the virtual disk in bytes
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsSsh',
+  'data': { 'location': 'BlockdevOptionsSsh',
+'size': 'size' } }
+
+##
 # @BlockdevCreateNotSupported:
 #
 # This is used for all drivers that don't support creating images.
@@ -3644,7 +3658,7 @@
   'rbd':'BlockdevCreateOptionsRbd',
   'replication':'BlockdevCreateNotSupported',
   'sheepdog':   'BlockdevCreateOptionsSheepdog',
-  'ssh':'BlockdevCreateNotSupported',
+  'ssh':'BlockdevCreateOptionsSsh',
   'throttle':   'BlockdevCreateNotSupported',
   'vdi':'BlockdevCreateNotSupported',
   'vhdx':   'BlockdevCreateNotSupported',
diff --git a/block/ssh.c b/block/ssh.c
index 80f59055cc..ab3acf0c22 100644
--- a/block/ssh.c
+++ b/block/ssh.c
@@ -854,59 +854,71 @@ static QemuOptsList ssh_create_opts = {
 }
 };
 
+static int ssh_co_create(BlockdevCreateOptions *options, Error **errp)
+{
+BlockdevCreateOptionsSsh *opts = &options->u.ssh;
+BDRVSSHState s;
+int ret;
+
+assert(options->driver == BLOCKDEV_DRIVER_SSH);
+
+ssh_state_init(&s);
+
+ret = connect_to_ssh(&s, opts->location,
+ LIBSSH2_FXF_READ|LIBSSH2_FXF_WRITE|
+ LIBSSH2_FXF_CREAT|LIBSSH2_FXF_TRUNC,
+ 0644, errp);
+if (ret < 0) {
+goto fail;
+}
+
+if (opts->size > 0) {
+ret = ssh_grow_file(&s, opts->size, errp);
+if (ret < 0) {
+goto fail;
+}
+}
+
+ret = 0;
+fail:
+ssh_state_free(&s);
+return ret;
+}
+
 static int coroutine_fn ssh_co_create_opts(const char *filename, QemuOpts 
*opts,
Error **errp)
 {
-int r, ret;
-int64_t total_size = 0;
+BlockdevCreateOptions *create_options;
+BlockdevCreateOptionsSsh *ssh_opts;
+int ret;
 QDict *uri_options = NULL;
-BlockdevOptionsSsh *ssh_opts = NULL;
-BDRVSSHState s;
 
-ssh_state_init(&s);
+create_options = g_new0(BlockdevCreateOptions, 1);
+create_options->driver = BLOCKDEV_DRIVER_SSH;
+ssh_opts = &create_options->u.ssh;
 
 /* Get desired file size. */
-total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
-  BDRV_SECTOR_SIZE);
-DPRINTF("total_size=%" PRIi64, total_size);
+ssh_opts->size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+  BDRV_SECTOR_SIZE);
+DPRINTF("total_size=%" PRIi64, ssh_opts->size);
 
 uri_options = qdict_new();
-r = parse_uri(filename, uri_options, errp);
-if (r < 0) {
-ret = r;
+ret = parse_uri(filename, uri_options, errp);
+if (ret < 0) {
 goto out;
 }
 
-ssh_opts = ssh_parse_options(uri_options, errp);
-if (ssh_opts == NULL) {
+ssh_opts->location = ssh_parse_options(uri_options, errp);
+if (ssh_opts->location == NULL) {
 ret = -EINVAL;
 goto out;
 }
 
-r = connect_to_ssh(&s, ssh_opts,
-   LIBSSH2_FXF_READ|LIBSSH2_FXF_WRITE|
-   LIBSSH2_FXF_CREAT|LIBSSH2_FXF_TRUNC,
-   0644, errp);
-if (r < 0) {
-ret = r;
-goto out;
-}
-
-if (total_size > 0) {
-ret = ssh_grow_file(&s, total_size, errp);
-if (ret < 0) {
-goto out;
-}
-}
-
-ret = 0;
+ret = ssh_co_create(create_options, errp);
 
  out:
-ssh_state_free(&s);
-if (uri_options != NULL) {
-QDECREF(uri_options);
-}
-qapi_free_BlockdevOptionsSsh(ssh_opts);
+QDECREF(uri_options);
+qapi_free_BlockdevCreateOptions(create_options);
 return ret;
 }
 
@@ -1268,6 +1280,7 @@ static BlockDriver bdrv_ssh = {
 .instance_size= sizeof(BDRVSSHState),
 .bdrv_parse_filename  = ssh_parse_filename,
 .bdrv_file_open   = ssh_file_open,
+.bdrv_co_create   = ssh_co_create,
 .bdrv_co_create_opts  = ssh_co_create_opts,
 .bdrv_close   = ssh_close,
 .bdrv_has_zero_init   = ssh_has_zero_init,
-- 
2.13.6

[Qemu-block] [PULL 43/56] sheepdog: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to sheepdog, which enables
image creation over QMP.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 qapi/block-core.json |  24 -
 block/sheepdog.c | 243 +++
 2 files changed, 192 insertions(+), 75 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index e590ab6c71..fd21fc 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3512,6 +3512,28 @@
 'erasure-coded': 'SheepdogRedundancyErasureCoded' } }
 
 ##
+# @BlockdevCreateOptionsSheepdog:
+#
+# Driver specific image creation options for Sheepdog.
+#
+# @location Where to store the new image file
+# @size Size of the virtual disk in bytes
+# @backing-file File name of a base image
+# @preallocationPreallocation mode (allowed values: off, full)
+# @redundancy   Redundancy of the image
+# @object-size  Object size of the image
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsSheepdog',
+  'data': { 'location': 'BlockdevOptionsSheepdog',
+'size': 'size',
+'*backing-file':'str',
+'*preallocation':   'PreallocMode',
+'*redundancy':  'SheepdogRedundancy',
+'*object-size': 'size' } }
+
+##
 # @BlockdevCreateNotSupported:
 #
 # This is used for all drivers that don't support creating images.
@@ -3562,7 +3584,7 @@
   'raw':'BlockdevCreateNotSupported',
   'rbd':'BlockdevCreateOptionsRbd',
   'replication':'BlockdevCreateNotSupported',
-  'sheepdog':   'BlockdevCreateNotSupported',
+  'sheepdog':   'BlockdevCreateOptionsSheepdog',
   'ssh':'BlockdevCreateNotSupported',
   'throttle':   'BlockdevCreateNotSupported',
   'vdi':'BlockdevCreateNotSupported',
diff --git a/block/sheepdog.c b/block/sheepdog.c
index 3966cd229a..8680b2926f 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -15,8 +15,10 @@
 #include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "qapi/qapi-visit-sockets.h"
+#include "qapi/qapi-visit-block-core.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qobject-input-visitor.h"
+#include "qapi/qobject-output-visitor.h"
 #include "qemu/uri.h"
 #include "qemu/error-report.h"
 #include "qemu/option.h"
@@ -533,23 +535,6 @@ static void sd_aio_setup(SheepdogAIOCB *acb, 
BDRVSheepdogState *s,
 qemu_co_mutex_unlock(&s->queue_lock);
 }
 
-static SocketAddress *sd_socket_address(const char *path,
-const char *host, const char *port)
-{
-SocketAddress *addr = g_new0(SocketAddress, 1);
-
-if (path) {
-addr->type = SOCKET_ADDRESS_TYPE_UNIX;
-addr->u.q_unix.path = g_strdup(path);
-} else {
-addr->type = SOCKET_ADDRESS_TYPE_INET;
-addr->u.inet.host = g_strdup(host ?: SD_DEFAULT_ADDR);
-addr->u.inet.port = g_strdup(port ?: stringify(SD_DEFAULT_PORT));
-}
-
-return addr;
-}
-
 static SocketAddress *sd_server_config(QDict *options, Error **errp)
 {
 QDict *server = NULL;
@@ -1882,6 +1867,44 @@ out_with_err_set:
 return ret;
 }
 
+static int sd_create_prealloc(BlockdevOptionsSheepdog *location, int64_t size,
+  Error **errp)
+{
+BlockDriverState *bs;
+Visitor *v;
+QObject *obj = NULL;
+QDict *qdict;
+Error *local_err = NULL;
+int ret;
+
+v = qobject_output_visitor_new(&obj);
+visit_type_BlockdevOptionsSheepdog(v, NULL, &location, &local_err);
+visit_free(v);
+
+if (local_err) {
+error_propagate(errp, local_err);
+qobject_decref(obj);
+return -EINVAL;
+}
+
+qdict = qobject_to_qdict(obj);
+qdict_flatten(qdict);
+
+qdict_put_str(qdict, "driver", "sheepdog");
+
+bs = bdrv_open(NULL, NULL, qdict, BDRV_O_PROTOCOL | BDRV_O_RDWR, errp);
+if (bs == NULL) {
+ret = -EIO;
+goto fail;
+}
+
+ret = sd_prealloc(bs, 0, size, errp);
+fail:
+bdrv_unref(bs);
+QDECREF(qdict);
+return ret;
+}
+
 static int parse_redundancy(BDRVSheepdogState *s, SheepdogRedundancy *opt)
 {
 struct SheepdogInode *inode = &s->inode;
@@ -1934,9 +1957,9 @@ static int parse_redundancy(BDRVSheepdogState *s, 
SheepdogRedundancy *opt)
  * # create a erasure coded vdi with x data strips and y parity strips
  * -o redundancy=x:y (x must be one of {2,4,8,16} and 1 <= y < SD_EC_MAX_STRIP)
  */
-static int parse_redundancy_str(BDRVSheepdogState *s, const char *opt)
+static SheepdogRedundancy *parse_redundancy_str(const char *opt)
 {
-struct SheepdogRedundancy redundancy;
+SheepdogRedundancy *redundancy;
 const char *n1, *n2;
 long copy, parity;
 char p[10];
@@ -1947,26 +1970,27 @@ static int parse_redundancy_str(BDRVSheepdogState *s, 
const char *opt)
 n2 = strtok(NULL, ":");
 
 if (!n1) {
-return -EINVAL;
+return NULL;
 }
 
   

[Qemu-block] [PULL 40/56] nfs: Use QAPI options in nfs_client_open()

[Kevin Wolf]

Using the QAPI visitor to turn all options into QAPI BlockdevOptionsNfs
simplifies the code a lot. It will also be useful for implementing the
QAPI based .bdrv_co_create callback.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 block/nfs.c | 176 ++--
 1 file changed, 53 insertions(+), 123 deletions(-)

diff --git a/block/nfs.c b/block/nfs.c
index 7433d25856..e402d643fe 100644
--- a/block/nfs.c
+++ b/block/nfs.c
@@ -367,49 +367,6 @@ static int coroutine_fn nfs_co_flush(BlockDriverState *bs)
 return task.ret;
 }
 
-static QemuOptsList runtime_opts = {
-.name = "nfs",
-.head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
-.desc = {
-{
-.name = "path",
-.type = QEMU_OPT_STRING,
-.help = "Path of the image on the host",
-},
-{
-.name = "user",
-.type = QEMU_OPT_NUMBER,
-.help = "UID value to use when talking to the server",
-},
-{
-.name = "group",
-.type = QEMU_OPT_NUMBER,
-.help = "GID value to use when talking to the server",
-},
-{
-.name = "tcp-syn-count",
-.type = QEMU_OPT_NUMBER,
-.help = "Number of SYNs to send during the session establish",
-},
-{
-.name = "readahead-size",
-.type = QEMU_OPT_NUMBER,
-.help = "Set the readahead size in bytes",
-},
-{
-.name = "page-cache-size",
-.type = QEMU_OPT_NUMBER,
-.help = "Set the pagecache size in bytes",
-},
-{
-.name = "debug",
-.type = QEMU_OPT_NUMBER,
-.help = "Set the NFS debug level (max 2)",
-},
-{ /* end of list */ }
-},
-};
-
 static void nfs_detach_aio_context(BlockDriverState *bs)
 {
 NFSClient *client = bs->opaque;
@@ -452,71 +409,16 @@ static void nfs_file_close(BlockDriverState *bs)
 nfs_client_close(client);
 }
 
-static NFSServer *nfs_config(QDict *options, Error **errp)
-{
-NFSServer *server = NULL;
-QDict *addr = NULL;
-QObject *crumpled_addr = NULL;
-Visitor *iv = NULL;
-Error *local_error = NULL;
-
-qdict_extract_subqdict(options, &addr, "server.");
-if (!qdict_size(addr)) {
-error_setg(errp, "NFS server address missing");
-goto out;
-}
-
-crumpled_addr = qdict_crumple(addr, errp);
-if (!crumpled_addr) {
-goto out;
-}
-
-/*
- * Caution: this works only because all scalar members of
- * NFSServer are QString in @crumpled_addr.  The visitor expects
- * @crumpled_addr to be typed according to the QAPI schema.  It
- * is when @options come from -blockdev or blockdev_add.  But when
- * they come from -drive, they're all QString.
- */
-iv = qobject_input_visitor_new(crumpled_addr);
-visit_type_NFSServer(iv, NULL, &server, &local_error);
-if (local_error) {
-error_propagate(errp, local_error);
-goto out;
-}
-
-out:
-QDECREF(addr);
-qobject_decref(crumpled_addr);
-visit_free(iv);
-return server;
-}
-
-
-static int64_t nfs_client_open(NFSClient *client, QDict *options,
+static int64_t nfs_client_open(NFSClient *client, BlockdevOptionsNfs *opts,
int flags, int open_flags, Error **errp)
 {
 int64_t ret = -EINVAL;
-QemuOpts *opts = NULL;
-Error *local_err = NULL;
 struct stat st;
 char *file = NULL, *strp = NULL;
 
 qemu_mutex_init(&client->mutex);
-opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
-qemu_opts_absorb_qdict(opts, options, &local_err);
-if (local_err) {
-error_propagate(errp, local_err);
-ret = -EINVAL;
-goto fail;
-}
 
-client->path = g_strdup(qemu_opt_get(opts, "path"));
-if (!client->path) {
-ret = -EINVAL;
-error_setg(errp, "No path was specified");
-goto fail;
-}
+client->path = g_strdup(opts->path);
 
 strp = strrchr(client->path, '/');
 if (strp == NULL) {
@@ -526,12 +428,10 @@ static int64_t nfs_client_open(NFSClient *client, QDict 
*options,
 file = g_strdup(strp);
 *strp = 0;
 
-/* Pop the config into our state object, Exit if invalid */
-client->server = nfs_config(options, errp);
-if (!client->server) {
-ret = -EINVAL;
-goto fail;
-}
+/* Steal the NFSServer object from opts; set the original pointer to NULL
+ * to avoid use after free and double free. */
+client->server = opts->server;
+opts->server = NULL;
 
 client->context = nfs_init_context();
 if (client->context == NULL) {
@@ -539,29 +439,29 @@ static int64_t nfs_client_open(NFSClient *client, QDict 
*options,
 goto fail;
 }
 
-if (qemu_opt_get(opts, "user")) {
-client->uid = qemu_opt_get_number(opts, "user", 0);
+if (opts->has_user

[Qemu-block] [PULL 36/56] rbd: Pass BlockdevOptionsRbd to qemu_rbd_connect()

[Kevin Wolf]

With the conversion to a QAPI options object, the function is now
prepared to be used in a .bdrv_co_create implementation.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 block/rbd.c | 115 +---
 1 file changed, 55 insertions(+), 60 deletions(-)

diff --git a/block/rbd.c b/block/rbd.c
index a979107f65..999fea105f 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -24,6 +24,8 @@
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qlist.h"
+#include "qapi/qobject-input-visitor.h"
+#include "qapi/qapi-visit-block-core.h"
 
 /*
  * When specifying the image filename use:
@@ -484,98 +486,71 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
 qemu_aio_unref(acb);
 }
 
-static char *qemu_rbd_mon_host(QDict *options, Error **errp)
+static char *qemu_rbd_mon_host(BlockdevOptionsRbd *opts, Error **errp)
 {
-const char **vals = g_new(const char *, qdict_size(options) + 1);
-char keybuf[32];
+const char **vals;
 const char *host, *port;
 char *rados_str;
-int i;
-
-for (i = 0;; i++) {
-sprintf(keybuf, "server.%d.host", i);
-host = qdict_get_try_str(options, keybuf);
-qdict_del(options, keybuf);
-sprintf(keybuf, "server.%d.port", i);
-port = qdict_get_try_str(options, keybuf);
-qdict_del(options, keybuf);
-if (!host && !port) {
-break;
-}
-if (!host) {
-error_setg(errp, "Parameter server.%d.host is missing", i);
-rados_str = NULL;
-goto out;
-}
+InetSocketAddressBaseList *p;
+int i, cnt;
+
+if (!opts->has_server) {
+return NULL;
+}
+
+for (cnt = 0, p = opts->server; p; p = p->next) {
+cnt++;
+}
+
+vals = g_new(const char *, cnt + 1);
+
+for (i = 0, p = opts->server; p; p = p->next, i++) {
+host = p->value->host;
+port = p->value->port;
 
 if (strchr(host, ':')) {
-vals[i] = port ? g_strdup_printf("[%s]:%s", host, port)
-: g_strdup_printf("[%s]", host);
+vals[i] = g_strdup_printf("[%s]:%s", host, port);
 } else {
-vals[i] = port ? g_strdup_printf("%s:%s", host, port)
-: g_strdup(host);
+vals[i] = g_strdup_printf("%s:%s", host, port);
 }
 }
 vals[i] = NULL;
 
 rados_str = i ? g_strjoinv(";", (char **)vals) : NULL;
-out:
 g_strfreev((char **)vals);
 return rados_str;
 }
 
 static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t *io_ctx,
 char **s_snap, char **s_image_name,
-QDict *options, bool cache,
+BlockdevOptionsRbd *opts, bool cache,
 const char *keypairs, const char *secretid,
 Error **errp)
 {
-QemuOpts *opts;
 char *mon_host = NULL;
-const char *pool, *snap, *conf, *user, *image_name;
 Error *local_err = NULL;
 int r;
 
-opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
-qemu_opts_absorb_qdict(opts, options, &local_err);
-if (local_err) {
-error_propagate(errp, local_err);
-r = -EINVAL;
-goto failed_opts;
-}
-
-mon_host = qemu_rbd_mon_host(options, &local_err);
+mon_host = qemu_rbd_mon_host(opts, &local_err);
 if (local_err) {
 error_propagate(errp, local_err);
 r = -EINVAL;
 goto failed_opts;
 }
 
-pool   = qemu_opt_get(opts, "pool");
-conf   = qemu_opt_get(opts, "conf");
-snap   = qemu_opt_get(opts, "snapshot");
-user   = qemu_opt_get(opts, "user");
-image_name = qemu_opt_get(opts, "image");
-
-if (!pool || !image_name) {
-error_setg(errp, "Parameters 'pool' and 'image' are required");
-r = -EINVAL;
-goto failed_opts;
-}
-
-r = rados_create(cluster, user);
+r = rados_create(cluster, opts->user);
 if (r < 0) {
 error_setg_errno(errp, -r, "error initializing");
 goto failed_opts;
 }
 
-*s_snap = g_strdup(snap);
-*s_image_name = g_strdup(image_name);
+*s_snap = g_strdup(opts->snapshot);
+*s_image_name = g_strdup(opts->image);
 
 /* try default location when conf=NULL, but ignore failure */
-r = rados_conf_read_file(*cluster, conf);
-if (conf && r < 0) {
-error_setg_errno(errp, -r, "error reading conf file %s", conf);
+r = rados_conf_read_file(*cluster, opts->conf);
+if (opts->has_conf && r < 0) {
+error_setg_errno(errp, -r, "error reading conf file %s", opts->conf);
 goto failed_shutdown;
 }
 
@@ -615,13 +590,12 @@ static int qemu_rbd_connect(rados_t *cluster, 
rados_ioctx_t *io_ctx,
 goto failed_shutdown;
 }
 
-r = rados_ioctx_create(*cluster, pool, io_ctx);
+r = rados_ioctx_create(*cluster, opts->pool, io_ctx);
 if (r < 0) {
- 

[Qemu-block] [PULL 44/56] ssh: Use QAPI BlockdevOptionsSsh object

[Kevin Wolf]

Create a BlockdevOptionsSsh object in connect_to_ssh() and take the
options from there. 'host_key_check' is still processed separately
because it's not in the schema yet.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 block/ssh.c | 137 +++-
 1 file changed, 62 insertions(+), 75 deletions(-)

diff --git a/block/ssh.c b/block/ssh.c
index ff9929497d..8b646c0ede 100644
--- a/block/ssh.c
+++ b/block/ssh.c
@@ -35,6 +35,7 @@
 #include "qemu/sockets.h"
 #include "qemu/uri.h"
 #include "qapi/qapi-visit-sockets.h"
+#include "qapi/qapi-visit-block-core.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qstring.h"
 #include "qapi/qobject-input-visitor.h"
@@ -543,21 +544,6 @@ static QemuOptsList ssh_runtime_opts = {
 .type = QEMU_OPT_NUMBER,
 .help = "Port to connect to",
 },
-{
-.name = "path",
-.type = QEMU_OPT_STRING,
-.help = "Path of the image on the host",
-},
-{
-.name = "user",
-.type = QEMU_OPT_STRING,
-.help = "User as which to connect",
-},
-{
-.name = "host_key_check",
-.type = QEMU_OPT_STRING,
-.help = "Defines how and what to check the host key against",
-},
 { /* end of list */ }
 },
 };
@@ -582,23 +568,31 @@ static bool ssh_process_legacy_socket_options(QDict 
*output_opts,
 return true;
 }
 
-static InetSocketAddress *ssh_config(QDict *options, Error **errp)
+static BlockdevOptionsSsh *ssh_parse_options(QDict *options, Error **errp)
 {
-InetSocketAddress *inet = NULL;
-QDict *addr = NULL;
-QObject *crumpled_addr = NULL;
-Visitor *iv = NULL;
-Error *local_error = NULL;
-
-qdict_extract_subqdict(options, &addr, "server.");
-if (!qdict_size(addr)) {
-error_setg(errp, "SSH server address missing");
-goto out;
+BlockdevOptionsSsh *result = NULL;
+QemuOpts *opts = NULL;
+Error *local_err = NULL;
+QObject *crumpled;
+const QDictEntry *e;
+Visitor *v;
+
+/* Translate legacy options */
+opts = qemu_opts_create(&ssh_runtime_opts, NULL, 0, &error_abort);
+qemu_opts_absorb_qdict(opts, options, &local_err);
+if (local_err) {
+error_propagate(errp, local_err);
+goto fail;
 }
 
-crumpled_addr = qdict_crumple(addr, errp);
-if (!crumpled_addr) {
-goto out;
+if (!ssh_process_legacy_socket_options(options, opts, errp)) {
+goto fail;
+}
+
+/* Create the QAPI object */
+crumpled = qdict_crumple(options, errp);
+if (crumpled == NULL) {
+goto fail;
 }
 
 /*
@@ -609,51 +603,50 @@ static InetSocketAddress *ssh_config(QDict *options, 
Error **errp)
  * but when they come from -drive, they're all QString.  The
  * visitor expects the former.
  */
-iv = qobject_input_visitor_new(crumpled_addr);
-visit_type_InetSocketAddress(iv, NULL, &inet, &local_error);
-if (local_error) {
-error_propagate(errp, local_error);
-goto out;
+v = qobject_input_visitor_new(crumpled);
+visit_type_BlockdevOptionsSsh(v, NULL, &result, &local_err);
+visit_free(v);
+qobject_decref(crumpled);
+
+if (local_err) {
+error_propagate(errp, local_err);
+goto fail;
 }
 
-out:
-QDECREF(addr);
-qobject_decref(crumpled_addr);
-visit_free(iv);
-return inet;
+/* Remove the processed options from the QDict (the visitor processes
+ * _all_ options in the QDict) */
+while ((e = qdict_first(options))) {
+qdict_del(options, e->key);
+}
+
+fail:
+qemu_opts_del(opts);
+return result;
 }
 
 static int connect_to_ssh(BDRVSSHState *s, QDict *options,
   int ssh_flags, int creat_mode, Error **errp)
 {
+BlockdevOptionsSsh *opts;
 int r, ret;
-QemuOpts *opts = NULL;
-Error *local_err = NULL;
-const char *user, *path, *host_key_check;
+const char *user, *host_key_check;
 long port = 0;
 
-opts = qemu_opts_create(&ssh_runtime_opts, NULL, 0, &error_abort);
-qemu_opts_absorb_qdict(opts, options, &local_err);
-if (local_err) {
-ret = -EINVAL;
-error_propagate(errp, local_err);
-goto err;
-}
-
-if (!ssh_process_legacy_socket_options(options, opts, errp)) {
-ret = -EINVAL;
-goto err;
+host_key_check = qdict_get_try_str(options, "host_key_check");
+if (!host_key_check) {
+host_key_check = "yes";
+} else {
+qdict_del(options, "host_key_check");
 }
 
-path = qemu_opt_get(opts, "path");
-if (!path) {
-ret = -EINVAL;
-error_setg(errp, "No path was specified");
-goto err;
+opts = ssh_parse_options(options, errp);
+if (opts == NULL) {
+return -EINVAL;
 }
 
-user = qemu_opt_get(opts, "user");
-if (!user) {
+if (opts->has_user) {
+u

Re: [Qemu-block] [PATCH v4 0/2] block latency histogram

[Vladimir Sementsov-Ogievskiy]

09.03.2018 18:40, Eric Blake wrote:

On 03/09/2018 09:33 AM, Vladimir Sementsov-Ogievskiy wrote:

v4:

Move block_latency_histogram_set from block/accounting.c to
blockdev.c, as it uses qapi type uint64List and this fact breaks
build.


Was the cross file motion necessary, or could you just fix the 
#includes according to the recent qapi header refactoring?




Looks like it's not as simple..

/tmp/qemu-test/src/include/block/accounting.h:30:10: fatal error: qapi-types.h: 
No such file or directory
 #include "qapi-types.h


what to do now? Looks like qapi-types are not generated for some builds. So, 
this version (v4) should take place.

--
Best regards,
Vladimir

Re: [Qemu-block] [PATCH 0/5] nbd server fixing and refactoring before BLOCK_STATUS

[Eric Blake]

On 03/08/2018 12:46 PM, Vladimir Sementsov-Ogievskiy wrote:

01 and 02 are splitted and updated "[PATCH] nbd/server: fix space read",
others are new.

Vladimir Sementsov-Ogievskiy (5):
   nbd/server: move nbd_co_send_structured_error up
   nbd/server: fix sparse read
   nbd/server: fix: check client->closing before reply sending
   nbd/server: refactor nbd_trip: cmd_read and generic reply
   nbd/server: refactor nbd_trip: split out nbd_handle_request


I had a tough time applying this one:

Applying: nbd/server: move nbd_co_send_structured_error up
Applying: nbd/server: fix: check client->closing before reply sending
Applying: nbd/server: refactor nbd_trip: cmd_read and generic reply
error: sha1 information is lacking or useless (nbd/server.c).
error: could not build fake ancestor
Patch failed at 0004 nbd/server: refactor nbd_trip: cmd_read and generic 
reply

The copy of the patch that failed is found in: .git/rebase-apply/patch
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".

I think I've resolved the conflicts correctly, but if I get through 
reviewing this series and posting it to my NBD queue, you may want to 
double-check things.


--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3266
Virtualization:  qemu.org | libvirt.org

[Qemu-block] [PULL 34/56] rbd: Factor out qemu_rbd_connect()

[Kevin Wolf]

The code to establish an RBD connection is duplicated between open and
create. In order to be able to share the code, factor out the code from
qemu_rbd_open() as a first step.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 block/rbd.c | 100 
 1 file changed, 60 insertions(+), 40 deletions(-)

diff --git a/block/rbd.c b/block/rbd.c
index c1025c8493..99fcc7ecdf 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -546,32 +546,17 @@ out:
 return rados_str;
 }
 
-static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
- Error **errp)
+static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t *io_ctx,
+char **s_snap, char **s_image_name,
+QDict *options, bool cache, Error **errp)
 {
-BDRVRBDState *s = bs->opaque;
-const char *pool, *snap, *conf, *user, *image_name, *keypairs;
-const char *secretid, *filename;
 QemuOpts *opts;
-Error *local_err = NULL;
 char *mon_host = NULL;
+const char *pool, *snap, *conf, *user, *image_name, *keypairs;
+const char *secretid;
+Error *local_err = NULL;
 int r;
 
-/* If we are given a filename, parse the filename, with precedence given to
- * filename encoded options */
-filename = qdict_get_try_str(options, "filename");
-if (filename) {
-warn_report("'filename' option specified. "
-"This is an unsupported option, and may be deprecated "
-"in the future");
-qemu_rbd_parse_filename(filename, options, &local_err);
-if (local_err) {
-r = -EINVAL;
-error_propagate(errp, local_err);
-goto exit;
-}
-}
-
 opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
 qemu_opts_absorb_qdict(opts, options, &local_err);
 if (local_err) {
@@ -602,35 +587,35 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict 
*options, int flags,
 goto failed_opts;
 }
 
-r = rados_create(&s->cluster, user);
+r = rados_create(cluster, user);
 if (r < 0) {
 error_setg_errno(errp, -r, "error initializing");
 goto failed_opts;
 }
 
-s->snap = g_strdup(snap);
-s->image_name = g_strdup(image_name);
+*s_snap = g_strdup(snap);
+*s_image_name = g_strdup(image_name);
 
 /* try default location when conf=NULL, but ignore failure */
-r = rados_conf_read_file(s->cluster, conf);
+r = rados_conf_read_file(*cluster, conf);
 if (conf && r < 0) {
 error_setg_errno(errp, -r, "error reading conf file %s", conf);
 goto failed_shutdown;
 }
 
-r = qemu_rbd_set_keypairs(s->cluster, keypairs, errp);
+r = qemu_rbd_set_keypairs(*cluster, keypairs, errp);
 if (r < 0) {
 goto failed_shutdown;
 }
 
 if (mon_host) {
-r = rados_conf_set(s->cluster, "mon_host", mon_host);
+r = rados_conf_set(*cluster, "mon_host", mon_host);
 if (r < 0) {
 goto failed_shutdown;
 }
 }
 
-if (qemu_rbd_set_auth(s->cluster, secretid, errp) < 0) {
+if (qemu_rbd_set_auth(*cluster, secretid, errp) < 0) {
 r = -EIO;
 goto failed_shutdown;
 }
@@ -642,24 +627,65 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict 
*options, int flags,
  * librbd defaults to no caching. If write through caching cannot
  * be set up, fall back to no caching.
  */
-if (flags & BDRV_O_NOCACHE) {
-rados_conf_set(s->cluster, "rbd_cache", "false");
+if (cache) {
+rados_conf_set(*cluster, "rbd_cache", "true");
 } else {
-rados_conf_set(s->cluster, "rbd_cache", "true");
+rados_conf_set(*cluster, "rbd_cache", "false");
 }
 
-r = rados_connect(s->cluster);
+r = rados_connect(*cluster);
 if (r < 0) {
 error_setg_errno(errp, -r, "error connecting");
 goto failed_shutdown;
 }
 
-r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
+r = rados_ioctx_create(*cluster, pool, io_ctx);
 if (r < 0) {
 error_setg_errno(errp, -r, "error opening pool %s", pool);
 goto failed_shutdown;
 }
 
+qemu_opts_del(opts);
+return 0;
+
+failed_shutdown:
+rados_shutdown(*cluster);
+g_free(*s_snap);
+g_free(*s_image_name);
+failed_opts:
+qemu_opts_del(opts);
+g_free(mon_host);
+return r;
+}
+
+static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
+ Error **errp)
+{
+BDRVRBDState *s = bs->opaque;
+Error *local_err = NULL;
+const char *filename;
+int r;
+
+/* If we are given a filename, parse the filename, with precedence given to
+ * filename encoded options */
+filename = qdict_get_try_str(options, "filename");
+if (filename) {
+warn_report("'filename' option specified. "
+"This is an unsupported option, and may be depr

Re: [Qemu-block] [Qemu-devel] [PATCH v5 0/2] block latency histogram

[no-reply]

Hi,

This series failed build test on s390x host. Please find the details below.

Type: series
Message-id: 20180309160224.78821-1-vsement...@virtuozzo.com
Subject: [Qemu-devel] [PATCH v5 0/2] block latency histogram

=== TEST SCRIPT BEGIN ===
#!/bin/bash
# Testing script will be invoked under the git checkout with
# HEAD pointing to a commit that has the patches applied on top of "base"
# branch
set -e
echo "=== ENV ==="
env
echo "=== PACKAGES ==="
rpm -qa
echo "=== TEST BEGIN ==="
CC=$HOME/bin/cc
INSTALL=$PWD/install
BUILD=$PWD/build
echo -n "Using CC: "
realpath $CC
mkdir -p $BUILD $INSTALL
SRC=$PWD
cd $BUILD
$SRC/configure --cc=$CC --prefix=$INSTALL
make -j4
# XXX: we need reliable clean up
# make check -j4 V=1
make install
=== TEST SCRIPT END ===

Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384
From https://github.com/patchew-project/qemu
   cffad426f5..b39b61e410  master -> master
Switched to a new branch 'test'
a15f5e3215 qapi: add block latency histogram interface
d57ae07d73 block/accounting: introduce latency histogram

=== OUTPUT BEGIN ===
=== ENV ===
LANG=en_US.UTF-8
XDG_SESSION_ID=89726
USER=fam
PWD=/var/tmp/patchew-tester-tmp-8r1fqmiu/src
HOME=/home/fam
SHELL=/bin/sh
SHLVL=2
PATCHEW=/home/fam/patchew/patchew-cli -s http://patchew.org --nodebug
LOGNAME=fam
DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1012/bus
XDG_RUNTIME_DIR=/run/user/1012
PATH=/usr/bin:/bin
_=/usr/bin/env
=== PACKAGES ===
gpg-pubkey-873529b8-54e386ff
glibc-debuginfo-common-2.24-10.fc25.s390x
fedora-release-26-1.noarch
dejavu-sans-mono-fonts-2.35-4.fc26.noarch
xemacs-filesystem-21.5.34-22.20170124hgf412e9f093d4.fc26.noarch
bash-4.4.12-7.fc26.s390x
freetype-2.7.1-9.fc26.s390x
libSM-1.2.2-5.fc26.s390x
libmpc-1.0.2-6.fc26.s390x
libaio-0.3.110-7.fc26.s390x
libverto-0.2.6-7.fc26.s390x
perl-Scalar-List-Utils-1.48-1.fc26.s390x
iptables-libs-1.6.1-2.fc26.s390x
p11-kit-trust-0.23.9-2.fc26.s390x
tcl-8.6.6-2.fc26.s390x
libxshmfence-1.2-4.fc26.s390x
expect-5.45-23.fc26.s390x
perl-Thread-Queue-3.12-1.fc26.noarch
perl-encoding-2.19-6.fc26.s390x
keyutils-1.5.10-1.fc26.s390x
gmp-devel-6.1.2-4.fc26.s390x
enchant-1.6.0-16.fc26.s390x
net-snmp-libs-5.7.3-17.fc26.s390x
python-gobject-base-3.24.1-1.fc26.s390x
python3-enchant-1.6.10-1.fc26.noarch
python-lockfile-0.11.0-6.fc26.noarch
python2-pyparsing-2.1.10-3.fc26.noarch
python2-lxml-4.1.1-1.fc26.s390x
librados2-10.2.7-2.fc26.s390x
trousers-lib-0.3.13-7.fc26.s390x
libpaper-1.1.24-14.fc26.s390x
libdatrie-0.2.9-4.fc26.s390x
libsoup-2.58.2-1.fc26.s390x
passwd-0.79-9.fc26.s390x
bind99-libs-9.9.10-3.P3.fc26.s390x
python3-rpm-4.13.0.2-1.fc26.s390x
systemd-233-7.fc26.s390x
virglrenderer-0.6.0-1.20170210git76b3da97b.fc26.s390x
s390utils-ziomon-1.36.1-3.fc26.s390x
s390utils-osasnmpd-1.36.1-3.fc26.s390x
libXrandr-1.5.1-2.fc26.s390x
libglvnd-glx-1.0.0-1.fc26.s390x
texlive-ifxetex-svn19685.0.5-33.fc26.2.noarch
texlive-psnfss-svn33946.9.2a-33.fc26.2.noarch
texlive-dvipdfmx-def-svn40328-33.fc26.2.noarch
texlive-natbib-svn20668.8.31b-33.fc26.2.noarch
texlive-xdvi-bin-svn40750-33.20160520.fc26.2.s390x
texlive-cm-svn32865.0-33.fc26.2.noarch
texlive-beton-svn15878.0-33.fc26.2.noarch
texlive-fpl-svn15878.1.002-33.fc26.2.noarch
texlive-mflogo-svn38628-33.fc26.2.noarch
texlive-texlive-docindex-svn41430-33.fc26.2.noarch
texlive-luaotfload-bin-svn34647.0-33.20160520.fc26.2.noarch
texlive-koma-script-svn41508-33.fc26.2.noarch
texlive-pst-tree-svn24142.1.12-33.fc26.2.noarch
texlive-breqn-svn38099.0.98d-33.fc26.2.noarch
texlive-xetex-svn41438-33.fc26.2.noarch
gstreamer1-plugins-bad-free-1.12.3-1.fc26.s390x
xorg-x11-font-utils-7.5-33.fc26.s390x
ghostscript-fonts-5.50-36.fc26.noarch
libXext-devel-1.3.3-5.fc26.s390x
libusbx-devel-1.0.21-2.fc26.s390x
libglvnd-devel-1.0.0-1.fc26.s390x
emacs-25.3-3.fc26.s390x
alsa-lib-devel-1.1.4.1-1.fc26.s390x
kbd-2.0.4-2.fc26.s390x
dconf-0.26.0-2.fc26.s390x
ccache-3.3.4-1.fc26.s390x
mc-4.8.19-5.fc26.s390x
doxygen-1.8.13-9.fc26.s390x
dpkg-1.18.24-1.fc26.s390x
libtdb-1.3.13-1.fc26.s390x
python2-pynacl-1.1.1-1.fc26.s390x
nss-sysinit-3.34.0-1.0.fc26.s390x
kernel-4.13.16-202.fc26.s390x
perl-Filter-1.58-1.fc26.s390x
python2-pip-9.0.1-11.fc26.noarch
dnf-2.7.5-2.fc26.noarch
sssd-common-1.16.0-4.fc26.s390x
python2-sssdconfig-1.16.0-4.fc26.noarch
bind-license-9.11.2-1.P1.fc26.noarch
libtasn1-4.13-1.fc26.s390x
glusterfs-fuse-3.10.10-1.fc26.s390x
cpp-7.3.1-2.fc26.s390x
pkgconf-1.3.12-2.fc26.s390x
python2-fedora-0.10.0-1.fc26.noarch
cmake-filesystem-3.10.1-11.fc26.s390x
selinux-policy-targeted-3.13.1-260.18.fc26.noarch
python3-requests-kerberos-0.12.0-1.fc26.noarch
libmicrohttpd-0.9.59-1.fc26.s390x
GeoIP-GeoLite-data-2018.01-1.fc26.noarch
glibc-debuginfo-2.24-10.fc25.s390x
dejavu-fonts-common-2.35-4.fc26.noarch
bind99-license-9.9.10-3.P3.fc26.noarch
ncurses-libs-6.0-8.20170212.fc26.s390x
libpng-1.6.28-2.fc26.s390x
libICE-1.0.9-9.fc26.s390x
perl-Text-ParseWords-3.30-366.fc26.noarch
libtool-ltdl-2.4.6-17.fc26.s390x
libselinux-utils-2.6-7.fc26.s390x
userspace-rcu-0.9.3-2.fc26.s390x
libXfont-1.5.2-5.fc26.s39

[Qemu-block] [PULL 33/56] rbd: Fix use after free in qemu_rbd_set_keypairs() error path

[Kevin Wolf]

If we want to include the invalid option name in the error message, we
can't free the string earlier than that.

Cc: qemu-sta...@nongnu.org
Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 block/rbd.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/block/rbd.c b/block/rbd.c
index c1275c1ec9..c1025c8493 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -268,13 +268,14 @@ static int qemu_rbd_set_keypairs(rados_t cluster, const 
char *keypairs_json,
 key = qstring_get_str(name);
 
 ret = rados_conf_set(cluster, key, qstring_get_str(value));
-QDECREF(name);
 QDECREF(value);
 if (ret < 0) {
 error_setg_errno(errp, -ret, "invalid conf option %s", key);
+QDECREF(name);
 ret = -EINVAL;
 break;
 }
+QDECREF(name);
 }
 
 QDECREF(keypairs);
-- 
2.13.6

[Qemu-block] [PULL 53/56] iotests: Mark all tests executable

[Kevin Wolf]

From: Eric Blake 

The majority of our iotests have the executable bit set; fix the
few outliers for consistency.

Signed-off-by: Eric Blake 
Message-id: 20180305161824.7188-1-ebl...@redhat.com
Signed-off-by: Max Reitz 
---
 tests/qemu-iotests/096 | 0
 tests/qemu-iotests/124 | 0
 tests/qemu-iotests/129 | 0
 tests/qemu-iotests/132 | 0
 tests/qemu-iotests/136 | 0
 tests/qemu-iotests/139 | 0
 tests/qemu-iotests/148 | 0
 tests/qemu-iotests/152 | 0
 tests/qemu-iotests/163 | 0
 tests/qemu-iotests/205 | 0
 10 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 tests/qemu-iotests/096
 mode change 100644 => 100755 tests/qemu-iotests/124
 mode change 100644 => 100755 tests/qemu-iotests/129
 mode change 100644 => 100755 tests/qemu-iotests/132
 mode change 100644 => 100755 tests/qemu-iotests/136
 mode change 100644 => 100755 tests/qemu-iotests/139
 mode change 100644 => 100755 tests/qemu-iotests/148
 mode change 100644 => 100755 tests/qemu-iotests/152
 mode change 100644 => 100755 tests/qemu-iotests/163
 mode change 100644 => 100755 tests/qemu-iotests/205

diff --git a/tests/qemu-iotests/096 b/tests/qemu-iotests/096
old mode 100644
new mode 100755
diff --git a/tests/qemu-iotests/124 b/tests/qemu-iotests/124
old mode 100644
new mode 100755
diff --git a/tests/qemu-iotests/129 b/tests/qemu-iotests/129
old mode 100644
new mode 100755
diff --git a/tests/qemu-iotests/132 b/tests/qemu-iotests/132
old mode 100644
new mode 100755
diff --git a/tests/qemu-iotests/136 b/tests/qemu-iotests/136
old mode 100644
new mode 100755
diff --git a/tests/qemu-iotests/139 b/tests/qemu-iotests/139
old mode 100644
new mode 100755
diff --git a/tests/qemu-iotests/148 b/tests/qemu-iotests/148
old mode 100644
new mode 100755
diff --git a/tests/qemu-iotests/152 b/tests/qemu-iotests/152
old mode 100644
new mode 100755
diff --git a/tests/qemu-iotests/163 b/tests/qemu-iotests/163
old mode 100644
new mode 100755
diff --git a/tests/qemu-iotests/205 b/tests/qemu-iotests/205
old mode 100644
new mode 100755
-- 
2.13.6

[Qemu-block] [PULL 50/56] qemu-iotests: Test qcow2 over file image creation with QMP

[Kevin Wolf]

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 tests/qemu-iotests/206 | 436 +
 tests/qemu-iotests/206.out | 209 ++
 tests/qemu-iotests/group   |   1 +
 3 files changed, 646 insertions(+)
 create mode 100755 tests/qemu-iotests/206
 create mode 100644 tests/qemu-iotests/206.out

diff --git a/tests/qemu-iotests/206 b/tests/qemu-iotests/206
new file mode 100755
index 00..0a18b2b19a
--- /dev/null
+++ b/tests/qemu-iotests/206
@@ -0,0 +1,436 @@
+#!/bin/bash
+#
+# Test qcow2 and file image creation
+#
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+#
+
+# creator
+owner=kw...@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1   # failure is the default!
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+function do_run_qemu()
+{
+echo Testing: "$@"
+$QEMU -nographic -qmp stdio -serial none "$@"
+echo
+}
+
+function run_qemu()
+{
+do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp \
+  | _filter_qemu | _filter_imgfmt \
+  | _filter_actual_image_size
+}
+
+echo
+echo "=== Successful image creation (defaults) ==="
+echo
+
+size=$((128 * 1024 * 1024))
+
+run_qemu <

[Qemu-block] [PULL 28/56] block: Make bdrv_is_whitelisted() public

[Kevin Wolf]

We'll use a separate source file for image creation, and we need to
check there whether the requested driver is whitelisted.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 include/block/block.h | 1 +
 block.c   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/include/block/block.h b/include/block/block.h
index 7805187b30..cdec3639a3 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -226,6 +226,7 @@ char *bdrv_perm_names(uint64_t perm);
 void bdrv_init(void);
 void bdrv_init_with_whitelist(void);
 bool bdrv_uses_whitelist(void);
+int bdrv_is_whitelisted(BlockDriver *drv, bool read_only);
 BlockDriver *bdrv_find_protocol(const char *filename,
 bool allow_protocol_prefix,
 Error **errp);
diff --git a/block.c b/block.c
index 4fc65f7621..00f94241fc 100644
--- a/block.c
+++ b/block.c
@@ -370,7 +370,7 @@ BlockDriver *bdrv_find_format(const char *format_name)
 return bdrv_do_find_format(format_name);
 }
 
-static int bdrv_is_whitelisted(BlockDriver *drv, bool read_only)
+int bdrv_is_whitelisted(BlockDriver *drv, bool read_only)
 {
 static const char *whitelist_rw[] = {
 CONFIG_BDRV_RW_WHITELIST
-- 
2.13.6

[Qemu-block] [PULL 49/56] block: Fail bdrv_truncate() with negative size

[Kevin Wolf]

Most callers have their own checks, but something like this should also
be checked centrally. As it happens, x-blockdev-create can pass negative
image sizes to format drivers (because there is no QAPI type that would
reject negative numbers) and triggers the check added by this patch.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 block.c | 5 +
 1 file changed, 5 insertions(+)

diff --git a/block.c b/block.c
index 00f94241fc..75a9fd49de 100644
--- a/block.c
+++ b/block.c
@@ -3719,6 +3719,11 @@ int bdrv_truncate(BdrvChild *child, int64_t offset, 
PreallocMode prealloc,
 error_setg(errp, "No medium inserted");
 return -ENOMEDIUM;
 }
+if (offset < 0) {
+error_setg(errp, "Image size cannot be negative");
+return -EINVAL;
+}
+
 if (!drv->bdrv_truncate) {
 if (bs->file && drv->is_filter) {
 return bdrv_truncate(bs->file, offset, prealloc, errp);
-- 
2.13.6

[Qemu-block] [PULL 52/56] iotests: Test creating overlay when guest running

[Kevin Wolf]

From: Fam Zheng 

Signed-off-by: Fam Zheng 
Message-id: 20171225025107.23985-1-f...@redhat.com
Reviewed-by: Eric Blake 
Signed-off-by: Max Reitz 
---
 tests/qemu-iotests/153 | 8 +---
 tests/qemu-iotests/153.out | 7 ---
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/qemu-iotests/153 b/tests/qemu-iotests/153
index fa25eb24bd..adfd02695b 100755
--- a/tests/qemu-iotests/153
+++ b/tests/qemu-iotests/153
@@ -32,6 +32,7 @@ _cleanup()
 {
 _cleanup_test_img
 rm -f "${TEST_IMG}.base"
+rm -f "${TEST_IMG}.overlay"
 rm -f "${TEST_IMG}.convert"
 rm -f "${TEST_IMG}.a"
 rm -f "${TEST_IMG}.b"
@@ -177,8 +178,6 @@ rm -f "${TEST_IMG}.lnk" &>/dev/null
 ln -s ${TEST_IMG} "${TEST_IMG}.lnk" || echo "Failed to create link"
 _run_qemu_with_images "${TEST_IMG}.lnk" "${TEST_IMG}"
 
-echo
-echo "== Closing an image should unlock it =="
 _launch_qemu
 
 _send_qemu_cmd $QEMU_HANDLE \
@@ -193,7 +192,10 @@ _send_qemu_cmd $QEMU_HANDLE \
 
 _run_cmd $QEMU_IO "${TEST_IMG}" -c 'write 0 512'
 
-echo "Closing drive"
+echo "Creating overlay with qemu-img when the guest is running should be 
allowed"
+_run_cmd $QEMU_IMG create -f $IMGFMT -b "${TEST_IMG}" "${TEST_IMG}.overlay"
+
+echo "== Closing an image should unlock it =="
 _send_qemu_cmd $QEMU_HANDLE \
 "{ 'execute': 'human-monitor-command',
'arguments': { 'command-line': 'drive_del d0' } }" \
diff --git a/tests/qemu-iotests/153.out b/tests/qemu-iotests/153.out
index 5b917b177c..34309cfb20 100644
--- a/tests/qemu-iotests/153.out
+++ b/tests/qemu-iotests/153.out
@@ -372,15 +372,16 @@ Is another process using the image?
 == Symbolic link ==
 QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2: Failed to get "write" lock
 Is another process using the image?
-
-== Closing an image should unlock it ==
 {"return": {}}
 Adding drive
 
 _qemu_io_wrapper TEST_DIR/t.qcow2 -c write 0 512
 can't open device TEST_DIR/t.qcow2: Failed to get "write" lock
 Is another process using the image?
-Closing drive
+Creating overlay with qemu-img when the guest is running should be allowed
+
+_qemu_img_wrapper create -f qcow2 -b TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.overlay
+== Closing an image should unlock it ==
 
 _qemu_io_wrapper TEST_DIR/t.qcow2 -c write 0 512
 Adding two and closing one
-- 
2.13.6

[Qemu-block] [PULL 45/56] ssh: QAPIfy host-key-check option

[Kevin Wolf]

This makes the host-key-check option available in blockdev-add.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 qapi/block-core.json | 63 +++--
 block/ssh.c  | 88 +---
 2 files changed, 117 insertions(+), 34 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index fd21fc..4814bb7db7 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -2553,6 +2553,63 @@
 '*encrypt': 'BlockdevQcow2Encryption' } }
 
 ##
+# @SshHostKeyCheckMode:
+#
+# @none Don't check the host key at all
+# @hash Compare the host key with a given hash
+# @known_hosts  Check the host key against the known_hosts file
+#
+# Since: 2.12
+##
+{ 'enum': 'SshHostKeyCheckMode',
+  'data': [ 'none', 'hash', 'known_hosts' ] }
+
+##
+# @SshHostKeyCheckHashType:
+#
+# @md5  The given hash is an md5 hash
+# @sha1 The given hash is an sha1 hash
+#
+# Since: 2.12
+##
+{ 'enum': 'SshHostKeyCheckHashType',
+  'data': [ 'md5', 'sha1' ] }
+
+##
+# @SshHostKeyHash:
+#
+# @type The hash algorithm used for the hash
+# @hash The expected hash value
+#
+# Since: 2.12
+##
+{ 'struct': 'SshHostKeyHash',
+  'data': { 'type': 'SshHostKeyCheckHashType',
+'hash': 'str' }}
+
+##
+# @SshHostKeyDummy:
+#
+# For those union branches that don't need additional fields.
+#
+# Since: 2.12
+##
+{ 'struct': 'SshHostKeyDummy',
+  'data': {} }
+
+##
+# @SshHostKeyCheck:
+#
+# Since: 2.12
+##
+{ 'union': 'SshHostKeyCheck',
+  'base': { 'mode': 'SshHostKeyCheckMode' },
+  'discriminator': 'mode',
+  'data': { 'none': 'SshHostKeyDummy',
+'hash': 'SshHostKeyHash',
+'known_hosts': 'SshHostKeyDummy' } }
+
+##
 # @BlockdevOptionsSsh:
 #
 # @server:  host address
@@ -2562,14 +2619,16 @@
 # @user:user as which to connect, defaults to current
 #   local user name
 #
-# TODO: Expose the host_key_check option in QMP
+# @host-key-check:  Defines how and what to check the host key against
+#   (default: known_hosts)
 #
 # Since: 2.9
 ##
 { 'struct': 'BlockdevOptionsSsh',
   'data': { 'server': 'InetSocketAddress',
 'path': 'str',
-'*user': 'str' } }
+'*user': 'str',
+'*host-key-check': 'SshHostKeyCheck' } }
 
 
 ##
diff --git a/block/ssh.c b/block/ssh.c
index 8b646c0ede..30cdf9a99f 100644
--- a/block/ssh.c
+++ b/block/ssh.c
@@ -431,31 +431,35 @@ check_host_key_hash(BDRVSSHState *s, const char *hash,
 }
 
 static int check_host_key(BDRVSSHState *s, const char *host, int port,
-  const char *host_key_check, Error **errp)
+  SshHostKeyCheck *hkc, Error **errp)
 {
-/* host_key_check=no */
-if (strcmp(host_key_check, "no") == 0) {
-return 0;
-}
+SshHostKeyCheckMode mode;
 
-/* host_key_check=md5:xx:yy:zz:... */
-if (strncmp(host_key_check, "md5:", 4) == 0) {
-return check_host_key_hash(s, &host_key_check[4],
-   LIBSSH2_HOSTKEY_HASH_MD5, 16, errp);
-}
-
-/* host_key_check=sha1:xx:yy:zz:... */
-if (strncmp(host_key_check, "sha1:", 5) == 0) {
-return check_host_key_hash(s, &host_key_check[5],
-   LIBSSH2_HOSTKEY_HASH_SHA1, 20, errp);
+if (hkc) {
+mode = hkc->mode;
+} else {
+mode = SSH_HOST_KEY_CHECK_MODE_KNOWN_HOSTS;
 }
 
-/* host_key_check=yes */
-if (strcmp(host_key_check, "yes") == 0) {
+switch (mode) {
+case SSH_HOST_KEY_CHECK_MODE_NONE:
+return 0;
+case SSH_HOST_KEY_CHECK_MODE_HASH:
+if (hkc->u.hash.type == SSH_HOST_KEY_CHECK_HASH_TYPE_MD5) {
+return check_host_key_hash(s, hkc->u.hash.hash,
+   LIBSSH2_HOSTKEY_HASH_MD5, 16, errp);
+} else if (hkc->u.hash.type == SSH_HOST_KEY_CHECK_HASH_TYPE_SHA1) {
+return check_host_key_hash(s, hkc->u.hash.hash,
+   LIBSSH2_HOSTKEY_HASH_SHA1, 20, errp);
+}
+g_assert_not_reached();
+break;
+case SSH_HOST_KEY_CHECK_MODE_KNOWN_HOSTS:
 return check_host_key_knownhosts(s, host, port, errp);
+default:
+g_assert_not_reached();
 }
 
-error_setg(errp, "unknown host_key_check setting (%s)", host_key_check);
 return -EINVAL;
 }
 
@@ -544,16 +548,22 @@ static QemuOptsList ssh_runtime_opts = {
 .type = QEMU_OPT_NUMBER,
 .help = "Port to connect to",
 },
+{
+.name = "host_key_check",
+.type = QEMU_OPT_STRING,
+.help = "Defines how and what to check the host key against",
+},
 { /* end of list */ }
 },
 };
 
-static bool ssh_process_legacy_socket_options(QDict *output_opts,
-  Q

[Qemu-block] [PULL 55/56] iotests: Tweak 030 in order to trigger a race condition with parallel jobs

[Kevin Wolf]

From: Alberto Garcia 

This patch tweaks TestParallelOps in iotest 030 so it allocates data
in smaller regions (256KB/512KB instead of 512KB/1MB) and the
block-stream job in test_stream_commit() only needs to copy data that
is at the very end of the image.

This way when the block-stream job is awakened it will finish right
away without any chance of being stopped by block_job_sleep_ns(). This
triggers the bug that was fixed by 3d5d319e1221082974711af1d09d82f and
1a63a907507fbbcfaee3f622907ec24 and is therefore a more useful test
case for parallel block jobs.

After this patch the aforementiond bug can also be reproduced with the
test_stream_parallel() test case.

Since with this change the stream job in test_stream_commit() finishes
early, this patch introduces a similar test case where both jobs are
slowed down so they can actually run in parallel.

Signed-off-by: Alberto Garcia 
Cc: John Snow 
Message-id: 20180306130121.30243-1-be...@igalia.com
Signed-off-by: Max Reitz 
---
 tests/qemu-iotests/030 | 52 ++
 tests/qemu-iotests/030.out |  4 ++--
 2 files changed, 45 insertions(+), 11 deletions(-)

diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030
index 457984b8e9..b5f88959aa 100755
--- a/tests/qemu-iotests/030
+++ b/tests/qemu-iotests/030
@@ -156,7 +156,7 @@ class TestSingleDrive(iotests.QMPTestCase):
 class TestParallelOps(iotests.QMPTestCase):
 num_ops = 4 # Number of parallel block-stream operations
 num_imgs = num_ops * 2 + 1
-image_len = num_ops * 1024 * 1024
+image_len = num_ops * 512 * 1024
 imgs = []
 
 def setUp(self):
@@ -176,14 +176,14 @@ class TestParallelOps(iotests.QMPTestCase):
  '-o', 'backing_file=%s' % self.imgs[i-1], self.imgs[i])
 
 # Put data into the images we are copying data from
-for i in range(self.num_imgs / 2):
-img_index = i * 2 + 1
-# Alternate between 512k and 1M.
+odd_img_indexes = [x for x in reversed(range(self.num_imgs)) if x % 2 
== 1]
+for i in range(len(odd_img_indexes)):
+# Alternate between 256KB and 512KB.
 # This way jobs will not finish in the same order they were created
-num_kb = 512 + 512 * (i % 2)
+num_kb = 256 + 256 * (i % 2)
 qemu_io('-f', iotests.imgfmt,
-'-c', 'write -P %d %d %d' % (i, i*1024*1024, num_kb * 
1024),
-self.imgs[img_index])
+'-c', 'write -P 0xFF %dk %dk' % (i * 512, num_kb),
+self.imgs[odd_img_indexes[i]])
 
 # Attach the drive to the VM
 self.vm = iotests.VM()
@@ -318,12 +318,14 @@ class TestParallelOps(iotests.QMPTestCase):
 self.wait_until_completed(drive='commit-drive0')
 
 # Test a block-stream and a block-commit job in parallel
-def test_stream_commit(self):
+# Here the stream job is supposed to finish quickly in order to reproduce
+# the scenario that triggers the bug fixed in 3d5d319e1221 and 1a63a907507
+def test_stream_commit_1(self):
 self.assertLessEqual(8, self.num_imgs)
 self.assert_no_active_block_jobs()
 
 # Stream from node0 into node2
-result = self.vm.qmp('block-stream', device='node2', job_id='node2')
+result = self.vm.qmp('block-stream', device='node2', 
base_node='node0', job_id='node2')
 self.assert_qmp(result, 'return', {})
 
 # Commit from the active layer into node3
@@ -348,6 +350,38 @@ class TestParallelOps(iotests.QMPTestCase):
 
 self.assert_no_active_block_jobs()
 
+# This is similar to test_stream_commit_1 but both jobs are slowed
+# down so they can run in parallel for a little while.
+def test_stream_commit_2(self):
+self.assertLessEqual(8, self.num_imgs)
+self.assert_no_active_block_jobs()
+
+# Stream from node0 into node4
+result = self.vm.qmp('block-stream', device='node4', 
base_node='node0', job_id='node4', speed=1024*1024)
+self.assert_qmp(result, 'return', {})
+
+# Commit from the active layer into node5
+result = self.vm.qmp('block-commit', device='drive0', 
base=self.imgs[5], speed=1024*1024)
+self.assert_qmp(result, 'return', {})
+
+# Wait for all jobs to be finished.
+pending_jobs = ['node4', 'drive0']
+while len(pending_jobs) > 0:
+for event in self.vm.get_qmp_events(wait=True):
+if event['event'] == 'BLOCK_JOB_COMPLETED':
+node_name = self.dictpath(event, 'data/device')
+self.assertTrue(node_name in pending_jobs)
+self.assert_qmp_absent(event, 'data/error')
+pending_jobs.remove(node_name)
+if event['event'] == 'BLOCK_JOB_READY':
+self.assert_qmp(event, 'data/device', 'drive0')
+self.assert_qmp(event, 'data/type', 'commit')
+   

[Qemu-block] [PULL 56/56] qemu-iotests: fix 203 migration completion race

[Kevin Wolf]

From: Stefan Hajnoczi 

There is a race between the test's 'query-migrate' QMP command after the
QMP 'STOP' event and completing the migration:

The test case invokes 'query-migrate' upon receiving 'STOP'.  At this
point the migration thread may still be in the process of completing.
Therefore 'query-migrate' can return 'status': 'active' for a brief
window of time instead of 'status': 'completed'.  This results in
qemu-iotests 203 hanging.

Solve the race by enabling the 'events' migration capability, which
causes QEMU to emit migration-specific QMP events that do not suffer
from this race condition.  Wait for the QMP 'MIGRATION' event with
'status': 'completed'.

Reported-by: Max Reitz 
Signed-off-by: Stefan Hajnoczi 
Message-id: 20180305155926.25858-1-stefa...@redhat.com
Reviewed-by: Max Reitz 
Signed-off-by: Max Reitz 
---
 tests/qemu-iotests/203 | 15 +++
 tests/qemu-iotests/203.out |  5 +
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/tests/qemu-iotests/203 b/tests/qemu-iotests/203
index 2c811917d8..4874a1a0d8 100755
--- a/tests/qemu-iotests/203
+++ b/tests/qemu-iotests/203
@@ -49,11 +49,18 @@ with iotests.FilePath('disk0.img') as disk0_img_path, \
node_name='drive1-node', iothread='iothread0',
force=True))
 
+iotests.log('Enabling migration QMP events...')
+iotests.log(vm.qmp('migrate-set-capabilities', capabilities=[
+{
+'capability': 'events',
+'state': True
+}
+]))
+
 iotests.log('Starting migration...')
 iotests.log(vm.qmp('migrate', uri='exec:cat >/dev/null'))
 while True:
-vm.get_qmp_event(wait=60.0)
-result = vm.qmp('query-migrate')
-status = result.get('return', {}).get('status', None)
-if status == 'completed':
+event = vm.event_wait('MIGRATION')
+iotests.log(event, filters=[iotests.filter_qmp_event])
+if event['data']['status'] == 'completed':
 break
diff --git a/tests/qemu-iotests/203.out b/tests/qemu-iotests/203.out
index 3f1ff900e4..1a11f0975c 100644
--- a/tests/qemu-iotests/203.out
+++ b/tests/qemu-iotests/203.out
@@ -2,5 +2,10 @@ Launching VM...
 Setting IOThreads...
 {u'return': {}}
 {u'return': {}}
+Enabling migration QMP events...
+{u'return': {}}
 Starting migration...
 {u'return': {}}
+{u'timestamp': {u'seconds': 'SECS', u'microseconds': 'USECS'}, u'data': 
{u'status': u'setup'}, u'event': u'MIGRATION'}
+{u'timestamp': {u'seconds': 'SECS', u'microseconds': 'USECS'}, u'data': 
{u'status': u'active'}, u'event': u'MIGRATION'}
+{u'timestamp': {u'seconds': 'SECS', u'microseconds': 'USECS'}, u'data': 
{u'status': u'completed'}, u'event': u'MIGRATION'}
-- 
2.13.6

[Qemu-block] [PULL 46/56] ssh: Pass BlockdevOptionsSsh to connect_to_ssh()

[Kevin Wolf]

Move the parsing of the QDict options up to the callers, in preparation
for the .bdrv_co_create implementation that directly gets a QAPI type.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 block/ssh.c | 34 +-
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/block/ssh.c b/block/ssh.c
index 30cdf9a99f..80f59055cc 100644
--- a/block/ssh.c
+++ b/block/ssh.c
@@ -656,19 +656,13 @@ fail:
 return result;
 }
 
-static int connect_to_ssh(BDRVSSHState *s, QDict *options,
+static int connect_to_ssh(BDRVSSHState *s, BlockdevOptionsSsh *opts,
   int ssh_flags, int creat_mode, Error **errp)
 {
-BlockdevOptionsSsh *opts;
 int r, ret;
 const char *user;
 long port = 0;
 
-opts = ssh_parse_options(options, errp);
-if (opts == NULL) {
-return -EINVAL;
-}
-
 if (opts->has_user) {
 user = opts->user;
 } else {
@@ -748,8 +742,6 @@ static int connect_to_ssh(BDRVSSHState *s, QDict *options,
 goto err;
 }
 
-qapi_free_BlockdevOptionsSsh(opts);
-
 r = libssh2_sftp_fstat(s->sftp_handle, &s->attrs);
 if (r < 0) {
 sftp_error_setg(errp, s, "failed to read file attributes");
@@ -775,8 +767,6 @@ static int connect_to_ssh(BDRVSSHState *s, QDict *options,
 }
 s->session = NULL;
 
-qapi_free_BlockdevOptionsSsh(opts);
-
 return ret;
 }
 
@@ -784,6 +774,7 @@ static int ssh_file_open(BlockDriverState *bs, QDict 
*options, int bdrv_flags,
  Error **errp)
 {
 BDRVSSHState *s = bs->opaque;
+BlockdevOptionsSsh *opts;
 int ret;
 int ssh_flags;
 
@@ -794,8 +785,13 @@ static int ssh_file_open(BlockDriverState *bs, QDict 
*options, int bdrv_flags,
 ssh_flags |= LIBSSH2_FXF_WRITE;
 }
 
+opts = ssh_parse_options(options, errp);
+if (opts == NULL) {
+return -EINVAL;
+}
+
 /* Start up SSH. */
-ret = connect_to_ssh(s, options, ssh_flags, 0, errp);
+ret = connect_to_ssh(s, opts, ssh_flags, 0, errp);
 if (ret < 0) {
 goto err;
 }
@@ -803,6 +799,8 @@ static int ssh_file_open(BlockDriverState *bs, QDict 
*options, int bdrv_flags,
 /* Go non-blocking. */
 libssh2_session_set_blocking(s->session, 0);
 
+qapi_free_BlockdevOptionsSsh(opts);
+
 return 0;
 
  err:
@@ -811,6 +809,8 @@ static int ssh_file_open(BlockDriverState *bs, QDict 
*options, int bdrv_flags,
 }
 s->sock = -1;
 
+qapi_free_BlockdevOptionsSsh(opts);
+
 return ret;
 }
 
@@ -860,6 +860,7 @@ static int coroutine_fn ssh_co_create_opts(const char 
*filename, QemuOpts *opts,
 int r, ret;
 int64_t total_size = 0;
 QDict *uri_options = NULL;
+BlockdevOptionsSsh *ssh_opts = NULL;
 BDRVSSHState s;
 
 ssh_state_init(&s);
@@ -876,7 +877,13 @@ static int coroutine_fn ssh_co_create_opts(const char 
*filename, QemuOpts *opts,
 goto out;
 }
 
-r = connect_to_ssh(&s, uri_options,
+ssh_opts = ssh_parse_options(uri_options, errp);
+if (ssh_opts == NULL) {
+ret = -EINVAL;
+goto out;
+}
+
+r = connect_to_ssh(&s, ssh_opts,
LIBSSH2_FXF_READ|LIBSSH2_FXF_WRITE|
LIBSSH2_FXF_CREAT|LIBSSH2_FXF_TRUNC,
0644, errp);
@@ -899,6 +906,7 @@ static int coroutine_fn ssh_co_create_opts(const char 
*filename, QemuOpts *opts,
 if (uri_options != NULL) {
 QDECREF(uri_options);
 }
+qapi_free_BlockdevOptionsSsh(ssh_opts);
 return ret;
 }
 
-- 
2.13.6

[Qemu-block] [PULL 42/56] sheepdog: QAPIfy "redundancy" create option

[Kevin Wolf]

The "redundancy" option for Sheepdog image creation is currently a
string that can encode one or two integers depending on its format,
which at the same time implicitly selects a mode.

This patch turns it into a QAPI union and converts the string into such
a QAPI object before interpreting the values.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 qapi/block-core.json | 45 +
 block/sheepdog.c | 94 +---
 2 files changed, 112 insertions(+), 27 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 39e53c7791..e590ab6c71 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3467,6 +3467,51 @@
 '*cluster-size' :   'size' } }
 
 ##
+# @SheepdogRedundancyType:
+#
+# @full Create a fully replicated vdi with x copies
+# @erasure-codedCreate an erasure coded vdi with x data strips and
+#   y parity strips
+#
+# Since: 2.12
+##
+{ 'enum': 'SheepdogRedundancyType',
+  'data': [ 'full', 'erasure-coded' ] }
+
+##
+# @SheepdogRedundancyFull:
+#
+# @copies   Number of copies to use (between 1 and 31)
+#
+# Since: 2.12
+##
+{ 'struct': 'SheepdogRedundancyFull',
+  'data': { 'copies': 'int' }}
+
+##
+# @SheepdogRedundancyErasureCoded:
+#
+# @data-strips  Number of data strips to use (one of {2,4,8,16})
+# @parity-stripsNumber of parity strips to use (between 1 and 15)
+#
+# Since: 2.12
+##
+{ 'struct': 'SheepdogRedundancyErasureCoded',
+  'data': { 'data-strips': 'int',
+'parity-strips': 'int' }}
+
+##
+# @SheepdogRedundancy:
+#
+# Since: 2.12
+##
+{ 'union': 'SheepdogRedundancy',
+  'base': { 'type': 'SheepdogRedundancyType' },
+  'discriminator': 'type',
+  'data': { 'full': 'SheepdogRedundancyFull',
+'erasure-coded': 'SheepdogRedundancyErasureCoded' } }
+
+##
 # @BlockdevCreateNotSupported:
 #
 # This is used for all drivers that don't support creating images.
diff --git a/block/sheepdog.c b/block/sheepdog.c
index d8c10b7cac..3966cd229a 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -1882,6 +1882,48 @@ out_with_err_set:
 return ret;
 }
 
+static int parse_redundancy(BDRVSheepdogState *s, SheepdogRedundancy *opt)
+{
+struct SheepdogInode *inode = &s->inode;
+
+switch (opt->type) {
+case SHEEPDOG_REDUNDANCY_TYPE_FULL:
+if (opt->u.full.copies > SD_MAX_COPIES || opt->u.full.copies < 1) {
+return -EINVAL;
+}
+inode->copy_policy = 0;
+inode->nr_copies = opt->u.full.copies;
+return 0;
+
+case SHEEPDOG_REDUNDANCY_TYPE_ERASURE_CODED:
+{
+int64_t copy = opt->u.erasure_coded.data_strips;
+int64_t parity = opt->u.erasure_coded.parity_strips;
+
+if (copy != 2 && copy != 4 && copy != 8 && copy != 16) {
+return -EINVAL;
+}
+
+if (parity >= SD_EC_MAX_STRIP || parity < 1) {
+return -EINVAL;
+}
+
+/*
+ * 4 bits for parity and 4 bits for data.
+ * We have to compress upper data bits because it can't represent 16
+ */
+inode->copy_policy = ((copy / 2) << 4) + parity;
+inode->nr_copies = copy + parity;
+return 0;
+}
+
+default:
+g_assert_not_reached();
+}
+
+return -EINVAL;
+}
+
 /*
  * Sheepdog support two kinds of redundancy, full replication and erasure
  * coding.
@@ -1892,12 +1934,13 @@ out_with_err_set:
  * # create a erasure coded vdi with x data strips and y parity strips
  * -o redundancy=x:y (x must be one of {2,4,8,16} and 1 <= y < SD_EC_MAX_STRIP)
  */
-static int parse_redundancy(BDRVSheepdogState *s, const char *opt)
+static int parse_redundancy_str(BDRVSheepdogState *s, const char *opt)
 {
-struct SheepdogInode *inode = &s->inode;
+struct SheepdogRedundancy redundancy;
 const char *n1, *n2;
 long copy, parity;
 char p[10];
+int ret;
 
 pstrcpy(p, sizeof(p), opt);
 n1 = strtok(p, ":");
@@ -1907,35 +1950,32 @@ static int parse_redundancy(BDRVSheepdogState *s, const 
char *opt)
 return -EINVAL;
 }
 
-copy = strtol(n1, NULL, 10);
-/* FIXME fix error checking by switching to qemu_strtol() */
-if (copy > SD_MAX_COPIES || copy < 1) {
-return -EINVAL;
-}
-if (!n2) {
-inode->copy_policy = 0;
-inode->nr_copies = copy;
-return 0;
+ret = qemu_strtol(n1, NULL, 10, ©);
+if (ret < 0) {
+return ret;
 }
 
-if (copy != 2 && copy != 4 && copy != 8 && copy != 16) {
-return -EINVAL;
-}
+if (!n2) {
+redundancy = (SheepdogRedundancy) {
+.type   = SHEEPDOG_REDUNDANCY_TYPE_FULL,
+.u.full.copies  = copy,
+};
+} else {
+ret = qemu_strtol(n2, NULL, 10, &parity);
+if (ret < 0) {
+return ret;
+}
 
-parity = strtol(n2, NULL, 10);
-/* FIXME fix error checking by switching to qemu_strtol() 

[Qemu-block] [PULL 21/56] qcow2: Use QCryptoBlockCreateOptions in qcow2_co_create()

[Kevin Wolf]

Instead of passing the encryption format name and the QemuOpts down, use
the QCryptoBlockCreateOptions contained in BlockdevCreateOptions.

Signed-off-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Reviewed-by: Max Reitz 
---
 block/qcow2.c | 62 +++
 1 file changed, 45 insertions(+), 17 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index b7df2d5cab..e1821eb3c8 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -2449,13 +2449,10 @@ static int qcow2_crypt_method_from_format(const char 
*encryptfmt)
 }
 }
 
-static int qcow2_set_up_encryption(BlockDriverState *bs, const char 
*encryptfmt,
-   QemuOpts *opts, Error **errp)
+static QCryptoBlockCreateOptions *
+qcow2_parse_encryption(const char *encryptfmt, QemuOpts *opts, Error **errp)
 {
-BDRVQcow2State *s = bs->opaque;
 QCryptoBlockCreateOptions *cryptoopts = NULL;
-QCryptoBlock *crypto = NULL;
-int ret = -EINVAL;
 QDict *options, *encryptopts;
 int fmt;
 
@@ -2478,10 +2475,31 @@ static int qcow2_set_up_encryption(BlockDriverState 
*bs, const char *encryptfmt,
 error_setg(errp, "Unknown encryption format '%s'", encryptfmt);
 break;
 }
-if (!cryptoopts) {
-ret = -EINVAL;
-goto out;
+
+QDECREF(encryptopts);
+return cryptoopts;
+}
+
+static int qcow2_set_up_encryption(BlockDriverState *bs,
+   QCryptoBlockCreateOptions *cryptoopts,
+   Error **errp)
+{
+BDRVQcow2State *s = bs->opaque;
+QCryptoBlock *crypto = NULL;
+int fmt, ret;
+
+switch (cryptoopts->format) {
+case Q_CRYPTO_BLOCK_FORMAT_LUKS:
+fmt = QCOW_CRYPT_LUKS;
+break;
+case Q_CRYPTO_BLOCK_FORMAT_QCOW:
+fmt = QCOW_CRYPT_AES;
+break;
+default:
+error_setg(errp, "Crypto format not supported in qcow2");
+return -EINVAL;
 }
+
 s->crypt_method_header = fmt;
 
 crypto = qcrypto_block_create(cryptoopts, "encrypt.",
@@ -2489,8 +2507,7 @@ static int qcow2_set_up_encryption(BlockDriverState *bs, 
const char *encryptfmt,
   qcow2_crypto_hdr_write_func,
   bs, errp);
 if (!crypto) {
-ret = -EINVAL;
-goto out;
+return -EINVAL;
 }
 
 ret = qcow2_update_header(bs);
@@ -2499,10 +2516,9 @@ static int qcow2_set_up_encryption(BlockDriverState *bs, 
const char *encryptfmt,
 goto out;
 }
 
+ret = 0;
  out:
-QDECREF(encryptopts);
 qcrypto_block_free(crypto);
-qapi_free_QCryptoBlockCreateOptions(cryptoopts);
 return ret;
 }
 
@@ -2768,8 +2784,7 @@ static uint64_t qcow2_opt_get_refcount_bits_del(QemuOpts 
*opts, int version,
 }
 
 static int coroutine_fn
-qcow2_co_create(BlockdevCreateOptions *create_options, QemuOpts *opts,
-const char *encryptfmt, Error **errp)
+qcow2_co_create(BlockdevCreateOptions *create_options, Error **errp)
 {
 BlockdevCreateOptionsQcow2 *qcow2_opts;
 QDict *options;
@@ -2999,8 +3014,8 @@ qcow2_co_create(BlockdevCreateOptions *create_options, 
QemuOpts *opts,
 }
 
 /* Want encryption? There you go. */
-if (encryptfmt) {
-ret = qcow2_set_up_encryption(blk_bs(blk), encryptfmt, opts, errp);
+if (qcow2_opts->has_encrypt) {
+ret = qcow2_set_up_encryption(blk_bs(blk), qcow2_opts->encrypt, errp);
 if (ret < 0) {
 goto out;
 }
@@ -3058,6 +3073,7 @@ static int coroutine_fn qcow2_co_create_opts(const char 
*filename, QemuOpts *opt
 int version;
 uint64_t refcount_bits;
 char *encryptfmt = NULL;
+QCryptoBlockCreateOptions *cryptoopts = NULL;
 BlockDriverState *bs = NULL;
 Error *local_err = NULL;
 int ret;
@@ -3074,6 +3090,7 @@ static int coroutine_fn qcow2_co_create_opts(const char 
*filename, QemuOpts *opt
 ret = -EINVAL;
 goto finish;
 }
+
 encryptfmt = qemu_opt_get_del(opts, BLOCK_OPT_ENCRYPT_FORMAT);
 if (encryptfmt) {
 if (qemu_opt_get(opts, BLOCK_OPT_ENCRYPT)) {
@@ -3085,6 +3102,14 @@ static int coroutine_fn qcow2_co_create_opts(const char 
*filename, QemuOpts *opt
 } else if (qemu_opt_get_bool_del(opts, BLOCK_OPT_ENCRYPT, false)) {
 encryptfmt = g_strdup("aes");
 }
+if (encryptfmt) {
+cryptoopts = qcow2_parse_encryption(encryptfmt, opts, errp);
+if (cryptoopts == NULL) {
+ret = -EINVAL;
+goto finish;
+}
+}
+
 cluster_size = qcow2_opt_get_cluster_size_del(opts, &local_err);
 if (local_err) {
 error_propagate(errp, local_err);
@@ -3158,6 +3183,8 @@ static int coroutine_fn qcow2_co_create_opts(const char 
*filename, QemuOpts *opt
 .backing_file   = backing_file,
 .has_backing_fmt= (backing_fmt != NULL),
 .backing_fmt= backing_drv,
+.has_encrypt= (encryptfmt != NULL

[Qemu-block] [PULL 20/56] qcow2: Use BlockdevRef in qcow2_co_create()

[Kevin Wolf]

Instead of passing a separate BlockDriverState* into qcow2_co_create(),
make use of the BlockdevRef that is included in BlockdevCreateOptions.

Signed-off-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Reviewed-by: Max Reitz 
---
 include/block/block.h |  1 +
 block.c   | 47 +++
 block/qcow2.c | 39 +--
 3 files changed, 73 insertions(+), 14 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index 8b6db952a2..7805187b30 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -246,6 +246,7 @@ BdrvChild *bdrv_open_child(const char *filename,
BlockDriverState* parent,
const BdrvChildRole *child_role,
bool allow_none, Error **errp);
+BlockDriverState *bdrv_open_blockdev_ref(BlockdevRef *ref, Error **errp);
 void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd,
  Error **errp);
 int bdrv_open_backing_file(BlockDriverState *bs, QDict *parent_options,
diff --git a/block.c b/block.c
index 8f1c43d037..4fc65f7621 100644
--- a/block.c
+++ b/block.c
@@ -34,6 +34,8 @@
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qstring.h"
+#include "qapi/qobject-output-visitor.h"
+#include "qapi/qapi-visit-block-core.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "qemu/notify.h"
@@ -2406,6 +2408,51 @@ BdrvChild *bdrv_open_child(const char *filename,
 return c;
 }
 
+/* TODO Future callers may need to specify parent/child_role in order for
+ * option inheritance to work. Existing callers use it for the root node. */
+BlockDriverState *bdrv_open_blockdev_ref(BlockdevRef *ref, Error **errp)
+{
+BlockDriverState *bs = NULL;
+Error *local_err = NULL;
+QObject *obj = NULL;
+QDict *qdict = NULL;
+const char *reference = NULL;
+Visitor *v = NULL;
+
+if (ref->type == QTYPE_QSTRING) {
+reference = ref->u.reference;
+} else {
+BlockdevOptions *options = &ref->u.definition;
+assert(ref->type == QTYPE_QDICT);
+
+v = qobject_output_visitor_new(&obj);
+visit_type_BlockdevOptions(v, NULL, &options, &local_err);
+if (local_err) {
+error_propagate(errp, local_err);
+goto fail;
+}
+visit_complete(v, &obj);
+
+qdict = qobject_to_qdict(obj);
+qdict_flatten(qdict);
+
+/* bdrv_open_inherit() defaults to the values in bdrv_flags (for
+ * compatibility with other callers) rather than what we want as the
+ * real defaults. Apply the defaults here instead. */
+qdict_set_default_str(qdict, BDRV_OPT_CACHE_DIRECT, "off");
+qdict_set_default_str(qdict, BDRV_OPT_CACHE_NO_FLUSH, "off");
+qdict_set_default_str(qdict, BDRV_OPT_READ_ONLY, "off");
+}
+
+bs = bdrv_open_inherit(NULL, reference, qdict, 0, NULL, NULL, errp);
+obj = NULL;
+
+fail:
+qobject_decref(obj);
+visit_free(v);
+return bs;
+}
+
 static BlockDriverState *bdrv_append_temp_snapshot(BlockDriverState *bs,
int flags,
QDict *snapshot_options,
diff --git a/block/qcow2.c b/block/qcow2.c
index 7679c28f57..b7df2d5cab 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -2768,8 +2768,8 @@ static uint64_t qcow2_opt_get_refcount_bits_del(QemuOpts 
*opts, int version,
 }
 
 static int coroutine_fn
-qcow2_co_create(BlockDriverState *bs, BlockdevCreateOptions *create_options,
-QemuOpts *opts, const char *encryptfmt, Error **errp)
+qcow2_co_create(BlockdevCreateOptions *create_options, QemuOpts *opts,
+const char *encryptfmt, Error **errp)
 {
 BlockdevCreateOptionsQcow2 *qcow2_opts;
 QDict *options;
@@ -2786,7 +2786,8 @@ qcow2_co_create(BlockDriverState *bs, 
BlockdevCreateOptions *create_options,
  * 2 GB for 64k clusters, and we don't want to have a 2 GB initial file
  * size for any qcow2 image.
  */
-BlockBackend *blk;
+BlockBackend *blk = NULL;
+BlockDriverState *bs = NULL;
 QCowHeader *header;
 size_t cluster_size;
 int version;
@@ -2795,10 +2796,15 @@ qcow2_co_create(BlockDriverState *bs, 
BlockdevCreateOptions *create_options,
 Error *local_err = NULL;
 int ret;
 
-/* Validate options and set default values */
 assert(create_options->driver == BLOCKDEV_DRIVER_QCOW2);
 qcow2_opts = &create_options->u.qcow2;
 
+bs = bdrv_open_blockdev_ref(qcow2_opts->file, errp);
+if (bs == NULL) {
+return -EIO;
+}
+
+/* Validate options and set default values */
 if (!QEMU_IS_ALIGNED(qcow2_opts->size, BDRV_SECTOR_SIZE)) {
 error_setg(errp, "Image size must be a multiple of 512 bytes");
 ret = -EINVAL;
@@ -2827,7 +2833,8 @@ qcow2_co_create(BlockDriverState *bs, 
BlockdevCrea

[Qemu-block] [PULL 39/56] rbd: Use qemu_rbd_connect() in qemu_rbd_do_create()

[Kevin Wolf]

This is almost exactly the same code. The differences are that
qemu_rbd_connect() supports BlockdevOptionsRbd.server and that the cache
mode is set explicitly.

Supporting 'server' is a welcome new feature for image creation.
Caching is disabled by default, so leave it that way.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 block/rbd.c | 54 ++
 1 file changed, 10 insertions(+), 44 deletions(-)

diff --git a/block/rbd.c b/block/rbd.c
index 2ac7ffca42..294ed07ac4 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -103,6 +103,11 @@ typedef struct BDRVRBDState {
 char *snap;
 } BDRVRBDState;
 
+static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t *io_ctx,
+BlockdevOptionsRbd *opts, bool cache,
+const char *keypairs, const char *secretid,
+Error **errp);
+
 static char *qemu_rbd_next_tok(char *src, char delim, char **p)
 {
 char *end;
@@ -351,12 +356,6 @@ static int qemu_rbd_do_create(BlockdevCreateOptions 
*options,
 return -EINVAL;
 }
 
-/* TODO Remove the limitation */
-if (opts->location->has_server) {
-error_setg(errp, "Can't specify server for image creation");
-return -EINVAL;
-}
-
 if (opts->has_cluster_size) {
 int64_t objsize = opts->cluster_size;
 if ((objsize - 1) & objsize) {/* not a power of 2? */
@@ -370,54 +369,21 @@ static int qemu_rbd_do_create(BlockdevCreateOptions 
*options,
 obj_order = ctz32(objsize);
 }
 
-ret = rados_create(&cluster, opts->location->user);
+ret = qemu_rbd_connect(&cluster, &io_ctx, opts->location, false, keypairs,
+   password_secret, errp);
 if (ret < 0) {
-error_setg_errno(errp, -ret, "error initializing");
 return ret;
 }
 
-/* try default location when conf=NULL, but ignore failure */
-ret = rados_conf_read_file(cluster, opts->location->conf);
-if (opts->location->conf && ret < 0) {
-error_setg_errno(errp, -ret, "error reading conf file %s",
- opts->location->conf);
-ret = -EIO;
-goto shutdown;
-}
-
-ret = qemu_rbd_set_keypairs(cluster, keypairs, errp);
-if (ret < 0) {
-ret = -EIO;
-goto shutdown;
-}
-
-if (qemu_rbd_set_auth(cluster, password_secret, errp) < 0) {
-ret = -EIO;
-goto shutdown;
-}
-
-ret = rados_connect(cluster);
-if (ret < 0) {
-error_setg_errno(errp, -ret, "error connecting");
-goto shutdown;
-}
-
-ret = rados_ioctx_create(cluster, opts->location->pool, &io_ctx);
-if (ret < 0) {
-error_setg_errno(errp, -ret, "error opening pool %s",
- opts->location->pool);
-goto shutdown;
-}
-
 ret = rbd_create(io_ctx, opts->location->image, opts->size, &obj_order);
 if (ret < 0) {
 error_setg_errno(errp, -ret, "error rbd create");
+goto out;
 }
 
-rados_ioctx_destroy(io_ctx);
-
 ret = 0;
-shutdown:
+out:
+rados_ioctx_destroy(io_ctx);
 rados_shutdown(cluster);
 return ret;
 }
-- 
2.13.6

[Qemu-block] [PULL 48/56] file-posix: Fix no-op bdrv_truncate() with falloc preallocation

[Kevin Wolf]

If bdrv_truncate() is called, but the requested size is the same as
before, don't call posix_fallocate(), which returns -EINVAL for length
zero and would therefore make bdrv_truncate() fail.

The problem can be triggered by creating a zero-sized raw image with
'falloc' preallocation mode.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 block/file-posix.c | 14 +-
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/block/file-posix.c b/block/file-posix.c
index fbc21a9921..d7fb772c14 100644
--- a/block/file-posix.c
+++ b/block/file-posix.c
@@ -1686,11 +1686,15 @@ static int raw_regular_truncate(int fd, int64_t offset, 
PreallocMode prealloc,
  * file systems that do not support fallocate(), trying to check if a
  * block is allocated before allocating it, so don't do that here.
  */
-result = -posix_fallocate(fd, current_length, offset - current_length);
-if (result != 0) {
-/* posix_fallocate() doesn't set errno. */
-error_setg_errno(errp, -result,
- "Could not preallocate new data");
+if (offset != current_length) {
+result = -posix_fallocate(fd, current_length, offset - 
current_length);
+if (result != 0) {
+/* posix_fallocate() doesn't set errno. */
+error_setg_errno(errp, -result,
+ "Could not preallocate new data");
+}
+} else {
+result = 0;
 }
 goto out;
 #endif
-- 
2.13.6

[Qemu-block] [PULL 41/56] nfs: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to nfs, which enables
image creation over QMP.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 qapi/block-core.json | 16 ++-
 block/nfs.c  | 76 +---
 2 files changed, 75 insertions(+), 17 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index d4351877fc..39e53c7791 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3393,6 +3393,20 @@
 '*preallocation':   'PreallocMode' } }
 
 ##
+# @BlockdevCreateOptionsNfs:
+#
+# Driver specific image creation options for NFS.
+#
+# @location Where to store the new image file
+# @size Size of the virtual disk in bytes
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsNfs',
+  'data': { 'location': 'BlockdevOptionsNfs',
+'size': 'size' } }
+
+##
 # @BlockdevQcow2Version:
 #
 # @v2:  The original QCOW2 format as introduced in qemu 0.10 (version 2)
@@ -3491,7 +3505,7 @@
   'iscsi':  'BlockdevCreateNotSupported',
   'luks':   'BlockdevCreateNotSupported',
   'nbd':'BlockdevCreateNotSupported',
-  'nfs':'BlockdevCreateNotSupported',
+  'nfs':'BlockdevCreateOptionsNfs',
   'null-aio':   'BlockdevCreateNotSupported',
   'null-co':'BlockdevCreateNotSupported',
   'nvme':   'BlockdevCreateNotSupported',
diff --git a/block/nfs.c b/block/nfs.c
index e402d643fe..2577df4b26 100644
--- a/block/nfs.c
+++ b/block/nfs.c
@@ -551,33 +551,45 @@ out:
 return ret;
 }
 
-static int64_t nfs_client_open_qdict(NFSClient *client, QDict *options,
- int flags, int open_flags, Error **errp)
+static BlockdevOptionsNfs *nfs_options_qdict_to_qapi(QDict *options,
+ Error **errp)
 {
 BlockdevOptionsNfs *opts = NULL;
 QObject *crumpled = NULL;
 Visitor *v;
 Error *local_err = NULL;
-int ret;
 
 crumpled = qdict_crumple(options, errp);
 if (crumpled == NULL) {
-return -EINVAL;
+return NULL;
 }
 
 v = qobject_input_visitor_new_keyval(crumpled);
 visit_type_BlockdevOptionsNfs(v, NULL, &opts, &local_err);
 visit_free(v);
+qobject_decref(crumpled);
 
 if (local_err) {
-error_propagate(errp, local_err);
+return NULL;
+}
+
+return opts;
+}
+
+static int64_t nfs_client_open_qdict(NFSClient *client, QDict *options,
+ int flags, int open_flags, Error **errp)
+{
+BlockdevOptionsNfs *opts;
+int ret;
+
+opts = nfs_options_qdict_to_qapi(options, errp);
+if (opts == NULL) {
 ret = -EINVAL;
 goto fail;
 }
 
 ret = nfs_client_open(client, opts, flags, open_flags, errp);
 fail:
-qobject_decref(crumpled);
 qapi_free_BlockdevOptionsNfs(opts);
 return ret;
 }
@@ -614,18 +626,43 @@ static QemuOptsList nfs_create_opts = {
 }
 };
 
-static int coroutine_fn nfs_file_co_create_opts(const char *url, QemuOpts 
*opts,
-Error **errp)
+static int nfs_file_co_create(BlockdevCreateOptions *options, Error **errp)
 {
-int64_t ret, total_size;
+BlockdevCreateOptionsNfs *opts = &options->u.nfs;
 NFSClient *client = g_new0(NFSClient, 1);
-QDict *options = NULL;
+int ret;
+
+assert(options->driver == BLOCKDEV_DRIVER_NFS);
 
 client->aio_context = qemu_get_aio_context();
 
+ret = nfs_client_open(client, opts->location, O_CREAT, 0, errp);
+if (ret < 0) {
+goto out;
+}
+ret = nfs_ftruncate(client->context, client->fh, opts->size);
+nfs_client_close(client);
+
+out:
+g_free(client);
+return ret;
+}
+
+static int coroutine_fn nfs_file_co_create_opts(const char *url, QemuOpts 
*opts,
+Error **errp)
+{
+BlockdevCreateOptions *create_options;
+BlockdevCreateOptionsNfs *nfs_opts;
+QDict *options;
+int ret;
+
+create_options = g_new0(BlockdevCreateOptions, 1);
+create_options->driver = BLOCKDEV_DRIVER_NFS;
+nfs_opts = &create_options->u.nfs;
+
 /* Read out options */
-total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
-  BDRV_SECTOR_SIZE);
+nfs_opts->size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+  BDRV_SECTOR_SIZE);
 
 options = qdict_new();
 ret = nfs_parse_uri(url, options, errp);
@@ -633,15 +670,21 @@ static int coroutine_fn nfs_file_co_create_opts(const 
char *url, QemuOpts *opts,
 goto out;
 }
 
-ret = nfs_client_open_qdict(client, options, O_CREAT, 0, errp);
+nfs_opts->location = nfs_options_qdict_to_qapi(options, errp);
+if (nfs_opts->location == NULL) {
+ret = -EINVAL;
+goto out;
+}
+
+ret = nfs_file_co_create(cre

[Qemu-block] [PULL 38/56] rbd: Assign s->snap/image_name in qemu_rbd_open()

[Kevin Wolf]

Now that the options are already available in qemu_rbd_open() and not
only parsed in qemu_rbd_connect(), we can assign s->snap and
s->image_name there instead of passing the fields by reference to
qemu_rbd_connect().

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 block/rbd.c | 14 +-
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/block/rbd.c b/block/rbd.c
index 1cd526bcea..2ac7ffca42 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -571,7 +571,6 @@ static char *qemu_rbd_mon_host(BlockdevOptionsRbd *opts, 
Error **errp)
 }
 
 static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t *io_ctx,
-char **s_snap, char **s_image_name,
 BlockdevOptionsRbd *opts, bool cache,
 const char *keypairs, const char *secretid,
 Error **errp)
@@ -593,9 +592,6 @@ static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t 
*io_ctx,
 goto failed_opts;
 }
 
-*s_snap = g_strdup(opts->snapshot);
-*s_image_name = g_strdup(opts->image);
-
 /* try default location when conf=NULL, but ignore failure */
 r = rados_conf_read_file(*cluster, opts->conf);
 if (opts->has_conf && r < 0) {
@@ -649,8 +645,6 @@ static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t 
*io_ctx,
 
 failed_shutdown:
 rados_shutdown(*cluster);
-g_free(*s_snap);
-g_free(*s_image_name);
 failed_opts:
 g_free(mon_host);
 return r;
@@ -711,13 +705,15 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict 
*options, int flags,
 goto out;
 }
 
-r = qemu_rbd_connect(&s->cluster, &s->io_ctx, &s->snap, &s->image_name,
- opts, !(flags & BDRV_O_NOCACHE), keypairs, secretid,
- errp);
+r = qemu_rbd_connect(&s->cluster, &s->io_ctx, opts,
+ !(flags & BDRV_O_NOCACHE), keypairs, secretid, errp);
 if (r < 0) {
 goto out;
 }
 
+s->snap = g_strdup(opts->snapshot);
+s->image_name = g_strdup(opts->image);
+
 /* rbd_open is always r/w */
 r = rbd_open(s->io_ctx, s->image_name, &s->image, s->snap);
 if (r < 0) {
-- 
2.13.6

[Qemu-block] [PULL 37/56] rbd: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to rbd, which enables
image creation over QMP.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 qapi/block-core.json |  19 ++-
 block/rbd.c  | 150 ++-
 2 files changed, 118 insertions(+), 51 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 9170fbf6e6..d4351877fc 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3436,6 +3436,23 @@
 '*refcount-bits':   'int' } }
 
 ##
+# @BlockdevCreateOptionsRbd:
+#
+# Driver specific image creation options for rbd/Ceph.
+#
+# @location Where to store the new image file. This location cannot
+#   point to a snapshot.
+# @size Size of the virtual disk in bytes
+# @cluster-size RBD object size
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsRbd',
+  'data': { 'location': 'BlockdevOptionsRbd',
+'size': 'size',
+'*cluster-size' :   'size' } }
+
+##
 # @BlockdevCreateNotSupported:
 #
 # This is used for all drivers that don't support creating images.
@@ -3484,7 +3501,7 @@
   'qed':'BlockdevCreateNotSupported',
   'quorum': 'BlockdevCreateNotSupported',
   'raw':'BlockdevCreateNotSupported',
-  'rbd':'BlockdevCreateNotSupported',
+  'rbd':'BlockdevCreateOptionsRbd',
   'replication':'BlockdevCreateNotSupported',
   'sheepdog':   'BlockdevCreateNotSupported',
   'ssh':'BlockdevCreateNotSupported',
diff --git a/block/rbd.c b/block/rbd.c
index 999fea105f..1cd526bcea 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -332,71 +332,55 @@ static QemuOptsList runtime_opts = {
 },
 };
 
-static int coroutine_fn qemu_rbd_co_create_opts(const char *filename,
-QemuOpts *opts,
-Error **errp)
+/* FIXME Deprecate and remove keypairs or make it available in QMP.
+ * password_secret should eventually be configurable in opts->location. Support
+ * for it in .bdrv_open will make it work here as well. */
+static int qemu_rbd_do_create(BlockdevCreateOptions *options,
+  const char *keypairs, const char 
*password_secret,
+  Error **errp)
 {
-Error *local_err = NULL;
-int64_t bytes = 0;
-int64_t objsize;
-int obj_order = 0;
-const char *pool, *image_name, *conf, *user, *keypairs;
-const char *secretid;
+BlockdevCreateOptionsRbd *opts = &options->u.rbd;
 rados_t cluster;
 rados_ioctx_t io_ctx;
-QDict *options = NULL;
-int ret = 0;
+int obj_order = 0;
+int ret;
+
+assert(options->driver == BLOCKDEV_DRIVER_RBD);
+if (opts->location->has_snapshot) {
+error_setg(errp, "Can't use snapshot name for image creation");
+return -EINVAL;
+}
 
-secretid = qemu_opt_get(opts, "password-secret");
+/* TODO Remove the limitation */
+if (opts->location->has_server) {
+error_setg(errp, "Can't specify server for image creation");
+return -EINVAL;
+}
 
-/* Read out options */
-bytes = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
- BDRV_SECTOR_SIZE);
-objsize = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE, 0);
-if (objsize) {
+if (opts->has_cluster_size) {
+int64_t objsize = opts->cluster_size;
 if ((objsize - 1) & objsize) {/* not a power of 2? */
 error_setg(errp, "obj size needs to be power of 2");
-ret = -EINVAL;
-goto exit;
+return -EINVAL;
 }
 if (objsize < 4096) {
 error_setg(errp, "obj size too small");
-ret = -EINVAL;
-goto exit;
+return -EINVAL;
 }
 obj_order = ctz32(objsize);
 }
 
-options = qdict_new();
-qemu_rbd_parse_filename(filename, options, &local_err);
-if (local_err) {
-ret = -EINVAL;
-error_propagate(errp, local_err);
-goto exit;
-}
-
-/*
- * Caution: while qdict_get_try_str() is fine, getting non-string
- * types would require more care.  When @options come from -blockdev
- * or blockdev_add, its members are typed according to the QAPI
- * schema, but when they come from -drive, they're all QString.
- */
-pool   = qdict_get_try_str(options, "pool");
-conf   = qdict_get_try_str(options, "conf");
-user   = qdict_get_try_str(options, "user");
-image_name = qdict_get_try_str(options, "image");
-keypairs   = qdict_get_try_str(options, "=keyvalue-pairs");
-
-ret = rados_create(&cluster, user);
+ret = rados_create(&cluster, opts->location->user);
 if (ret < 0) {
 error_setg_errno(errp, -ret, "error initializing");
-goto exit;
+return ret;
 }
 
 

[Qemu-block] [PULL 35/56] rbd: Remove non-schema options from runtime_opts

[Kevin Wolf]

Instead of the QemuOpts in qemu_rbd_connect(), we want to use QAPI
objects. As a preparation, fetch those options directly from the QDict
that .bdrv_open() supports in the rbd driver and that are not in the
schema.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
---
 block/rbd.c | 55 ---
 1 file changed, 24 insertions(+), 31 deletions(-)

diff --git a/block/rbd.c b/block/rbd.c
index 99fcc7ecdf..a979107f65 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -326,28 +326,6 @@ static QemuOptsList runtime_opts = {
 /*
  * server.* extracted manually, see qemu_rbd_mon_host()
  */
-{
-.name = "password-secret",
-.type = QEMU_OPT_STRING,
-.help = "ID of secret providing the password",
-},
-
-/*
- * Keys for qemu_rbd_parse_filename(), not in the QAPI schema
- */
-{
-/*
- * HACK: name starts with '=' so that qemu_opts_parse()
- * can't set it
- */
-.name = "=keyvalue-pairs",
-.type = QEMU_OPT_STRING,
-.help = "Legacy rados key/value option parameters",
-},
-{
-.name = "filename",
-.type = QEMU_OPT_STRING,
-},
 { /* end of list */ }
 },
 };
@@ -548,12 +526,13 @@ out:
 
 static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t *io_ctx,
 char **s_snap, char **s_image_name,
-QDict *options, bool cache, Error **errp)
+QDict *options, bool cache,
+const char *keypairs, const char *secretid,
+Error **errp)
 {
 QemuOpts *opts;
 char *mon_host = NULL;
-const char *pool, *snap, *conf, *user, *image_name, *keypairs;
-const char *secretid;
+const char *pool, *snap, *conf, *user, *image_name;
 Error *local_err = NULL;
 int r;
 
@@ -572,14 +551,11 @@ static int qemu_rbd_connect(rados_t *cluster, 
rados_ioctx_t *io_ctx,
 goto failed_opts;
 }
 
-secretid = qemu_opt_get(opts, "password-secret");
-
 pool   = qemu_opt_get(opts, "pool");
 conf   = qemu_opt_get(opts, "conf");
 snap   = qemu_opt_get(opts, "snapshot");
 user   = qemu_opt_get(opts, "user");
 image_name = qemu_opt_get(opts, "image");
-keypairs   = qemu_opt_get(opts, "=keyvalue-pairs");
 
 if (!pool || !image_name) {
 error_setg(errp, "Parameters 'pool' and 'image' are required");
@@ -664,6 +640,7 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict 
*options, int flags,
 BDRVRBDState *s = bs->opaque;
 Error *local_err = NULL;
 const char *filename;
+char *keypairs, *secretid;
 int r;
 
 /* If we are given a filename, parse the filename, with precedence given to
@@ -674,16 +651,28 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict 
*options, int flags,
 "This is an unsupported option, and may be deprecated "
 "in the future");
 qemu_rbd_parse_filename(filename, options, &local_err);
+qdict_del(options, "filename");
 if (local_err) {
 error_propagate(errp, local_err);
 return -EINVAL;
 }
 }
 
+keypairs = g_strdup(qdict_get_try_str(options, "=keyvalue-pairs"));
+if (keypairs) {
+qdict_del(options, "=keyvalue-pairs");
+}
+
+secretid = g_strdup(qdict_get_try_str(options, "password-secret"));
+if (secretid) {
+qdict_del(options, "password-secret");
+}
+
 r = qemu_rbd_connect(&s->cluster, &s->io_ctx, &s->snap, &s->image_name,
- options, !(flags & BDRV_O_NOCACHE), errp);
+ options, !(flags & BDRV_O_NOCACHE), keypairs, 
secretid,
+ errp);
 if (r < 0) {
-return r;
+goto out;
 }
 
 /* rbd_open is always r/w */
@@ -710,13 +699,17 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict 
*options, int flags,
 }
 }
 
-return 0;
+r = 0;
+goto out;
 
 failed_open:
 rados_ioctx_destroy(s->io_ctx);
 g_free(s->snap);
 g_free(s->image_name);
 rados_shutdown(s->cluster);
+out:
+g_free(keypairs);
+g_free(secretid);
 return r;
 }
 
-- 
2.13.6

[Qemu-block] [PULL 32/56] gluster: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to gluster, which enables
image creation over QMP.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 qapi/block-core.json |  18 ++-
 block/gluster.c  | 135 ++-
 2 files changed, 108 insertions(+), 45 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 41955b097f..9170fbf6e6 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3377,6 +3377,22 @@
 '*nocow':   'bool' } }
 
 ##
+# @BlockdevCreateOptionsGluster:
+#
+# Driver specific image creation options for gluster.
+#
+# @location Where to store the new image file
+# @size Size of the virtual disk in bytes
+# @preallocationPreallocation mode for the new image (default: off)
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsGluster',
+  'data': { 'location': 'BlockdevOptionsGluster',
+'size': 'size',
+'*preallocation':   'PreallocMode' } }
+
+##
 # @BlockdevQcow2Version:
 #
 # @v2:  The original QCOW2 format as introduced in qemu 0.10 (version 2)
@@ -3450,7 +3466,7 @@
   'file':   'BlockdevCreateOptionsFile',
   'ftp':'BlockdevCreateNotSupported',
   'ftps':   'BlockdevCreateNotSupported',
-  'gluster':'BlockdevCreateNotSupported',
+  'gluster':'BlockdevCreateOptionsGluster',
   'host_cdrom': 'BlockdevCreateNotSupported',
   'host_device':'BlockdevCreateNotSupported',
   'http':   'BlockdevCreateNotSupported',
diff --git a/block/gluster.c b/block/gluster.c
index 79b4cfdf74..63d3c37d4c 100644
--- a/block/gluster.c
+++ b/block/gluster.c
@@ -655,9 +655,11 @@ out:
 return -errno;
 }
 
-static struct glfs *qemu_gluster_init(BlockdevOptionsGluster *gconf,
-  const char *filename,
-  QDict *options, Error **errp)
+/* Converts options given in @filename and the @options QDict into the QAPI
+ * object @gconf. */
+static int qemu_gluster_parse(BlockdevOptionsGluster *gconf,
+  const char *filename,
+  QDict *options, Error **errp)
 {
 int ret;
 if (filename) {
@@ -668,8 +670,7 @@ static struct glfs 
*qemu_gluster_init(BlockdevOptionsGluster *gconf,
 "[host[:port]]volume/path[?socket=...]"
 "[,file.debug=N]"
 "[,file.logfile=/path/filename.log]\n");
-errno = -ret;
-return NULL;
+return ret;
 }
 } else {
 ret = qemu_gluster_parse_json(gconf, options, errp);
@@ -685,10 +686,23 @@ static struct glfs 
*qemu_gluster_init(BlockdevOptionsGluster *gconf,
  "file.server.1.transport=unix,"
  "file.server.1.socket=/var/run/glusterd.socket 
..."
  "\n");
-errno = -ret;
-return NULL;
+return ret;
 }
+}
 
+return 0;
+}
+
+static struct glfs *qemu_gluster_init(BlockdevOptionsGluster *gconf,
+  const char *filename,
+  QDict *options, Error **errp)
+{
+int ret;
+
+ret = qemu_gluster_parse(gconf, filename, options, errp);
+if (ret < 0) {
+errno = -ret;
+return NULL;
 }
 
 return qemu_gluster_glfs_init(gconf, errp);
@@ -1021,20 +1035,72 @@ static int qemu_gluster_do_truncate(struct glfs_fd *fd, 
int64_t offset,
 return 0;
 }
 
+static int qemu_gluster_co_create(BlockdevCreateOptions *options,
+  Error **errp)
+{
+BlockdevCreateOptionsGluster *opts = &options->u.gluster;
+struct glfs *glfs;
+struct glfs_fd *fd = NULL;
+int ret = 0;
+
+assert(options->driver == BLOCKDEV_DRIVER_GLUSTER);
+
+glfs = qemu_gluster_glfs_init(opts->location, errp);
+if (!glfs) {
+ret = -errno;
+goto out;
+}
+
+fd = glfs_creat(glfs, opts->location->path,
+O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR | 
S_IWUSR);
+if (!fd) {
+ret = -errno;
+goto out;
+}
+
+ret = qemu_gluster_do_truncate(fd, opts->size, opts->preallocation, errp);
+
+out:
+if (fd) {
+if (glfs_close(fd) != 0 && ret == 0) {
+ret = -errno;
+}
+}
+glfs_clear_preopened(glfs);
+return ret;
+}
+
 static int coroutine_fn qemu_gluster_co_create_opts(const char *filename,
 QemuOpts *opts,
 Error **errp)
 {
+BlockdevCreateOptions *options;
+BlockdevCreateOptionsGluster *gopts;
 BlockdevOptionsGluster *gconf;
-struct glfs *glfs;
-struct glfs_fd *fd = NULL;
-int ret 

[Qemu-block] [PULL 29/56] block: x-blockdev-create QMP command

[Kevin Wolf]

This adds a synchronous x-blockdev-create QMP command that can create
qcow2 images on a given node name.

We don't want to block while creating an image, so this is not the final
interface in all aspects, but BlockdevCreateOptionsQcow2 and
.bdrv_co_create() are what they actually might look like in the end. In
any case, this should be good enough to test whether we interpret
BlockdevCreateOptions as we should.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 qapi/block-core.json  | 12 
 include/block/block_int.h |  5 +++-
 block/create.c| 76 +++
 block/qcow2.c |  1 +
 block/Makefile.objs   |  2 +-
 5 files changed, 94 insertions(+), 2 deletions(-)
 create mode 100644 block/create.c

diff --git a/qapi/block-core.json b/qapi/block-core.json
index dfea7b0102..88d7a8678d 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3464,6 +3464,18 @@
   } }
 
 ##
+# @x-blockdev-create:
+#
+# Create an image format on a given node.
+# TODO Replace with something asynchronous (block job?)
+#
+# Since: 2.12
+##
+{ 'command': 'x-blockdev-create',
+  'data': 'BlockdevCreateOptions',
+  'boxed': true }
+
+##
 # @blockdev-open-tray:
 #
 # Opens a block device's tray. If there is a block driver state tree inserted 
as
diff --git a/include/block/block_int.h b/include/block/block_int.h
index a84cc04d55..27e17addba 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -129,8 +129,11 @@ struct BlockDriver {
 int (*bdrv_file_open)(BlockDriverState *bs, QDict *options, int flags,
   Error **errp);
 void (*bdrv_close)(BlockDriverState *bs);
-int coroutine_fn (*bdrv_co_create_opts)(const char *filename, QemuOpts 
*opts,
+int coroutine_fn (*bdrv_co_create)(BlockdevCreateOptions *opts,
Error **errp);
+int coroutine_fn (*bdrv_co_create_opts)(const char *filename,
+QemuOpts *opts,
+Error **errp);
 int (*bdrv_make_empty)(BlockDriverState *bs);
 
 void (*bdrv_refresh_filename)(BlockDriverState *bs, QDict *options);
diff --git a/block/create.c b/block/create.c
new file mode 100644
index 00..8bd8a03719
--- /dev/null
+++ b/block/create.c
@@ -0,0 +1,76 @@
+/*
+ * Block layer code related to image creation
+ *
+ * Copyright (c) 2018 Kevin Wolf 
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to 
deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "block/block_int.h"
+#include "qapi/qapi-commands-block-core.h"
+#include "qapi/error.h"
+
+typedef struct BlockdevCreateCo {
+BlockDriver *drv;
+BlockdevCreateOptions *opts;
+int ret;
+Error **errp;
+} BlockdevCreateCo;
+
+static void coroutine_fn bdrv_co_create_co_entry(void *opaque)
+{
+BlockdevCreateCo *cco = opaque;
+cco->ret = cco->drv->bdrv_co_create(cco->opts, cco->errp);
+}
+
+void qmp_x_blockdev_create(BlockdevCreateOptions *options, Error **errp)
+{
+const char *fmt = BlockdevDriver_str(options->driver);
+BlockDriver *drv = bdrv_find_format(fmt);
+Coroutine *co;
+BlockdevCreateCo cco;
+
+/* If the driver is in the schema, we know that it exists. But it may not
+ * be whitelisted. */
+assert(drv);
+if (bdrv_uses_whitelist() && !bdrv_is_whitelisted(drv, false)) {
+error_setg(errp, "Driver is not whitelisted");
+return;
+}
+
+/* Call callback if it exists */
+if (!drv->bdrv_co_create) {
+error_setg(errp, "Driver does not support blockdev-create");
+return;
+}
+
+cco = (BlockdevCreateCo) {
+.drv = drv,
+.opts = options,
+.ret = -EINPROGRESS,
+.errp = errp,
+};
+
+co = qemu_coroutine_create(bdrv_co_create_co_entry, &cco);
+qemu_coroutine_enter(co);
+while (cco.ret == -EINPROGRESS) {
+aio_poll(

[Qemu-block] [PULL 19/56] qcow2: Pass BlockdevCreateOptions to qcow2_co_create()

[Kevin Wolf]

All of the simple options are now passed to qcow2_co_create() in a
BlockdevCreateOptions object. Still missing: node-name and the
encryption options.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 block/qcow2.c | 189 ++
 1 file changed, 151 insertions(+), 38 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index 7a11874d22..7679c28f57 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -2700,19 +2700,26 @@ static int64_t qcow2_calc_prealloc_size(int64_t 
total_size,
 return meta_size + aligned_total_size;
 }
 
-static size_t qcow2_opt_get_cluster_size_del(QemuOpts *opts, Error **errp)
+static bool validate_cluster_size(size_t cluster_size, Error **errp)
 {
-size_t cluster_size;
-int cluster_bits;
-
-cluster_size = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE,
- DEFAULT_CLUSTER_SIZE);
-cluster_bits = ctz32(cluster_size);
+int cluster_bits = ctz32(cluster_size);
 if (cluster_bits < MIN_CLUSTER_BITS || cluster_bits > MAX_CLUSTER_BITS ||
 (1 << cluster_bits) != cluster_size)
 {
 error_setg(errp, "Cluster size must be a power of two between %d and "
"%dk", 1 << MIN_CLUSTER_BITS, 1 << (MAX_CLUSTER_BITS - 10));
+return false;
+}
+return true;
+}
+
+static size_t qcow2_opt_get_cluster_size_del(QemuOpts *opts, Error **errp)
+{
+size_t cluster_size;
+
+cluster_size = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE,
+ DEFAULT_CLUSTER_SIZE);
+if (!validate_cluster_size(cluster_size, errp)) {
 return 0;
 }
 return cluster_size;
@@ -2761,12 +2768,10 @@ static uint64_t 
qcow2_opt_get_refcount_bits_del(QemuOpts *opts, int version,
 }
 
 static int coroutine_fn
-qcow2_co_create(BlockDriverState *bs, int64_t total_size,
-const char *backing_file, const char *backing_format,
-int flags, size_t cluster_size, PreallocMode prealloc,
-QemuOpts *opts, int version, int refcount_order,
-const char *encryptfmt, Error **errp)
+qcow2_co_create(BlockDriverState *bs, BlockdevCreateOptions *create_options,
+QemuOpts *opts, const char *encryptfmt, Error **errp)
 {
+BlockdevCreateOptionsQcow2 *qcow2_opts;
 QDict *options;
 
 /*
@@ -2783,10 +2788,92 @@ qcow2_co_create(BlockDriverState *bs, int64_t 
total_size,
  */
 BlockBackend *blk;
 QCowHeader *header;
+size_t cluster_size;
+int version;
+int refcount_order;
 uint64_t* refcount_table;
 Error *local_err = NULL;
 int ret;
 
+/* Validate options and set default values */
+assert(create_options->driver == BLOCKDEV_DRIVER_QCOW2);
+qcow2_opts = &create_options->u.qcow2;
+
+if (!QEMU_IS_ALIGNED(qcow2_opts->size, BDRV_SECTOR_SIZE)) {
+error_setg(errp, "Image size must be a multiple of 512 bytes");
+ret = -EINVAL;
+goto out;
+}
+
+if (qcow2_opts->has_version) {
+switch (qcow2_opts->version) {
+case BLOCKDEV_QCOW2_VERSION_V2:
+version = 2;
+break;
+case BLOCKDEV_QCOW2_VERSION_V3:
+version = 3;
+break;
+default:
+g_assert_not_reached();
+}
+} else {
+version = 3;
+}
+
+if (qcow2_opts->has_cluster_size) {
+cluster_size = qcow2_opts->cluster_size;
+} else {
+cluster_size = DEFAULT_CLUSTER_SIZE;
+}
+
+if (!validate_cluster_size(cluster_size, errp)) {
+return -EINVAL;
+}
+
+if (!qcow2_opts->has_preallocation) {
+qcow2_opts->preallocation = PREALLOC_MODE_OFF;
+}
+if (qcow2_opts->has_backing_file &&
+qcow2_opts->preallocation != PREALLOC_MODE_OFF)
+{
+error_setg(errp, "Backing file and preallocation cannot be used at "
+   "the same time");
+return -EINVAL;
+}
+if (qcow2_opts->has_backing_fmt && !qcow2_opts->has_backing_file) {
+error_setg(errp, "Backing format cannot be used without backing file");
+return -EINVAL;
+}
+
+if (!qcow2_opts->has_lazy_refcounts) {
+qcow2_opts->lazy_refcounts = false;
+}
+if (version < 3 && qcow2_opts->lazy_refcounts) {
+error_setg(errp, "Lazy refcounts only supported with compatibility "
+   "level 1.1 and above (use compat=1.1 or greater)");
+return -EINVAL;
+}
+
+if (!qcow2_opts->has_refcount_bits) {
+qcow2_opts->refcount_bits = 16;
+}
+if (qcow2_opts->refcount_bits > 64 ||
+!is_power_of_2(qcow2_opts->refcount_bits))
+{
+error_setg(errp, "Refcount width must be a power of two and may not "
+   "exceed 64 bits");
+return -EINVAL;
+}
+if (version < 3 && qcow2_opts->refcount_bits != 16) {
+error_setg(errp, "Different refcount widt

[Qemu-block] [PULL 18/56] qcow2: Let qcow2_create() handle protocol layer

[Kevin Wolf]

Currently, qcow2_create() only parses the QemuOpts and then calls
qcow2_co_create() for the actual image creation, which includes both the
creation of the actual file on the file system and writing a valid empty
qcow2 image into that file.

The plan is that qcow2_co_create() becomes the function that implements
the functionality for a future 'blockdev-create' QMP command, which only
creates the qcow2 layer on an already opened file node.

This is a first step towards that goal: Let's move out anything that
deals with the protocol layer from qcow2_co_create() into
qcow2_create().  This means that qcow2_co_create() doesn't need a file
name any more.

Signed-off-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Reviewed-by: Max Reitz 
---
 block/qcow2.c | 64 +++
 1 file changed, 38 insertions(+), 26 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index b7354a27a1..7a11874d22 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -2761,7 +2761,7 @@ static uint64_t qcow2_opt_get_refcount_bits_del(QemuOpts 
*opts, int version,
 }
 
 static int coroutine_fn
-qcow2_co_create(const char *filename, int64_t total_size,
+qcow2_co_create(BlockDriverState *bs, int64_t total_size,
 const char *backing_file, const char *backing_format,
 int flags, size_t cluster_size, PreallocMode prealloc,
 QemuOpts *opts, int version, int refcount_order,
@@ -2787,28 +2787,11 @@ qcow2_co_create(const char *filename, int64_t 
total_size,
 Error *local_err = NULL;
 int ret;
 
-if (prealloc == PREALLOC_MODE_FULL || prealloc == PREALLOC_MODE_FALLOC) {
-int64_t prealloc_size =
-qcow2_calc_prealloc_size(total_size, cluster_size, refcount_order);
-qemu_opt_set_number(opts, BLOCK_OPT_SIZE, prealloc_size, &error_abort);
-qemu_opt_set(opts, BLOCK_OPT_PREALLOC, PreallocMode_str(prealloc),
- &error_abort);
-}
-
-ret = bdrv_create_file(filename, opts, &local_err);
+blk = blk_new(BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
+ret = blk_insert_bs(blk, bs, errp);
 if (ret < 0) {
-error_propagate(errp, local_err);
-return ret;
-}
-
-blk = blk_new_open(filename, NULL, NULL,
-   BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_PROTOCOL,
-   &local_err);
-if (blk == NULL) {
-error_propagate(errp, local_err);
-return -EIO;
+goto out;
 }
-
 blk_set_allow_write_beyond_eof(blk, true);
 
 /* Write the header */
@@ -2863,7 +2846,8 @@ qcow2_co_create(const char *filename, int64_t total_size,
  */
 options = qdict_new();
 qdict_put_str(options, "driver", "qcow2");
-blk = blk_new_open(filename, NULL, options,
+qdict_put_str(options, "file", bs->node_name);
+blk = blk_new_open(NULL, NULL, options,
BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_NO_FLUSH,
&local_err);
 if (blk == NULL) {
@@ -2935,7 +2919,8 @@ qcow2_co_create(const char *filename, int64_t total_size,
  */
 options = qdict_new();
 qdict_put_str(options, "driver", "qcow2");
-blk = blk_new_open(filename, NULL, options,
+qdict_put_str(options, "file", bs->node_name);
+blk = blk_new_open(NULL, NULL, options,
BDRV_O_RDWR | BDRV_O_NO_BACKING | BDRV_O_NO_IO,
&local_err);
 if (blk == NULL) {
@@ -2966,6 +2951,7 @@ static int coroutine_fn qcow2_co_create_opts(const char 
*filename, QemuOpts *opt
 uint64_t refcount_bits;
 int refcount_order;
 char *encryptfmt = NULL;
+BlockDriverState *bs = NULL;
 Error *local_err = NULL;
 int ret;
 
@@ -3034,12 +3020,38 @@ static int coroutine_fn qcow2_co_create_opts(const char 
*filename, QemuOpts *opt
 
 refcount_order = ctz32(refcount_bits);
 
-ret = qcow2_co_create(filename, size, backing_file, backing_fmt, flags,
+/* Create and open the file (protocol layer) */
+if (prealloc == PREALLOC_MODE_FULL || prealloc == PREALLOC_MODE_FALLOC) {
+int64_t prealloc_size =
+qcow2_calc_prealloc_size(size, cluster_size, refcount_order);
+qemu_opt_set_number(opts, BLOCK_OPT_SIZE, prealloc_size, &error_abort);
+qemu_opt_set(opts, BLOCK_OPT_PREALLOC, PreallocMode_str(prealloc),
+ &error_abort);
+}
+
+ret = bdrv_create_file(filename, opts, errp);
+if (ret < 0) {
+goto finish;
+}
+
+bs = bdrv_open(filename, NULL, NULL,
+   BDRV_O_RDWR | BDRV_O_RESIZE | BDRV_O_PROTOCOL, errp);
+if (bs == NULL) {
+ret = -EIO;
+goto finish;
+}
+
+/* Create the qcow2 image (format layer) */
+ret = qcow2_co_create(bs, size, backing_file, backing_fmt, flags,
   cluster_size, prealloc, opts, version, 
refcount_order,
-  encryptfmt, &local_err);
-error_propagate(errp, local_err);
+

[Qemu-block] [PULL 22/56] qcow2: Handle full/falloc preallocation in qcow2_co_create()

[Kevin Wolf]

Once qcow2_co_create() can be called directly on an already existing
node, we must provide the 'full' and 'falloc' preallocation modes
outside of creating the image on the protocol layer. Fortunately, we
have preallocated truncate now which can provide this functionality.

Signed-off-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Reviewed-by: Max Reitz 
---
 block/qcow2.c | 28 +++-
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index e1821eb3c8..933c612754 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -2908,6 +2908,25 @@ qcow2_co_create(BlockdevCreateOptions *create_options, 
Error **errp)
 }
 blk_set_allow_write_beyond_eof(blk, true);
 
+/* Clear the protocol layer and preallocate it if necessary */
+ret = blk_truncate(blk, 0, PREALLOC_MODE_OFF, errp);
+if (ret < 0) {
+goto out;
+}
+
+if (qcow2_opts->preallocation == PREALLOC_MODE_FULL ||
+qcow2_opts->preallocation == PREALLOC_MODE_FALLOC)
+{
+int64_t prealloc_size =
+qcow2_calc_prealloc_size(qcow2_opts->size, cluster_size,
+ refcount_order);
+
+ret = blk_truncate(blk, prealloc_size, qcow2_opts->preallocation, 
errp);
+if (ret < 0) {
+goto out;
+}
+}
+
 /* Write the header */
 QEMU_BUILD_BUG_ON((1 << MIN_CLUSTER_BITS) < sizeof(*header));
 header = g_malloc0(cluster_size);
@@ -3145,15 +3164,6 @@ static int coroutine_fn qcow2_co_create_opts(const char 
*filename, QemuOpts *opt
 
 
 /* Create and open the file (protocol layer) */
-if (prealloc == PREALLOC_MODE_FULL || prealloc == PREALLOC_MODE_FALLOC) {
-int refcount_order = ctz32(refcount_bits);
-int64_t prealloc_size =
-qcow2_calc_prealloc_size(size, cluster_size, refcount_order);
-qemu_opt_set_number(opts, BLOCK_OPT_SIZE, prealloc_size, &error_abort);
-qemu_opt_set(opts, BLOCK_OPT_PREALLOC, PreallocMode_str(prealloc),
- &error_abort);
-}
-
 ret = bdrv_create_file(filename, opts, errp);
 if (ret < 0) {
 goto finish;
-- 
2.13.6

[Qemu-block] [PULL 31/56] file-win32: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to file-win32, which
enables image creation over QMP.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 block/file-win32.c | 47 ++-
 1 file changed, 38 insertions(+), 9 deletions(-)

diff --git a/block/file-win32.c b/block/file-win32.c
index 4a430d45f1..2e2f746bb1 100644
--- a/block/file-win32.c
+++ b/block/file-win32.c
@@ -553,30 +553,59 @@ static int64_t 
raw_get_allocated_file_size(BlockDriverState *bs)
 return st.st_size;
 }
 
-static int coroutine_fn raw_co_create_opts(const char *filename, QemuOpts 
*opts,
-   Error **errp)
+static int raw_co_create(BlockdevCreateOptions *options, Error **errp)
 {
+BlockdevCreateOptionsFile *file_opts;
 int fd;
-int64_t total_size = 0;
 
-strstart(filename, "file:", &filename);
+assert(options->driver == BLOCKDEV_DRIVER_FILE);
+file_opts = &options->u.file;
 
-/* Read out options */
-total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
-  BDRV_SECTOR_SIZE);
+if (file_opts->has_preallocation) {
+error_setg(errp, "Preallocation is not supported on Windows");
+return -EINVAL;
+}
+if (file_opts->has_nocow) {
+error_setg(errp, "nocow is not supported on Windows");
+return -EINVAL;
+}
 
-fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY,
+fd = qemu_open(file_opts->filename, O_WRONLY | O_CREAT | O_TRUNC | 
O_BINARY,
0644);
 if (fd < 0) {
 error_setg_errno(errp, errno, "Could not create file");
 return -EIO;
 }
 set_sparse(fd);
-ftruncate(fd, total_size);
+ftruncate(fd, file_opts->size);
 qemu_close(fd);
+
 return 0;
 }
 
+static int coroutine_fn raw_co_create_opts(const char *filename, QemuOpts 
*opts,
+   Error **errp)
+{
+BlockdevCreateOptions options;
+int64_t total_size = 0;
+
+strstart(filename, "file:", &filename);
+
+/* Read out options */
+total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+  BDRV_SECTOR_SIZE);
+
+options = (BlockdevCreateOptions) {
+.driver = BLOCKDEV_DRIVER_FILE,
+.u.file = {
+.filename   = (char *) filename,
+.size   = total_size,
+.has_preallocation  = false,
+.has_nocow  = false,
+},
+};
+return raw_co_create(&options, errp);
+}
 
 static QemuOptsList raw_create_opts = {
 .name = "raw-create-opts",
-- 
2.13.6

[Qemu-block] [PULL 23/56] util: Add qemu_opts_to_qdict_filtered()

[Kevin Wolf]

This allows, given a QemuOpts for a QemuOptsList that was merged from
multiple QemuOptsList, to only consider those options that exist in one
specific list. Block drivers need this to separate format-layer create
options from protocol-level options.

Signed-off-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Reviewed-by: Max Reitz 
---
 include/qemu/option.h |  2 ++
 util/qemu-option.c| 42 +-
 2 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/include/qemu/option.h b/include/qemu/option.h
index b127fb6db6..306fdb5f7a 100644
--- a/include/qemu/option.h
+++ b/include/qemu/option.h
@@ -124,6 +124,8 @@ void qemu_opts_set_defaults(QemuOptsList *list, const char 
*params,
 int permit_abbrev);
 QemuOpts *qemu_opts_from_qdict(QemuOptsList *list, const QDict *qdict,
Error **errp);
+QDict *qemu_opts_to_qdict_filtered(QemuOpts *opts, QDict *qdict,
+   QemuOptsList *list, bool del);
 QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict);
 void qemu_opts_absorb_qdict(QemuOpts *opts, QDict *qdict, Error **errp);
 
diff --git a/util/qemu-option.c b/util/qemu-option.c
index a401e936da..2b412eff5e 100644
--- a/util/qemu-option.c
+++ b/util/qemu-option.c
@@ -1007,14 +1007,23 @@ void qemu_opts_absorb_qdict(QemuOpts *opts, QDict 
*qdict, Error **errp)
 }
 
 /*
- * Convert from QemuOpts to QDict.
- * The QDict values are of type QString.
+ * Convert from QemuOpts to QDict. The QDict values are of type QString.
+ *
+ * If @list is given, only add those options to the QDict that are contained in
+ * the list. If @del is true, any options added to the QDict are removed from
+ * the QemuOpts, otherwise they remain there.
+ *
+ * If two options in @opts have the same name, they are processed in order
+ * so that the last one wins (consistent with the reverse iteration in
+ * qemu_opt_find()), but all of them are deleted if @del is true.
+ *
  * TODO We'll want to use types appropriate for opt->desc->type, but
  * this is enough for now.
  */
-QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict)
+QDict *qemu_opts_to_qdict_filtered(QemuOpts *opts, QDict *qdict,
+   QemuOptsList *list, bool del)
 {
-QemuOpt *opt;
+QemuOpt *opt, *next;
 
 if (!qdict) {
 qdict = qdict_new();
@@ -1022,12 +1031,35 @@ QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict)
 if (opts->id) {
 qdict_put_str(qdict, "id", opts->id);
 }
-QTAILQ_FOREACH(opt, &opts->head, next) {
+QTAILQ_FOREACH_SAFE(opt, &opts->head, next, next) {
+if (list) {
+QemuOptDesc *desc;
+bool found = false;
+for (desc = list->desc; desc->name; desc++) {
+if (!strcmp(desc->name, opt->name)) {
+found = true;
+break;
+}
+}
+if (!found) {
+continue;
+}
+}
 qdict_put_str(qdict, opt->name, opt->str);
+if (del) {
+qemu_opt_del(opt);
+}
 }
 return qdict;
 }
 
+/* Copy all options in a QemuOpts to the given QDict. See
+ * qemu_opts_to_qdict_filtered() for details. */
+QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict)
+{
+return qemu_opts_to_qdict_filtered(opts, qdict, NULL, false);
+}
+
 /* Validate parsed opts against descriptions where no
  * descriptions were provided in the QemuOptsList.
  */
-- 
2.13.6

[Qemu-block] [PULL 27/56] qcow2: Use visitor for options in qcow2_create()

[Kevin Wolf]

Instead of manually creating the BlockdevCreateOptions object, use a
visitor to parse the given options into the QAPI object.

This involves translation from the old command line syntax to the syntax
mandated by the QAPI schema. Option names are still checked against
qcow2_create_opts, so only the old option names are allowed on the
command line, even if they are translated in qcow2_create().

In contrast, new option values are optionally recognised besides the old
values: 'compat' accepts 'v2'/'v3' as an alias for '0.10'/'1.1', and
'encrypt.format' accepts 'qcow' as an alias for 'aes' now.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 block/qcow2.c  | 218 -
 tests/qemu-iotests/049.out |   8 +-
 tests/qemu-iotests/112.out |   4 +-
 3 files changed, 84 insertions(+), 146 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index 933c612754..37b0e36c1e 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -37,7 +37,8 @@
 #include "qemu/option_int.h"
 #include "qemu/cutils.h"
 #include "qemu/bswap.h"
-#include "qapi/opts-visitor.h"
+#include "qapi/qobject-input-visitor.h"
+#include "qapi/qapi-visit-block-core.h"
 #include "block/crypto.h"
 
 /*
@@ -2449,37 +2450,6 @@ static int qcow2_crypt_method_from_format(const char 
*encryptfmt)
 }
 }
 
-static QCryptoBlockCreateOptions *
-qcow2_parse_encryption(const char *encryptfmt, QemuOpts *opts, Error **errp)
-{
-QCryptoBlockCreateOptions *cryptoopts = NULL;
-QDict *options, *encryptopts;
-int fmt;
-
-options = qemu_opts_to_qdict(opts, NULL);
-qdict_extract_subqdict(options, &encryptopts, "encrypt.");
-QDECREF(options);
-
-fmt = qcow2_crypt_method_from_format(encryptfmt);
-
-switch (fmt) {
-case QCOW_CRYPT_LUKS:
-cryptoopts = block_crypto_create_opts_init(
-Q_CRYPTO_BLOCK_FORMAT_LUKS, encryptopts, errp);
-break;
-case QCOW_CRYPT_AES:
-cryptoopts = block_crypto_create_opts_init(
-Q_CRYPTO_BLOCK_FORMAT_QCOW, encryptopts, errp);
-break;
-default:
-error_setg(errp, "Unknown encryption format '%s'", encryptfmt);
-break;
-}
-
-QDECREF(encryptopts);
-return cryptoopts;
-}
-
 static int qcow2_set_up_encryption(BlockDriverState *bs,
QCryptoBlockCreateOptions *cryptoopts,
Error **errp)
@@ -2874,7 +2844,7 @@ qcow2_co_create(BlockdevCreateOptions *create_options, 
Error **errp)
 }
 if (version < 3 && qcow2_opts->lazy_refcounts) {
 error_setg(errp, "Lazy refcounts only supported with compatibility "
-   "level 1.1 and above (use compat=1.1 or greater)");
+   "level 1.1 and above (use version=v3 or greater)");
 ret = -EINVAL;
 goto out;
 }
@@ -2892,7 +2862,7 @@ qcow2_co_create(BlockdevCreateOptions *create_options, 
Error **errp)
 }
 if (version < 3 && qcow2_opts->refcount_bits != 16) {
 error_setg(errp, "Different refcount widths than 16 bits require "
-   "compatibility level 1.1 or above (use compat=1.1 or "
+   "compatibility level 1.1 or above (use version=v3 or "
"greater)");
 ret = -EINVAL;
 goto out;
@@ -3080,144 +3050,112 @@ out:
 static int coroutine_fn qcow2_co_create_opts(const char *filename, QemuOpts 
*opts,
  Error **errp)
 {
-BlockdevCreateOptions create_options;
-char *backing_file = NULL;
-char *backing_fmt = NULL;
-BlockdevDriver backing_drv;
-char *buf = NULL;
-uint64_t size = 0;
-int flags = 0;
-size_t cluster_size = DEFAULT_CLUSTER_SIZE;
-PreallocMode prealloc;
-int version;
-uint64_t refcount_bits;
-char *encryptfmt = NULL;
-QCryptoBlockCreateOptions *cryptoopts = NULL;
+BlockdevCreateOptions *create_options = NULL;
+QDict *qdict = NULL;
+QObject *qobj;
+Visitor *v;
 BlockDriverState *bs = NULL;
 Error *local_err = NULL;
+const char *val;
 int ret;
 
-/* Read out options */
-size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
-BDRV_SECTOR_SIZE);
-backing_file = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FILE);
-backing_fmt = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FMT);
-backing_drv = qapi_enum_parse(&BlockdevDriver_lookup, backing_fmt,
-  0, &local_err);
-if (local_err) {
-error_propagate(errp, local_err);
+/* Only the keyval visitor supports the dotted syntax needed for
+ * encryption, so go through a QDict before getting a QAPI type. Ignore
+ * options meant for the protocol layer so that the visitor doesn't
+ * complain. */
+qdict = qemu_opts_to_qdict_filtered(opts, NULL, bdrv_qcow2.create_opts,
+true);
+
+/* Handle encryption optio

[Qemu-block] [PULL 24/56] test-qemu-opts: Test qemu_opts_append()

[Kevin Wolf]

Basic test for merging two QemuOptsLists.

Signed-off-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Reviewed-by: Max Reitz 
---
 tests/test-qemu-opts.c | 128 +
 1 file changed, 128 insertions(+)

diff --git a/tests/test-qemu-opts.c b/tests/test-qemu-opts.c
index 5d5a3daa7b..6c3183390b 100644
--- a/tests/test-qemu-opts.c
+++ b/tests/test-qemu-opts.c
@@ -23,6 +23,8 @@ static QemuOptsList opts_list_01 = {
 {
 .name = "str1",
 .type = QEMU_OPT_STRING,
+.help = "Help texts are preserved in qemu_opts_append",
+.def_value_str = "default",
 },{
 .name = "str2",
 .type = QEMU_OPT_STRING,
@@ -32,6 +34,7 @@ static QemuOptsList opts_list_01 = {
 },{
 .name = "number1",
 .type = QEMU_OPT_NUMBER,
+.help = "Having help texts only for some options is okay",
 },{
 .name = "number2",
 .type = QEMU_OPT_NUMBER,
@@ -743,6 +746,129 @@ static void test_opts_parse_size(void)
 qemu_opts_reset(&opts_list_02);
 }
 
+static void append_verify_list_01(QemuOptDesc *desc, bool with_overlapping)
+{
+int i = 0;
+
+if (with_overlapping) {
+g_assert_cmpstr(desc[i].name, ==, "str1");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_STRING);
+g_assert_cmpstr(desc[i].help, ==,
+"Help texts are preserved in qemu_opts_append");
+g_assert_cmpstr(desc[i].def_value_str, ==, "default");
+i++;
+
+g_assert_cmpstr(desc[i].name, ==, "str2");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_STRING);
+g_assert_cmpstr(desc[i].help, ==, NULL);
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+i++;
+}
+
+g_assert_cmpstr(desc[i].name, ==, "str3");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_STRING);
+g_assert_cmpstr(desc[i].help, ==, NULL);
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+i++;
+
+g_assert_cmpstr(desc[i].name, ==, "number1");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_NUMBER);
+g_assert_cmpstr(desc[i].help, ==,
+"Having help texts only for some options is okay");
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+i++;
+
+g_assert_cmpstr(desc[i].name, ==, "number2");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_NUMBER);
+g_assert_cmpstr(desc[i].help, ==, NULL);
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+i++;
+
+g_assert_cmpstr(desc[i].name, ==, NULL);
+}
+
+static void append_verify_list_02(QemuOptDesc *desc)
+{
+int i = 0;
+
+g_assert_cmpstr(desc[i].name, ==, "str1");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_STRING);
+g_assert_cmpstr(desc[i].help, ==, NULL);
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+i++;
+
+g_assert_cmpstr(desc[i].name, ==, "str2");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_STRING);
+g_assert_cmpstr(desc[i].help, ==, NULL);
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+i++;
+
+g_assert_cmpstr(desc[i].name, ==, "bool1");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_BOOL);
+g_assert_cmpstr(desc[i].help, ==, NULL);
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+i++;
+
+g_assert_cmpstr(desc[i].name, ==, "bool2");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_BOOL);
+g_assert_cmpstr(desc[i].help, ==, NULL);
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+i++;
+
+g_assert_cmpstr(desc[i].name, ==, "size1");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_SIZE);
+g_assert_cmpstr(desc[i].help, ==, NULL);
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+i++;
+
+g_assert_cmpstr(desc[i].name, ==, "size2");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_SIZE);
+g_assert_cmpstr(desc[i].help, ==, NULL);
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+i++;
+
+g_assert_cmpstr(desc[i].name, ==, "size3");
+g_assert_cmpint(desc[i].type, ==, QEMU_OPT_SIZE);
+g_assert_cmpstr(desc[i].help, ==, NULL);
+g_assert_cmpstr(desc[i].def_value_str, ==, NULL);
+}
+
+static void test_opts_append_to_null(void)
+{
+QemuOptsList *merged;
+
+merged = qemu_opts_append(NULL, &opts_list_01);
+g_assert(merged != &opts_list_01);
+
+g_assert_cmpstr(merged->name, ==, NULL);
+g_assert_cmpstr(merged->implied_opt_name, ==, NULL);
+g_assert_false(merged->merge_lists);
+
+append_verify_list_01(merged->desc, true);
+
+qemu_opts_free(merged);
+}
+
+static void test_opts_append(void)
+{
+QemuOptsList *first, *merged;
+
+first = qemu_opts_append(NULL, &opts_list_02);
+merged = qemu_opts_append(first, &opts_list_01);
+g_assert(first != &opts_list_02);
+g_assert(merged != &opts_list_01);
+
+g_assert_cmpstr(merged->name, ==, NULL);
+g_assert_cmpstr(merged->implied_opt_name, ==, NULL);
+g_assert_false(merged->merge_lists);
+
+append_verify_list_

[Qemu-block] [PULL 26/56] qdict: Introduce qdict_rename_keys()

[Kevin Wolf]

A few block drivers will need to rename .bdrv_create options for their
QAPIfication, so let's have a helper function for that.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 include/qapi/qmp/qdict.h |   6 +++
 qobject/qdict.c  |  34 +
 tests/check-qdict.c  | 129 +++
 3 files changed, 169 insertions(+)

diff --git a/include/qapi/qmp/qdict.h b/include/qapi/qmp/qdict.h
index ff6f7842c3..7c6d844549 100644
--- a/include/qapi/qmp/qdict.h
+++ b/include/qapi/qmp/qdict.h
@@ -81,4 +81,10 @@ QObject *qdict_crumple(const QDict *src, Error **errp);
 
 void qdict_join(QDict *dest, QDict *src, bool overwrite);
 
+typedef struct QDictRenames {
+const char *from;
+const char *to;
+} QDictRenames;
+bool qdict_rename_keys(QDict *qdict, const QDictRenames *renames, Error 
**errp);
+
 #endif /* QDICT_H */
diff --git a/qobject/qdict.c b/qobject/qdict.c
index 23df84f9cd..229b8c840b 100644
--- a/qobject/qdict.c
+++ b/qobject/qdict.c
@@ -1072,3 +1072,37 @@ void qdict_join(QDict *dest, QDict *src, bool overwrite)
 entry = next;
 }
 }
+
+/**
+ * qdict_rename_keys(): Rename keys in qdict according to the replacements
+ * specified in the array renames. The array must be terminated by an entry
+ * with from = NULL.
+ *
+ * The renames are performed individually in the order of the array, so entries
+ * may be renamed multiple times and may or may not conflict depending on the
+ * order of the renames array.
+ *
+ * Returns true for success, false in error cases.
+ */
+bool qdict_rename_keys(QDict *qdict, const QDictRenames *renames, Error **errp)
+{
+QObject *qobj;
+
+while (renames->from) {
+if (qdict_haskey(qdict, renames->from)) {
+if (qdict_haskey(qdict, renames->to)) {
+error_setg(errp, "'%s' and its alias '%s' can't be used at the 
"
+   "same time", renames->to, renames->from);
+return false;
+}
+
+qobj = qdict_get(qdict, renames->from);
+qobject_incref(qobj);
+qdict_put_obj(qdict, renames->to, qobj);
+qdict_del(qdict, renames->from);
+}
+
+renames++;
+}
+return true;
+}
diff --git a/tests/check-qdict.c b/tests/check-qdict.c
index ec628f3453..a3faea8bfc 100644
--- a/tests/check-qdict.c
+++ b/tests/check-qdict.c
@@ -665,6 +665,133 @@ static void qdict_crumple_test_empty(void)
 QDECREF(dst);
 }
 
+static int qdict_count_entries(QDict *dict)
+{
+const QDictEntry *e;
+int count = 0;
+
+for (e = qdict_first(dict); e; e = qdict_next(dict, e)) {
+count++;
+}
+
+return count;
+}
+
+static void qdict_rename_keys_test(void)
+{
+QDict *dict = qdict_new();
+QDict *copy;
+QDictRenames *renames;
+Error *local_err = NULL;
+
+qdict_put_str(dict, "abc", "foo");
+qdict_put_str(dict, "abcdef", "bar");
+qdict_put_int(dict, "number", 42);
+qdict_put_bool(dict, "flag", true);
+qdict_put_null(dict, "nothing");
+
+/* Empty rename list */
+renames = (QDictRenames[]) {
+{ NULL, "this can be anything" }
+};
+copy = qdict_clone_shallow(dict);
+qdict_rename_keys(copy, renames, &error_abort);
+
+g_assert_cmpstr(qdict_get_str(copy, "abc"), ==, "foo");
+g_assert_cmpstr(qdict_get_str(copy, "abcdef"), ==, "bar");
+g_assert_cmpint(qdict_get_int(copy, "number"), ==, 42);
+g_assert_cmpint(qdict_get_bool(copy, "flag"), ==, true);
+g_assert(qobject_type(qdict_get(copy, "nothing")) == QTYPE_QNULL);
+g_assert_cmpint(qdict_count_entries(copy), ==, 5);
+
+QDECREF(copy);
+
+/* Simple rename of all entries */
+renames = (QDictRenames[]) {
+{ "abc","str1" },
+{ "abcdef", "str2" },
+{ "number", "int" },
+{ "flag",   "bool" },
+{ "nothing","null" },
+{ NULL , NULL }
+};
+copy = qdict_clone_shallow(dict);
+qdict_rename_keys(copy, renames, &error_abort);
+
+g_assert(!qdict_haskey(copy, "abc"));
+g_assert(!qdict_haskey(copy, "abcdef"));
+g_assert(!qdict_haskey(copy, "number"));
+g_assert(!qdict_haskey(copy, "flag"));
+g_assert(!qdict_haskey(copy, "nothing"));
+
+g_assert_cmpstr(qdict_get_str(copy, "str1"), ==, "foo");
+g_assert_cmpstr(qdict_get_str(copy, "str2"), ==, "bar");
+g_assert_cmpint(qdict_get_int(copy, "int"), ==, 42);
+g_assert_cmpint(qdict_get_bool(copy, "bool"), ==, true);
+g_assert(qobject_type(qdict_get(copy, "null")) == QTYPE_QNULL);
+g_assert_cmpint(qdict_count_entries(copy), ==, 5);
+
+QDECREF(copy);
+
+/* Renames are processed top to bottom */
+renames = (QDictRenames[]) {
+{ "abc","tmp" },
+{ "abcdef", "abc" },
+{ "number", "abcdef" },
+{ "flag",   "number" },
+{ "nothing","flag" },
+{ "tmp","nothing" },
+{ NULL , NULL }

[Qemu-block] [PULL 17/56] qcow2: Rename qcow2_co_create2() to qcow2_co_create()

[Kevin Wolf]

The functions originally known as qcow2_create() and qcow2_create2()
are now called qcow2_co_create_opts() and qcow2_co_create(), which
matches the names of the BlockDriver callbacks that they will implement
at the end of this patch series.

Signed-off-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Reviewed-by: Max Reitz 
---
 block/qcow2.c | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index 369e374a9b..b7354a27a1 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -2761,11 +2761,11 @@ static uint64_t 
qcow2_opt_get_refcount_bits_del(QemuOpts *opts, int version,
 }
 
 static int coroutine_fn
-qcow2_co_create2(const char *filename, int64_t total_size,
- const char *backing_file, const char *backing_format,
- int flags, size_t cluster_size, PreallocMode prealloc,
- QemuOpts *opts, int version, int refcount_order,
- const char *encryptfmt, Error **errp)
+qcow2_co_create(const char *filename, int64_t total_size,
+const char *backing_file, const char *backing_format,
+int flags, size_t cluster_size, PreallocMode prealloc,
+QemuOpts *opts, int version, int refcount_order,
+const char *encryptfmt, Error **errp)
 {
 QDict *options;
 
@@ -3034,9 +3034,9 @@ static int coroutine_fn qcow2_co_create_opts(const char 
*filename, QemuOpts *opt
 
 refcount_order = ctz32(refcount_bits);
 
-ret = qcow2_co_create2(filename, size, backing_file, backing_fmt, flags,
-   cluster_size, prealloc, opts, version, 
refcount_order,
-   encryptfmt, &local_err);
+ret = qcow2_co_create(filename, size, backing_file, backing_fmt, flags,
+  cluster_size, prealloc, opts, version, 
refcount_order,
+  encryptfmt, &local_err);
 error_propagate(errp, local_err);
 
 finish:
-- 
2.13.6

[Qemu-block] [PULL 12/56] qcow2: Check snapshot L1 table in qcow2_snapshot_goto()

[Kevin Wolf]

From: Alberto Garcia 

This function copies a snapshot's L1 table into the active one without
validating it first.

We now have a function to take care of this, so let's use it.

Cc: Eric Blake 
Signed-off-by: Alberto Garcia 
Reviewed-by: Eric Blake 
Signed-off-by: Kevin Wolf 
---
 block/qcow2-snapshot.c | 9 +
 tests/qemu-iotests/080 | 2 ++
 tests/qemu-iotests/080.out | 4 
 3 files changed, 15 insertions(+)

diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c
index bf94f4f434..0faf728dc4 100644
--- a/block/qcow2-snapshot.c
+++ b/block/qcow2-snapshot.c
@@ -465,6 +465,7 @@ int qcow2_snapshot_goto(BlockDriverState *bs, const char 
*snapshot_id)
 {
 BDRVQcow2State *s = bs->opaque;
 QCowSnapshot *sn;
+Error *local_err = NULL;
 int i, snapshot_index;
 int cur_l1_bytes, sn_l1_bytes;
 int ret;
@@ -477,6 +478,14 @@ int qcow2_snapshot_goto(BlockDriverState *bs, const char 
*snapshot_id)
 }
 sn = &s->snapshots[snapshot_index];
 
+ret = qcow2_validate_table(bs, sn->l1_table_offset, sn->l1_size,
+   sizeof(uint64_t), QCOW_MAX_L1_SIZE,
+   "Snapshot L1 table", &local_err);
+if (ret < 0) {
+error_report_err(local_err);
+goto fail;
+}
+
 if (sn->disk_size != bs->total_sectors * BDRV_SECTOR_SIZE) {
 error_report("qcow2: Loading snapshots with different disk "
 "size is not implemented");
diff --git a/tests/qemu-iotests/080 b/tests/qemu-iotests/080
index 018f815697..538857310f 100755
--- a/tests/qemu-iotests/080
+++ b/tests/qemu-iotests/080
@@ -180,6 +180,7 @@ poke_file "$TEST_IMG" "$offset_snap1_l1_offset" 
"\x00\x00\x00\x00\x00\x40\x02\x0
 { $QEMU_IMG amend -o compat=0.10 $TEST_IMG; } 2>&1 | _filter_testdir
 { $QEMU_IO -c "open -o overlap-check.inactive-l2=on $TEST_IMG" \
-c 'write 0 4k'; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IMG snapshot -a test $TEST_IMG; } 2>&1 | _filter_testdir
 
 echo
 echo "== Invalid snapshot L1 table size =="
@@ -191,6 +192,7 @@ poke_file "$TEST_IMG" "$offset_snap1_l1_size" 
"\x10\x00\x00\x00"
 { $QEMU_IMG amend -o compat=0.10 $TEST_IMG; } 2>&1 | _filter_testdir
 { $QEMU_IO -c "open -o overlap-check.inactive-l2=on $TEST_IMG" \
-c 'write 0 4k'; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IMG snapshot -a test $TEST_IMG; } 2>&1 | _filter_testdir
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/080.out b/tests/qemu-iotests/080.out
index d622ac06c0..1525e1b087 100644
--- a/tests/qemu-iotests/080.out
+++ b/tests/qemu-iotests/080.out
@@ -68,6 +68,8 @@ qemu-img: Snapshot L1 table offset invalid
 qemu-img: Error while amending options: Invalid argument
 Failed to flush the refcount block cache: Invalid argument
 write failed: Invalid argument
+qemu-img: Snapshot L1 table offset invalid
+qemu-img: Could not apply snapshot 'test': Failed to load snapshot: Invalid 
argument
 
 == Invalid snapshot L1 table size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
@@ -78,4 +80,6 @@ qemu-img: Snapshot L1 table too large
 qemu-img: Error while amending options: File too large
 Failed to flush the refcount block cache: File too large
 write failed: File too large
+qemu-img: Snapshot L1 table too large
+qemu-img: Could not apply snapshot 'test': Failed to load snapshot: File too 
large
 *** done
-- 
2.13.6

[Qemu-block] [PULL 16/56] block/qapi: Add qcow2 create options to schema

[Kevin Wolf]

Signed-off-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Reviewed-by: Max Reitz 
---
 qapi/block-core.json | 45 -
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index bb2db662f7..dfea7b0102 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3359,6 +3359,49 @@
 { 'command': 'blockdev-del', 'data': { 'node-name': 'str' } }
 
 ##
+# @BlockdevQcow2Version:
+#
+# @v2:  The original QCOW2 format as introduced in qemu 0.10 (version 2)
+# @v3:  The extended QCOW2 format as introduced in qemu 1.1 (version 3)
+#
+# Since: 2.12
+##
+{ 'enum': 'BlockdevQcow2Version',
+  'data': [ 'v2', 'v3' ] }
+
+
+##
+# @BlockdevCreateOptionsQcow2:
+#
+# Driver specific image creation options for qcow2.
+#
+# @file Node to create the image format on
+# @size Size of the virtual disk in bytes
+# @version  Compatibility level (default: v3)
+# @backing-file File name of the backing file if a backing file
+#   should be used
+# @backing-fmt  Name of the block driver to use for the backing file
+# @encrypt  Encryption options if the image should be encrypted
+# @cluster-size qcow2 cluster size in bytes (default: 65536)
+# @preallocationPreallocation mode for the new image (default: off)
+# @lazy-refcounts   True if refcounts may be updated lazily (default: off)
+# @refcount-bitsWidth of reference counts in bits (default: 16)
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsQcow2',
+  'data': { 'file': 'BlockdevRef',
+'size': 'size',
+'*version': 'BlockdevQcow2Version',
+'*backing-file':'str',
+'*backing-fmt': 'BlockdevDriver',
+'*encrypt': 'QCryptoBlockCreateOptions',
+'*cluster-size':'size',
+'*preallocation':   'PreallocMode',
+'*lazy-refcounts':  'bool',
+'*refcount-bits':   'int' } }
+
+##
 # @BlockdevCreateNotSupported:
 #
 # This is used for all drivers that don't support creating images.
@@ -3402,7 +3445,7 @@
   'null-co':'BlockdevCreateNotSupported',
   'nvme':   'BlockdevCreateNotSupported',
   'parallels':  'BlockdevCreateNotSupported',
-  'qcow2':  'BlockdevCreateNotSupported',
+  'qcow2':  'BlockdevCreateOptionsQcow2',
   'qcow':   'BlockdevCreateNotSupported',
   'qed':'BlockdevCreateNotSupported',
   'quorum': 'BlockdevCreateNotSupported',
-- 
2.13.6

[Qemu-block] [PULL 15/56] block/qapi: Introduce BlockdevCreateOptions

[Kevin Wolf]

This creates a BlockdevCreateOptions union type that will contain all of
the options for image creation. We'll start out with an empty struct
type BlockdevCreateNotSupported for all drivers.

Signed-off-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Reviewed-by: Max Reitz 
---
 qapi/block-core.json | 62 
 1 file changed, 62 insertions(+)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 00475f08d4..bb2db662f7 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3359,6 +3359,68 @@
 { 'command': 'blockdev-del', 'data': { 'node-name': 'str' } }
 
 ##
+# @BlockdevCreateNotSupported:
+#
+# This is used for all drivers that don't support creating images.
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateNotSupported', 'data': {}}
+
+##
+# @BlockdevCreateOptions:
+#
+# Options for creating an image format on a given node.
+#
+# @driver   block driver to create the image format
+#
+# Since: 2.12
+##
+{ 'union': 'BlockdevCreateOptions',
+  'base': {
+  'driver': 'BlockdevDriver' },
+  'discriminator': 'driver',
+  'data': {
+  'blkdebug':   'BlockdevCreateNotSupported',
+  'blkverify':  'BlockdevCreateNotSupported',
+  'bochs':  'BlockdevCreateNotSupported',
+  'cloop':  'BlockdevCreateNotSupported',
+  'dmg':'BlockdevCreateNotSupported',
+  'file':   'BlockdevCreateNotSupported',
+  'ftp':'BlockdevCreateNotSupported',
+  'ftps':   'BlockdevCreateNotSupported',
+  'gluster':'BlockdevCreateNotSupported',
+  'host_cdrom': 'BlockdevCreateNotSupported',
+  'host_device':'BlockdevCreateNotSupported',
+  'http':   'BlockdevCreateNotSupported',
+  'https':  'BlockdevCreateNotSupported',
+  'iscsi':  'BlockdevCreateNotSupported',
+  'luks':   'BlockdevCreateNotSupported',
+  'nbd':'BlockdevCreateNotSupported',
+  'nfs':'BlockdevCreateNotSupported',
+  'null-aio':   'BlockdevCreateNotSupported',
+  'null-co':'BlockdevCreateNotSupported',
+  'nvme':   'BlockdevCreateNotSupported',
+  'parallels':  'BlockdevCreateNotSupported',
+  'qcow2':  'BlockdevCreateNotSupported',
+  'qcow':   'BlockdevCreateNotSupported',
+  'qed':'BlockdevCreateNotSupported',
+  'quorum': 'BlockdevCreateNotSupported',
+  'raw':'BlockdevCreateNotSupported',
+  'rbd':'BlockdevCreateNotSupported',
+  'replication':'BlockdevCreateNotSupported',
+  'sheepdog':   'BlockdevCreateNotSupported',
+  'ssh':'BlockdevCreateNotSupported',
+  'throttle':   'BlockdevCreateNotSupported',
+  'vdi':'BlockdevCreateNotSupported',
+  'vhdx':   'BlockdevCreateNotSupported',
+  'vmdk':   'BlockdevCreateNotSupported',
+  'vpc':'BlockdevCreateNotSupported',
+  'vvfat':  'BlockdevCreateNotSupported',
+  'vxhs':   'BlockdevCreateNotSupported'
+  } }
+
+##
 # @blockdev-open-tray:
 #
 # Opens a block device's tray. If there is a block driver state tree inserted 
as
-- 
2.13.6

[Qemu-block] [PULL 30/56] file-posix: Support .bdrv_co_create

[Kevin Wolf]

This adds the .bdrv_co_create driver callback to file, which enables
image creation over QMP.

Signed-off-by: Kevin Wolf 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
---
 qapi/block-core.json | 20 -
 block/file-posix.c   | 79 +---
 2 files changed, 75 insertions(+), 24 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 88d7a8678d..41955b097f 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -3359,6 +3359,24 @@
 { 'command': 'blockdev-del', 'data': { 'node-name': 'str' } }
 
 ##
+# @BlockdevCreateOptionsFile:
+#
+# Driver specific image creation options for file.
+#
+# @filename Filename for the new image file
+# @size Size of the virtual disk in bytes
+# @preallocationPreallocation mode for the new image (default: off)
+# @nocowTurn off copy-on-write (valid only on btrfs; default: off)
+#
+# Since: 2.12
+##
+{ 'struct': 'BlockdevCreateOptionsFile',
+  'data': { 'filename': 'str',
+'size': 'size',
+'*preallocation':   'PreallocMode',
+'*nocow':   'bool' } }
+
+##
 # @BlockdevQcow2Version:
 #
 # @v2:  The original QCOW2 format as introduced in qemu 0.10 (version 2)
@@ -3429,7 +3447,7 @@
   'bochs':  'BlockdevCreateNotSupported',
   'cloop':  'BlockdevCreateNotSupported',
   'dmg':'BlockdevCreateNotSupported',
-  'file':   'BlockdevCreateNotSupported',
+  'file':   'BlockdevCreateOptionsFile',
   'ftp':'BlockdevCreateNotSupported',
   'ftps':   'BlockdevCreateNotSupported',
   'gluster':'BlockdevCreateNotSupported',
diff --git a/block/file-posix.c b/block/file-posix.c
index 7f2cc63c60..fbc21a9921 100644
--- a/block/file-posix.c
+++ b/block/file-posix.c
@@ -1982,34 +1982,25 @@ static int64_t 
raw_get_allocated_file_size(BlockDriverState *bs)
 return (int64_t)st.st_blocks * 512;
 }
 
-static int coroutine_fn raw_co_create_opts(const char *filename, QemuOpts 
*opts,
-   Error **errp)
+static int raw_co_create(BlockdevCreateOptions *options, Error **errp)
 {
+BlockdevCreateOptionsFile *file_opts;
 int fd;
 int result = 0;
-int64_t total_size = 0;
-bool nocow = false;
-PreallocMode prealloc;
-char *buf = NULL;
-Error *local_err = NULL;
 
-strstart(filename, "file:", &filename);
+/* Validate options and set default values */
+assert(options->driver == BLOCKDEV_DRIVER_FILE);
+file_opts = &options->u.file;
 
-/* Read out options */
-total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
-  BDRV_SECTOR_SIZE);
-nocow = qemu_opt_get_bool(opts, BLOCK_OPT_NOCOW, false);
-buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
-prealloc = qapi_enum_parse(&PreallocMode_lookup, buf,
-   PREALLOC_MODE_OFF, &local_err);
-g_free(buf);
-if (local_err) {
-error_propagate(errp, local_err);
-result = -EINVAL;
-goto out;
+if (!file_opts->has_nocow) {
+file_opts->nocow = false;
+}
+if (!file_opts->has_preallocation) {
+file_opts->preallocation = PREALLOC_MODE_OFF;
 }
 
-fd = qemu_open(filename, O_RDWR | O_CREAT | O_TRUNC | O_BINARY,
+/* Create file */
+fd = qemu_open(file_opts->filename, O_RDWR | O_CREAT | O_TRUNC | O_BINARY,
0644);
 if (fd < 0) {
 result = -errno;
@@ -2017,7 +2008,7 @@ static int coroutine_fn raw_co_create_opts(const char 
*filename, QemuOpts *opts,
 goto out;
 }
 
-if (nocow) {
+if (file_opts->nocow) {
 #ifdef __linux__
 /* Set NOCOW flag to solve performance issue on fs like btrfs.
  * This is an optimisation. The FS_IOC_SETFLAGS ioctl return value
@@ -2032,7 +2023,8 @@ static int coroutine_fn raw_co_create_opts(const char 
*filename, QemuOpts *opts,
 #endif
 }
 
-result = raw_regular_truncate(fd, total_size, prealloc, errp);
+result = raw_regular_truncate(fd, file_opts->size, 
file_opts->preallocation,
+  errp);
 if (result < 0) {
 goto out_close;
 }
@@ -2046,6 +2038,46 @@ out:
 return result;
 }
 
+static int coroutine_fn raw_co_create_opts(const char *filename, QemuOpts 
*opts,
+   Error **errp)
+{
+BlockdevCreateOptions options;
+int64_t total_size = 0;
+bool nocow = false;
+PreallocMode prealloc;
+char *buf = NULL;
+Error *local_err = NULL;
+
+/* Skip file: protocol prefix */
+strstart(filename, "file:", &filename);
+
+/* Read out options */
+total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+  BDRV_SECTOR_SIZE);
+nocow = qemu_opt_get_bool(opts, BLOCK_OPT_NOCOW, false);
+buf = qemu_opt_get_del(opts, BLO

[Qemu-block] [PULL 25/56] test-qemu-opts: Test qemu_opts_to_qdict_filtered()

[Kevin Wolf]

Signed-off-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Reviewed-by: Max Reitz 
---
 tests/test-qemu-opts.c | 125 +
 1 file changed, 125 insertions(+)

diff --git a/tests/test-qemu-opts.c b/tests/test-qemu-opts.c
index 6c3183390b..2c422abcd4 100644
--- a/tests/test-qemu-opts.c
+++ b/tests/test-qemu-opts.c
@@ -10,6 +10,7 @@
 #include "qemu/osdep.h"
 #include "qemu/cutils.h"
 #include "qemu/option.h"
+#include "qemu/option_int.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qstring.h"
@@ -868,6 +869,127 @@ static void test_opts_append(void)
 qemu_opts_free(merged);
 }
 
+static void test_opts_to_qdict_basic(void)
+{
+QemuOpts *opts;
+QDict *dict;
+
+opts = qemu_opts_parse(&opts_list_01, "str1=foo,str2=,str3=bar,number1=42",
+   false, &error_abort);
+g_assert(opts != NULL);
+
+dict = qemu_opts_to_qdict(opts, NULL);
+g_assert(dict != NULL);
+
+g_assert_cmpstr(qdict_get_str(dict, "str1"), ==, "foo");
+g_assert_cmpstr(qdict_get_str(dict, "str2"), ==, "");
+g_assert_cmpstr(qdict_get_str(dict, "str3"), ==, "bar");
+g_assert_cmpstr(qdict_get_str(dict, "number1"), ==, "42");
+g_assert_false(qdict_haskey(dict, "number2"));
+
+QDECREF(dict);
+qemu_opts_del(opts);
+}
+
+static void test_opts_to_qdict_filtered(void)
+{
+QemuOptsList *first, *merged;
+QemuOpts *opts;
+QDict *dict;
+
+first = qemu_opts_append(NULL, &opts_list_02);
+merged = qemu_opts_append(first, &opts_list_01);
+
+opts = qemu_opts_parse(merged,
+   "str1=foo,str2=,str3=bar,bool1=off,number1=42",
+   false, &error_abort);
+g_assert(opts != NULL);
+
+/* Convert to QDict without deleting from opts */
+dict = qemu_opts_to_qdict_filtered(opts, NULL, &opts_list_01, false);
+g_assert(dict != NULL);
+g_assert_cmpstr(qdict_get_str(dict, "str1"), ==, "foo");
+g_assert_cmpstr(qdict_get_str(dict, "str2"), ==, "");
+g_assert_cmpstr(qdict_get_str(dict, "str3"), ==, "bar");
+g_assert_cmpstr(qdict_get_str(dict, "number1"), ==, "42");
+g_assert_false(qdict_haskey(dict, "number2"));
+g_assert_false(qdict_haskey(dict, "bool1"));
+QDECREF(dict);
+
+dict = qemu_opts_to_qdict_filtered(opts, NULL, &opts_list_02, false);
+g_assert(dict != NULL);
+g_assert_cmpstr(qdict_get_str(dict, "str1"), ==, "foo");
+g_assert_cmpstr(qdict_get_str(dict, "str2"), ==, "");
+g_assert_cmpstr(qdict_get_str(dict, "bool1"), ==, "off");
+g_assert_false(qdict_haskey(dict, "str3"));
+g_assert_false(qdict_haskey(dict, "number1"));
+g_assert_false(qdict_haskey(dict, "number2"));
+QDECREF(dict);
+
+/* Now delete converted options from opts */
+dict = qemu_opts_to_qdict_filtered(opts, NULL, &opts_list_01, true);
+g_assert(dict != NULL);
+g_assert_cmpstr(qdict_get_str(dict, "str1"), ==, "foo");
+g_assert_cmpstr(qdict_get_str(dict, "str2"), ==, "");
+g_assert_cmpstr(qdict_get_str(dict, "str3"), ==, "bar");
+g_assert_cmpstr(qdict_get_str(dict, "number1"), ==, "42");
+g_assert_false(qdict_haskey(dict, "number2"));
+g_assert_false(qdict_haskey(dict, "bool1"));
+QDECREF(dict);
+
+dict = qemu_opts_to_qdict_filtered(opts, NULL, &opts_list_02, true);
+g_assert(dict != NULL);
+g_assert_cmpstr(qdict_get_str(dict, "bool1"), ==, "off");
+g_assert_false(qdict_haskey(dict, "str1"));
+g_assert_false(qdict_haskey(dict, "str2"));
+g_assert_false(qdict_haskey(dict, "str3"));
+g_assert_false(qdict_haskey(dict, "number1"));
+g_assert_false(qdict_haskey(dict, "number2"));
+QDECREF(dict);
+
+g_assert_true(QTAILQ_EMPTY(&opts->head));
+
+qemu_opts_del(opts);
+qemu_opts_free(merged);
+}
+
+static void test_opts_to_qdict_duplicates(void)
+{
+QemuOpts *opts;
+QemuOpt *opt;
+QDict *dict;
+
+opts = qemu_opts_parse(&opts_list_03, "foo=a,foo=b", false, &error_abort);
+g_assert(opts != NULL);
+
+/* Verify that opts has two options with the same name */
+opt = QTAILQ_FIRST(&opts->head);
+g_assert_cmpstr(opt->name, ==, "foo");
+g_assert_cmpstr(opt->str , ==, "a");
+
+opt = QTAILQ_NEXT(opt, next);
+g_assert_cmpstr(opt->name, ==, "foo");
+g_assert_cmpstr(opt->str , ==, "b");
+
+opt = QTAILQ_NEXT(opt, next);
+g_assert(opt == NULL);
+
+/* In the conversion to QDict, the last one wins */
+dict = qemu_opts_to_qdict(opts, NULL);
+g_assert(dict != NULL);
+g_assert_cmpstr(qdict_get_str(dict, "foo"), ==, "b");
+QDECREF(dict);
+
+/* The last one still wins if entries are deleted, and both are deleted */
+dict = qemu_opts_to_qdict_filtered(opts, NULL, NULL, true);
+g_assert(dict != NULL);
+g_assert_cmpstr(qdict_get_str(dict, "foo"), ==, "b");
+QDECREF(dict);
+
+g_assert_true(QTAILQ_EMPTY(&opts->head));
+
+qemu_opts_del(opts);
+}
 
 int main(int argc, char *argv[]

[Qemu-block] [PULL 13/56] qcow2: Check snapshot L1 table in qcow2_snapshot_delete()

[Kevin Wolf]

From: Alberto Garcia 

This function deletes a snapshot from disk, removing its entry from
the snapshot table, freeing its L1 table and decreasing the refcounts
of all clusters.

The L1 table offset and size are however not validated. If we use
invalid values in this function we'll probably corrupt the image even
more, so we should return an error instead.

We now have a function to take care of this, so let's use it.

Signed-off-by: Alberto Garcia 
Reviewed-by: Eric Blake 
Signed-off-by: Kevin Wolf 
---
 block/qcow2-snapshot.c | 7 +++
 tests/qemu-iotests/080 | 2 ++
 tests/qemu-iotests/080.out | 2 ++
 3 files changed, 11 insertions(+)

diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c
index 0faf728dc4..74293be470 100644
--- a/block/qcow2-snapshot.c
+++ b/block/qcow2-snapshot.c
@@ -611,6 +611,13 @@ int qcow2_snapshot_delete(BlockDriverState *bs,
 }
 sn = s->snapshots[snapshot_index];
 
+ret = qcow2_validate_table(bs, sn.l1_table_offset, sn.l1_size,
+   sizeof(uint64_t), QCOW_MAX_L1_SIZE,
+   "Snapshot L1 table", errp);
+if (ret < 0) {
+return ret;
+}
+
 /* Remove it from the snapshot list */
 memmove(s->snapshots + snapshot_index,
 s->snapshots + snapshot_index + 1,
diff --git a/tests/qemu-iotests/080 b/tests/qemu-iotests/080
index 538857310f..f8e7d6f4df 100755
--- a/tests/qemu-iotests/080
+++ b/tests/qemu-iotests/080
@@ -181,6 +181,7 @@ poke_file "$TEST_IMG" "$offset_snap1_l1_offset" 
"\x00\x00\x00\x00\x00\x40\x02\x0
 { $QEMU_IO -c "open -o overlap-check.inactive-l2=on $TEST_IMG" \
-c 'write 0 4k'; } 2>&1 | _filter_qemu_io | _filter_testdir
 { $QEMU_IMG snapshot -a test $TEST_IMG; } 2>&1 | _filter_testdir
+{ $QEMU_IMG snapshot -d test $TEST_IMG; } 2>&1 | _filter_testdir
 
 echo
 echo "== Invalid snapshot L1 table size =="
@@ -193,6 +194,7 @@ poke_file "$TEST_IMG" "$offset_snap1_l1_size" 
"\x10\x00\x00\x00"
 { $QEMU_IO -c "open -o overlap-check.inactive-l2=on $TEST_IMG" \
-c 'write 0 4k'; } 2>&1 | _filter_qemu_io | _filter_testdir
 { $QEMU_IMG snapshot -a test $TEST_IMG; } 2>&1 | _filter_testdir
+{ $QEMU_IMG snapshot -d test $TEST_IMG; } 2>&1 | _filter_testdir
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/080.out b/tests/qemu-iotests/080.out
index 1525e1b087..89bcd27172 100644
--- a/tests/qemu-iotests/080.out
+++ b/tests/qemu-iotests/080.out
@@ -70,6 +70,7 @@ Failed to flush the refcount block cache: Invalid argument
 write failed: Invalid argument
 qemu-img: Snapshot L1 table offset invalid
 qemu-img: Could not apply snapshot 'test': Failed to load snapshot: Invalid 
argument
+qemu-img: Could not delete snapshot 'test': Snapshot L1 table offset invalid
 
 == Invalid snapshot L1 table size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
@@ -82,4 +83,5 @@ Failed to flush the refcount block cache: File too large
 write failed: File too large
 qemu-img: Snapshot L1 table too large
 qemu-img: Could not apply snapshot 'test': Failed to load snapshot: File too 
large
+qemu-img: Could not delete snapshot 'test': Snapshot L1 table too large
 *** done
-- 
2.13.6

[Qemu-block] [PULL 08/56] qcow2: Generalize validate_table_offset() into qcow2_validate_table()

[Kevin Wolf]

From: Alberto Garcia 

This function checks that the offset and size of a table are valid.

While the offset checks are fine, the size check is too generic, since
it only verifies that the total size in bytes fits in a 64-bit
integer. In practice all tables used in qcow2 have much smaller size
limits, so the size needs to be checked again for each table using its
actual limit.

This patch generalizes this function by allowing the caller to specify
the maximum size for that table. In addition to that it allows passing
an Error variable.

The function is also renamed and made public since we're going to use
it in other parts of the code.

Signed-off-by: Alberto Garcia 
Reviewed-by: Eric Blake 
Signed-off-by: Kevin Wolf 
---
 block/qcow2.h  | 10 +++---
 block/qcow2.c  | 77 ++
 tests/qemu-iotests/080.out | 16 +-
 3 files changed, 43 insertions(+), 60 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index 965846f7af..ccb92a9696 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -485,11 +485,6 @@ static inline int64_t qcow2_vm_state_offset(BDRVQcow2State 
*s)
 return (int64_t)s->l1_vm_state_index << (s->cluster_bits + s->l2_bits);
 }
 
-static inline uint64_t qcow2_max_refcount_clusters(BDRVQcow2State *s)
-{
-return QCOW_MAX_REFTABLE_SIZE >> s->cluster_bits;
-}
-
 static inline QCow2ClusterType qcow2_get_cluster_type(uint64_t l2_entry)
 {
 if (l2_entry & QCOW_OFLAG_COMPRESSED) {
@@ -547,6 +542,11 @@ void qcow2_signal_corruption(BlockDriverState *bs, bool 
fatal, int64_t offset,
  int64_t size, const char *message_format, ...)
  GCC_FMT_ATTR(5, 6);
 
+int qcow2_validate_table(BlockDriverState *bs, uint64_t offset,
+ uint64_t entries, size_t entry_len,
+ int64_t max_size_bytes, const char *table_name,
+ Error **errp);
+
 /* qcow2-refcount.c functions */
 int qcow2_refcount_init(BlockDriverState *bs);
 void qcow2_refcount_close(BlockDriverState *bs);
diff --git a/block/qcow2.c b/block/qcow2.c
index 9fb00f2373..369e374a9b 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -573,26 +573,23 @@ static int coroutine_fn qcow2_co_check(BlockDriverState 
*bs,
 return ret;
 }
 
-static int validate_table_offset(BlockDriverState *bs, uint64_t offset,
- uint64_t entries, size_t entry_len)
+int qcow2_validate_table(BlockDriverState *bs, uint64_t offset,
+ uint64_t entries, size_t entry_len,
+ int64_t max_size_bytes, const char *table_name,
+ Error **errp)
 {
 BDRVQcow2State *s = bs->opaque;
-uint64_t size;
 
-/* Use signed INT64_MAX as the maximum even for uint64_t header fields,
- * because values will be passed to qemu functions taking int64_t. */
-if (entries > INT64_MAX / entry_len) {
-return -EINVAL;
-}
-
-size = entries * entry_len;
-
-if (INT64_MAX - size < offset) {
-return -EINVAL;
+if (entries > max_size_bytes / entry_len) {
+error_setg(errp, "%s too large", table_name);
+return -EFBIG;
 }
 
-/* Tables must be cluster aligned */
-if (offset_into_cluster(s, offset) != 0) {
+/* Use signed INT64_MAX as the maximum even for uint64_t header fields,
+ * because values will be passed to qemu functions taking int64_t. */
+if ((INT64_MAX - entries * entry_len < offset) ||
+(offset_into_cluster(s, offset) != 0)) {
+error_setg(errp, "%s offset invalid", table_name);
 return -EINVAL;
 }
 
@@ -1323,47 +1320,42 @@ static int coroutine_fn qcow2_do_open(BlockDriverState 
*bs, QDict *options,
 s->refcount_table_size =
 header.refcount_table_clusters << (s->cluster_bits - 3);
 
-if (header.refcount_table_clusters > qcow2_max_refcount_clusters(s)) {
-error_setg(errp, "Reference count table too large");
-ret = -EINVAL;
-goto fail;
-}
-
 if (header.refcount_table_clusters == 0 && !(flags & BDRV_O_CHECK)) {
 error_setg(errp, "Image does not contain a reference count table");
 ret = -EINVAL;
 goto fail;
 }
 
-ret = validate_table_offset(bs, s->refcount_table_offset,
-s->refcount_table_size, sizeof(uint64_t));
+ret = qcow2_validate_table(bs, s->refcount_table_offset,
+   header.refcount_table_clusters,
+   s->cluster_size, QCOW_MAX_REFTABLE_SIZE,
+   "Reference count table", errp);
 if (ret < 0) {
-error_setg(errp, "Invalid reference count table offset");
 goto fail;
 }
 
-/* Snapshot table offset/length */
-if (header.nb_snapshots > QCOW_MAX_SNAPSHOTS) {
-error_setg(errp, "Too many snapshots");
-ret = -EINVAL;
-goto fail;
-}
-
-ret = valida

[Qemu-block] [PULL 11/56] qcow2: Check snapshot L1 tables in qcow2_check_metadata_overlap()

[Kevin Wolf]

From: Alberto Garcia 

The inactive-l2 overlap check iterates uses the L1 tables from all
snapshots, but it does not validate them first.

We now have a function to take care of this, so let's use it.

Signed-off-by: Alberto Garcia 
Reviewed-by: Eric Blake 
Signed-off-by: Kevin Wolf 
---
 block/qcow2-refcount.c | 10 +-
 tests/qemu-iotests/080 |  4 
 tests/qemu-iotests/080.out |  4 
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
index 2f7e710fa6..b18ea0ca98 100644
--- a/block/qcow2-refcount.c
+++ b/block/qcow2-refcount.c
@@ -2642,9 +2642,17 @@ int qcow2_check_metadata_overlap(BlockDriverState *bs, 
int ign, int64_t offset,
 uint64_t l1_ofs = s->snapshots[i].l1_table_offset;
 uint32_t l1_sz  = s->snapshots[i].l1_size;
 uint64_t l1_sz2 = l1_sz * sizeof(uint64_t);
-uint64_t *l1 = g_try_malloc(l1_sz2);
+uint64_t *l1;
 int ret;
 
+ret = qcow2_validate_table(bs, l1_ofs, l1_sz, sizeof(uint64_t),
+   QCOW_MAX_L1_SIZE, "", NULL);
+if (ret < 0) {
+return ret;
+}
+
+l1 = g_try_malloc(l1_sz2);
+
 if (l1_sz2 && l1 == NULL) {
 return -ENOMEM;
 }
diff --git a/tests/qemu-iotests/080 b/tests/qemu-iotests/080
index 5622604f83..018f815697 100755
--- a/tests/qemu-iotests/080
+++ b/tests/qemu-iotests/080
@@ -178,6 +178,8 @@ _make_test_img 64M
 poke_file "$TEST_IMG" "$offset_snap1_l1_offset" 
"\x00\x00\x00\x00\x00\x40\x02\x00"
 { $QEMU_IMG convert -s test $TEST_IMG $TEST_IMG.snap; } 2>&1 | _filter_testdir
 { $QEMU_IMG amend -o compat=0.10 $TEST_IMG; } 2>&1 | _filter_testdir
+{ $QEMU_IO -c "open -o overlap-check.inactive-l2=on $TEST_IMG" \
+   -c 'write 0 4k'; } 2>&1 | _filter_qemu_io | _filter_testdir
 
 echo
 echo "== Invalid snapshot L1 table size =="
@@ -187,6 +189,8 @@ _make_test_img 64M
 poke_file "$TEST_IMG" "$offset_snap1_l1_size" "\x10\x00\x00\x00"
 { $QEMU_IMG convert -s test $TEST_IMG $TEST_IMG.snap; } 2>&1 | _filter_testdir
 { $QEMU_IMG amend -o compat=0.10 $TEST_IMG; } 2>&1 | _filter_testdir
+{ $QEMU_IO -c "open -o overlap-check.inactive-l2=on $TEST_IMG" \
+   -c 'write 0 4k'; } 2>&1 | _filter_qemu_io | _filter_testdir
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/080.out b/tests/qemu-iotests/080.out
index 315edbc26f..d622ac06c0 100644
--- a/tests/qemu-iotests/080.out
+++ b/tests/qemu-iotests/080.out
@@ -66,6 +66,8 @@ wrote 512/512 bytes at offset 0
 qemu-img: Failed to load snapshot: Snapshot L1 table offset invalid
 qemu-img: Snapshot L1 table offset invalid
 qemu-img: Error while amending options: Invalid argument
+Failed to flush the refcount block cache: Invalid argument
+write failed: Invalid argument
 
 == Invalid snapshot L1 table size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
@@ -74,4 +76,6 @@ wrote 512/512 bytes at offset 0
 qemu-img: Failed to load snapshot: Snapshot L1 table too large
 qemu-img: Snapshot L1 table too large
 qemu-img: Error while amending options: File too large
+Failed to flush the refcount block cache: File too large
+write failed: File too large
 *** done
-- 
2.13.6

[Qemu-block] [PULL 04/56] qcow2: make qcow2_do_open a coroutine_fn

[Kevin Wolf]

From: Paolo Bonzini 

It is called from qcow2_invalidate_cache in coroutine context (incoming
migration runs in a coroutine), so it's cleaner if metadata is always
loaded from a coroutine.

Signed-off-by: Paolo Bonzini 
Message-Id: <1516279431-30424-4-git-send-email-pbonz...@redhat.com>
Signed-off-by: Paolo Bonzini 
Signed-off-by: Kevin Wolf 
---
 block/qcow2.c | 46 +-
 1 file changed, 41 insertions(+), 5 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index ee040a49cd..4ac1a45e61 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -1118,8 +1118,9 @@ static int qcow2_update_options(BlockDriverState *bs, 
QDict *options,
 return ret;
 }
 
-static int qcow2_do_open(BlockDriverState *bs, QDict *options, int flags,
- Error **errp)
+/* Called with s->lock held.  */
+static int coroutine_fn qcow2_do_open(BlockDriverState *bs, QDict *options,
+  int flags, Error **errp)
 {
 BDRVQcow2State *s = bs->opaque;
 unsigned int len, i;
@@ -1498,8 +1499,6 @@ static int qcow2_do_open(BlockDriverState *bs, QDict 
*options, int flags,
 }
 }
 
-/* Initialise locks */
-qemu_co_mutex_init(&s->lock);
 bs->supported_zero_flags = header.version >= 3 ? BDRV_REQ_MAY_UNMAP : 0;
 
 /* Repair image if dirty */
@@ -1545,16 +1544,53 @@ static int qcow2_do_open(BlockDriverState *bs, QDict 
*options, int flags,
 return ret;
 }
 
+typedef struct QCow2OpenCo {
+BlockDriverState *bs;
+QDict *options;
+int flags;
+Error **errp;
+int ret;
+} QCow2OpenCo;
+
+static void coroutine_fn qcow2_open_entry(void *opaque)
+{
+QCow2OpenCo *qoc = opaque;
+BDRVQcow2State *s = qoc->bs->opaque;
+
+qemu_co_mutex_lock(&s->lock);
+qoc->ret = qcow2_do_open(qoc->bs, qoc->options, qoc->flags, qoc->errp);
+qemu_co_mutex_unlock(&s->lock);
+}
+
 static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
   Error **errp)
 {
+BDRVQcow2State *s = bs->opaque;
+QCow2OpenCo qoc = {
+.bs = bs,
+.options = options,
+.flags = flags,
+.errp = errp,
+.ret = -EINPROGRESS
+};
+
 bs->file = bdrv_open_child(NULL, options, "file", bs, &child_file,
false, errp);
 if (!bs->file) {
 return -EINVAL;
 }
 
-return qcow2_do_open(bs, options, flags, errp);
+/* Initialise locks */
+qemu_co_mutex_init(&s->lock);
+
+if (qemu_in_coroutine()) {
+/* From bdrv_co_create.  */
+qcow2_open_entry(&qoc);
+} else {
+qemu_coroutine_enter(qemu_coroutine_create(qcow2_open_entry, &qoc));
+BDRV_POLL_WHILE(bs, qoc.ret == -EINPROGRESS);
+}
+return qoc.ret;
 }
 
 static void qcow2_refresh_limits(BlockDriverState *bs, Error **errp)
-- 
2.13.6

[Qemu-block] [PULL 06/56] block: convert bdrv_invalidate_cache callback to coroutine_fn

[Kevin Wolf]

From: Paolo Bonzini 

QED's bdrv_invalidate_cache implementation would like to reuse functions
that acquire/release the metadata locks.  Call it from coroutine context
to simplify the logic.

Signed-off-by: Paolo Bonzini 
Message-Id: <1516279431-30424-6-git-send-email-pbonz...@redhat.com>
Signed-off-by: Paolo Bonzini 
Signed-off-by: Kevin Wolf 
---
 include/block/block_int.h |  3 ++-
 block.c   | 41 +
 block/iscsi.c |  6 +++---
 block/nfs.c   |  6 +++---
 block/qcow2.c |  7 +--
 block/qed.c   | 13 +
 block/rbd.c   |  6 +++---
 7 files changed, 58 insertions(+), 24 deletions(-)

diff --git a/include/block/block_int.h b/include/block/block_int.h
index 64a5700f2b..e391cae53b 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -224,7 +224,8 @@ struct BlockDriver {
 /*
  * Invalidate any cached meta-data.
  */
-void (*bdrv_invalidate_cache)(BlockDriverState *bs, Error **errp);
+void coroutine_fn (*bdrv_co_invalidate_cache)(BlockDriverState *bs,
+  Error **errp);
 int (*bdrv_inactivate)(BlockDriverState *bs);
 
 /*
diff --git a/block.c b/block.c
index 4f76714f6b..603befb325 100644
--- a/block.c
+++ b/block.c
@@ -4209,7 +4209,8 @@ void bdrv_init_with_whitelist(void)
 bdrv_init();
 }
 
-void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp)
+static void coroutine_fn bdrv_co_invalidate_cache(BlockDriverState *bs,
+  Error **errp)
 {
 BdrvChild *child, *parent;
 uint64_t perm, shared_perm;
@@ -4225,7 +4226,7 @@ void bdrv_invalidate_cache(BlockDriverState *bs, Error 
**errp)
 }
 
 QLIST_FOREACH(child, &bs->children, next) {
-bdrv_invalidate_cache(child->bs, &local_err);
+bdrv_co_invalidate_cache(child->bs, &local_err);
 if (local_err) {
 error_propagate(errp, local_err);
 return;
@@ -4255,8 +4256,8 @@ void bdrv_invalidate_cache(BlockDriverState *bs, Error 
**errp)
 }
 bdrv_set_perm(bs, perm, shared_perm);
 
-if (bs->drv->bdrv_invalidate_cache) {
-bs->drv->bdrv_invalidate_cache(bs, &local_err);
+if (bs->drv->bdrv_co_invalidate_cache) {
+bs->drv->bdrv_co_invalidate_cache(bs, &local_err);
 if (local_err) {
 bs->open_flags |= BDRV_O_INACTIVE;
 error_propagate(errp, local_err);
@@ -4282,6 +4283,38 @@ void bdrv_invalidate_cache(BlockDriverState *bs, Error 
**errp)
 }
 }
 
+typedef struct InvalidateCacheCo {
+BlockDriverState *bs;
+Error **errp;
+bool done;
+} InvalidateCacheCo;
+
+static void coroutine_fn bdrv_invalidate_cache_co_entry(void *opaque)
+{
+InvalidateCacheCo *ico = opaque;
+bdrv_co_invalidate_cache(ico->bs, ico->errp);
+ico->done = true;
+}
+
+void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp)
+{
+Coroutine *co;
+InvalidateCacheCo ico = {
+.bs = bs,
+.done = false,
+.errp = errp
+};
+
+if (qemu_in_coroutine()) {
+/* Fast-path if already in coroutine context */
+bdrv_invalidate_cache_co_entry(&ico);
+} else {
+co = qemu_coroutine_create(bdrv_invalidate_cache_co_entry, &ico);
+qemu_coroutine_enter(co);
+BDRV_POLL_WHILE(bs, !ico.done);
+}
+}
+
 void bdrv_invalidate_cache_all(Error **errp)
 {
 BlockDriverState *bs;
diff --git a/block/iscsi.c b/block/iscsi.c
index 8bf0e87244..a82170f16e 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -2177,8 +2177,8 @@ static int iscsi_get_info(BlockDriverState *bs, 
BlockDriverInfo *bdi)
 return 0;
 }
 
-static void iscsi_invalidate_cache(BlockDriverState *bs,
-   Error **errp)
+static void coroutine_fn iscsi_co_invalidate_cache(BlockDriverState *bs,
+   Error **errp)
 {
 IscsiLun *iscsilun = bs->opaque;
 iscsi_allocmap_invalidate(iscsilun);
@@ -2209,7 +2209,7 @@ static BlockDriver bdrv_iscsi = {
 .create_opts= &iscsi_create_opts,
 .bdrv_reopen_prepare= iscsi_reopen_prepare,
 .bdrv_reopen_commit = iscsi_reopen_commit,
-.bdrv_invalidate_cache  = iscsi_invalidate_cache,
+.bdrv_co_invalidate_cache = iscsi_co_invalidate_cache,
 
 .bdrv_getlength  = iscsi_getlength,
 .bdrv_get_info   = iscsi_get_info,
diff --git a/block/nfs.c b/block/nfs.c
index 1d82ff5042..7433d25856 100644
--- a/block/nfs.c
+++ b/block/nfs.c
@@ -876,8 +876,8 @@ static void nfs_refresh_filename(BlockDriverState *bs, 
QDict *options)
 }
 
 #ifdef LIBNFS_FEATURE_PAGECACHE
-static void nfs_invalidate_cache(BlockDriverState *bs,
- Error **errp)
+static void coroutine_fn nfs_co_invalidate_cache(BlockDriverState *bs,
+ Error **errp)
 {
 NFSClient *client = bs->op