date:20200609

This was mostly boilerplate conversion, but in one case I needed to
define several differently named char* to take the place of a single
char *tmp that was re-used multiple times, and in another place there
was a single char* that was used at the toplevel of the function, and
then later used repeatedly inside a for loop, so I defined a new
separate char* inside the loop.

Signed-off-by: Laine Stump 
---

This should be applied on top of Dan's IPv6 NAT patch series (it was
reviewing that series that showed me this file hadn't yet been
converted).

 src/conf/network_conf.c | 613 +---
 src/conf/network_conf.h |   1 +
 src/util/virxml.h   |   1 +
 3 files changed, 259 insertions(+), 356 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 1b89e2985d..ebe8a03262 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -306,8 +306,7 @@ virNetworkDefCopy(virNetworkDefPtr def,
   virNetworkXMLOptionPtr xmlopt,
   unsigned int flags)
 {
-char *xml = NULL;
-virNetworkDefPtr newDef = NULL;
+g_autofree char *xml = NULL;
 
 if (!def) {
 virReportError(VIR_ERR_INTERNAL_ERROR,
@@ -317,11 +316,9 @@ virNetworkDefCopy(virNetworkDefPtr def,
 
 /* deep copy with a format/parse cycle */
 if (!(xml = virNetworkDefFormat(def, xmlopt, flags)))
-goto cleanup;
-newDef = virNetworkDefParseString(xml, xmlopt);
- cleanup:
-VIR_FREE(xml);
-return newDef;
+   return NULL;
+
+return virNetworkDefParseString(xml, xmlopt);
 }
 
 
@@ -465,48 +462,43 @@ virNetworkDHCPRangeDefParseXML(const char *networkName,
 {
 virSocketAddrRangePtr addr = >addr;
 xmlNodePtr cur = node->children;
-char *start = NULL, *end = NULL;
-int ret = -1;
+g_autofree char *start = NULL;
+g_autofree char *end = NULL;
 
 if (!(start = virXMLPropString(node, "start"))) {
 virReportError(VIR_ERR_XML_ERROR,
_("Missing 'start' attribute in dhcp range for network 
'%s'"),
networkName);
-goto cleanup;
+return -1;
 }
 if (virSocketAddrParse(>start, start, AF_UNSPEC) < 0)
-goto cleanup;
+return -1;
 
 if (!(end = virXMLPropString(node, "end"))) {
 virReportError(VIR_ERR_XML_ERROR,
_("Missing 'end' attribute in dhcp range for network 
'%s'"),
networkName);
-goto cleanup;
+return -1;
 }
 if (virSocketAddrParse(>end, end, AF_UNSPEC) < 0)
-goto cleanup;
+return -1;
 
 /* do a sanity check of the range */
 if (virSocketAddrGetRange(>start, >end, >address,
   virNetworkIPDefPrefix(ipdef)) < 0)
-goto cleanup;
+return -1;
 
 while (cur != NULL) {
 if (cur->type == XML_ELEMENT_NODE &&
 virXMLNodeNameEqual(cur, "lease")) {
 
 if (virNetworkDHCPLeaseTimeDefParseXML(>lease, cur) < 0)
-goto cleanup;
+return -1;
 }
 cur = cur->next;
 }
 
-ret = 0;
-
- cleanup:
-VIR_FREE(start);
-VIR_FREE(end);
-return ret;
+return 0;
 }
 
 
@@ -517,11 +509,13 @@ virNetworkDHCPHostDefParseXML(const char *networkName,
   virNetworkDHCPHostDefPtr host,
   bool partialOkay)
 {
-char *mac = NULL, *name = NULL, *ip = NULL, *id = NULL;
+g_autofree char *mac = NULL;
+g_autofree char *name = NULL;
+g_autofree char *ip = NULL;
+g_autofree char *id = NULL;
 virMacAddr addr;
 virSocketAddr inaddr;
 xmlNodePtr cur = node->children;
-int ret = -1;
 
 mac = virXMLPropString(node, "mac");
 if (mac != NULL) {
@@ -530,20 +524,20 @@ virNetworkDHCPHostDefParseXML(const char *networkName,
_("Invalid to specify MAC address '%s' "
  "in network '%s' IPv6 static host definition"),
mac, networkName);
-goto cleanup;
+return -1;
 }
 if (virMacAddrParse(mac, ) < 0) {
 virReportError(VIR_ERR_XML_ERROR,
_("Cannot parse MAC address '%s' in network '%s'"),
mac, networkName);
-goto cleanup;
+return -1;
 }
 if (virMacAddrIsMulticast()) {
 virReportError(VIR_ERR_XML_ERROR,
_("expected unicast mac address, found "
  "multicast '%s' in network '%s'"),
(const char *)mac, networkName);
-goto cleanup;
+return -1;
 }
 }
 
@@ -554,7 +548,7 @@ virNetworkDHCPHostDefParseXML(const char *networkName,
 virReportError(VIR_ERR_XML_ERROR,
_("Invalid character '%c' in id '%s' of network 
'%s'"),

Re: [libvirt PATCH v2 2/3] conf: add an attribute to turn on NAT for IPv6 virtual networks


On 6/9/20 12:17 PM, Daniel P. Berrangé wrote:

Historically IPv6 did not support NAT, so when IPv6 was added to
libvirt's virtual networks, when requesting 
libvirt will NOT apply NAT to IPv6 traffic, only IPv4 traffic.

This is an annoying historical design decision as it means we
cannot enable IPv6 automatically. We thus need to introduce a
new attribute


  


The new attribute is a tri-state, so it leaves open the possibility of
us intentionally changing the default behaviour in future to honour
NAT for IPv6.



Any network defined with an older libvirt, and even newly defined 
networks that don't have ipv6 set explicitly in the input XML, won't 
have an explicit value saved in the config. So if we change the default, 
those existing networks will have different behavior. I don't see how we 
could change the default without those existing networks getting 
automatically changed to ipv6='yes'.





Signed-off-by: Daniel P. Berrangé 
---
  docs/formatnetwork.html.in| 14 +
  docs/schemas/network.rng  |  5 
  src/conf/network_conf.c   | 30 +--
  src/conf/network_conf.h   |  2 ++
  .../nat-network-forward-nat-ipv6.xml  | 10 +++
  .../nat-network-forward-nat-ipv6.xml  | 10 +++
  tests/networkxml2xmltest.c|  1 +
  7 files changed, 69 insertions(+), 3 deletions(-)
  create mode 100644 tests/networkxml2xmlin/nat-network-forward-nat-ipv6.xml
  create mode 100644 tests/networkxml2xmlout/nat-network-forward-nat-ipv6.xml

diff --git a/docs/formatnetwork.html.in b/docs/formatnetwork.html.in
index 0383e2d891..fb740111b1 100644
--- a/docs/formatnetwork.html.in
+++ b/docs/formatnetwork.html.in
@@ -276,6 +276,20 @@
  /nat
/forward
  ...
+
+
+  Since 6.5.0 it is possible to
+  enable NAT with IPv6 networking. As noted above, IPv6
+  has historically done plain forwarding and thus to avoid
+  breaking historical compatibility, IPv6 NAT must be
+  explicitly requested.
+
+
+...
+  forward mode='nat'
+nat ipv6='yes'/
+  /forward
+...

  
route

diff --git a/docs/schemas/network.rng b/docs/schemas/network.rng
index 88b6f4dfdd..3a5eb3ced4 100644
--- a/docs/schemas/network.rng
+++ b/docs/schemas/network.rng
@@ -181,6 +181,11 @@


  
+  
+
+  
+
+  

  

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index f1d22b25b1..1b89e2985d 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -1358,6 +1358,7 @@ virNetworkForwardNatDefParseXML(const char *networkName,
  int nNatAddrs, nNatPorts;
  char *addrStart = NULL;
  char *addrEnd = NULL;
+char *ipv6 = NULL;



Eww. I just noticed this file isn't doing autofree memory allocation on  
many of its pointers...




  VIR_XPATH_NODE_AUTORESTORE(ctxt);
  
  ctxt->node = node;

@@ -1369,6 +1370,20 @@ virNetworkForwardNatDefParseXML(const char *networkName,
  goto cleanup;
  }
  
+ipv6 = virXMLPropString(node, "ipv6");

+if (ipv6) {
+int natIPv6;
+if ((natIPv6 = virTristateBoolTypeFromString(ipv6)) <= 0) {
+virReportError(VIR_ERR_XML_ERROR,
+   _("Invalid ipv6 setting '%s' "
+ "in network '%s' NAT"),
+   ipv6, networkName);
+goto cleanup;



...which leads to a memory leak here, since the code at the cleanup 
label doesn't free ipv6.



For some reason I felt like spending an hour doing something mindless 
just now, so I made a patch to convert all the char* and xmlNodePtr* in 
network_conf.c to g_autofree (along with a few other g_autoptr things, 
and the standard cleanup/error label removals). I'll post that as a 
patch based on your patches; you can either just assume that my patch 
will go in after yours and fix the memory leak, or you can just change 
ipv6 to a g_autofree yourself and I'll adjust my patch accordingly.




+}
+def->natIPv6 = natIPv6;
+VIR_FREE(ipv6);
+}
+
  /* addresses for SNAT */
  nNatAddrs = virXPathNodeSet("./address", ctxt, );
  if (nNatAddrs < 0) {
@@ -2516,10 +2531,18 @@ virNetworkForwardNatDefFormat(virBufferPtr buf,
  goto cleanup;
  }
  
-if (!addrEnd && !addrStart && !fwd->port.start && !fwd->port.end)

+if (!addrEnd && !addrStart && !fwd->port.start && !fwd->port.end && 
!fwd->natIPv6)
  return 0;
  
-virBufferAddLit(buf, "\n");

+virBufferAddLit(buf, "natIPv6)
+virBufferAsprintf(buf, " ipv6='%s'", 
virTristateBoolTypeToString(fwd->natIPv6));
+
+if (!addrEnd && !addrStart &&

Re: [libvirt PATCH v2 3/3] network: wire up support for IPv6 NAT rules


On 6/9/20 12:17 PM, Daniel P. Berrangé wrote:

Now that we have support for IPv6 in the iptables helpers, and a new
option in the XML schema, we can wire up support for it in the network
driver.

Signed-off-by: Daniel P. Berrangé 
---
  src/network/bridge_driver_linux.c |  23 +-
  .../nat-ipv6-masquerade-linux.args| 228 ++
  .../nat-ipv6-masquerade.xml   |  17 ++
  tests/networkxml2firewalltest.c   |   1 +
  4 files changed, 262 insertions(+), 7 deletions(-)
  create mode 100644 
tests/networkxml2firewalldata/nat-ipv6-masquerade-linux.args
  create mode 100644 tests/networkxml2firewalldata/nat-ipv6-masquerade.xml

diff --git a/src/network/bridge_driver_linux.c 
b/src/network/bridge_driver_linux.c
index b0bd207250..fcb3803965 100644
--- a/src/network/bridge_driver_linux.c
+++ b/src/network/bridge_driver_linux.c
@@ -307,7 +307,8 @@ int networkCheckRouteCollision(virNetworkDefPtr def)
  return ret;
  }
  
-static const char networkLocalMulticast[] = "224.0.0.0/24";

+static const char networkLocalMulticastIPv4[] = "224.0.0.0/24";
+static const char networkLocalMulticastIPv6[] = "ffx2::/16";



Once I got everything built and tried starting a network with ipv6 nat, 
I got this error message:



virsh net-start ipv6 error: Failed to start network ipv6 error: 
COMMAND_FAILED: '/usr/sbin/ip6tables -w10 -w --table nat --insert 
LIBVIRT_PRT --source 2001:4978:2ac:5::/80 --destination ffx2::/16 --jump 
RETURN' failed: ip6tables v1.8.3 (legacy): host/network `ffx2::' not 
found Try `ip6tables -h' or 'ip6tables --help' for more information.



Do we need to do something different for multicast traffic in the case 
of IPv6?



Other than that it all looks good, so


Reviewed-by: Laine Stump 


once the problem with multicast ffx2::/16 as the destination of a rule 
is resolved.




  static const char networkLocalBroadcast[] = "255.255.255.255/32";
  
  static int

@@ -317,6 +318,7 @@ networkAddMasqueradingFirewallRules(virFirewallPtr fw,
  {
  int prefix = virNetworkIPDefPrefix(ipdef);
  const char *forwardIf = virNetworkDefForwardIf(def, 0);
+bool isIPv4 = VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET);
  
  if (prefix < 0) {

  virReportError(VIR_ERR_INTERNAL_ERROR,
@@ -406,7 +408,8 @@ networkAddMasqueradingFirewallRules(virFirewallPtr fw,
  return -1;
  
  /* exempt local network broadcast address as destination */

-if (iptablesAddDontMasquerade(fw,
+if (isIPv4 &&
+iptablesAddDontMasquerade(fw,
>address,
prefix,
forwardIf,
@@ -418,7 +421,8 @@ networkAddMasqueradingFirewallRules(virFirewallPtr fw,
>address,
prefix,
forwardIf,
-  networkLocalMulticast) < 0)
+  isIPv4 ? networkLocalMulticastIPv4 :
+  networkLocalMulticastIPv6) < 0)
  return -1;
  
  return 0;

@@ -431,6 +435,7 @@ networkRemoveMasqueradingFirewallRules(virFirewallPtr fw,
  {
  int prefix = virNetworkIPDefPrefix(ipdef);
  const char *forwardIf = virNetworkDefForwardIf(def, 0);
+bool isIPv4 = VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET);
  
  if (prefix < 0)

  return 0;
@@ -439,10 +444,12 @@ networkRemoveMasqueradingFirewallRules(virFirewallPtr fw,
   >address,
   prefix,
   forwardIf,
- networkLocalMulticast) < 0)
+ isIPv4 ? networkLocalMulticastIPv4 :
+ networkLocalMulticastIPv6) < 0)
  return -1;
  
-if (iptablesRemoveDontMasquerade(fw,

+if (isIPv4 &&
+iptablesRemoveDontMasquerade(fw,
   >address,
   prefix,
   forwardIf,
@@ -769,7 +776,8 @@ networkAddIPSpecificFirewallRules(virFirewallPtr fw,
   */
  
  if (def->forward.type == VIR_NETWORK_FORWARD_NAT) {

-if (VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET))
+if (VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET) ||
+def->forward.natIPv6 == VIR_TRISTATE_BOOL_YES)
  return networkAddMasqueradingFirewallRules(fw, def, ipdef);
  else if (VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET6))
  return networkAddRoutingFirewallRules(fw, def, ipdef);
@@ -786,7 +794,8 @@ networkRemoveIPSpecificFirewallRules(virFirewallPtr fw,
   virNetworkIPDefPtr ipdef)
  {
  if (def->forward.type == VIR_NETWORK_FORWARD_NAT) {
-if (VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET))
+if

[RFC 17/21] conf: Replace virNetworkDNSForwarderParseXML(hardcoded) with namesake(generated)

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 41 +
 src/conf/network_conf.h |  6 +++---
 2 files changed, 4 insertions(+), 43 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index e1790bc..8610fbc 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -174,13 +174,6 @@ virNetworkIPDefClear(virNetworkIPDefPtr def)
 }
 
 
-static void
-virNetworkDNSForwarderClear(virNetworkDNSForwarderPtr def)
-{
-VIR_FREE(def->domain);
-}
-
-
 static void
 virNetworkDNSDefClear(virNetworkDNSDefPtr def)
 {
@@ -888,7 +881,7 @@ virNetworkDNSTxtDefParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
 }
 
 
-static int
+int
 virNetworkDNSForwarderParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
virNetworkDNSForwarderPtr def,
const char *instname G_GNUC_UNUSED,
@@ -907,38 +900,6 @@ virNetworkDNSForwarderParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
 }
 
 
-static int
-virNetworkDNSForwarderParseXML(xmlNodePtr node,
-   virNetworkDNSForwarderPtr def,
-   const char *networkName,
-   void *opaque)
-{
-char *addr = virXMLPropString(node, "addr");
-
-if (addr && virSocketAddrParse(>addr,
-   addr, AF_UNSPEC) < 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("Invalid forwarder IP address '%s' "
- "in network '%s'"),
-   addr, networkName);
-VIR_FREE(addr);
-goto cleanup;
-}
-def->domain = virXMLPropString(node, "domain");
-
-if (virNetworkDNSForwarderParseXMLHook(node, def, networkName, opaque,
-   addr, def->domain) < 0)
-goto cleanup;
-
-VIR_FREE(addr);
-
-return 0;
-
- cleanup:
-return -1;
-}
-
-
 static int
 virNetworkDNSDefParseXML(const char *networkName,
  xmlNodePtr node,
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index a5a4939..f14eef2 100644
--- a/src/conf/network_conf.h
+++ b/src/conf/network_conf.h
@@ -158,9 +158,9 @@ struct _virNetworkDNSHostDef {  /* genparse:withhook, 
genformat */
 
 typedef struct _virNetworkDNSForwarder virNetworkDNSForwarder;
 typedef virNetworkDNSForwarder *virNetworkDNSForwarderPtr;
-struct _virNetworkDNSForwarder {
-virSocketAddr addr;
-char *domain;
+struct _virNetworkDNSForwarder {/* genparse:withhook */
+virSocketAddr addr; /* xmlattr */
+char *domain;   /* xmlattr */
 };
 
 typedef struct _virNetworkDNSDef virNetworkDNSDef;
-- 
2.17.1

[RFC 16/21] conf: Extract virNetworkDNSForwarderParseXML from virNetworkDNSParseXML

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 74 +++--
 1 file changed, 56 insertions(+), 18 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 1fea580..e1790bc 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -888,6 +888,57 @@ virNetworkDNSTxtDefParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
 }
 
 
+static int
+virNetworkDNSForwarderParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
+   virNetworkDNSForwarderPtr def,
+   const char *instname G_GNUC_UNUSED,
+   void *opaque G_GNUC_UNUSED,
+   const char *addr,
+   const char *domain G_GNUC_UNUSED)
+{
+if (!(addr || def->domain)) {
+virReportError(VIR_ERR_XML_ERROR, "%s",
+   _("Invalid forwarder element, must contain "
+ "at least one of addr or domain"));
+return -1;
+}
+
+return 0;
+}
+
+
+static int
+virNetworkDNSForwarderParseXML(xmlNodePtr node,
+   virNetworkDNSForwarderPtr def,
+   const char *networkName,
+   void *opaque)
+{
+char *addr = virXMLPropString(node, "addr");
+
+if (addr && virSocketAddrParse(>addr,
+   addr, AF_UNSPEC) < 0) {
+virReportError(VIR_ERR_XML_ERROR,
+   _("Invalid forwarder IP address '%s' "
+ "in network '%s'"),
+   addr, networkName);
+VIR_FREE(addr);
+goto cleanup;
+}
+def->domain = virXMLPropString(node, "domain");
+
+if (virNetworkDNSForwarderParseXMLHook(node, def, networkName, opaque,
+   addr, def->domain) < 0)
+goto cleanup;
+
+VIR_FREE(addr);
+
+return 0;
+
+ cleanup:
+return -1;
+}
+
+
 static int
 virNetworkDNSDefParseXML(const char *networkName,
  xmlNodePtr node,
@@ -943,25 +994,12 @@ virNetworkDNSDefParseXML(const char *networkName,
 goto cleanup;
 
 for (i = 0; i < nfwds; i++) {
-char *addr = virXMLPropString(fwdNodes[i], "addr");
-
-if (addr && virSocketAddrParse(>forwarders[i].addr,
-   addr, AF_UNSPEC) < 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("Invalid forwarder IP address '%s' "
- "in network '%s'"),
-   addr, networkName);
-VIR_FREE(addr);
+if (virNetworkDNSForwarderParseXML(fwdNodes[i],
+   >forwarders[i],
+   networkName,
+   NULL) < 0)
 goto cleanup;
-}
-def->forwarders[i].domain = virXMLPropString(fwdNodes[i], 
"domain");
-if (!(addr || def->forwarders[i].domain)) {
-virReportError(VIR_ERR_XML_ERROR, "%s",
-   _("Invalid forwarder element, must contain "
- "at least one of addr or domain"));
-goto cleanup;
-}
-VIR_FREE(addr);
+
 def->nfwds++;
 }
 }
-- 
2.17.1

[RFC 21/21] conf: Generate virNetworkDNSDefFormatBuf

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 77 ++---
 src/conf/network_conf.h | 10 +++---
 2 files changed, 8 insertions(+), 79 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index db2b75f..604e589 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -881,10 +881,10 @@ virNetworkDNSDefParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
  void *opaque G_GNUC_UNUSED,
  const char *enable G_GNUC_UNUSED,
  const char *forwardPlainNames G_GNUC_UNUSED,
+ int nfwds,
  int ntxts,
- int nhosts,
  int nsrvs,
- int nfwds)
+ int nhosts)
 {
 if (def->enable == VIR_TRISTATE_BOOL_NO &&
 (nfwds || nhosts || nsrvs || ntxts)) {
@@ -2024,77 +2024,6 @@ virNetworkDefParseNode(xmlDocPtr xml,
 }
 
 
-static int
-virNetworkDNSDefFormat(virBufferPtr buf,
-   const virNetworkDNSDef *def)
-{
-size_t i;
-
-if (!(def->enable || def->forwardPlainNames || def->nforwarders || 
def->nhosts ||
-  def->nsrvs || def->ntxts))
-return 0;
-
-virBufferAddLit(buf, "enable) {
-const char *fwd = virTristateBoolTypeToString(def->enable);
-
-if (!fwd) {
-virReportError(VIR_ERR_INTERNAL_ERROR,
-   _("Unknown enable type %d in network"),
-   def->enable);
-return -1;
-}
-virBufferAsprintf(buf, " enable='%s'", fwd);
-}
-if (def->forwardPlainNames) {
-const char *fwd = virTristateBoolTypeToString(def->forwardPlainNames);
-
-if (!fwd) {
-virReportError(VIR_ERR_INTERNAL_ERROR,
-   _("Unknown forwardPlainNames type %d in network"),
-   def->forwardPlainNames);
-return -1;
-}
-virBufferAsprintf(buf, " forwardPlainNames='%s'", fwd);
-}
-if (!(def->nforwarders || def->nhosts || def->nsrvs || def->ntxts)) {
-virBufferAddLit(buf, "/>\n");
-return 0;
-}
-
-virBufferAddLit(buf, ">\n");
-virBufferAdjustIndent(buf, 2);
-
-for (i = 0; i < def->nforwarders; i++) {
-if (virNetworkDNSForwarderFormatBuf(buf, "forwarder",
->forwarders[i], NULL) < 0)
-return -1;
-}
-
-for (i = 0; i < def->ntxts; i++) {
-if (virNetworkDNSTxtDefFormatBuf(buf, "txt", >txts[i], NULL) < 0)
-return -1;
-}
-
-for (i = 0; i < def->nsrvs; i++) {
-if (def->srvs[i].service && def->srvs[i].protocol) {
-if (virNetworkDNSSrvDefFormatBuf(buf, "srv", >srvs[i], NULL) 
< 0)
-return -1;
-}
-}
-
-if (def->nhosts) {
-for (i = 0; i < def->nhosts; i++) {
-if (virNetworkDNSHostDefFormatBuf(buf, "host", >hosts[i], 
NULL) < 0)
-return -1;
-}
-}
-virBufferAdjustIndent(buf, -2);
-virBufferAddLit(buf, "\n");
-return 0;
-}
-
-
 static int
 virNetworkIPDefFormat(virBufferPtr buf,
   const virNetworkIPDef *def)
@@ -2502,7 +2431,7 @@ virNetworkDefFormatBuf(virBufferPtr buf,
 virBufferAddLit(buf, "/>\n");
 }
 
-if (virNetworkDNSDefFormat(buf, >dns) < 0)
+if (virNetworkDNSDefFormatBuf(buf, "dns", >dns, NULL) < 0)
 return -1;
 
 if (virNetDevVlanFormat(>vlan, buf) < 0)
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index 462d97d..cfbd7c9 100644
--- a/src/conf/network_conf.h
+++ b/src/conf/network_conf.h
@@ -165,17 +165,17 @@ struct _virNetworkDNSForwarder {/* genparse:withhook, 
genformat */
 
 typedef struct _virNetworkDNSDef virNetworkDNSDef;
 typedef virNetworkDNSDef *virNetworkDNSDefPtr;
-struct _virNetworkDNSDef {  /* genparse:withhook */
+struct _virNetworkDNSDef {  /* genparse:withhook, genformat */
 virTristateBool enable; /* xmlattr */
 virTristateBool forwardPlainNames;  /* xmlattr */
+size_t nforwarders;
+virNetworkDNSForwarderPtr forwarders;   /* xmlelem, array */
 size_t ntxts;
 virNetworkDNSTxtDefPtr txts;/* xmlelem, array */
-size_t nhosts;
-virNetworkDNSHostDefPtr hosts;  /* xmlelem, array */
 size_t nsrvs;
 virNetworkDNSSrvDefPtr srvs;/* xmlelem, array */
-size_t nforwarders;
-virNetworkDNSForwarderPtr forwarders;   /* xmlelem, array */
+size_t nhosts;
+virNetworkDNSHostDefPtr hosts;  /* xmlelem, array */
 };
 
 typedef struct _virNetworkIPDef virNetworkIPDef;
-- 
2.17.1

[RFC 20/21] conf: Replace virNetworkDNSDefParseXML(hardcoded) with namesake(generated)

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 178 +---
 src/conf/network_conf.h |  16 ++--
 src/network/bridge_driver.c |   2 +-
 3 files changed, 14 insertions(+), 182 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 4c0751f..db2b75f 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -174,32 +174,6 @@ virNetworkIPDefClear(virNetworkIPDefPtr def)
 }
 
 
-static void
-virNetworkDNSDefClear(virNetworkDNSDefPtr def)
-{
-if (def->forwarders) {
-while (def->nfwds)
-virNetworkDNSForwarderClear(>forwarders[--def->nfwds]);
-VIR_FREE(def->forwarders);
-}
-if (def->txts) {
-while (def->ntxts)
-virNetworkDNSTxtDefClear(>txts[--def->ntxts]);
-VIR_FREE(def->txts);
-}
-if (def->hosts) {
-while (def->nhosts)
-virNetworkDNSHostDefClear(>hosts[--def->nhosts]);
-VIR_FREE(def->hosts);
-}
-if (def->srvs) {
-while (def->nsrvs)
-virNetworkDNSSrvDefClear(>srvs[--def->nsrvs]);
-VIR_FREE(def->srvs);
-}
-}
-
-
 static void
 virNetworkForwardDefClear(virNetworkForwardDefPtr def)
 {
@@ -900,7 +874,7 @@ virNetworkDNSForwarderParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
 }
 
 
-static int
+int
 virNetworkDNSDefParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
  virNetworkDNSDefPtr def,
  const char *networkName,
@@ -924,148 +898,6 @@ virNetworkDNSDefParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
 }
 
 
-static int
-virNetworkDNSDefParseXML(const char *networkName,
- xmlNodePtr node,
- xmlXPathContextPtr ctxt,
- virNetworkDNSDefPtr def)
-{
-xmlNodePtr *hostNodes = NULL;
-xmlNodePtr *srvNodes = NULL;
-xmlNodePtr *txtNodes = NULL;
-xmlNodePtr *fwdNodes = NULL;
-char *forwardPlainNames = NULL;
-char *enable = NULL;
-int nhosts, nsrvs, ntxts, nfwds;
-size_t i;
-int ret = -1;
-VIR_XPATH_NODE_AUTORESTORE(ctxt);
-
-ctxt->node = node;
-
-enable = virXPathString("string(./@enable)", ctxt);
-if (enable) {
-def->enable = virTristateBoolTypeFromString(enable);
-if (def->enable <= 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("Invalid dns enable setting '%s' "
- "in network '%s'"),
-   enable, networkName);
-goto cleanup;
-}
-}
-
-forwardPlainNames = virXPathString("string(./@forwardPlainNames)", ctxt);
-if (forwardPlainNames) {
-def->forwardPlainNames = 
virTristateBoolTypeFromString(forwardPlainNames);
-if (def->forwardPlainNames <= 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("Invalid dns forwardPlainNames setting '%s' "
- "in network '%s'"),
-   forwardPlainNames, networkName);
-goto cleanup;
-}
-}
-
-nfwds = virXPathNodeSet("./forwarder", ctxt, );
-if (nfwds < 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("invalid  element found in  of 
network %s"),
-   networkName);
-goto cleanup;
-}
-if (nfwds > 0) {
-if (VIR_ALLOC_N(def->forwarders, nfwds) < 0)
-goto cleanup;
-
-for (i = 0; i < nfwds; i++) {
-if (virNetworkDNSForwarderParseXML(fwdNodes[i],
-   >forwarders[i],
-   networkName,
-   NULL) < 0)
-goto cleanup;
-
-def->nfwds++;
-}
-}
-
-nhosts = virXPathNodeSet("./host", ctxt, );
-if (nhosts < 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("invalid  element found in  of network 
%s"),
-   networkName);
-goto cleanup;
-}
-if (nhosts > 0) {
-if (VIR_ALLOC_N(def->hosts, nhosts) < 0)
-goto cleanup;
-
-for (i = 0; i < nhosts; i++) {
-if (virNetworkDNSHostDefParseXML(hostNodes[i], 
>hosts[def->nhosts],
- networkName, NULL) < 0) {
-goto cleanup;
-}
-def->nhosts++;
-}
-}
-
-nsrvs = virXPathNodeSet("./srv", ctxt, );
-if (nsrvs < 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("invalid  element found in  of network %s"),
-   networkName);
-goto cleanup;
-}
-if (nsrvs > 0) {
-if (VIR_ALLOC_N(def->srvs, nsrvs) < 0)
-goto cleanup;
-
-for (i = 0; i < nsrvs; i++) {
-if (virNetworkDNSSrvDefParseXML(srvNodes[i], 
>srvs[def->nsrvs],
-networkName, NULL) < 0) {
-

[RFC 14/21] conf: Replace virNetworkDNSHostDefParseXML(hardcoded) with namesake(generated)

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 72 -
 src/conf/network_conf.h |  6 ++--
 2 files changed, 10 insertions(+), 68 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 7a3dcd4..29ef3cf 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -174,15 +174,6 @@ virNetworkIPDefClear(virNetworkIPDefPtr def)
 }
 
 
-static void
-virNetworkDNSHostDefClear(virNetworkDNSHostDefPtr def)
-{
-while (def->nnames)
-VIR_FREE(def->names[--def->nnames]);
-VIR_FREE(def->names);
-}
-
-
 static void
 virNetworkDNSForwarderClear(virNetworkDNSForwarderPtr def)
 {
@@ -691,12 +682,12 @@ virNetworkDHCPDefParseXML(const char *networkName,
 }
 
 
-static int
+int
 virNetworkDNSHostDefParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
  virNetworkDNSHostDefPtr def,
  const char *networkName,
  void *opaque,
- char *ip,
+ const char *ip,
  int nHostnameNodes G_GNUC_UNUSED)
 {
 bool partialOkay = false;
@@ -732,57 +723,6 @@ virNetworkDNSHostDefParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
 }
 
 
-static int
-virNetworkDNSHostDefParseXML(const char *networkName,
- xmlNodePtr node,
- virNetworkDNSHostDefPtr def,
- bool partialOkay)
-{
-xmlNodePtr cur;
-g_autofree char *ip = NULL;
-
-ip = virXMLPropString(node, "ip");
-if (ip && (virSocketAddrParse(>ip, ip, AF_UNSPEC) < 0)) {
-virReportError(VIR_ERR_XML_DETAIL,
-   _("Invalid IP address in network '%s' DNS HOST record"),
-   networkName);
-goto error;
-}
-
-cur = node->children;
-while (cur != NULL) {
-if (cur->type == XML_ELEMENT_NODE &&
-virXMLNodeNameEqual(cur, "hostname")) {
-  if (cur->children != NULL) {
-  char *name = (char *) xmlNodeGetContent(cur);
-
-  if (!name) {
-  virReportError(VIR_ERR_XML_DETAIL,
- _("Missing hostname in network '%s' DNS 
HOST record"),
- networkName);
-  goto error;
-  }
-  if (VIR_APPEND_ELEMENT(def->names, def->nnames, name) < 0) {
-  VIR_FREE(name);
-  goto error;
-  }
-  }
-}
-cur = cur->next;
-}
-
-if (virNetworkDNSHostDefParseXMLHook(node, def, networkName, ,
- ip, def->nnames) < 0)
-goto error;
-
-return 0;
-
- error:
-virNetworkDNSHostDefClear(def);
-return -1;
-}
-
-
 /* This includes all characters used in the names of current
  * /etc/services and /etc/protocols files (on Fedora 20), except ".",
  * which we can't allow because it would conflict with the use of "."
@@ -1038,8 +978,8 @@ virNetworkDNSDefParseXML(const char *networkName,
 goto cleanup;
 
 for (i = 0; i < nhosts; i++) {
-if (virNetworkDNSHostDefParseXML(networkName, hostNodes[i],
- >hosts[def->nhosts], false) 
< 0) {
+if (virNetworkDNSHostDefParseXML(hostNodes[i], 
>hosts[def->nhosts],
+ networkName, NULL) < 0) {
 goto cleanup;
 }
 def->nhosts++;
@@ -3498,6 +3438,7 @@ virNetworkDefUpdateDNSHost(virNetworkDefPtr def,
 bool isAdd = (command == VIR_NETWORK_UPDATE_COMMAND_ADD_FIRST ||
   command == VIR_NETWORK_UPDATE_COMMAND_ADD_LAST);
 int foundCt = 0;
+bool notAdd;
 
 memset(, 0, sizeof(host));
 
@@ -3511,7 +3452,8 @@ virNetworkDefUpdateDNSHost(virNetworkDefPtr def,
 if (virNetworkDefUpdateCheckElementName(def, ctxt->node, "host") < 0)
 goto cleanup;
 
-if (virNetworkDNSHostDefParseXML(def->name, ctxt->node, , !isAdd) < 0)
+notAdd = !isAdd;
+if (virNetworkDNSHostDefParseXML(ctxt->node, , def->name, ) < 
0)
 goto cleanup;
 
 for (i = 0; i < dns->nhosts; i++) {
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index c867f27..b715dc3 100644
--- a/src/conf/network_conf.h
+++ b/src/conf/network_conf.h
@@ -149,10 +149,10 @@ struct _virNetworkDNSSrvDef {   /* genparse:withhook, 
genformat */
 
 typedef struct _virNetworkDNSHostDef virNetworkDNSHostDef;
 typedef virNetworkDNSHostDef *virNetworkDNSHostDefPtr;
-struct _virNetworkDNSHostDef {
-virSocketAddr ip;
+struct _virNetworkDNSHostDef {  /* genparse:withhook */
+virSocketAddr ip;   /* xmlattr */
 size_t nnames;
-char **names;
+char **names;   /* xmlelem:hostname, array */
 };
 
 
-- 
2.17.1

[RFC 05/21] util: Add two helper functions virXMLChildNode and virXMLChildNodeSet

Add these helper functions to parse xml without using xmlXPathContext.

Signed-off-by: Shi Lei 
---
 src/util/virxml.c | 57 +++
 src/util/virxml.h |  3 +++
 2 files changed, 60 insertions(+)

diff --git a/src/util/virxml.c b/src/util/virxml.c
index 02b59ea..a2edbc1 100644
--- a/src/util/virxml.c
+++ b/src/util/virxml.c
@@ -1405,3 +1405,60 @@ virXMLNamespaceRegister(xmlXPathContextPtr ctxt,
 
 return 0;
 }
+
+
+/**
+ * virXMLChildNode:
+ * @node: Parent XML dom node pointer
+ * @name: Name of the child element
+ *
+ * Convenience function to return the child element of a XML node.
+ *
+ * Returns the pointer of child element node or NULL in case of failure.
+ * If there are many nodes match condition, it only returns the first node.
+ */
+xmlNodePtr
+virXMLChildNode(xmlNodePtr node, const char *name)
+{
+xmlNodePtr cur = node->children;
+while (cur) {
+if (cur->type == XML_ELEMENT_NODE && virXMLNodeNameEqual(cur, name))
+return cur;
+cur = cur->next;
+}
+return NULL;
+}
+
+
+/**
+ * virXMLChildNodeSet:
+ * @node: Parent XML dom node pointer
+ * @name: Name of the children element
+ * @list: the returned list of nodes (or NULL if only count matters)
+ *
+ * Convenience function to evaluate a set of children elements
+ *
+ * Returns the number of nodes found in which case @list is set (and
+ * must be freed) or -1 if the evaluation failed.
+ */
+int
+virXMLChildNodeSet(xmlNodePtr node, const char *name, xmlNodePtr **list)
+{
+size_t count = 0;
+if (list != NULL)
+*list = NULL;
+
+xmlNodePtr cur = node->children;
+while (cur) {
+if (cur->type == XML_ELEMENT_NODE && virXMLNodeNameEqual(cur, name)) {
+if (list != NULL) {
+if (VIR_APPEND_ELEMENT_COPY(*list, count, cur) < 0)
+return -1;
+} else {
+count++;
+}
+}
+cur = cur->next;
+}
+return count;
+}
diff --git a/src/util/virxml.h b/src/util/virxml.h
index 26ab9f9..0abde44 100644
--- a/src/util/virxml.h
+++ b/src/util/virxml.h
@@ -79,6 +79,9 @@ char * virXMLPropStringLimit(xmlNodePtr node,
 char *   virXMLNodeContentString(xmlNodePtr node);
 long virXMLChildElementCount(xmlNodePtr node);
 
+xmlNodePtr virXMLChildNode(xmlNodePtr node, const char *name);
+int virXMLChildNodeSet(xmlNodePtr node, const char *name, xmlNodePtr **list);
+
 /* Internal function; prefer the macros below.  */
 xmlDocPtr  virXMLParseHelper(int domcode,
  const char *filename,
-- 
2.17.1

[RFC 19/21] conf: Extract error-checking code from virNetworkDNSDefParseXML

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 33 +++--
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 3b93772..4c0751f 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -900,6 +900,30 @@ virNetworkDNSForwarderParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
 }
 
 
+static int
+virNetworkDNSDefParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
+ virNetworkDNSDefPtr def,
+ const char *networkName,
+ void *opaque G_GNUC_UNUSED,
+ const char *enable G_GNUC_UNUSED,
+ const char *forwardPlainNames G_GNUC_UNUSED,
+ int ntxts,
+ int nhosts,
+ int nsrvs,
+ int nfwds)
+{
+if (def->enable == VIR_TRISTATE_BOOL_NO &&
+(nfwds || nhosts || nsrvs || ntxts)) {
+virReportError(VIR_ERR_XML_ERROR,
+   _("Extra data in disabled network '%s'"),
+   networkName);
+return -1;
+}
+
+return 0;
+}
+
+
 static int
 virNetworkDNSDefParseXML(const char *networkName,
  xmlNodePtr node,
@@ -1025,13 +1049,10 @@ virNetworkDNSDefParseXML(const char *networkName,
 }
 }
 
-if (def->enable == VIR_TRISTATE_BOOL_NO &&
-(nfwds || nhosts || nsrvs || ntxts)) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("Extra data in disabled network '%s'"),
-   networkName);
+if (virNetworkDNSDefParseXMLHook(node, def, networkName, NULL,
+ enable, forwardPlainNames,
+ ntxts, nhosts, nsrvs, nfwds) < 0)
 goto cleanup;
-}
 
 ret = 0;
  cleanup:
-- 
2.17.1

[RFC 04/21] maint: Add helper macro VIR_USED

The macro VIR_USED is used in generated parse/format functions to
avoid args-unused warnings.

Signed-off-by: Shi Lei 
---
 src/internal.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/internal.h b/src/internal.h
index e181218..315f12d 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -502,3 +502,5 @@ enum {
 # define fprintf(fh, ...) g_fprintf(fh, __VA_ARGS__)
 
 #endif /* VIR_NO_GLIB_STDIO */
+
+#define VIR_USED(var) do { break; } while(var)
-- 
2.17.1

[RFC 02/21] maint: Check libclang and its python3 binding

Make sure libclang and its python3 binding have been installed.

Signed-off-by: Shi Lei 
---
 configure.ac | 12 
 1 file changed, 12 insertions(+)

diff --git a/configure.ac b/configure.ac
index 6c8ac2f..747e52a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -707,6 +707,18 @@ if test -z "$FLAKE8"; then
 AC_MSG_WARN(['flake8' binary is required to check python code style])
 fi
 
+dnl Need libclang and its python3 binding to generate some functions
+LIBCLANG_LINE=`ldconfig -p | grep libclang`
+LIBCLANG_PATH=`expr "$LIBCLANG_LINE" : '.* => \(.*\)$'`
+if test -z "$LIBCLANG_PATH"; then
+AC_MSG_ERROR([libclang is required by libvirt])
+fi
+HAVE_PY3_CLANG=`python3 -c 'import clang.cindex;print("ok")' 2>/dev/null`
+if test -z "$HAVE_PY3_CLANG"; then
+AC_MSG_ERROR(['python3-clang' is required. Execute 'pip3 install clang'.])
+fi
+AC_SUBST(LIBCLANG_PATH)
+
 dnl Python3 < 3.7 treats the C locale as 7-bit only.
 dnl We must force env vars so it treats it as UTF-8
 dnl regardless of the user's locale.
-- 
2.17.1

[RFC 18/21] conf: Generate virNetworkDNSForwarderFormatBuf

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 21 -
 src/conf/network_conf.h |  4 ++--
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 8610fbc..3b93772 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -886,8 +886,8 @@ virNetworkDNSForwarderParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
virNetworkDNSForwarderPtr def,
const char *instname G_GNUC_UNUSED,
void *opaque G_GNUC_UNUSED,
-   const char *addr,
-   const char *domain G_GNUC_UNUSED)
+   const char *domain G_GNUC_UNUSED,
+   const char *addr)
 {
 if (!(addr || def->domain)) {
 virReportError(VIR_ERR_XML_ERROR, "%s",
@@ -2213,22 +2213,9 @@ virNetworkDNSDefFormat(virBufferPtr buf,
 virBufferAdjustIndent(buf, 2);
 
 for (i = 0; i < def->nfwds; i++) {
-
-virBufferAddLit(buf, "forwarders[i].domain) {
-virBufferEscapeString(buf, " domain='%s'",
-  def->forwarders[i].domain);
-}
-if (VIR_SOCKET_ADDR_VALID(>forwarders[i].addr)) {
-char *addr = virSocketAddrFormat(>forwarders[i].addr);
-
-if (!addr)
+if (virNetworkDNSForwarderFormatBuf(buf, "forwarder",
+>forwarders[i], NULL) < 0)
 return -1;
-
-virBufferAsprintf(buf, " addr='%s'", addr);
-VIR_FREE(addr);
-}
-virBufferAddLit(buf, "/>\n");
 }
 
 for (i = 0; i < def->ntxts; i++) {
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index f14eef2..1313c3e 100644
--- a/src/conf/network_conf.h
+++ b/src/conf/network_conf.h
@@ -158,9 +158,9 @@ struct _virNetworkDNSHostDef {  /* genparse:withhook, 
genformat */
 
 typedef struct _virNetworkDNSForwarder virNetworkDNSForwarder;
 typedef virNetworkDNSForwarder *virNetworkDNSForwarderPtr;
-struct _virNetworkDNSForwarder {/* genparse:withhook */
-virSocketAddr addr; /* xmlattr */
+struct _virNetworkDNSForwarder {/* genparse:withhook, genformat */
 char *domain;   /* xmlattr */
+virSocketAddr addr; /* xmlattr */
 };
 
 typedef struct _virNetworkDNSDef virNetworkDNSDef;
-- 
2.17.1

[RFC 11/21] conf: Generate virNetworkDNSSrvDefFormatBuf

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 18 ++
 src/conf/network_conf.h |  4 ++--
 2 files changed, 4 insertions(+), 18 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index bfdc10b..e8e7922 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -2275,22 +2275,8 @@ virNetworkDNSDefFormat(virBufferPtr buf,
 
 for (i = 0; i < def->nsrvs; i++) {
 if (def->srvs[i].service && def->srvs[i].protocol) {
-virBufferEscapeString(buf, "srvs[i].service);
-virBufferEscapeString(buf, "protocol='%s'", def->srvs[i].protocol);
-
-if (def->srvs[i].domain)
-virBufferEscapeString(buf, " domain='%s'", 
def->srvs[i].domain);
-if (def->srvs[i].target)
-virBufferEscapeString(buf, " target='%s'", 
def->srvs[i].target);
-if (def->srvs[i].port)
-virBufferAsprintf(buf, " port='%d'", def->srvs[i].port);
-if (def->srvs[i].priority)
-virBufferAsprintf(buf, " priority='%d'", 
def->srvs[i].priority);
-if (def->srvs[i].weight)
-virBufferAsprintf(buf, " weight='%d'", def->srvs[i].weight);
-
-virBufferAddLit(buf, "/>\n");
+if (virNetworkDNSSrvDefFormatBuf(buf, "srv", >srvs[i], NULL) 
< 0)
+return -1;
 }
 }
 
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index 3a4d829..c867f27 100644
--- a/src/conf/network_conf.h
+++ b/src/conf/network_conf.h
@@ -137,10 +137,10 @@ struct _virNetworkDNSTxtDef {   /* genparse:concisehook, 
genformat */
 
 typedef struct _virNetworkDNSSrvDef virNetworkDNSSrvDef;
 typedef virNetworkDNSSrvDef *virNetworkDNSSrvDefPtr;
-struct _virNetworkDNSSrvDef {   /* genparse:withhook */
-char *domain;   /* xmlattr */
+struct _virNetworkDNSSrvDef {   /* genparse:withhook, genformat */
 char *service;  /* xmlattr */
 char *protocol; /* xmlattr */
+char *domain;   /* xmlattr */
 char *target;   /* xmlattr */
 unsigned int port;  /* xmlattr */
 unsigned int priority;  /* xmlattr */
-- 
2.17.1

[RFC 08/21] conf: Generate virNetworkDNSTxtDefFormatBuf

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 4 ++--
 src/conf/network_conf.h | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 964a8a7..b807bac 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -2280,8 +2280,8 @@ virNetworkDNSDefFormat(virBufferPtr buf,
 }
 
 for (i = 0; i < def->ntxts; i++) {
-virBufferEscapeString(buf, "txts[i].name);
-virBufferEscapeString(buf, "value='%s'/>\n", def->txts[i].value);
+if (virNetworkDNSTxtDefFormatBuf(buf, "txt", >txts[i], NULL) < 0)
+return -1;
 }
 
 for (i = 0; i < def->nsrvs; i++) {
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index eac8a76..b3c2895 100644
--- a/src/conf/network_conf.h
+++ b/src/conf/network_conf.h
@@ -130,7 +130,7 @@ struct _virNetworkDHCPHostDef {
 
 typedef struct _virNetworkDNSTxtDef virNetworkDNSTxtDef;
 typedef virNetworkDNSTxtDef *virNetworkDNSTxtDefPtr;
-struct _virNetworkDNSTxtDef {   /* genparse:concisehook */
+struct _virNetworkDNSTxtDef {   /* genparse:concisehook, genformat */
 char *name; /* xmlattr, required */
 char *value;/* xmlattr */
 };
-- 
2.17.1

[RFC 15/21] conf: Generate virNetworkDNSHostDefFormatBuf

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 15 +++
 src/conf/network_conf.h |  2 +-
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 29ef3cf..1fea580 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -2176,7 +2176,7 @@ static int
 virNetworkDNSDefFormat(virBufferPtr buf,
const virNetworkDNSDef *def)
 {
-size_t i, j;
+size_t i;
 
 if (!(def->enable || def->forwardPlainNames || def->nfwds || def->nhosts ||
   def->nsrvs || def->ntxts))
@@ -2246,17 +2246,8 @@ virNetworkDNSDefFormat(virBufferPtr buf,
 
 if (def->nhosts) {
 for (i = 0; i < def->nhosts; i++) {
-char *ip = virSocketAddrFormat(>hosts[i].ip);
-
-virBufferAsprintf(buf, "\n", ip);
-virBufferAdjustIndent(buf, 2);
-for (j = 0; j < def->hosts[i].nnames; j++)
-virBufferEscapeString(buf, "%s\n",
-  def->hosts[i].names[j]);
-
-virBufferAdjustIndent(buf, -2);
-virBufferAddLit(buf, "\n");
-VIR_FREE(ip);
+if (virNetworkDNSHostDefFormatBuf(buf, "host", >hosts[i], 
NULL) < 0)
+return -1;
 }
 }
 virBufferAdjustIndent(buf, -2);
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index b715dc3..a5a4939 100644
--- a/src/conf/network_conf.h
+++ b/src/conf/network_conf.h
@@ -149,7 +149,7 @@ struct _virNetworkDNSSrvDef {   /* genparse:withhook, 
genformat */
 
 typedef struct _virNetworkDNSHostDef virNetworkDNSHostDef;
 typedef virNetworkDNSHostDef *virNetworkDNSHostDefPtr;
-struct _virNetworkDNSHostDef {  /* genparse:withhook */
+struct _virNetworkDNSHostDef {  /* genparse:withhook, genformat */
 virSocketAddr ip;   /* xmlattr */
 size_t nnames;
 char **names;   /* xmlelem:hostname, array */
-- 
2.17.1

[RFC 12/21] util: Add parsexml/formatbuf helper functions for virSocketAddr

Implement the parsexml/formatbuf functions for virSocketAddr.

Signed-off-by: Shi Lei 
---
 src/util/virsocketaddr.c | 38 ++
 src/util/virsocketaddr.h | 22 ++
 2 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/src/util/virsocketaddr.c b/src/util/virsocketaddr.c
index a858a69..fd29678 100644
--- a/src/util/virsocketaddr.c
+++ b/src/util/virsocketaddr.c
@@ -157,6 +157,14 @@ int virSocketAddrParse(virSocketAddrPtr addr, const char 
*val, int family)
 return len;
 }
 
+int virSocketAddrParseXML(const char *val,
+  virSocketAddrPtr addr,
+  const char *instname G_GNUC_UNUSED,
+  void *opaque G_GNUC_UNUSED)
+{
+return virSocketAddrParse(addr, val, AF_UNSPEC);
+}
+
 /**
  * virSocketAddrParseAny:
  * @addr: where to store the return value, optional.
@@ -1316,3 +1324,33 @@ virSocketAddrFree(virSocketAddrPtr addr)
 {
 VIR_FREE(addr);
 }
+
+void
+virSocketAddrClear(virSocketAddrPtr addr)
+{
+memset(addr, 0, sizeof(virSocketAddr));
+}
+
+int
+virSocketAddrFormatBuf(virBufferPtr buf,
+   const char *fmt,
+   const virSocketAddr *addr,
+   void *opaque G_GNUC_UNUSED)
+{
+g_autofree char *str = NULL;
+if (!VIR_SOCKET_ADDR_VALID(addr))
+return 0;
+
+str = virSocketAddrFormatFull(addr, false, NULL);
+if (!str)
+return -1;
+
+virBufferAsprintf(buf, fmt, str);
+return 0;
+}
+
+bool
+virSocketAddrCheck(const virSocketAddr *addr, void *opaque G_GNUC_UNUSED)
+{
+return VIR_SOCKET_ADDR_VALID(addr);
+}
diff --git a/src/util/virsocketaddr.h b/src/util/virsocketaddr.h
index d06e751..7ef4f35 100644
--- a/src/util/virsocketaddr.h
+++ b/src/util/virsocketaddr.h
@@ -18,11 +18,14 @@
 
 #pragma once
 
+#include "virbuffer.h"
 #include "virsocket.h"
 
 #define VIR_LOOPBACK_IPV4_ADDR "127.0.0.1"
 
-typedef struct {
+typedef struct _virSocketAddr virSocketAddr;
+typedef virSocketAddr *virSocketAddrPtr;
+struct _virSocketAddr {
 union {
 struct sockaddr sa;
 struct sockaddr_storage stor;
@@ -33,7 +36,7 @@ typedef struct {
 #endif
 } data;
 socklen_t len;
-} virSocketAddr;
+};
 
 #define VIR_SOCKET_ADDR_VALID(s) \
 ((s)->data.sa.sa_family != AF_UNSPEC)
@@ -50,8 +53,6 @@ typedef struct {
 #define VIR_SOCKET_ADDR_IPV4_ARPA "in-addr.arpa"
 #define VIR_SOCKET_ADDR_IPV6_ARPA "ip6.arpa"
 
-typedef virSocketAddr *virSocketAddrPtr;
-
 typedef struct _virSocketAddrRange virSocketAddrRange;
 typedef virSocketAddrRange *virSocketAddrRangePtr;
 struct _virSocketAddrRange {
@@ -70,6 +71,11 @@ int virSocketAddrParse(virSocketAddrPtr addr,
const char *val,
int family);
 
+int virSocketAddrParseXML(const char *val,
+  virSocketAddrPtr addr,
+  const char *instname,
+  void *opaque);
+
 int virSocketAddrParseAny(virSocketAddrPtr addr,
   const char *val,
   int family,
@@ -93,6 +99,11 @@ char *virSocketAddrFormatFull(const virSocketAddr *addr,
   bool withService,
   const char *separator);
 
+int virSocketAddrFormatBuf(virBufferPtr buf,
+   const char *fmt,
+   const virSocketAddr *addr,
+   void *opaque);
+
 char *virSocketAddrGetPath(virSocketAddrPtr addr);
 
 int virSocketAddrSetPort(virSocketAddrPtr addr, int port);
@@ -145,5 +156,8 @@ int virSocketAddrPTRDomain(const virSocketAddr *addr,
 ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(3);
 
 void virSocketAddrFree(virSocketAddrPtr addr);
+void virSocketAddrClear(virSocketAddrPtr addr);
 
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(virSocketAddr, virSocketAddrFree);
+
+bool virSocketAddrCheck(const virSocketAddr *addr, void *opaque);
-- 
2.17.1

[RFC 10/21] conf: Replace virNetworkDNSSrvDefParseXML(hardcoded) with namesake(generated)

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 88 +++--
 src/conf/network_conf.h | 16 
 2 files changed, 14 insertions(+), 90 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 21b13ad..bfdc10b 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -183,16 +183,6 @@ virNetworkDNSHostDefClear(virNetworkDNSHostDefPtr def)
 }
 
 
-static void
-virNetworkDNSSrvDefClear(virNetworkDNSSrvDefPtr def)
-{
-VIR_FREE(def->domain);
-VIR_FREE(def->service);
-VIR_FREE(def->protocol);
-VIR_FREE(def->target);
-}
-
-
 static void
 virNetworkDNSForwarderClear(virNetworkDNSForwarderPtr def)
 {
@@ -784,7 +774,7 @@ virNetworkDNSHostDefParseXML(const char *networkName,
 "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" \
 "_-+/*"
 
-static int
+int
 virNetworkDNSSrvDefParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
 virNetworkDNSSrvDefPtr def,
 const char *networkName,
@@ -894,74 +884,6 @@ virNetworkDNSSrvDefParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
 }
 
 
-static int
-virNetworkDNSSrvDefParseXML(const char *networkName,
-xmlNodePtr node,
-xmlXPathContextPtr ctxt,
-virNetworkDNSSrvDefPtr def,
-bool partialOkay)
-{
-g_autofree char *portStr = NULL;
-g_autofree char *priorityStr = NULL;
-g_autofree char *weightStr = NULL;
-VIR_XPATH_NODE_AUTORESTORE(ctxt);
-
-ctxt->node = node;
-
-def->service = virXMLPropString(node, "service");
-def->protocol = virXMLPropString(node, "protocol");
-
-/* Following attributes are optional */
-def->domain = virXMLPropString(node, "domain");
-def->target = virXMLPropString(node, "target");
-
-portStr = virXMLPropString(node, "port");
-if (portStr) {
-if (virStrToLong_uip(portStr, NULL, 0, >port) < 0) {
-virReportError(VIR_ERR_XML_DETAIL,
-   _("invalid DNS SRV port attribute "
- "for service '%s' in network '%s'"),
-   def->service, networkName);
-goto error;
-}
-}
-
-priorityStr = virXMLPropString(node, "priority");
-if (priorityStr) {
-if (virStrToLong_uip(priorityStr, NULL, 0, >priority) < 0) {
-virReportError(VIR_ERR_XML_DETAIL,
-   _("Invalid DNS SRV priority attribute "
- "for service '%s' in network '%s'"),
-   def->service, networkName);
-goto error;
-}
-}
-
-weightStr = virXMLPropString(node, "weight");
-if (weightStr) {
-if (virStrToLong_uip(weightStr, NULL, 0, >weight) < 0) {
-virReportError(VIR_ERR_XML_DETAIL,
-   _("invalid DNS SRV weight attribute "
- "for service '%s' in network '%s'"),
-   def->service, networkName);
-goto error;
-}
-}
-
-if (virNetworkDNSSrvDefParseXMLHook(node, def, networkName, ,
-def->domain, def->service,
-def->protocol, def->target,
-portStr, priorityStr, weightStr) < 0)
-goto error;
-
-return 0;
-
- error:
-virNetworkDNSSrvDefClear(def);
-return -1;
-}
-
-
 int
 virNetworkDNSTxtDefParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
 virNetworkDNSTxtDefPtr def,
@@ -1112,8 +1034,8 @@ virNetworkDNSDefParseXML(const char *networkName,
 goto cleanup;
 
 for (i = 0; i < nsrvs; i++) {
-if (virNetworkDNSSrvDefParseXML(networkName, srvNodes[i], ctxt,
->srvs[def->nsrvs], false) < 
0) {
+if (virNetworkDNSSrvDefParseXML(srvNodes[i], 
>srvs[def->nsrvs],
+networkName, NULL) < 0) {
 goto cleanup;
 }
 def->nsrvs++;
@@ -3661,6 +3583,7 @@ virNetworkDefUpdateDNSSrv(virNetworkDefPtr def,
 bool isAdd = (command == VIR_NETWORK_UPDATE_COMMAND_ADD_FIRST ||
   command == VIR_NETWORK_UPDATE_COMMAND_ADD_LAST);
 int foundCt = 0;
+bool notAdd;
 
 memset(, 0, sizeof(srv));
 
@@ -3674,7 +3597,8 @@ virNetworkDefUpdateDNSSrv(virNetworkDefPtr def,
 if (virNetworkDefUpdateCheckElementName(def, ctxt->node, "srv") < 0)
 goto cleanup;
 
-if (virNetworkDNSSrvDefParseXML(def->name, ctxt->node, ctxt, , !isAdd) 
< 0)
+notAdd = !isAdd;
+if (virNetworkDNSSrvDefParseXML(ctxt->node, , def->name, ) < 0)
 goto cleanup;
 
 for (i = 0; i < dns->nsrvs; i++) {
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index b3c2895..3a4d829 100644
--- a/src/conf/network_conf.h
+++

[RFC 13/21] conf: Extract error-checking code from virNetworkDNSHostDefParseXML

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 64 -
 1 file changed, 44 insertions(+), 20 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index e8e7922..7a3dcd4 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -692,29 +692,62 @@ virNetworkDHCPDefParseXML(const char *networkName,
 
 
 static int
-virNetworkDNSHostDefParseXML(const char *networkName,
- xmlNodePtr node,
- virNetworkDNSHostDefPtr def,
- bool partialOkay)
+virNetworkDNSHostDefParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
+ virNetworkDNSHostDefPtr def,
+ const char *networkName,
+ void *opaque,
+ char *ip,
+ int nHostnameNodes G_GNUC_UNUSED)
 {
-xmlNodePtr cur;
-char *ip;
+bool partialOkay = false;
 
-if (!(ip = virXMLPropString(node, "ip")) && !partialOkay) {
+if (opaque)
+partialOkay = *((bool *) opaque);
+
+if (!ip && !partialOkay) {
 virReportError(VIR_ERR_XML_DETAIL,
_("Missing IP address in network '%s' DNS HOST record"),
networkName);
 goto error;
 }
 
+if (def->nnames == 0 && !partialOkay) {
+virReportError(VIR_ERR_XML_DETAIL,
+   _("Missing hostname in network '%s' DNS HOST record"),
+   networkName);
+goto error;
+}
+
+if (!VIR_SOCKET_ADDR_VALID(>ip) && def->nnames == 0) {
+virReportError(VIR_ERR_XML_DETAIL,
+   _("Missing ip and hostname in network '%s' DNS HOST 
record"),
+   networkName);
+goto error;
+}
+
+return 0;
+
+ error:
+return -1;
+}
+
+
+static int
+virNetworkDNSHostDefParseXML(const char *networkName,
+ xmlNodePtr node,
+ virNetworkDNSHostDefPtr def,
+ bool partialOkay)
+{
+xmlNodePtr cur;
+g_autofree char *ip = NULL;
+
+ip = virXMLPropString(node, "ip");
 if (ip && (virSocketAddrParse(>ip, ip, AF_UNSPEC) < 0)) {
 virReportError(VIR_ERR_XML_DETAIL,
_("Invalid IP address in network '%s' DNS HOST record"),
networkName);
-VIR_FREE(ip);
 goto error;
 }
-VIR_FREE(ip);
 
 cur = node->children;
 while (cur != NULL) {
@@ -737,19 +770,10 @@ virNetworkDNSHostDefParseXML(const char *networkName,
 }
 cur = cur->next;
 }
-if (def->nnames == 0 && !partialOkay) {
-virReportError(VIR_ERR_XML_DETAIL,
-   _("Missing hostname in network '%s' DNS HOST record"),
-   networkName);
-goto error;
-}
 
-if (!VIR_SOCKET_ADDR_VALID(>ip) && def->nnames == 0) {
-virReportError(VIR_ERR_XML_DETAIL,
-   _("Missing ip and hostname in network '%s' DNS HOST 
record"),
-   networkName);
+if (virNetworkDNSHostDefParseXMLHook(node, def, networkName, ,
+ ip, def->nnames) < 0)
 goto error;
-}
 
 return 0;
 
-- 
2.17.1

[RFC 06/21] conf: Extract error-checking code from virNetworkDNSTxtDefParseXML

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 48 +++--
 1 file changed, 37 insertions(+), 11 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index f1d22b2..47aaef3 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -904,26 +904,25 @@ virNetworkDNSSrvDefParseXML(const char *networkName,
 
 
 static int
-virNetworkDNSTxtDefParseXML(const char *networkName,
-xmlNodePtr node,
-virNetworkDNSTxtDefPtr def,
-bool partialOkay)
+virNetworkDNSTxtDefParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
+virNetworkDNSTxtDefPtr def,
+const char *networkName,
+void *opaque)
 {
 const char *bad = " ,";
+bool partialOkay = false;
+
+if (opaque)
+partialOkay = *((bool *) opaque);
 
-if (!(def->name = virXMLPropString(node, "name"))) {
-virReportError(VIR_ERR_XML_DETAIL,
-   _("missing required name attribute in DNS TXT record "
- "of network %s"), networkName);
-goto error;
-}
 if (strcspn(def->name, bad) != strlen(def->name)) {
 virReportError(VIR_ERR_XML_DETAIL,
_("prohibited character in DNS TXT record "
  "name '%s' of network %s"), def->name, networkName);
 goto error;
 }
-if (!(def->value = virXMLPropString(node, "value")) && !partialOkay) {
+
+if (!def->value && !partialOkay) {
 virReportError(VIR_ERR_XML_DETAIL,
_("missing required value attribute in DNS TXT record "
  "named '%s' of network %s"), def->name, networkName);
@@ -936,6 +935,33 @@ virNetworkDNSTxtDefParseXML(const char *networkName,
  "in DNS TXT record of network %s"), networkName);
 goto error;
 }
+
+return 0;
+
+ error:
+return -1;
+}
+
+
+static int
+virNetworkDNSTxtDefParseXML(const char *networkName,
+xmlNodePtr node,
+virNetworkDNSTxtDefPtr def,
+bool partialOkay)
+{
+if (!(def->name = virXMLPropString(node, "name"))) {
+virReportError(VIR_ERR_XML_DETAIL,
+   _("missing required name attribute in DNS TXT record "
+ "of network %s"), networkName);
+goto error;
+}
+
+def->value = virXMLPropString(node, "value");
+
+if (virNetworkDNSTxtDefParseXMLHook(node, def, networkName,
+) < 0)
+goto error;
+
 return 0;
 
  error:
-- 
2.17.1

[RFC 07/21] conf: Replace virNetworkDNSTxtDefParseXML(hardcoded) with namesake(generated)

Signed-off-by: Shi Lei 
---
 po/POTFILES.in   |  1 +
 src/conf/Makefile.inc.am |  2 ++
 src/conf/network_conf.c  | 47 ++--
 src/conf/network_conf.h  |  8 ---
 4 files changed, 15 insertions(+), 43 deletions(-)

diff --git a/po/POTFILES.in b/po/POTFILES.in
index 2d7eb1e..a0bd358 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -3,6 +3,7 @@
 @BUILDDIR@/src/access/viraccessapicheckqemu.c
 @BUILDDIR@/src/admin/admin_client.h
 @BUILDDIR@/src/admin/admin_server_dispatch_stubs.h
+@BUILDDIR@/src/conf/network_conf.generated.c
 @BUILDDIR@/src/remote/remote_client_bodies.h
 @BUILDDIR@/src/remote/remote_daemon_dispatch_stubs.h
 @SRCDIR@/build-aux/generator/directive.py
diff --git a/src/conf/Makefile.inc.am b/src/conf/Makefile.inc.am
index 3bd2199..0b36a8b 100644
--- a/src/conf/Makefile.inc.am
+++ b/src/conf/Makefile.inc.am
@@ -161,6 +161,8 @@ DEVICE_CONF_SOURCES = \
$(NULL)
 
 CONF_GENERATED_SOURCES = \
+   conf/network_conf.generated.c \
+   conf/network_conf.generated.h \
$(NULL)
 
 CONF_SOURCES = \
diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 47aaef3..964a8a7 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -174,14 +174,6 @@ virNetworkIPDefClear(virNetworkIPDefPtr def)
 }
 
 
-static void
-virNetworkDNSTxtDefClear(virNetworkDNSTxtDefPtr def)
-{
-VIR_FREE(def->name);
-VIR_FREE(def->value);
-}
-
-
 static void
 virNetworkDNSHostDefClear(virNetworkDNSHostDefPtr def)
 {
@@ -903,7 +895,7 @@ virNetworkDNSSrvDefParseXML(const char *networkName,
 }
 
 
-static int
+int
 virNetworkDNSTxtDefParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
 virNetworkDNSTxtDefPtr def,
 const char *networkName,
@@ -943,33 +935,6 @@ virNetworkDNSTxtDefParseXMLHook(xmlNodePtr node 
G_GNUC_UNUSED,
 }
 
 
-static int
-virNetworkDNSTxtDefParseXML(const char *networkName,
-xmlNodePtr node,
-virNetworkDNSTxtDefPtr def,
-bool partialOkay)
-{
-if (!(def->name = virXMLPropString(node, "name"))) {
-virReportError(VIR_ERR_XML_DETAIL,
-   _("missing required name attribute in DNS TXT record "
- "of network %s"), networkName);
-goto error;
-}
-
-def->value = virXMLPropString(node, "value");
-
-if (virNetworkDNSTxtDefParseXMLHook(node, def, networkName,
-) < 0)
-goto error;
-
-return 0;
-
- error:
-virNetworkDNSTxtDefClear(def);
-return -1;
-}
-
-
 static int
 virNetworkDNSDefParseXML(const char *networkName,
  xmlNodePtr node,
@@ -1100,10 +1065,10 @@ virNetworkDNSDefParseXML(const char *networkName,
 goto cleanup;
 
 for (i = 0; i < ntxts; i++) {
-if (virNetworkDNSTxtDefParseXML(networkName, txtNodes[i],
->txts[def->ntxts], false) < 
0) {
+if (virNetworkDNSTxtDefParseXML(txtNodes[i], 
>txts[def->ntxts],
+networkName, NULL) < 0)
 goto cleanup;
-}
+
 def->ntxts++;
 }
 }
@@ -3714,6 +3679,7 @@ virNetworkDefUpdateDNSTxt(virNetworkDefPtr def,
 virNetworkDNSTxtDef txt;
 bool isAdd = (command == VIR_NETWORK_UPDATE_COMMAND_ADD_FIRST ||
   command == VIR_NETWORK_UPDATE_COMMAND_ADD_LAST);
+bool notAdd;
 
 memset(, 0, sizeof(txt));
 
@@ -3727,7 +3693,8 @@ virNetworkDefUpdateDNSTxt(virNetworkDefPtr def,
 if (virNetworkDefUpdateCheckElementName(def, ctxt->node, "txt") < 0)
 goto cleanup;
 
-if (virNetworkDNSTxtDefParseXML(def->name, ctxt->node, , !isAdd) < 0)
+notAdd = !isAdd;
+if (virNetworkDNSTxtDefParseXML(ctxt->node, , def->name, ) < 0)
 goto cleanup;
 
 for (foundIdx = 0; foundIdx < dns->ntxts; foundIdx++) {
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index f2dc388..eac8a76 100644
--- a/src/conf/network_conf.h
+++ b/src/conf/network_conf.h
@@ -130,9 +130,9 @@ struct _virNetworkDHCPHostDef {
 
 typedef struct _virNetworkDNSTxtDef virNetworkDNSTxtDef;
 typedef virNetworkDNSTxtDef *virNetworkDNSTxtDefPtr;
-struct _virNetworkDNSTxtDef {
-char *name;
-char *value;
+struct _virNetworkDNSTxtDef {   /* genparse:concisehook */
+char *name; /* xmlattr, required */
+char *value;/* xmlattr */
 };
 
 typedef struct _virNetworkDNSSrvDef virNetworkDNSSrvDef;
@@ -440,3 +440,5 @@ virNetworkDefUpdateSection(virNetworkDefPtr def,
unsigned int flags);  /* virNetworkUpdateFlags */
 
 VIR_ENUM_DECL(virNetworkTaint);
+
+#include "network_conf.generated.h"
-- 
2.17.1

[RFC 09/21] conf: Extract error-checking code from virNetworkDNSSrvDefParseXML

Signed-off-by: Shi Lei 
---
 src/conf/network_conf.c | 115 +++-
 1 file changed, 91 insertions(+), 24 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index b807bac..21b13ad 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -785,23 +785,29 @@ virNetworkDNSHostDefParseXML(const char *networkName,
 "_-+/*"
 
 static int
-virNetworkDNSSrvDefParseXML(const char *networkName,
-xmlNodePtr node,
-xmlXPathContextPtr ctxt,
-virNetworkDNSSrvDefPtr def,
-bool partialOkay)
+virNetworkDNSSrvDefParseXMLHook(xmlNodePtr node G_GNUC_UNUSED,
+virNetworkDNSSrvDefPtr def,
+const char *networkName,
+void *opaque,
+const char *domainStr G_GNUC_UNUSED,
+const char *serviceStr G_GNUC_UNUSED,
+const char *protocolStr G_GNUC_UNUSED,
+const char *targetStr G_GNUC_UNUSED,
+const char *portStr,
+const char *priorityStr,
+const char *weightStr)
 {
-int ret;
-VIR_XPATH_NODE_AUTORESTORE(ctxt);
-
-ctxt->node = node;
+bool partialOkay = false;
+if (opaque)
+partialOkay = *((bool *) opaque);
 
-if (!(def->service = virXMLPropString(node, "service")) && !partialOkay) {
+if (!def->service && !partialOkay) {
 virReportError(VIR_ERR_XML_DETAIL,
_("missing required service attribute in DNS SRV record 
"
  "of network '%s'"), networkName);
 goto error;
 }
+
 if (def->service) {
 if (strlen(def->service) > DNS_RECORD_LENGTH_SRV) {
 virReportError(VIR_ERR_XML_DETAIL,
@@ -819,13 +825,14 @@ virNetworkDNSSrvDefParseXML(const char *networkName,
 }
 }
 
-if (!(def->protocol = virXMLPropString(node, "protocol")) && !partialOkay) 
{
+if (!def->protocol && !partialOkay) {
 virReportError(VIR_ERR_XML_DETAIL,
_("missing required protocol attribute "
  "in DNS SRV record '%s' of network '%s'"),
def->service, networkName);
 goto error;
 }
+
 if (def->protocol &&
 strspn(def->protocol, PROTOCOL_CHARS) < strlen(def->protocol)) {
 virReportError(VIR_ERR_XML_DETAIL,
@@ -835,19 +842,14 @@ virNetworkDNSSrvDefParseXML(const char *networkName,
 goto error;
 }
 
-/* Following attributes are optional */
-def->domain = virXMLPropString(node, "domain");
-def->target = virXMLPropString(node, "target");
-
-ret = virXPathUInt("string(./@port)", ctxt, >port);
-if (ret >= 0 && !def->target) {
+if (portStr && !def->target) {
 virReportError(VIR_ERR_XML_DETAIL,
_("DNS SRV port attribute not permitted without "
  "target for service '%s' in network '%s'"),
def->service, networkName);
 goto error;
 }
-if (ret == -2 || (ret >= 0 && (def->port < 1 || def->port > 65535))) {
+if (portStr && (def->port < 1 || def->port > 65535)) {
 virReportError(VIR_ERR_XML_DETAIL,
_("invalid DNS SRV port attribute "
  "for service '%s' in network '%s'"),
@@ -855,15 +857,14 @@ virNetworkDNSSrvDefParseXML(const char *networkName,
 goto error;
 }
 
-ret = virXPathUInt("string(./@priority)", ctxt, >priority);
-if (ret >= 0 && !def->target) {
+if (priorityStr && !def->target) {
 virReportError(VIR_ERR_XML_DETAIL,
_("DNS SRV priority attribute not permitted without "
  "target for service '%s' in network '%s'"),
def->service, networkName);
 goto error;
 }
-if (ret == -2 || (ret >= 0 && def->priority > 65535)) {
+if (priorityStr && def->priority > 65535) {
 virReportError(VIR_ERR_XML_DETAIL,
_("Invalid DNS SRV priority attribute "
  "for service '%s' in network '%s'"),
@@ -871,15 +872,14 @@ virNetworkDNSSrvDefParseXML(const char *networkName,
 goto error;
 }
 
-ret = virXPathUInt("string(./@weight)", ctxt, >weight);
-if (ret >= 0 && !def->target) {
+if (weightStr && !def->target) {
 virReportError(VIR_ERR_XML_DETAIL,
_("DNS SRV weight attribute not permitted without "
  "target for service '%s' in network '%s'"),
def->service, networkName);
 goto error;
 }
-if (ret == -2 || (ret >= 0 && def->weight > 65535)) {
+if (weightStr && def->weight > 65535) {

[RFC 01/21] build-aux: Add a tool to generate xml parse/format functions

This tool is used to generate parsexml/formatbuf functions for structs.
It is based on libclang and its python-binding.
Some directives (such as genparse, xmlattr, etc.) need to be added on
the declarations of structs to direct the tool.

Signed-off-by: Shi Lei 
---
 build-aux/generator/directive.py | 839 +++
 build-aux/generator/go   |  14 +
 build-aux/generator/main.py  | 416 +++
 build-aux/generator/utils.py | 100 
 po/POTFILES.in   |   1 +
 5 files changed, 1370 insertions(+)
 create mode 100644 build-aux/generator/directive.py
 create mode 100755 build-aux/generator/go
 create mode 100755 build-aux/generator/main.py
 create mode 100644 build-aux/generator/utils.py

diff --git a/build-aux/generator/directive.py b/build-aux/generator/directive.py
new file mode 100644
index 000..c0d3c61
--- /dev/null
+++ b/build-aux/generator/directive.py
@@ -0,0 +1,839 @@
+#
+# Copyright (C) 2020 Shandong Massclouds Co.,Ltd.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see
+# .
+#
+
+import json
+from collections import OrderedDict
+from utils import singleton
+from utils import dedup, counterName
+from utils import BlockAssembler
+from utils import Terms, singleline, indent, render, renderByDict
+
+BUILTIN_TYPES = {
+'Bool': {},
+'String': {},
+'Chars': {
+'conv': 'virStrcpyStatic(def->${name}, ${name}Str)'
+},
+'UChars': {
+'conv': 'virStrcpyStatic((char *)def->${name}, ${mdvar})'
+},
+'Int': {
+'fmt': '%d',
+'conv': 'virStrToLong_i(${mdvar}, NULL, 0, >${name})'
+},
+'UInt': {
+'fmt': '%u',
+'conv': 'virStrToLong_uip(${mdvar}, NULL, 0, >${name})'
+},
+'ULong': {
+'fmt': '%lu',
+'conv': 'virStrToLong_ulp(${mdvar}, NULL, 0, >${name})'
+},
+'ULongLong': {
+'fmt': '%llu',
+'conv': 'virStrToLong_ullp(${mdvar}, NULL, 0, >${name})'
+},
+'U8': {
+'fmt': '%u',
+'conv': 'virStrToLong_u8p(${mdvar}, NULL, 0, >${name})'
+},
+'U32': {
+'fmt': '%u',
+'conv': 'virStrToLong_uip(${mdvar}, NULL, 0, >${name})'
+},
+}
+
+
+@singleton
+class TypeTable(OrderedDict):
+def __init__(self):
+OrderedDict.__init__(self)
+for name, kvs in BUILTIN_TYPES.items():
+kvs['name'] = name
+kvs['meta'] = 'Builtin'
+self[name] = kvs
+
+def register(self, kvs):
+name = kvs['name']
+if name not in self:
+self[name] = kvs
+return name
+
+def get(self, name):
+if name in self:
+return self[name]
+return {'meta': 'Struct', 'name': name, 'external': True}
+
+
+T_NAMESPACE_PARSE = '''
+if (xmlopt)
+def->ns = xmlopt->ns;
+if (def->ns.parse) {
+if (virXMLNamespaceRegister(ctxt, >ns) < 0)
+goto error;
+if ((def->ns.parse)(ctxt, >namespaceData) < 0)
+goto error;
+}
+'''
+
+T_NAMESPACE_FORMAT_BEGIN = '''
+if (def->namespaceData && def->ns.format)
+virXMLNamespaceFormatNS(buf, >ns);
+'''
+
+T_NAMESPACE_FORMAT_END = '''
+if (def->namespaceData && def->ns.format) {
+if ((def->ns.format)(buf, def->namespaceData) < 0)
+return -1;
+}
+'''
+
+T_CLEAR_FUNC_IMPL = '''
+void
+${typename}Clear(${typename}Ptr def)
+{
+if (!def)
+return;
+
+${body}
+}
+'''
+
+T_CLEAR_FUNC_DECL = '''
+void
+${typename}Clear(${typename}Ptr def);
+'''
+
+
+def clearMember(member):
+mtype = TypeTable().get(member['type'])
+
+refname = 'def->%s' % member['name']
+if member.get('array'):
+refname += '[i]'
+
+code = ''
+if mtype['meta'] == 'Struct':
+if member['pointer'] and not mtype.get('external'):
+code = '%sClear(%s);' % (mtype['name'], refname)
+code += '\nVIR_FREE(%s);' % refname
+else:
+code = '%sClear(&%s);' % (mtype['name'], refname)
+elif mtype['name'] == 'String':
+code = 'VIR_FREE(%s);' % refname
+elif mtype['name'] in ['Chars', 'UChars']:
+code = 'memset(%s, 0, sizeof(%s));' % (refname, refname)
+elif not member.get('array'):
+code = '%s = 0;' % refname
+
+if member.get('specified'):
+assert not member.get('array'), "'specified' can't come with 'array'."
+code += '\n%s_specified = false;' % refname
+
+

[RFC 00/21] RFC: Generate parsexml/formatbuf functions based on directives



Last RFC: [https://www.redhat.com/archives/libvir-list/2020-April/msg00970.html]
In last RFC, I suggested we can generate object-model code based on relax-ng
files and Daniel gave it some comments.

Follow the suggestion from Daniel, I make another try to generate 
parsexml/formatbuf
functions automatically.



 Directives


Still, we need several directives that can direct a tool to generate functions.
These directives work on the declarations of structs. For example:

typedef struct _virNetworkDNSTxtDef virNetworkDNSTxtDef;
typedef virNetworkDNSTxtDef *virNetworkDNSTxtDefPtr;
struct _virNetworkDNSTxtDef {   /* genparse:concisehook, genformat */
char *name; /* xmlattr, required */
char *value;/* xmlattr */
};

typedef struct _virNetworkDNSSrvDef virNetworkDNSSrvDef;
typedef virNetworkDNSSrvDef *virNetworkDNSSrvDefPtr;
struct _virNetworkDNSSrvDef {   /* genparse:withhook, genformat */
char *service;  /* xmlattr */
char *protocol; /* xmlattr */
char *domain;   /* xmlattr */
char *target;   /* xmlattr */
unsigned int port;  /* xmlattr */
unsigned int priority;  /* xmlattr */
unsigned int weight;/* xmlattr */
};

typedef struct _virNetworkDNSHostDef virNetworkDNSHostDef;
typedef virNetworkDNSHostDef *virNetworkDNSHostDefPtr;
struct _virNetworkDNSHostDef {  /* genparse:withhook, genformat */
virSocketAddr ip;   /* xmlattr */
size_t nnames;
char **names;   /* xmlelem:hostname, array */
};

typedef struct _virNetworkDNSForwarder virNetworkDNSForwarder;
typedef virNetworkDNSForwarder *virNetworkDNSForwarderPtr;
struct _virNetworkDNSForwarder {/* genparse:withhook, genformat */
char *domain;   /* xmlattr */
virSocketAddr addr; /* xmlattr */
};

typedef struct _virNetworkDNSDef virNetworkDNSDef;
typedef virNetworkDNSDef *virNetworkDNSDefPtr;
struct _virNetworkDNSDef {  /* genparse:withhook, genformat */
virTristateBool enable; /* xmlattr */
virTristateBool forwardPlainNames;  /* xmlattr */
size_t nforwarders;
virNetworkDNSForwarderPtr forwarders;   /* xmlelem, array */
size_t ntxts;
virNetworkDNSTxtDefPtr txts;/* xmlelem, array */
size_t nsrvs;
virNetworkDNSSrvDefPtr srvs;/* xmlelem, array */
size_t nhosts;
virNetworkDNSHostDefPtr hosts;  /* xmlelem, array */
};


Explanation for these directives:


- genparse[:withhook|concisehook]

  Only work on a struct.
  Generate parsexml function for this struct only if 'genparse' is 
specified.
  The function name is based on the struct-name and suffixed with 
'ParseXML'.
  E.g., for struct virNetworkDNSDef, its parsexml function is
  virNetworkDNSDefParseXML.

  If a parsexml function includes some error-checking code, it needs a
  post-process hook to hold them. Specify 'withhook' or 'concisehook' to
  setup a hook point in the parsexml function.

  Executing the tool manually can show the declaration of hook function.
  E.g. check declaration of 'virNetworkDNSDefParseXMLHook'.

# ./build-aux/generator/go show virNetworkDNSDef -kp

int
virNetworkDNSDefParseXMLHook(xmlNodePtr node,
 virNetworkDNSDefPtr def,
 const char *instname,
 void *opaque,
 const char *enableStr,
 const char *forwardPlainNamesStr,
 int nForwarderNodes,
 int nTxtNodes,
 int nSrvNodes,
 int nHostNodes);

  Some helper arguments (such as 'enableStr', 'nTxtNodes', etc.) are
  passed in to indicate the existence of fields.
  If these arguments are useless, specify 'concisehook' to omit them.

  When 'genparse' is specified, clear function for this struct is also
  generated implicitly, it is because the generated parsexml function needs
  to call the clear function.


- genformat

  Only work on a struct.
  Generate formatbuf function for this struct only if 'genformat' is 
specified.
  The function name is based on struct-name and suffixed with 'FormatBuf'.


- xmlattr[:thename]

  Parse/Format the field as an XML attribute.
  If 'thename' is specified, use it as the XML attribute name;
  or use the filed name.


- xmlelem[:thename]
  
  Parse/Format the field as a child struct.
  If 'thename' is specified, use it as the XML element name;

[RFC 03/21] maint: Call generator automatically when c-head-files change

Let makefile call the generator-tool whenever the c header files
change. Only check those header files under src/conf and src/util.

Signed-off-by: Shi Lei 
---
 src/Makefile.am | 15 +++
 src/access/Makefile.inc.am  |  2 +-
 src/conf/Makefile.inc.am| 11 ++-
 src/esx/Makefile.inc.am |  2 +-
 src/interface/Makefile.inc.am   |  2 +-
 src/lxc/Makefile.inc.am |  1 +
 src/network/Makefile.inc.am |  2 +-
 src/node_device/Makefile.inc.am |  2 +-
 src/nwfilter/Makefile.inc.am|  2 +-
 src/qemu/Makefile.inc.am|  1 +
 src/remote/Makefile.inc.am  |  2 +-
 src/secret/Makefile.inc.am  |  2 +-
 src/storage/Makefile.inc.am |  2 +-
 src/test/Makefile.inc.am|  2 +-
 src/util/Makefile.inc.am| 12 +---
 src/vbox/Makefile.inc.am|  2 +-
 tests/Makefile.am   |  2 ++
 tools/Makefile.am   |  2 ++
 18 files changed, 51 insertions(+), 15 deletions(-)

diff --git a/src/Makefile.am b/src/Makefile.am
index 12dd6b8..224d786 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -24,6 +24,7 @@ AM_CPPFLAGS = -I$(top_srcdir) \
-I$(top_srcdir)/include \
-I$(srcdir)/util \
-I./util \
+   -I./conf \
-DIN_LIBVIRT \
-Dabs_top_builddir="\"$(abs_top_builddir)\"" \
-Dabs_top_srcdir="\"$(abs_top_srcdir)\"" \
@@ -87,6 +88,20 @@ sbin_PROGRAMS =
 bin_PROGRAMS =
 DRIVER_SOURCES =
 
+GENERATED_FILES_STAMP = .generated_files.stamp
+UTIL_HEADS = $(wildcard $(top_srcdir)/src/util/*.h)
+CONF_HEADS = $(wildcard $(top_srcdir)/src/conf/*.h)
+GENERATOR_SRC = $(wildcard $(top_srcdir)/build-aux/generator/*.py)
+
+$(GENERATED_FILES_STAMP): $(UTIL_HEADS) $(CONF_HEADS) $(GENERATOR_SRC)
+   $(AM_V_GEN)topdir=$(top_srcdir) builddir=$(top_builddir) \
+   libclang_path=$(LIBCLANG_PATH) \
+   $(PYTHON) -B $(top_srcdir)/build-aux/generator/main.py \
+   generate -k cpf && touch $@
+
+MAINTAINERCLEANFILES += $(GENERATED_FILES_STAMP)
+CLEANFILES += $(GENERATED_FILES_STAMP)
+
 COMMON_UNIT_VARS = \
-e 's|[@]runstatedir[@]|$(runstatedir)|g' \
-e 's|[@]sbindir[@]|$(sbindir)|g' \
diff --git a/src/access/Makefile.inc.am b/src/access/Makefile.inc.am
index 11f87c6..7832558 100644
--- a/src/access/Makefile.inc.am
+++ b/src/access/Makefile.inc.am
@@ -55,7 +55,7 @@ nodist_libvirt_driver_access_la_SOURCES = \
 noinst_LTLIBRARIES += libvirt_driver_access.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_access.la
 libvirt_driver_access_la_CFLAGS = \
-   -I$(srcdir)/conf \
+   -I$(srcdir)/conf -I./conf \
$(AM_CFLAGS) \
$(NULL)
 libvirt_driver_access_la_LDFLAGS = $(AM_LDFLAGS)
diff --git a/src/conf/Makefile.inc.am b/src/conf/Makefile.inc.am
index debc6f4..3bd2199 100644
--- a/src/conf/Makefile.inc.am
+++ b/src/conf/Makefile.inc.am
@@ -160,7 +160,11 @@ DEVICE_CONF_SOURCES = \
conf/device_conf.h \
$(NULL)
 
+CONF_GENERATED_SOURCES = \
+   $(NULL)
+
 CONF_SOURCES = \
+   $(CONF_GENERATED_SOURCES) \
$(NETDEV_CONF_SOURCES) \
$(DOMAIN_CONF_SOURCES) \
$(OBJECT_EVENT_SOURCES) \
@@ -180,11 +184,16 @@ CONF_SOURCES = \
$(DEVICE_CONF_SOURCES) \
$(NULL)
 
+BUILT_SOURCES += $(CONF_GENERATED_SOURCES)
+$(CONF_GENERATED_SOURCES): $(GENERATED_FILES_STAMP)
+MAINTAINERCLEANFILES += $(CONF_GENERATED_SOURCES)
+CLEANFILES += $(CONF_GENERATED_SOURCES)
+
 noinst_LTLIBRARIES += libvirt_conf.la
 libvirt_la_BUILT_LIBADD += libvirt_conf.la
 libvirt_conf_la_SOURCES = $(CONF_SOURCES)
 libvirt_conf_la_CFLAGS = \
-   -I$(srcdir)/conf \
+   -I$(srcdir)/conf -I./conf \
$(AM_CFLAGS) \
$(NULL)
 libvirt_conf_la_LDFLAGS = $(AM_LDFLAGS)
diff --git a/src/esx/Makefile.inc.am b/src/esx/Makefile.inc.am
index d53cef1..1df9a54 100644
--- a/src/esx/Makefile.inc.am
+++ b/src/esx/Makefile.inc.am
@@ -78,7 +78,7 @@ noinst_LTLIBRARIES += libvirt_driver_esx.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_esx.la
 libvirt_driver_esx_la_CFLAGS = \
$(CURL_CFLAGS) \
-   -I$(srcdir)/conf \
+   -I$(srcdir)/conf -I./conf \
-I$(builddir)/esx \
-I$(srcdir)/vmx \
$(AM_CFLAGS) \
diff --git a/src/interface/Makefile.inc.am b/src/interface/Makefile.inc.am
index 46a43e6..03ccef1 100644
--- a/src/interface/Makefile.inc.am
+++ b/src/interface/Makefile.inc.am
@@ -23,7 +23,7 @@ mod_LTLIBRARIES += libvirt_driver_interface.la
 libvirt_driver_interface_la_CFLAGS = \
-I$(srcdir)/access \
-I$(builddir)/access \
-   -I$(srcdir)/conf \
+   -I$(srcdir)/conf -I./conf \
$(AM_CFLAGS) \
$(LIBNL_CFLAGS) \
$(NULL)
diff --git a/src/lxc/Makefile.inc.am b/src/lxc/Makefile.inc.am
index b8c2e1e..fdbefb5 100644
--- a/src/lxc/Makefile.inc.am
+++ b/src/lxc/Makefile.inc.am
@@ -95,6 +95,7 @@ libvirt_driver_lxc_impl_la_CFLAGS = \
-I$(srcdir)/access \
-I$(builddir)/access \

Re: [PATCH v5 0/4] introduction of migration_version attribute for VFIO live migration

2020-06-09 Thread Yan Zhao

On Fri, Jun 05, 2020 at 03:39:50PM +0100, Dr. David Alan Gilbert wrote:
> > > > I tried to simplify the problem a bit, but we keep going backwards.  If
> > > > the requirement is that potentially any source device can migrate to any
> > > > target device and we cannot provide any means other than writing an
> > > > opaque source string into a version attribute on the target and
> > > > evaluating the result to determine compatibility, then we're requiring
> > > > userspace to do an exhaustive search to find a potential match.  That
> > > > sucks.   
> > >
hi Alex and Dave,
do you think it's good for us to put aside physical devices and mdev aggregation
for the moment, and use Alex's original idea that

+  Userspace should regard two mdev devices compatible when ALL of below
+  conditions are met:
+  (0) The mdev devices are of the same type
+  (1) success when reading from migration_version attribute of one mdev device.
+  (2) success when writing migration_version string of one mdev device to
+  migration_version attribute of the other mdev device.

and what about adding another sysfs attribute for vendors to put
recommended migration compatible device type. e.g.
#cat 
/sys/bus/pci/devices/:00:02.0/mdev_supported_types/i915-GVTg_V5_8/migration_compatible_devices
parent id: 8086 591d
mdev_type: i915-GVTg_V5_8

vendors are free to define the format and conent of this 
migration_compatible_devices
and it's even not to be a full list.

before libvirt or user to do live migration, they have to read and test
migration_version attributes of src/target devices to check migration 
compatibility.

Thanks
Yan


> > > Why is the mechanism a 'write and test' why isn't it a 'write and ask'?
> > > i.e. the destination tells the driver what type it's received from the
> > > source, and the driver replies with a set of compatible configurations
> > > (in some preferred order).
> > 
> > A 'write and ask' interface would imply some sort of session in order
> > to not be racy with concurrent users.  More likely this would imply an
> > ioctl interface, which I don't think we have in sysfs.  Where do we
> > host this ioctl?
> 
> Or one fd?
>   f=open()
>   write(f, "The ID I want")
>   do {
>  read(f, ...)  -> The IDs we're offering that are compatible
>   } while (!eof)
> 
> > > It's also not clear to me why the name has to be that opaque;
> > > I agree it's only got to be understood by the driver but that doesn't
> > > seem to be a reason for the driver to make it purposely obfuscated.
> > > I wouldn't expect a user to be able to parse it necessarily; but would
> > > expect something that would be useful for an error message.
> > 
> > If the name is not opaque, then we're going to rat hole on the format
> > and the fields and evolving that format for every feature a vendor
> > decides they want the user to be able to parse out of the version
> > string.  Then we require a full specification of the string in order
> > that it be parsed according to a standard such that we don't break
> > users inferring features in subtly different ways.
> > 
> > This is a lot like the problems with mdev description attributes,
> > libvirt complains they can't use description because there's no
> > standard formatting, but even with two vendors describing the same class
> > of device we don't have an agreed set of things to expose in the
> > description attribute.  Thanks,
> 
> I'm not suggesting anything in anyway machine parsable; just something
> human readable that you can present in a menu/choice/configuration/error
> message.  The text would be down to the vendor, and I'd suggest it start
> with the vendor name just as a disambiguator and to make it obvious when
> we get it grossly wrong.
> 
> Dave
> 
> > Alex
> --
> Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK
> 
> ___
> intel-gvt-dev mailing list
> intel-gvt-...@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/intel-gvt-dev

Re: [libvirt PATCH v2 1/3] util: add support for IPv6 masquerade rules

(or: "Remove hardcoding to IPv4 in function that creates masquerade 
rules" :-)


On 6/9/20 12:17 PM, Daniel P. Berrangé wrote:

IPv6 does support masquerade since Linux 3.9.0 / ip6tables 1.4.18,
which is Fedora 18 / RHEL-7 vintage, which covers all our supported
Linux versions.

Signed-off-by: Daniel P. Berrangé 
---
  src/util/viriptables.c | 33 +++--
  1 file changed, 11 insertions(+), 22 deletions(-)

diff --git a/src/util/viriptables.c b/src/util/viriptables.c
index e6a1ded8d5..8ccce835b2 100644
--- a/src/util/viriptables.c
+++ b/src/util/viriptables.c
@@ -854,29 +854,24 @@ iptablesForwardMasquerade(virFirewallPtr fw,
  g_autofree char *portRangeStr = NULL;
  g_autofree char *natRangeStr = NULL;
  virFirewallRulePtr rule;
+int af = VIR_SOCKET_ADDR_FAMILY(netaddr);
+virFirewallLayer layer = af == AF_INET ?
+VIR_FIREWALL_LAYER_IPV4 : VIR_FIREWALL_LAYER_IPV6;
  
  if (!(networkstr = iptablesFormatNetwork(netaddr, prefix)))

  return -1;
  
-if (!VIR_SOCKET_ADDR_IS_FAMILY(netaddr, AF_INET)) {

-/* Higher level code *should* guaranteee it's impossible to get here. 
*/
-virReportError(VIR_ERR_INTERNAL_ERROR,
-   _("Attempted to NAT '%s'. NAT is only supported for 
IPv4."),
-   networkstr);
-return -1;
-}
-
-if (VIR_SOCKET_ADDR_IS_FAMILY(>start, AF_INET)) {
+if (VIR_SOCKET_ADDR_IS_FAMILY(>start, af)) {
  if (!(addrStartStr = virSocketAddrFormat(>start)))
  return -1;
-if (VIR_SOCKET_ADDR_IS_FAMILY(>end, AF_INET)) {
+if (VIR_SOCKET_ADDR_IS_FAMILY(>end, af)) {
  if (!(addrEndStr = virSocketAddrFormat(>end)))
  return -1;
  }
  }
  
  if (protocol && protocol[0]) {

-rule = virFirewallAddRule(fw, VIR_FIREWALL_LAYER_IPV4,
+rule = virFirewallAddRule(fw, layer,
"--table", "nat",
action == ADD ? "--insert" : "--delete",
pvt ? "LIBVIRT_PRT" : "POSTROUTING",
@@ -885,7 +880,7 @@ iptablesForwardMasquerade(virFirewallPtr fw,
"!", "--destination", networkstr,
NULL);
  } else {
-rule = virFirewallAddRule(fw, VIR_FIREWALL_LAYER_IPV4,
+rule = virFirewallAddRule(fw, layer,
"--table", "nat",
action == ADD ? "--insert" : "--delete",
pvt ? "LIBVIRT_PRT" : "POSTROUTING",
@@ -1004,20 +999,14 @@ iptablesForwardDontMasquerade(virFirewallPtr fw,
int action)
  {
  g_autofree char *networkstr = NULL;
+virFirewallLayer layer = VIR_SOCKET_ADDR_FAMILY(netaddr) == AF_INET ?
+VIR_FIREWALL_LAYER_IPV4 : VIR_FIREWALL_LAYER_IPV6;
  
  if (!(networkstr = iptablesFormatNetwork(netaddr, prefix)))

  return -1;
  
-if (!VIR_SOCKET_ADDR_IS_FAMILY(netaddr, AF_INET)) {

-/* Higher level code *should* guaranteee it's impossible to get here. 
*/
-virReportError(VIR_ERR_INTERNAL_ERROR,
-   _("Attempted to NAT '%s'. NAT is only supported for 
IPv4."),
-   networkstr);
-return -1;
-}
-
  if (physdev && physdev[0])
-virFirewallAddRule(fw, VIR_FIREWALL_LAYER_IPV4,
+virFirewallAddRule(fw, layer,
 "--table", "nat",
 action == ADD ? "--insert" : "--delete",
 pvt ? "LIBVIRT_PRT" : "POSTROUTING",
@@ -1027,7 +1016,7 @@ iptablesForwardDontMasquerade(virFirewallPtr fw,
 "--jump", "RETURN",
 NULL);
  else
-virFirewallAddRule(fw, VIR_FIREWALL_LAYER_IPV4,
+virFirewallAddRule(fw, layer,
 "--table", "nat",
 action == ADD ? "--insert" : "--delete",
 pvt ? "LIBVIRT_PRT" : "POSTROUTING",



It's nice that adding capability is done by *removing* code rather than 
adding it!



Reviewed-by: Laine Stump

[libvirt PATCH v2 08/10] nodedev: add mdev support to virNodeDeviceDestroy()

Add the ability to destroy mdev node devices via the mdevctl utility.

Signed-off-by: Jonathon Jongsma 
---
 src/node_device/node_device_driver.c | 46 
 src/node_device/node_device_driver.h |  3 ++
 2 files changed, 49 insertions(+)

diff --git a/src/node_device/node_device_driver.c 
b/src/node_device/node_device_driver.c
index dbc7eb4d1e..c956bb55fc 100644
--- a/src/node_device/node_device_driver.c
+++ b/src/node_device/node_device_driver.c
@@ -790,6 +790,45 @@ nodeDeviceCreateXML(virConnectPtr conn,
 }
 
 
+virCommandPtr
+nodeDeviceGetMdevctlStopCommand(const char *uuid,
+bool persist)
+{
+g_autofree char *mdevctl = virFindFileInPath(MDEVCTL);
+const char *subcommand;
+
+if (!mdevctl)
+return NULL;
+
+if (persist)
+subcommand = "undefine";
+else
+subcommand = "stop";
+
+virCommandPtr cmd = virCommandNewArgList(mdevctl,
+ subcommand,
+ "-u",
+ uuid,
+ NULL);
+
+return cmd;
+}
+
+static int
+virMdevctlStop(virNodeDeviceDefPtr def)
+{
+int status;
+g_autoptr(virCommand) cmd = NULL;
+
+cmd = nodeDeviceGetMdevctlStopCommand(def->caps->data.mdev.uuid, false);
+
+if (virCommandRun(cmd, ) < 0 || status != 0)
+return -1;
+
+return 0;
+}
+
+
 int
 nodeDeviceDestroy(virNodeDevicePtr device)
 {
@@ -836,6 +875,13 @@ nodeDeviceDestroy(virNodeDevicePtr device)
 if (virVHBAManageVport(parent_host, wwpn, wwnn, VPORT_DELETE) < 0)
 goto cleanup;
 
+ret = 0;
+} else if (nodeDeviceHasCapability(def, VIR_NODE_DEV_CAP_MDEV)) {
+if (virMdevctlStop(def) < 0) {
+virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+   _("Unable to stop mediated device"));
+goto cleanup;
+}
 ret = 0;
 } else {
 virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
diff --git a/src/node_device/node_device_driver.h 
b/src/node_device/node_device_driver.h
index 576f75375f..794cb5dc1c 100644
--- a/src/node_device/node_device_driver.h
+++ b/src/node_device/node_device_driver.h
@@ -122,3 +122,6 @@ virCommandPtr
 nodeDeviceGetMdevctlStartCommand(virNodeDeviceDefPtr def,
  bool persist,
  char **uuid_out);
+virCommandPtr
+nodeDeviceGetMdevctlStopCommand(const char *uuid,
+bool persist);
-- 
2.21.3

[libvirt PATCH v2 05/10] nodedev: add mdev support to virNodeDeviceCreateXML()

With recent additions to the node device xml schema, an xml schema can
now describe a mdev device sufficiently for libvirt to create and start
the device using the mdevctl utility.

Note that some of the the configuration for a mediated device must be
passed to mdevctl as a JSON-formatted file. In order to avoid creating
and cleaning up temporary files, the JSON is instead fed to stdin and we
pass the filename /dev/stdin to mdevctl. While this may not be portable,
neither are mediated devices, so I don't believe it should cause any
problems.

Signed-off-by: Jonathon Jongsma 
---
 libvirt.spec.in  |   2 +
 m4/virt-external-programs.m4 |   3 +
 src/conf/virnodedeviceobj.c  |  34 +
 src/conf/virnodedeviceobj.h  |   3 +
 src/libvirt_private.syms |   1 +
 src/node_device/node_device_driver.c | 203 +++
 src/node_device/node_device_driver.h |   6 +
 7 files changed, 252 insertions(+)

diff --git a/libvirt.spec.in b/libvirt.spec.in
index 262e66f3cc..2b3a4d2e71 100644
--- a/libvirt.spec.in
+++ b/libvirt.spec.in
@@ -522,6 +522,8 @@ Requires: libvirt-daemon = %{version}-%{release}
 Requires: libvirt-libs = %{version}-%{release}
 # needed for device enumeration
 Requires: systemd >= 185
+# For managing persistent mediated devices
+Requires: mdevctl
 
 %description daemon-driver-nodedev
 The nodedev driver plugin for the libvirtd daemon, providing
diff --git a/m4/virt-external-programs.m4 b/m4/virt-external-programs.m4
index 9046e3bf07..bd3cb1f757 100644
--- a/m4/virt-external-programs.m4
+++ b/m4/virt-external-programs.m4
@@ -65,6 +65,7 @@ AC_DEFUN([LIBVIRT_CHECK_EXTERNAL_PROGRAMS], [
   AC_PATH_PROG([OVSVSCTL], [ovs-vsctl], [ovs-vsctl], [$LIBVIRT_SBIN_PATH])
   AC_PATH_PROG([SCRUB], [scrub], [scrub], [$LIBVIRT_SBIN_PATH])
   AC_PATH_PROG([ADDR2LINE], [addr2line], [addr2line], [$LIBVIRT_SBIN_PATH])
+  AC_PATH_PROG([MDEVCTL], [mdevctl], [mdevctl], [$LIBVIRT_SBIN_PATH])
 
   AC_DEFINE_UNQUOTED([DMIDECODE], ["$DMIDECODE"],
  [Location or name of the dmidecode program])
@@ -88,6 +89,8 @@ AC_DEFUN([LIBVIRT_CHECK_EXTERNAL_PROGRAMS], [
  [Location or name of the scrub program (for wiping 
algorithms)])
   AC_DEFINE_UNQUOTED([ADDR2LINE], ["$ADDR2LINE"],
  [Location of addr2line program])
+  AC_DEFINE_UNQUOTED([MDEVCTL], ["$MDEVCTL"],
+ [Location or name of the mdevctl program])
 
   AC_PATH_PROG([IP_PATH], [ip], [/sbin/ip], [$LIBVIRT_SBIN_PATH])
   AC_DEFINE_UNQUOTED([IP_PATH], ["$IP_PATH"], [path to ip binary])
diff --git a/src/conf/virnodedeviceobj.c b/src/conf/virnodedeviceobj.c
index 3a34a324ca..fd20d5f9e2 100644
--- a/src/conf/virnodedeviceobj.c
+++ b/src/conf/virnodedeviceobj.c
@@ -399,6 +399,40 @@ 
virNodeDeviceObjListFindSCSIHostByWWNs(virNodeDeviceObjListPtr devs,
   );
 }
 
+static int
+virNodeDeviceObjListFindMediatedDeviceByUUIDCallback(const void *payload,
+ const void *name 
G_GNUC_UNUSED,
+ const void *opaque 
G_GNUC_UNUSED)
+{
+virNodeDeviceObjPtr obj = (virNodeDeviceObjPtr) payload;
+const char *uuid = (const char *) opaque;
+virNodeDevCapsDefPtr cap;
+int want = 0;
+
+virObjectLock(obj);
+
+for (cap = obj->def->caps; cap != NULL; cap = cap->next) {
+if (cap->data.type == VIR_NODE_DEV_CAP_MDEV) {
+if (STREQ(cap->data.mdev.uuid, uuid)) {
+want = 1;
+break;
+}
+}
+ }
+
+virObjectUnlock(obj);
+return want;
+}
+
+
+virNodeDeviceObjPtr
+virNodeDeviceObjListFindMediatedDeviceByUUID(virNodeDeviceObjListPtr devs,
+ const char *uuid)
+{
+return virNodeDeviceObjListSearch(devs,
+  
virNodeDeviceObjListFindMediatedDeviceByUUIDCallback,
+  uuid);
+}
 
 static void
 virNodeDeviceObjListDispose(void *obj)
diff --git a/src/conf/virnodedeviceobj.h b/src/conf/virnodedeviceobj.h
index c9df8dedab..6efdb23d36 100644
--- a/src/conf/virnodedeviceobj.h
+++ b/src/conf/virnodedeviceobj.h
@@ -118,3 +118,6 @@ virNodeDeviceObjListExport(virConnectPtr conn,
 void
 virNodeDeviceObjSetSkipUpdateCaps(virNodeDeviceObjPtr obj,
   bool skipUpdateCaps);
+virNodeDeviceObjPtr
+virNodeDeviceObjListFindMediatedDeviceByUUID(virNodeDeviceObjListPtr devs,
+ const char *uuid);
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index b8e6f058c3..532169d93d 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -1178,6 +1178,7 @@ virNodeDeviceObjListAssignDef;
 virNodeDeviceObjListExport;
 virNodeDeviceObjListFindByName;
 virNodeDeviceObjListFindBySysfsPath;
+virNodeDeviceObjListFindMediatedDeviceByUUID;

[libvirt PATCH v2 04/10] nodedev: store mdev UUID in mdev caps

In order to allow libvirt to create and start new mediated devices, we
need to be able to verify that the device has been started. In order to
do this, we'll need to save the UUID of newly-discovered devices within
the virNodeDevCapMdev structure. This allows us to search the device
list by UUID and verify whether the expected device has been started.

Signed-off-by: Jonathon Jongsma 
---
 src/conf/node_device_conf.c| 1 +
 src/conf/node_device_conf.h| 1 +
 src/node_device/node_device_udev.c | 5 ++---
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/conf/node_device_conf.c b/src/conf/node_device_conf.c
index 045d146433..7bf00bb5bc 100644
--- a/src/conf/node_device_conf.c
+++ b/src/conf/node_device_conf.c
@@ -,6 +,7 @@ virNodeDevCapsDefFree(virNodeDevCapsDefPtr caps)
 break;
 case VIR_NODE_DEV_CAP_MDEV:
 VIR_FREE(data->mdev.type);
+VIR_FREE(data->mdev.uuid);
 for (i = 0; i < data->mdev.nattributes; i++)
 virMediatedDeviceAttrFree(data->mdev.attributes[i]);
 VIR_FREE(data->mdev.attributes);
diff --git a/src/conf/node_device_conf.h b/src/conf/node_device_conf.h
index e3e1e788d4..9b8c7aadea 100644
--- a/src/conf/node_device_conf.h
+++ b/src/conf/node_device_conf.h
@@ -141,6 +141,7 @@ typedef virNodeDevCapMdev *virNodeDevCapMdevPtr;
 struct _virNodeDevCapMdev {
 char *type;
 unsigned int iommuGroupNumber;
+char *uuid;
 virMediatedDeviceAttrPtr *attributes;
 size_t nattributes;
 };
diff --git a/src/node_device/node_device_udev.c 
b/src/node_device/node_device_udev.c
index 386f23ef3a..bdf0b03add 100644
--- a/src/node_device/node_device_udev.c
+++ b/src/node_device/node_device_udev.c
@@ -1013,7 +1013,6 @@ udevProcessMediatedDevice(struct udev_device *dev,
   virNodeDeviceDefPtr def)
 {
 int ret = -1;
-const char *uuidstr = NULL;
 int iommugrp = -1;
 char *linkpath = NULL;
 char *canonicalpath = NULL;
@@ -1041,8 +1040,8 @@ udevProcessMediatedDevice(struct udev_device *dev,
 
 data->type = g_path_get_basename(canonicalpath);
 
-uuidstr = udev_device_get_sysname(dev);
-if ((iommugrp = virMediatedDeviceGetIOMMUGroupNum(uuidstr)) < 0)
+data->uuid = g_strdup(udev_device_get_sysname(dev));
+if ((iommugrp = virMediatedDeviceGetIOMMUGroupNum(data->uuid)) < 0)
 goto cleanup;
 
 if (udevGenerateDeviceName(dev, def, NULL) != 0)
-- 
2.21.3

[libvirt PATCH v2 03/10] nodedev: refactor nodeDeviceFindNewDevice()

In preparation for creating mediated devices in libvirt, we will need to
wait for new mediated devices to be created as well. Refactor
nodeDeviceFindNewDevice() so that we can re-use the main logic from this
function to wait for different device types by passing a different
'find' function.

Signed-off-by: Jonathon Jongsma 
---
 src/node_device/node_device_driver.c | 39 +---
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/src/node_device/node_device_driver.c 
b/src/node_device/node_device_driver.c
index ba7ea50e5b..629d4bcf91 100644
--- a/src/node_device/node_device_driver.c
+++ b/src/node_device/node_device_driver.c
@@ -447,6 +447,10 @@ nodeDeviceGetTime(time_t *t)
 }
 
 
+typedef virNodeDevicePtr (*nodeDeviceFindNewDeviceFunc)(virConnectPtr conn,
+const void* opaque);
+
+
 /* When large numbers of devices are present on the host, it's
  * possible for udev not to realize that it has work to do before we
  * get here.  We thus keep trying to find the new device we just
@@ -462,8 +466,8 @@ nodeDeviceGetTime(time_t *t)
  */
 static virNodeDevicePtr
 nodeDeviceFindNewDevice(virConnectPtr conn,
-const char *wwnn,
-const char *wwpn)
+nodeDeviceFindNewDeviceFunc func,
+const void *opaque)
 {
 virNodeDevicePtr device = NULL;
 time_t start = 0, now = 0;
@@ -474,7 +478,7 @@ nodeDeviceFindNewDevice(virConnectPtr conn,
 
 virWaitForDevices();
 
-device = nodeDeviceLookupSCSIHostByWWN(conn, wwnn, wwpn, 0);
+device = func(conn, opaque);
 
 if (device != NULL)
 break;
@@ -488,6 +492,33 @@ nodeDeviceFindNewDevice(virConnectPtr conn,
 }
 
 
+typedef struct _NewSCSIHostFuncData NewSCSIHostFuncData;
+struct _NewSCSIHostFuncData
+{
+const char *wwnn;
+const char *wwpn;
+};
+
+
+static virNodeDevicePtr
+nodeDeviceFindNewSCSIHostFunc(virConnectPtr conn,
+  const void *opaque)
+{
+const NewSCSIHostFuncData *data = opaque;
+return nodeDeviceLookupSCSIHostByWWN(conn, data->wwnn, data->wwpn, 0);
+}
+
+
+static virNodeDevicePtr
+nodeDeviceFindNewSCSIHost(virConnectPtr conn,
+  const char *wwnn,
+  const char *wwpn)
+{
+NewSCSIHostFuncData data = { .wwnn = wwnn, .wwpn = wwpn};
+return nodeDeviceFindNewDevice(conn, nodeDeviceFindNewSCSIHostFunc, );
+}
+
+
 static bool
 nodeDeviceHasCapability(virNodeDeviceDefPtr def, virNodeDevCapType type)
 {
@@ -538,7 +569,7 @@ nodeDeviceCreateXML(virConnectPtr conn,
 if (virVHBAManageVport(parent_host, wwpn, wwnn, VPORT_CREATE) < 0)
 return NULL;
 
-device = nodeDeviceFindNewDevice(conn, wwnn, wwpn);
+device = nodeDeviceFindNewSCSIHost(conn, wwnn, wwpn);
 /* We don't check the return value, because one way or another,
  * we're returning what we get... */
 
-- 
2.21.3

[libvirt PATCH v2 01/10] nodedev: factor out nodeDeviceHasCapability()

Currently nodeDeviceCreateXML() and nodeDeviceDestroy() only support
NPIV HBAs, but we want to be able to create mdev devices as well. This
is a first step to enabling that support.

Signed-off-by: Jonathon Jongsma 
---
 src/node_device/node_device_driver.c | 91 ++--
 1 file changed, 58 insertions(+), 33 deletions(-)

diff --git a/src/node_device/node_device_driver.c 
b/src/node_device/node_device_driver.c
index ee175e1095..ba7ea50e5b 100644
--- a/src/node_device/node_device_driver.c
+++ b/src/node_device/node_device_driver.c
@@ -488,6 +488,21 @@ nodeDeviceFindNewDevice(virConnectPtr conn,
 }
 
 
+static bool
+nodeDeviceHasCapability(virNodeDeviceDefPtr def, virNodeDevCapType type)
+{
+virNodeDevCapsDefPtr cap = def->caps;
+
+while (cap != NULL) {
+if (cap->data.type == type)
+return true;
+cap = cap->next;
+}
+
+return false;
+}
+
+
 virNodeDevicePtr
 nodeDeviceCreateXML(virConnectPtr conn,
 const char *xmlDesc,
@@ -513,24 +528,29 @@ nodeDeviceCreateXML(virConnectPtr conn,
 if (virNodeDeviceCreateXMLEnsureACL(conn, def) < 0)
 return NULL;
 
-if (virNodeDeviceGetWWNs(def, , ) == -1)
-return NULL;
+if (nodeDeviceHasCapability(def, VIR_NODE_DEV_CAP_SCSI_HOST)) {
+if (virNodeDeviceGetWWNs(def, , ) == -1)
+return NULL;
 
-if ((parent_host = virNodeDeviceObjListGetParentHost(driver->devs, def)) < 
0)
-return NULL;
+if ((parent_host = virNodeDeviceObjListGetParentHost(driver->devs, 
def)) < 0)
+return NULL;
 
-if (virVHBAManageVport(parent_host, wwpn, wwnn, VPORT_CREATE) < 0)
-return NULL;
+if (virVHBAManageVport(parent_host, wwpn, wwnn, VPORT_CREATE) < 0)
+return NULL;
 
-device = nodeDeviceFindNewDevice(conn, wwnn, wwpn);
-/* We don't check the return value, because one way or another,
- * we're returning what we get... */
+device = nodeDeviceFindNewDevice(conn, wwnn, wwpn);
+/* We don't check the return value, because one way or another,
+ * we're returning what we get... */
 
-if (device == NULL)
-virReportError(VIR_ERR_NO_NODE_DEVICE,
-   _("no node device for '%s' with matching "
- "wwnn '%s' and wwpn '%s'"),
-   def->name, wwnn, wwpn);
+if (device == NULL)
+virReportError(VIR_ERR_NO_NODE_DEVICE,
+   _("no node device for '%s' with matching "
+ "wwnn '%s' and wwpn '%s'"),
+   def->name, wwnn, wwpn);
+} else {
+virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+   _("Unsupported device type"));
+}
 
 return device;
 }
@@ -557,31 +577,36 @@ nodeDeviceDestroy(virNodeDevicePtr device)
 if (virNodeDeviceDestroyEnsureACL(device->conn, def) < 0)
 goto cleanup;
 
-if (virNodeDeviceGetWWNs(def, , ) < 0)
-goto cleanup;
+if (nodeDeviceHasCapability(def, VIR_NODE_DEV_CAP_SCSI_HOST)) {
+if (virNodeDeviceGetWWNs(def, , ) < 0)
+goto cleanup;
 
-/* Because we're about to release the lock and thus run into a race
- * possibility (however improbable) with a udevAddOneDevice change
- * event which would essentially free the existing @def (obj->def) and
- * replace it with something new, we need to grab the parent field
- * and then find the parent obj in order to manage the vport */
-parent = g_strdup(def->parent);
+/* Because we're about to release the lock and thus run into a race
+ * possibility (however improbable) with a udevAddOneDevice change
+ * event which would essentially free the existing @def (obj->def) and
+ * replace it with something new, we need to grab the parent field
+ * and then find the parent obj in order to manage the vport */
+parent = g_strdup(def->parent);
 
-virNodeDeviceObjEndAPI();
+virNodeDeviceObjEndAPI();
 
-if (!(obj = virNodeDeviceObjListFindByName(driver->devs, parent))) {
-virReportError(VIR_ERR_INTERNAL_ERROR,
-   _("cannot find parent '%s' definition"), parent);
-goto cleanup;
-}
+if (!(obj = virNodeDeviceObjListFindByName(driver->devs, parent))) {
+virReportError(VIR_ERR_INTERNAL_ERROR,
+   _("cannot find parent '%s' definition"), parent);
+goto cleanup;
+}
 
-if (virSCSIHostGetNumber(parent, _host) < 0)
-goto cleanup;
+if (virSCSIHostGetNumber(parent, _host) < 0)
+goto cleanup;
 
-if (virVHBAManageVport(parent_host, wwpn, wwnn, VPORT_DELETE) < 0)
-goto cleanup;
+if (virVHBAManageVport(parent_host, wwpn, wwnn, VPORT_DELETE) < 0)
+goto cleanup;
 
-ret = 0;
+ret = 0;
+} else {
+virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",

[libvirt PATCH v2 07/10] nodedev: Add testing for 'mdevctl start'

Test that we run 'mdevctl' with the proper arguments when creating new
mediated devices with virNodeDeviceCreateXML().

Signed-off-by: Jonathon Jongsma 
---
 build-aux/syntax-check.mk |   2 +-
 tests/Makefile.am |  14 +
 ...019_36ea_4111_8f0a_8c9a70e21366-start.argv |   1 +
 ...019_36ea_4111_8f0a_8c9a70e21366-start.json |   1 +
 ...d39_495e_4243_ad9f_beb3f14c23d9-start.argv |   1 +
 ...d39_495e_4243_ad9f_beb3f14c23d9-start.json |   1 +
 ...916_1ca8_49ac_b176_871d16c13076-start.argv |   1 +
 ...916_1ca8_49ac_b176_871d16c13076-start.json |   1 +
 tests/nodedevmdevctltest.c| 257 ++
 ...v_d069d019_36ea_4111_8f0a_8c9a70e21366.xml |   8 +
 ...v_d2441d39_495e_4243_ad9f_beb3f14c23d9.xml |  10 +
 ...v_fedc4916_1ca8_49ac_b176_871d16c13076.xml |   9 +
 12 files changed, 305 insertions(+), 1 deletion(-)
 create mode 100644 
tests/nodedevmdevctldata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366-start.argv
 create mode 100644 
tests/nodedevmdevctldata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366-start.json
 create mode 100644 
tests/nodedevmdevctldata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9-start.argv
 create mode 100644 
tests/nodedevmdevctldata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9-start.json
 create mode 100644 
tests/nodedevmdevctldata/mdev_fedc4916_1ca8_49ac_b176_871d16c13076-start.argv
 create mode 100644 
tests/nodedevmdevctldata/mdev_fedc4916_1ca8_49ac_b176_871d16c13076-start.json
 create mode 100644 tests/nodedevmdevctltest.c
 create mode 100644 
tests/nodedevschemadata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366.xml
 create mode 100644 
tests/nodedevschemadata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9.xml
 create mode 100644 
tests/nodedevschemadata/mdev_fedc4916_1ca8_49ac_b176_871d16c13076.xml

diff --git a/build-aux/syntax-check.mk b/build-aux/syntax-check.mk
index bf8832a2a5..d47a92b530 100644
--- a/build-aux/syntax-check.mk
+++ b/build-aux/syntax-check.mk
@@ -2015,7 +2015,7 @@ exclude_file_name_regexp--sc_prohibit_close = \
   
(\.p[yl]$$|\.spec\.in$$|^docs/|^(src/util/vir(file|event)\.c|src/libvirt-stream\.c|tests/(vir.+mock\.c|commandhelper\.c|qemusecuritymock\.c)|tools/nss/libvirt_nss_(leases|macs)\.c)$$)
 
 exclude_file_name_regexp--sc_prohibit_empty_lines_at_EOF = \
-  
(^tests/(virhostcpu|virpcitest)data/|docs/js/.*\.js|docs/fonts/.*\.woff|\.diff|tests/virconfdata/no-newline\.conf$$)
+  
(^tests/(nodedevmdevctl|virhostcpu|virpcitest)data/|docs/js/.*\.js|docs/fonts/.*\.woff|\.diff|tests/virconfdata/no-newline\.conf$$)
 
 exclude_file_name_regexp--sc_prohibit_fork_wrappers = \
   
(^(src/(util/(vircommand|virdaemon)|lxc/lxc_controller)|tests/testutils)\.c$$)
diff --git a/tests/Makefile.am b/tests/Makefile.am
index f5766a7790..13cbdbb31e 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -388,6 +388,10 @@ test_programs += storagevolxml2xmltest
 
 test_programs += nodedevxml2xmltest
 
+if WITH_NODE_DEVICES
+test_programs += nodedevmdevctltest
+endif WITH_NODE_DEVICES
+
 test_programs += interfacexml2xmltest
 
 test_programs += cputest
@@ -970,6 +974,16 @@ nodedevxml2xmltest_SOURCES = \
testutils.c testutils.h
 nodedevxml2xmltest_LDADD = $(LDADDS)
 
+if WITH_NODE_DEVICES
+nodedevmdevctltest_SOURCES = \
+   nodedevmdevctltest.c \
+   testutils.c testutils.h
+
+nodedevmdevctltest_LDADD = \
+   ../src/libvirt_driver_nodedev_impl.la \
+   $(LDADDS)
+endif WITH_NODE_DEVICES
+
 interfacexml2xmltest_SOURCES = \
interfacexml2xmltest.c \
testutils.c testutils.h
diff --git 
a/tests/nodedevmdevctldata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366-start.argv 
b/tests/nodedevmdevctldata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366-start.argv
new file mode 100644
index 00..dae6dedf7f
--- /dev/null
+++ 
b/tests/nodedevmdevctldata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366-start.argv
@@ -0,0 +1 @@
+/usr/sbin/mdevctl start -p :00:02.0 --jsonfile /dev/stdin
diff --git 
a/tests/nodedevmdevctldata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366-start.json 
b/tests/nodedevmdevctldata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366-start.json
new file mode 100644
index 00..bfc6dcace3
--- /dev/null
+++ 
b/tests/nodedevmdevctldata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366-start.json
@@ -0,0 +1 @@
+{"mdev_type":"i915-GVTg_V5_8","start":"manual"}
\ No newline at end of file
diff --git 
a/tests/nodedevmdevctldata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9-start.argv 
b/tests/nodedevmdevctldata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9-start.argv
new file mode 100644
index 00..dae6dedf7f
--- /dev/null
+++ 
b/tests/nodedevmdevctldata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9-start.argv
@@ -0,0 +1 @@
+/usr/sbin/mdevctl start -p :00:02.0 --jsonfile /dev/stdin
diff --git 
a/tests/nodedevmdevctldata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9-start.json 
b/tests/nodedevmdevctldata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9-start.json
new file mode 100644
index 00..e5b22b2c44
--- /dev/null
+++

[libvirt PATCH v2 10/10] docs: note node device fields that are read-only

As noted by Erik Skultety, we use the same XML schema to report
existing devices and to define new devices. However, some schema
elements are "read-only". In other words, they are used to report
information from the node device driver and cannot be used to define a
new device. Note these in the documentation.

Signed-off-by: Jonathon Jongsma 
---
 docs/formatnode.html.in | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/formatnode.html.in b/docs/formatnode.html.in
index a46b73254b..651411502c 100644
--- a/docs/formatnode.html.in
+++ b/docs/formatnode.html.in
@@ -31,11 +31,16 @@
 name is just the bus type and address, as in
 "pci__00_02_1" or "usb_1_5_3", but some devices are able
 to provide more specific names, such as
-"net_eth1_00_27_13_6a_fe_00".
+"net_eth1_00_27_13_6a_fe_00". This is a read-only field that is
+reported by the device driver. If this element is set when defining a
+new device, it will be ignored.
+
   
   path
   
-Fully qualified sysfs path to the device.
+Fully qualified sysfs path to the device. This is a read-only field
+that is reported by the device driver. If this element is set when
+defining a new device, it will be ignored.
   
   parent
   
-- 
2.21.3

[libvirt PATCH v2 02/10] nodedev: add support for mdev attributes

Mediated devices support arbitrary vendor-specific attributes that can
be attached to a mediated device. These attributes are ordered, and are
written to sysfs in order after a device is created. This patch adds
support for these attributes to the mdev data types and XML schema.

Signed-off-by: Jonathon Jongsma 
---
 docs/formatnode.html.in |  7 +
 docs/schemas/nodedev.rng|  6 
 src/conf/node_device_conf.c | 59 +++--
 src/conf/node_device_conf.h |  2 ++
 src/libvirt_private.syms|  2 ++
 src/util/virmdev.c  | 12 
 src/util/virmdev.h  | 11 +++
 7 files changed, 96 insertions(+), 3 deletions(-)

diff --git a/docs/formatnode.html.in b/docs/formatnode.html.in
index 76eae928de..a46b73254b 100644
--- a/docs/formatnode.html.in
+++ b/docs/formatnode.html.in
@@ -393,6 +393,13 @@
 which holds the IOMMU group number the mediated device belongs
   to.
   
+  attr
+  
+This optional element can occur multiple times. It represents a
+vendor-specific attribute that is used to configure this
+mediated device. It has two required attributes:
+name and value.
+  
 
   
   ccw
diff --git a/docs/schemas/nodedev.rng b/docs/schemas/nodedev.rng
index fe6ffa0b53..a1ce09af54 100644
--- a/docs/schemas/nodedev.rng
+++ b/docs/schemas/nodedev.rng
@@ -634,6 +634,12 @@
 
   
 
+
+  
+
+
+  
+
   
 
   
diff --git a/src/conf/node_device_conf.c b/src/conf/node_device_conf.c
index bccdbd0af8..045d146433 100644
--- a/src/conf/node_device_conf.c
+++ b/src/conf/node_device_conf.c
@@ -500,6 +500,21 @@ virNodeDeviceCapStorageDefFormat(virBufferPtr buf,
 virBufferAddLit(buf, "\n");
 }
 
+static void
+virNodeDeviceCapMdevDefFormat(virBufferPtr buf,
+  const virNodeDevCapData *data)
+{
+size_t i;
+
+virBufferEscapeString(buf, "\n", data->mdev.type);
+virBufferAsprintf(buf, "\n",
+  data->mdev.iommuGroupNumber);
+for (i = 0; i < data->mdev.nattributes; i++) {
+virMediatedDeviceAttrPtr attr = data->mdev.attributes[i];
+virBufferAsprintf(buf, "\n",
+  attr->name, attr->value);
+}
+}
 
 char *
 virNodeDeviceDefFormat(const virNodeDeviceDef *def)
@@ -583,9 +598,7 @@ virNodeDeviceDefFormat(const virNodeDeviceDef *def)
 virBufferEscapeString(, "%s\n", 
virNodeDevDRMTypeToString(data->drm.type));
 break;
 case VIR_NODE_DEV_CAP_MDEV:
-virBufferEscapeString(, "\n", data->mdev.type);
-virBufferAsprintf(, "\n",
-  data->mdev.iommuGroupNumber);
+virNodeDeviceCapMdevDefFormat(, data);
 break;
 case VIR_NODE_DEV_CAP_CCW_DEV:
 virBufferAsprintf(, "0x%x\n",
@@ -1757,6 +1770,35 @@ virNodeDevCapSystemParseXML(xmlXPathContextPtr ctxt,
 return ret;
 }
 
+static int
+virNodeDevCapMdevAttributeParseXML(xmlXPathContextPtr ctxt,
+   xmlNodePtr node,
+   virNodeDevCapMdevPtr mdev)
+{
+xmlNodePtr orig_node = node;
+ctxt->node = node;
+int ret = -1;
+g_autoptr(virMediatedDeviceAttr) attr = virMediatedDeviceAttrNew();
+
+attr->name = virXPathString("string(./@name)", ctxt);
+attr->value = virXPathString("string(./@value)", ctxt);
+if (!attr->name || !attr->value) {
+virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+   _("mdev attribute missing name or value"));
+goto cleanup;
+}
+
+if (VIR_APPEND_ELEMENT(mdev->attributes,
+   mdev->nattributes,
+   attr) < 0)
+  goto cleanup;
+
+ret = 0;
+
+ cleanup:
+ctxt->node = orig_node;
+return ret;
+}
 
 static int
 virNodeDevCapMdevParseXML(xmlXPathContextPtr ctxt,
@@ -1766,6 +1808,9 @@ virNodeDevCapMdevParseXML(xmlXPathContextPtr ctxt,
 {
 VIR_XPATH_NODE_AUTORESTORE(ctxt);
 int ret = -1;
+int nattrs = 0;
+xmlNodePtr *attrs;
+size_t i;
 
 ctxt->node = node;
 
@@ -1783,6 +1828,11 @@ virNodeDevCapMdevParseXML(xmlXPathContextPtr ctxt,
   "'%s'")) < 0)
 goto out;
 
+if ((nattrs = virXPathNodeSet("./attr", ctxt, )) < 0)
+goto out;
+for (i = 0; i < nattrs; i++)
+virNodeDevCapMdevAttributeParseXML(ctxt, attrs[i], mdev);
+
 ret = 0;
  out:
 return ret;
@@ -2172,6 +,9 @@ virNodeDevCapsDefFree(virNodeDevCapsDefPtr caps)
 break;
 case VIR_NODE_DEV_CAP_MDEV:
 VIR_FREE(data->mdev.type);
+for (i = 0; i < data->mdev.nattributes; i++)
+virMediatedDeviceAttrFree(data->mdev.attributes[i]);
+VIR_FREE(data->mdev.attributes);
 break;
 case

[libvirt PATCH v2 09/10] nodedev: Add testing for 'mdevctl stop'

Test that we run 'mdevctl' with the proper arguments when we destroy
mediated devices with virNodeDeviceDestroy()

Signed-off-by: Jonathon Jongsma 
---
 tests/nodedevmdevctldata/mdevctl-stop.argv |  1 +
 tests/nodedevmdevctltest.c | 43 ++
 2 files changed, 44 insertions(+)
 create mode 100644 tests/nodedevmdevctldata/mdevctl-stop.argv

diff --git a/tests/nodedevmdevctldata/mdevctl-stop.argv 
b/tests/nodedevmdevctldata/mdevctl-stop.argv
new file mode 100644
index 00..25ee7145ce
--- /dev/null
+++ b/tests/nodedevmdevctldata/mdevctl-stop.argv
@@ -0,0 +1 @@
+/usr/sbin/mdevctl stop -u e2451f73-c95b-4124-b900-e008af37c576
diff --git a/tests/nodedevmdevctltest.c b/tests/nodedevmdevctltest.c
index 32a22246c2..58ebf976e2 100644
--- a/tests/nodedevmdevctltest.c
+++ b/tests/nodedevmdevctltest.c
@@ -98,6 +98,42 @@ testMdevctlStartHelper(const void *data)
 jsonfile);
 }
 
+static int
+testMdevctlStopHelper(const void *data)
+{
+const char *uuid = data;
+virBuffer buf = VIR_BUFFER_INITIALIZER;
+const char *actualCmdline = NULL;
+int ret = -1;
+g_autoptr(virCommand) cmd = NULL;
+
+g_autofree char *cmdlinefile =
+g_strdup_printf("%s/nodedevmdevctldata/mdevctl-stop.argv",
+abs_srcdir);
+
+cmd = nodeDeviceGetMdevctlStopCommand(uuid, false);
+
+if (!cmd)
+goto cleanup;
+
+virCommandSetDryRun(, NULL, NULL);
+if (virCommandRun(cmd, NULL) < 0)
+goto cleanup;
+
+if (!(actualCmdline = virBufferCurrentContent()))
+goto cleanup;
+
+if (virTestCompareToFile(actualCmdline, cmdlinefile) < 0)
+goto cleanup;
+
+ret = 0;
+
+ cleanup:
+virBufferFreeAndReset();
+virCommandSetDryRun(NULL, NULL, NULL);
+return ret;
+}
+
 static void
 nodedevTestDriverFree(virNodeDeviceDriverStatePtr drv)
 {
@@ -248,6 +284,13 @@ mymain(void)
 DO_TEST("mdev_fedc4916_1ca8_49ac_b176_871d16c13076");
 DO_TEST("mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9");
 
+// Test mdevctl stop command, pass an arbitrary uuid
+if (virTestRun("mdevctl stop", testMdevctlStopHelper,
+   "e2451f73-c95b-4124-b900-e008af37c576") < 0)
+
+
+ret = -1;
+
  done:
 nodedevTestDriverFree(driver);
 
-- 
2.21.3

[libvirt PATCH v2 00/10] Add ability to create mediated devices in libvirt

Apologies for the delay in posting the second version of this series.

This is the first portion of an effort to support persistent mediated devices
with libvirt. This first series simply enables creating and destroying
non-persistent mediated devices via the virNodeDeviceCreateXML() and
virNodeDeviceDestroy() functions. The 'mdevctl' utility[1] provides the backend
implementation.

Changes in v2:
 - review findings fixed
 - Added Unit testing
 - passed JSON config via stdin instead of a temporary file (see portability
   note in patch 5)

[1] https://github.com/mdevctl/mdevctl

Jonathon Jongsma (10):
  nodedev: factor out nodeDeviceHasCapability()
  nodedev: add support for mdev attributes
  nodedev: refactor nodeDeviceFindNewDevice()
  nodedev: store mdev UUID in mdev caps
  nodedev: add mdev support to virNodeDeviceCreateXML()
  nodedev: Build a non-loadable driver lib
  nodedev: Add testing for 'mdevctl start'
  nodedev: add mdev support to virNodeDeviceDestroy()
  nodedev: Add testing for 'mdevctl stop'
  docs: note node device fields that are read-only

 build-aux/syntax-check.mk |   2 +-
 docs/formatnode.html.in   |  16 +-
 docs/schemas/nodedev.rng  |   6 +
 libvirt.spec.in   |   2 +
 m4/virt-external-programs.m4  |   3 +
 src/conf/node_device_conf.c   |  60 ++-
 src/conf/node_device_conf.h   |   3 +
 src/conf/virnodedeviceobj.c   |  34 ++
 src/conf/virnodedeviceobj.h   |   3 +
 src/libvirt_private.syms  |   3 +
 src/node_device/Makefile.inc.am   |  23 +-
 src/node_device/node_device_driver.c  | 377 --
 src/node_device/node_device_driver.h  |   9 +
 src/node_device/node_device_udev.c|   5 +-
 src/util/virmdev.c|  12 +
 src/util/virmdev.h|  11 +
 tests/Makefile.am |  14 +
 ...019_36ea_4111_8f0a_8c9a70e21366-start.argv |   1 +
 ...019_36ea_4111_8f0a_8c9a70e21366-start.json |   1 +
 ...d39_495e_4243_ad9f_beb3f14c23d9-start.argv |   1 +
 ...d39_495e_4243_ad9f_beb3f14c23d9-start.json |   1 +
 ...916_1ca8_49ac_b176_871d16c13076-start.argv |   1 +
 ...916_1ca8_49ac_b176_871d16c13076-start.json |   1 +
 tests/nodedevmdevctldata/mdevctl-stop.argv|   1 +
 tests/nodedevmdevctltest.c| 300 ++
 ...v_d069d019_36ea_4111_8f0a_8c9a70e21366.xml |   8 +
 ...v_d2441d39_495e_4243_ad9f_beb3f14c23d9.xml |  10 +
 ...v_fedc4916_1ca8_49ac_b176_871d16c13076.xml |   9 +
 28 files changed, 862 insertions(+), 55 deletions(-)
 create mode 100644 
tests/nodedevmdevctldata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366-start.argv
 create mode 100644 
tests/nodedevmdevctldata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366-start.json
 create mode 100644 
tests/nodedevmdevctldata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9-start.argv
 create mode 100644 
tests/nodedevmdevctldata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9-start.json
 create mode 100644 
tests/nodedevmdevctldata/mdev_fedc4916_1ca8_49ac_b176_871d16c13076-start.argv
 create mode 100644 
tests/nodedevmdevctldata/mdev_fedc4916_1ca8_49ac_b176_871d16c13076-start.json
 create mode 100644 tests/nodedevmdevctldata/mdevctl-stop.argv
 create mode 100644 tests/nodedevmdevctltest.c
 create mode 100644 
tests/nodedevschemadata/mdev_d069d019_36ea_4111_8f0a_8c9a70e21366.xml
 create mode 100644 
tests/nodedevschemadata/mdev_d2441d39_495e_4243_ad9f_beb3f14c23d9.xml
 create mode 100644 
tests/nodedevschemadata/mdev_fedc4916_1ca8_49ac_b176_871d16c13076.xml

-- 
2.21.3

[libvirt PATCH v2 06/10] nodedev: Build a non-loadable driver lib

In order to test the nodedev driver, we need to link against a
non-loadable module. Similar to other loadable modules already in the
repository, create an _impl library that can be linked against the unit
tests and then create a loadable module from that.

Signed-off-by: Jonathon Jongsma 
---
 src/node_device/Makefile.inc.am | 23 +--
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/node_device/Makefile.inc.am b/src/node_device/Makefile.inc.am
index 788563665f..5993165b56 100644
--- a/src/node_device/Makefile.inc.am
+++ b/src/node_device/Makefile.inc.am
@@ -34,34 +34,37 @@ EXTRA_DIST += \
 
 if WITH_NODE_DEVICES
 # Needed to keep automake quiet about conditionals
+noinst_LTLIBRARIES += libvirt_driver_nodedev_impl.la
+libvirt_driver_nodedev_la_SOURCES =
+libvirt_driver_nodedev_la_LIBADD = libvirt_driver_nodedev_impl.la
 mod_LTLIBRARIES += libvirt_driver_nodedev.la
-libvirt_driver_nodedev_la_SOURCES = $(NODE_DEVICE_DRIVER_SOURCES)
+libvirt_driver_nodedev_impl_la_SOURCES = $(NODE_DEVICE_DRIVER_SOURCES)
 
-libvirt_driver_nodedev_la_CFLAGS = \
+libvirt_driver_nodedev_impl_la_CFLAGS = \
-I$(srcdir)/access \
-I$(builddir)/access \
-I$(srcdir)/conf \
$(AM_CFLAGS) \
$(LIBNL_CFLAGS) \
$(NULL)
-libvirt_driver_nodedev_la_LDFLAGS = $(AM_LDFLAGS_MOD_NOUNDEF)
-libvirt_driver_nodedev_la_LIBADD = \
+libvirt_driver_nodedev_impl_la_LDFLAGS = $(AM_LDFLAGS_MOD_NOUNDEF)
+libvirt_driver_nodedev_impl_la_LIBADD = \
libvirt.la \
$(GLIB_LIBS) \
$(NULL)
 
 if WITH_HAL
-libvirt_driver_nodedev_la_SOURCES += $(NODE_DEVICE_DRIVER_HAL_SOURCES)
-libvirt_driver_nodedev_la_CFLAGS += $(HAL_CFLAGS)
-libvirt_driver_nodedev_la_LIBADD += $(HAL_LIBS)
+libvirt_driver_nodedev_impl_la_SOURCES += $(NODE_DEVICE_DRIVER_HAL_SOURCES)
+libvirt_driver_nodedev_impl_la_CFLAGS += $(HAL_CFLAGS)
+libvirt_driver_nodedev_impl_la_LIBADD += $(HAL_LIBS)
 endif WITH_HAL
 if WITH_UDEV
-libvirt_driver_nodedev_la_SOURCES += $(NODE_DEVICE_DRIVER_UDEV_SOURCES)
-libvirt_driver_nodedev_la_CFLAGS += \
+libvirt_driver_nodedev_impl_la_SOURCES += $(NODE_DEVICE_DRIVER_UDEV_SOURCES)
+libvirt_driver_nodedev_impl_la_CFLAGS += \
$(UDEV_CFLAGS) \
$(PCIACCESS_CFLAGS) \
$(NULL)
-libvirt_driver_nodedev_la_LIBADD += \
+libvirt_driver_nodedev_impl_la_LIBADD += \
$(UDEV_LIBS) \
$(PCIACCESS_LIBS) \
$(NULL)
-- 
2.21.3

Re: [libvirt PATCH 0/3] network: support NAT with IPv6


On 6/9/20 12:27 PM, Daniel P. Berrangé wrote:

On Mon, Jun 08, 2020 at 11:05:00PM -0400, Laine Stump wrote:

On 6/8/20 10:51 AM, Daniel P. Berrangé wrote:

The virtual network has never supported NAT with IPv6 since this feature
didn't exist at the time. NAT has been available since RHEL-7 vintage
though, and it is desirable to be able to use it.

This series enables it with


  


I've had this lurking on my "this is something I should do" list for a long
time, but couldn't decide on the best name in XML (and also figured that the
problem with accept_ra needed to be fixed first), so it never got to the
top. So I'm glad to see you've done it, disappointed in myself that I never
did it :-/

I like your XML knob naming better than what I'd considered. I had thought
of having  (or some other more reasonable extra
mode), but your proposal is more orthogonal and matches with the existing
ipv6='yes' at the toplevel of  (which is used to enable ipv6
traffic between guests on the bridge even when there are no IPv6 addresses
configured for the network.)

I considered  mode="nat6" as an alternative, but it would have meant
updating many switch() statements, and is a somewhat misleading as a
name.



Yeah, the way you have it is much better.







Conceptually this means

   - Try to gimme a subnet with IPv4 and DHCP
   - Try to gimme a subnet with IPv6 and RAs

Now when we start the virtual network

   - If IPv4 is not enabled on host, don't assign addr

What will we use to check for this? Not just "no IP addresses configured", I
guess, since it may be the case that libvirt has just happened to come up
before NM or whoever has started any networks. (or maybe someone wants to
use IPv6 on a libvirt virtual network, but have no IPv6 connectivity beyond
the host).

IIUC, we can simply check whether it is possible to create a socket
with AF_INET or AF_INET6.  If the kernel supports it, then this
should suceed, even if network manager isn't running yet.


   - Else
 - Iterate N=3D1..254 to find a free range for IPv4
 - Use 192.168.N.0/24 for subnet
 - Use 192.168.N.1 for host IP
 - Use 192.168.N.2 -> 192.168.N.254 for guest DHCP

   - If IPv6 is not enabled on host, don't assign addr
   - Else
 - Generate : as 4 random bytes
 - Use fd00:add:f00d::::0/64 for IPv6 subnet
 - Use fd00:add:f00d::::1 for host IP
 - Use route advertizement for IPv6 zero-conf

With :, even with 1000 guests running, we have just a 0.02%
chance of clashing with a guest for IPv6.

The "live" XML would always reflect the currently assigned addresses

Proactively monitor the address allocations of the host. If we see
a conflicting address appear, take down the dnsmasq intance, generate
a new subnet, bring dnsmasq back online.

Hmm. How would you see this monitoring happening? We couldn't do it with an
external script like I had done for simple "shut down on conflict" without
adding extra functionality to libvirt's network driver. We *could* go back
to the idea of monitoring netlink change messages ourselves within libvirtd
and doing it all internally ourselves. Or maybe the NM script I proposed
could go beyond simply destroying conflicting networks, and also restart any
network that had autoaddr='yes'; to make this fully functional we would need
to finally put in the proper stuff so that tap devices (and the underlying
emulated NICs) would be set offline when their connected network was
destroyed, and then reconnected/set online when the network was re-started.
Getting the networks to behave this way would be useful in general anyway,
even without thinking about the conflicting-networks problem. The one
downside of externally controlling renumbering-on-conflict using an external
script is that it would only work with NetworkManager...

Yeah, I'm trying to remember now why we went the NM hook route, rather
than listening for netlink events. I guess NM is much simpler to hook
into.



You mentioned the NM hook idea (I hadn't even known they had such hooks) 
when we were talking about libvirtd using activation sockets (unrelated 
to network conflict stuff), and I said that needing to have a thread 
always listening for netlink change messages was in conflict with the 
idea of having libvirtd use activation sockets and exit after a timeout. 
Once we decide that we're responsible for watching all netlink change 
messages, we have to have a process that's always running (also we would 
have to read the entire state of all interfaces each time that process 
was started, and respond to any changes that had taken place since the 
last time the process exited).



(Probably a NM hook script is also much easier than monitoring netlink 
message, since there's no need to figure out which field in which type 
of messages to look at, and you don't have to setup the thread to listen 
for it, etc. Just a simple python program that gets called with all 
relevant info in text format

Re: [libvirt PATCH 0/9] qemu: Add support for -cpu host, migratable=on|off

2020-06-09 Thread Jiri Denemark

On Tue, Jun 09, 2020 at 12:35:02 +0200, Michal Privoznik wrote:
> On 6/5/20 8:31 PM, Jiri Denemark wrote:
> > 
> > Jiri Denemark (9):
> >conf: Use g_auto* in virCPUDefParseXML
> >qemu: Probe for .migratable property of a CPU
> >qemu: Probe for migrtability support in CPU expansion
> >qemu: Avoid probing unsupported migratable CPU expansion
> >conf: Introduce migratable attribute for the  element
> >conf: Advertise migratable attribute for CPU in domcaps
> >qemu: Advertise migratable attribute for CPU in domcaps
> >qemu: Fill default value in //cpu/@migratable attribute
> >qemu: Pass migratable=on|off property for -cpu host
...
> 
> Is there a BZ link that can be put into one of the commit messages?

No, there's no BZ for this.

> Reviewed-by: Michal Privoznik 

Thanks.

Jirka

[libvirt-dockerfiles PATCH] Drop libosinfo images

As of

  
https://gitlab.com/libosinfo/osinfo-db-tools/-/commit/c185dd44ea6d8bfb9cb77b9ad85ef435816cd6e7
  
https://gitlab.com/libosinfo/osinfo-db/-/commit/84be97d141a5989589dfb5c4f14f6103a9e01cbb
  
https://gitlab.com/libosinfo/libosinfo/-/commit/ff1812e0c538f15bc192403782e92aa452147771

all projects under the libosinfo umbrella use the GitLab container
registry for their CI, so we no longer need to build these on Quay.

Signed-off-by: Andrea Bolognani 
---
Pushed under the Dockerfile refresh rule.

 buildenv-libosinfo-centos-7.zip  | Bin 1795 -> 0 bytes
 buildenv-libosinfo-centos-8.zip  | Bin 726 -> 0 bytes
 buildenv-libosinfo-debian-10.zip | Bin 792 -> 0 bytes
 buildenv-libosinfo-debian-9.zip  | Bin 814 -> 0 bytes
 buildenv-libosinfo-debian-sid.zip| Bin 792 -> 0 bytes
 buildenv-libosinfo-fedora-31.zip | Bin 680 -> 0 bytes
 buildenv-libosinfo-fedora-32.zip | Bin 680 -> 0 bytes
 ...nv-libosinfo-fedora-rawhide-cross-mingw32.zip | Bin 783 -> 0 bytes
 ...nv-libosinfo-fedora-rawhide-cross-mingw64.zip | Bin 789 -> 0 bytes
 buildenv-libosinfo-fedora-rawhide.zip| Bin 699 -> 0 bytes
 buildenv-libosinfo-opensuse-151.zip  | Bin 704 -> 0 bytes
 buildenv-libosinfo-ubuntu-1804.zip   | Bin 818 -> 0 bytes
 buildenv-libosinfo-ubuntu-2004.zip   | Bin 795 -> 0 bytes
 13 files changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 buildenv-libosinfo-centos-7.zip
 delete mode 100644 buildenv-libosinfo-centos-8.zip
 delete mode 100644 buildenv-libosinfo-debian-10.zip
 delete mode 100644 buildenv-libosinfo-debian-9.zip
 delete mode 100644 buildenv-libosinfo-debian-sid.zip
 delete mode 100644 buildenv-libosinfo-fedora-31.zip
 delete mode 100644 buildenv-libosinfo-fedora-32.zip
 delete mode 100644 buildenv-libosinfo-fedora-rawhide-cross-mingw32.zip
 delete mode 100644 buildenv-libosinfo-fedora-rawhide-cross-mingw64.zip
 delete mode 100644 buildenv-libosinfo-fedora-rawhide.zip
 delete mode 100644 buildenv-libosinfo-opensuse-151.zip
 delete mode 100644 buildenv-libosinfo-ubuntu-1804.zip
 delete mode 100644 buildenv-libosinfo-ubuntu-2004.zip

-- 
2.25.4

Re: [libvirt PATCH 0/3] network: support NAT with IPv6

On Mon, Jun 08, 2020 at 11:05:00PM -0400, Laine Stump wrote:
> On 6/8/20 10:51 AM, Daniel P. Berrangé wrote:
> > The virtual network has never supported NAT with IPv6 since this feature
> > didn't exist at the time. NAT has been available since RHEL-7 vintage
> > though, and it is desirable to be able to use it.
> > 
> > This series enables it with
> > 
> >
> >  
> >
> 
> I've had this lurking on my "this is something I should do" list for a long
> time, but couldn't decide on the best name in XML (and also figured that the
> problem with accept_ra needed to be fixed first), so it never got to the
> top. So I'm glad to see you've done it, disappointed in myself that I never
> did it :-/
> 
> I like your XML knob naming better than what I'd considered. I had thought
> of having  (or some other more reasonable extra
> mode), but your proposal is more orthogonal and matches with the existing
> ipv6='yes' at the toplevel of  (which is used to enable ipv6
> traffic between guests on the bridge even when there are no IPv6 addresses
> configured for the network.)

I considered  mode="nat6" as an alternative, but it would have meant
updating many switch() statements, and is a somewhat misleading as a
name. 

> >
> > 
> > Conceptually this means
> > 
> >   - Try to gimme a subnet with IPv4 and DHCP
> >   - Try to gimme a subnet with IPv6 and RAs
> > 
> > Now when we start the virtual network
> > 
> >   - If IPv4 is not enabled on host, don't assign addr
> 
> What will we use to check for this? Not just "no IP addresses configured", I
> guess, since it may be the case that libvirt has just happened to come up
> before NM or whoever has started any networks. (or maybe someone wants to
> use IPv6 on a libvirt virtual network, but have no IPv6 connectivity beyond
> the host).

IIUC, we can simply check whether it is possible to create a socket
with AF_INET or AF_INET6.  If the kernel supports it, then this
should suceed, even if network manager isn't running yet.

> >   - Else
> > - Iterate N=3D1..254 to find a free range for IPv4
> > - Use 192.168.N.0/24 for subnet
> > - Use 192.168.N.1 for host IP
> > - Use 192.168.N.2 -> 192.168.N.254 for guest DHCP
> > 
> >   - If IPv6 is not enabled on host, don't assign addr
> >   - Else
> > - Generate : as 4 random bytes
> > - Use fd00:add:f00d::::0/64 for IPv6 subnet
> > - Use fd00:add:f00d::::1 for host IP
> > - Use route advertizement for IPv6 zero-conf
> > 
> > With :, even with 1000 guests running, we have just a 0.02%
> > chance of clashing with a guest for IPv6.
> > 
> > The "live" XML would always reflect the currently assigned addresses
> > 
> > Proactively monitor the address allocations of the host. If we see
> > a conflicting address appear, take down the dnsmasq intance, generate
> > a new subnet, bring dnsmasq back online.
> 
> Hmm. How would you see this monitoring happening? We couldn't do it with an
> external script like I had done for simple "shut down on conflict" without
> adding extra functionality to libvirt's network driver. We *could* go back
> to the idea of monitoring netlink change messages ourselves within libvirtd
> and doing it all internally ourselves. Or maybe the NM script I proposed
> could go beyond simply destroying conflicting networks, and also restart any
> network that had autoaddr='yes'; to make this fully functional we would need
> to finally put in the proper stuff so that tap devices (and the underlying
> emulated NICs) would be set offline when their connected network was
> destroyed, and then reconnected/set online when the network was re-started.
> Getting the networks to behave this way would be useful in general anyway,
> even without thinking about the conflicting-networks problem. The one
> downside of externally controlling renumbering-on-conflict using an external
> script is that it would only work with NetworkManager...

Yeah, I'm trying to remember now why we went the NM hook route, rather
than listening for netlink events. I guess NM is much simpler to hook
into.  I'd honestly not thought about this too much though - just having
an automatically numbered network will already be a huge step forward
compared to current day.

In particular if we insituted a rule that if we are NOT on a hypervisor,
we count from N=254 -> 0, when picking 192.168.N.0, and count from
N=0 -> 254 when we are on a hypervisor, then we'll trivially avoid the
host/guest clash in simple case, even if network is not yet online.

Don't anyone dare mention nested virt with 3 levels of libvirt... 

Seriously though, even without automatic teardown & restart, we'd
be way better off by simply not hardcoding 192.168.N.0 at RPM
install time when the network env is not the same as the run time
network env. eg cloud images

> > Ideally we would have to bring the guest network links offline and
> > then online again to force DHCP re-assignment immediately.
> 
> Yeah, I think it

[libvirt PATCH v2 1/3] util: add support for IPv6 masquerade rules

IPv6 does support masquerade since Linux 3.9.0 / ip6tables 1.4.18,
which is Fedora 18 / RHEL-7 vintage, which covers all our supported
Linux versions.

Signed-off-by: Daniel P. Berrangé 
---
 src/util/viriptables.c | 33 +++--
 1 file changed, 11 insertions(+), 22 deletions(-)

diff --git a/src/util/viriptables.c b/src/util/viriptables.c
index e6a1ded8d5..8ccce835b2 100644
--- a/src/util/viriptables.c
+++ b/src/util/viriptables.c
@@ -854,29 +854,24 @@ iptablesForwardMasquerade(virFirewallPtr fw,
 g_autofree char *portRangeStr = NULL;
 g_autofree char *natRangeStr = NULL;
 virFirewallRulePtr rule;
+int af = VIR_SOCKET_ADDR_FAMILY(netaddr);
+virFirewallLayer layer = af == AF_INET ?
+VIR_FIREWALL_LAYER_IPV4 : VIR_FIREWALL_LAYER_IPV6;
 
 if (!(networkstr = iptablesFormatNetwork(netaddr, prefix)))
 return -1;
 
-if (!VIR_SOCKET_ADDR_IS_FAMILY(netaddr, AF_INET)) {
-/* Higher level code *should* guaranteee it's impossible to get here. 
*/
-virReportError(VIR_ERR_INTERNAL_ERROR,
-   _("Attempted to NAT '%s'. NAT is only supported for 
IPv4."),
-   networkstr);
-return -1;
-}
-
-if (VIR_SOCKET_ADDR_IS_FAMILY(>start, AF_INET)) {
+if (VIR_SOCKET_ADDR_IS_FAMILY(>start, af)) {
 if (!(addrStartStr = virSocketAddrFormat(>start)))
 return -1;
-if (VIR_SOCKET_ADDR_IS_FAMILY(>end, AF_INET)) {
+if (VIR_SOCKET_ADDR_IS_FAMILY(>end, af)) {
 if (!(addrEndStr = virSocketAddrFormat(>end)))
 return -1;
 }
 }
 
 if (protocol && protocol[0]) {
-rule = virFirewallAddRule(fw, VIR_FIREWALL_LAYER_IPV4,
+rule = virFirewallAddRule(fw, layer,
   "--table", "nat",
   action == ADD ? "--insert" : "--delete",
   pvt ? "LIBVIRT_PRT" : "POSTROUTING",
@@ -885,7 +880,7 @@ iptablesForwardMasquerade(virFirewallPtr fw,
   "!", "--destination", networkstr,
   NULL);
 } else {
-rule = virFirewallAddRule(fw, VIR_FIREWALL_LAYER_IPV4,
+rule = virFirewallAddRule(fw, layer,
   "--table", "nat",
   action == ADD ? "--insert" : "--delete",
   pvt ? "LIBVIRT_PRT" : "POSTROUTING",
@@ -1004,20 +999,14 @@ iptablesForwardDontMasquerade(virFirewallPtr fw,
   int action)
 {
 g_autofree char *networkstr = NULL;
+virFirewallLayer layer = VIR_SOCKET_ADDR_FAMILY(netaddr) == AF_INET ?
+VIR_FIREWALL_LAYER_IPV4 : VIR_FIREWALL_LAYER_IPV6;
 
 if (!(networkstr = iptablesFormatNetwork(netaddr, prefix)))
 return -1;
 
-if (!VIR_SOCKET_ADDR_IS_FAMILY(netaddr, AF_INET)) {
-/* Higher level code *should* guaranteee it's impossible to get here. 
*/
-virReportError(VIR_ERR_INTERNAL_ERROR,
-   _("Attempted to NAT '%s'. NAT is only supported for 
IPv4."),
-   networkstr);
-return -1;
-}
-
 if (physdev && physdev[0])
-virFirewallAddRule(fw, VIR_FIREWALL_LAYER_IPV4,
+virFirewallAddRule(fw, layer,
"--table", "nat",
action == ADD ? "--insert" : "--delete",
pvt ? "LIBVIRT_PRT" : "POSTROUTING",
@@ -1027,7 +1016,7 @@ iptablesForwardDontMasquerade(virFirewallPtr fw,
"--jump", "RETURN",
NULL);
 else
-virFirewallAddRule(fw, VIR_FIREWALL_LAYER_IPV4,
+virFirewallAddRule(fw, layer,
"--table", "nat",
action == ADD ? "--insert" : "--delete",
pvt ? "LIBVIRT_PRT" : "POSTROUTING",
-- 
2.26.2

[libvirt PATCH v2 2/3] conf: add an attribute to turn on NAT for IPv6 virtual networks

Historically IPv6 did not support NAT, so when IPv6 was added to
libvirt's virtual networks, when requesting 
libvirt will NOT apply NAT to IPv6 traffic, only IPv4 traffic.

This is an annoying historical design decision as it means we
cannot enable IPv6 automatically. We thus need to introduce a
new attribute

   
 
   

The new attribute is a tri-state, so it leaves open the possibility of
us intentionally changing the default behaviour in future to honour
NAT for IPv6.

Signed-off-by: Daniel P. Berrangé 
---
 docs/formatnetwork.html.in| 14 +
 docs/schemas/network.rng  |  5 
 src/conf/network_conf.c   | 30 +--
 src/conf/network_conf.h   |  2 ++
 .../nat-network-forward-nat-ipv6.xml  | 10 +++
 .../nat-network-forward-nat-ipv6.xml  | 10 +++
 tests/networkxml2xmltest.c|  1 +
 7 files changed, 69 insertions(+), 3 deletions(-)
 create mode 100644 tests/networkxml2xmlin/nat-network-forward-nat-ipv6.xml
 create mode 100644 tests/networkxml2xmlout/nat-network-forward-nat-ipv6.xml

diff --git a/docs/formatnetwork.html.in b/docs/formatnetwork.html.in
index 0383e2d891..fb740111b1 100644
--- a/docs/formatnetwork.html.in
+++ b/docs/formatnetwork.html.in
@@ -276,6 +276,20 @@
 /nat
   /forward
 ...
+
+
+  Since 6.5.0 it is possible to
+  enable NAT with IPv6 networking. As noted above, IPv6
+  has historically done plain forwarding and thus to avoid
+  breaking historical compatibility, IPv6 NAT must be
+  explicitly requested.
+
+
+...
+  forward mode='nat'
+nat ipv6='yes'/
+  /forward
+...
   
 
   route
diff --git a/docs/schemas/network.rng b/docs/schemas/network.rng
index 88b6f4dfdd..3a5eb3ced4 100644
--- a/docs/schemas/network.rng
+++ b/docs/schemas/network.rng
@@ -181,6 +181,11 @@
   
   
 
+  
+
+  
+
+  
   
 
   
diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index f1d22b25b1..1b89e2985d 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -1358,6 +1358,7 @@ virNetworkForwardNatDefParseXML(const char *networkName,
 int nNatAddrs, nNatPorts;
 char *addrStart = NULL;
 char *addrEnd = NULL;
+char *ipv6 = NULL;
 VIR_XPATH_NODE_AUTORESTORE(ctxt);
 
 ctxt->node = node;
@@ -1369,6 +1370,20 @@ virNetworkForwardNatDefParseXML(const char *networkName,
 goto cleanup;
 }
 
+ipv6 = virXMLPropString(node, "ipv6");
+if (ipv6) {
+int natIPv6;
+if ((natIPv6 = virTristateBoolTypeFromString(ipv6)) <= 0) {
+virReportError(VIR_ERR_XML_ERROR,
+   _("Invalid ipv6 setting '%s' "
+ "in network '%s' NAT"),
+   ipv6, networkName);
+goto cleanup;
+}
+def->natIPv6 = natIPv6;
+VIR_FREE(ipv6);
+}
+
 /* addresses for SNAT */
 nNatAddrs = virXPathNodeSet("./address", ctxt, );
 if (nNatAddrs < 0) {
@@ -2516,10 +2531,18 @@ virNetworkForwardNatDefFormat(virBufferPtr buf,
 goto cleanup;
 }
 
-if (!addrEnd && !addrStart && !fwd->port.start && !fwd->port.end)
+if (!addrEnd && !addrStart && !fwd->port.start && !fwd->port.end && 
!fwd->natIPv6)
 return 0;
 
-virBufferAddLit(buf, "\n");
+virBufferAddLit(buf, "natIPv6)
+virBufferAsprintf(buf, " ipv6='%s'", 
virTristateBoolTypeToString(fwd->natIPv6));
+
+if (!addrEnd && !addrStart && !fwd->port.start && !fwd->port.end) {
+virBufferAddLit(buf, "/>\n");
+return 0;
+}
+virBufferAddLit(buf, ">\n");
 virBufferAdjustIndent(buf, 2);
 
 if (addrStart) {
@@ -2627,7 +2650,8 @@ virNetworkDefFormatBuf(virBufferPtr buf,
  || def->forward.port.start
  || def->forward.port.end
  || (def->forward.driverName
- != VIR_NETWORK_FORWARD_DRIVER_NAME_DEFAULT));
+ != VIR_NETWORK_FORWARD_DRIVER_NAME_DEFAULT)
+ || def->forward.natIPv6);
 virBufferAsprintf(buf, "%s>\n", shortforward ? "/" : "");
 virBufferAdjustIndent(buf, 2);
 
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index f2dc388ef0..e3a61c62ea 100644
--- a/src/conf/network_conf.h
+++ b/src/conf/network_conf.h
@@ -244,6 +244,8 @@ struct _virNetworkForwardDef {
 /* ranges for NAT */
 virSocketAddrRange addr;
 virPortRange port;
+
+virTristateBool natIPv6;
 };
 
 typedef struct _virPortGroupDef virPortGroupDef;
diff --git a/tests/networkxml2xmlin/nat-network-forward-nat-ipv6.xml

[libvirt PATCH v2 0/3] network: support NAT with IPv6

The virtual network has never supported NAT with IPv6 since this feature
didn't exist at the time. NAT has been available since RHEL-7 vintage
though, and it is desirable to be able to use it.

This series enables it with

  

  

Note that I do NOT actually change the default.xml to enable use of IPv6
because that will cause failure if the user has force disabled IPv6 on
their host kernel.

Of course our current default.xml is already broken if someone has done
the reverse and force disabled IPv4.

We've also long had a problem with guests bringing up the default
network with the same subnet as the host. We'll have this same issue
with IPv6 too.

On my prompting Laine proposed a way to deal with the clash by tearing
down a network, if we see a real host NIC get assigned the same subnet.

Meanwhile we also have complaints about the fact that libvirt does
anything todo with networking in the %post of the RPM.

I'm thinking that we can do something entirely different by introducing
a concept of "automatic subnet selection" into the virtual network.

Consider if we made default.xml be able to contain only:

  
default


  


  

Conceptually this means

 - Try to gimme a subnet with IPv4 and DHCP
 - Try to gimme a subnet with IPv6 and RAs

Now when we start the virtual network

 - If IPv4 is not enabled on host, don't assign addr
 - Else
   - Iterate N=3D1..254 to find a free range for IPv4
   - Use 192.168.N.0/24 for subnet
   - Use 192.168.N.1 for host IP
   - Use 192.168.N.2 -> 192.168.N.254 for guest DHCP

 - If IPv6 is not enabled on host, don't assign addr
 - Else
   - Generate : as 4 random bytes
   - Use fd00:add:f00d::::0/64 for IPv6 subnet
   - Use fd00:add:f00d::::1 for host IP
   - Use route advertizement for IPv6 zero-conf

With :, even with 1000 guests running, we have just a 0.02%
chance of clashing with a guest for IPv6.

The "live" XML would always reflect the currently assigned addresses

Proactively monitor the address allocations of the host. If we see
a conflicting address appear, take down the dnsmasq intance, generate
a new subnet, bring dnsmasq back online.

Ideally we would have to bring the guest network links offline and
then online again to force DHCP re-assignment immediately.

In v2:

 - Fix short circuit XML closing tag
 - Fix error checking in enum conversion
 - Fix docs typos
 - Improve RNG schema

Daniel P. Berrang=C3=A9 (3):
  util: add support for IPv6 masquerade rules
  conf: add an attribute to turn on NAT for IPv6 virtual networks
  network: wire up support for IPv6 NAT rules

 docs/formatnetwork.html.in|  14 ++
 docs/schemas/network.rng  |   5 +
 src/conf/network_conf.c   |  30 ++-
 src/conf/network_conf.h   |   2 +
 src/network/bridge_driver_linux.c |  23 +-
 src/util/viriptables.c|  33 +--
 .../nat-ipv6-masquerade-linux.args| 228 ++
 .../nat-ipv6-masquerade.xml   |  17 ++
 tests/networkxml2firewalltest.c   |   1 +
 .../nat-network-forward-nat-ipv6.xml  |  10 +
 .../nat-network-forward-nat-ipv6.xml  |  10 +
 tests/networkxml2xmltest.c|   1 +
 12 files changed, 342 insertions(+), 32 deletions(-)
 create mode 100644 tests/networkxml2firewalldata/nat-ipv6-masquerade-linux.a=
rgs
 create mode 100644 tests/networkxml2firewalldata/nat-ipv6-masquerade.xml
 create mode 100644 tests/networkxml2xmlin/nat-network-forward-nat-ipv6.xml
 create mode 100644 tests/networkxml2xmlout/nat-network-forward-nat-ipv6.xml

--=20
2.26.2

[libvirt PATCH v2 3/3] network: wire up support for IPv6 NAT rules

Now that we have support for IPv6 in the iptables helpers, and a new
option in the XML schema, we can wire up support for it in the network
driver.

Signed-off-by: Daniel P. Berrangé 
---
 src/network/bridge_driver_linux.c |  23 +-
 .../nat-ipv6-masquerade-linux.args| 228 ++
 .../nat-ipv6-masquerade.xml   |  17 ++
 tests/networkxml2firewalltest.c   |   1 +
 4 files changed, 262 insertions(+), 7 deletions(-)
 create mode 100644 tests/networkxml2firewalldata/nat-ipv6-masquerade-linux.args
 create mode 100644 tests/networkxml2firewalldata/nat-ipv6-masquerade.xml

diff --git a/src/network/bridge_driver_linux.c 
b/src/network/bridge_driver_linux.c
index b0bd207250..fcb3803965 100644
--- a/src/network/bridge_driver_linux.c
+++ b/src/network/bridge_driver_linux.c
@@ -307,7 +307,8 @@ int networkCheckRouteCollision(virNetworkDefPtr def)
 return ret;
 }
 
-static const char networkLocalMulticast[] = "224.0.0.0/24";
+static const char networkLocalMulticastIPv4[] = "224.0.0.0/24";
+static const char networkLocalMulticastIPv6[] = "ffx2::/16";
 static const char networkLocalBroadcast[] = "255.255.255.255/32";
 
 static int
@@ -317,6 +318,7 @@ networkAddMasqueradingFirewallRules(virFirewallPtr fw,
 {
 int prefix = virNetworkIPDefPrefix(ipdef);
 const char *forwardIf = virNetworkDefForwardIf(def, 0);
+bool isIPv4 = VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET);
 
 if (prefix < 0) {
 virReportError(VIR_ERR_INTERNAL_ERROR,
@@ -406,7 +408,8 @@ networkAddMasqueradingFirewallRules(virFirewallPtr fw,
 return -1;
 
 /* exempt local network broadcast address as destination */
-if (iptablesAddDontMasquerade(fw,
+if (isIPv4 &&
+iptablesAddDontMasquerade(fw,
   >address,
   prefix,
   forwardIf,
@@ -418,7 +421,8 @@ networkAddMasqueradingFirewallRules(virFirewallPtr fw,
   >address,
   prefix,
   forwardIf,
-  networkLocalMulticast) < 0)
+  isIPv4 ? networkLocalMulticastIPv4 :
+  networkLocalMulticastIPv6) < 0)
 return -1;
 
 return 0;
@@ -431,6 +435,7 @@ networkRemoveMasqueradingFirewallRules(virFirewallPtr fw,
 {
 int prefix = virNetworkIPDefPrefix(ipdef);
 const char *forwardIf = virNetworkDefForwardIf(def, 0);
+bool isIPv4 = VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET);
 
 if (prefix < 0)
 return 0;
@@ -439,10 +444,12 @@ networkRemoveMasqueradingFirewallRules(virFirewallPtr fw,
  >address,
  prefix,
  forwardIf,
- networkLocalMulticast) < 0)
+ isIPv4 ? networkLocalMulticastIPv4 :
+ networkLocalMulticastIPv6) < 0)
 return -1;
 
-if (iptablesRemoveDontMasquerade(fw,
+if (isIPv4 &&
+iptablesRemoveDontMasquerade(fw,
  >address,
  prefix,
  forwardIf,
@@ -769,7 +776,8 @@ networkAddIPSpecificFirewallRules(virFirewallPtr fw,
  */
 
 if (def->forward.type == VIR_NETWORK_FORWARD_NAT) {
-if (VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET))
+if (VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET) ||
+def->forward.natIPv6 == VIR_TRISTATE_BOOL_YES)
 return networkAddMasqueradingFirewallRules(fw, def, ipdef);
 else if (VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET6))
 return networkAddRoutingFirewallRules(fw, def, ipdef);
@@ -786,7 +794,8 @@ networkRemoveIPSpecificFirewallRules(virFirewallPtr fw,
  virNetworkIPDefPtr ipdef)
 {
 if (def->forward.type == VIR_NETWORK_FORWARD_NAT) {
-if (VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET))
+if (VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET) ||
+def->forward.natIPv6 == VIR_TRISTATE_BOOL_YES)
 return networkRemoveMasqueradingFirewallRules(fw, def, ipdef);
 else if (VIR_SOCKET_ADDR_IS_FAMILY(>address, AF_INET6))
 return networkRemoveRoutingFirewallRules(fw, def, ipdef);
diff --git a/tests/networkxml2firewalldata/nat-ipv6-masquerade-linux.args 
b/tests/networkxml2firewalldata/nat-ipv6-masquerade-linux.args
new file mode 100644
index 00..4ba4c3da30
--- /dev/null
+++ b/tests/networkxml2firewalldata/nat-ipv6-masquerade-linux.args
@@ -0,0 +1,228 @@
+iptables \
+--table filter \
+--insert LIBVIRT_INP \
+--in-interface virbr0 \
+--protocol tcp \
+--destination-port 67 \
+--jump ACCEPT
+iptables \
+--table filter \
+--insert LIBVIRT_INP \
+--in-interface virbr0 \
+--protocol

[RFC PATCH 20/41] qemuBlockBitmapChainIsValid: Adjust to new semantics of bitmaps

Reject duplicates and other problematic bitmaps according to the new
semantics of bitmap use in libvirt.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_block.c | 32 ++--
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
index 6f9c7071c9..f42fd200a3 100644
--- a/src/qemu/qemu_block.c
+++ b/src/qemu/qemu_block.c
@@ -2851,40 +2851,36 @@ qemuBlockGetNamedNodeData(virDomainObjPtr vm,
  * qemuBlockBitmapChainIsValid:
  *
  * Validates that the backing chain of @src contains proper consistent bitmap
- * data for a chain of bitmaps named @bitmapname.
+ * @bitmapname.
  *
- * A valid chain:
- * 1) bitmaps of same name are in a consecutive subset of images without gap
- * 2) don't have any inconsistent bitmaps
+ * A valid bitmap:
+ * 1) There's only one such bitmap in the backing chain
+ * 2) It's persistent.
+ * 3) It's active
+ * 4) isn't incosistent
  */
 bool
 qemuBlockBitmapChainIsValid(virStorageSourcePtr src,
 const char *bitmapname,
 virHashTablePtr blockNamedNodeData)
 {
-qemuBlockNamedNodeDataBitmapPtr bitmap;
 virStorageSourcePtr n;
-bool chain_started = false;
-bool chain_ended = false;
+bool found = false;

-for (n = src; n; n = n->backingStore) {
-if (!(bitmap = 
qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData, n, bitmapname))) {
-if (chain_started)
-chain_ended = true;
+for (n = src; virStorageSourceIsBacking(n); n = n->backingStore) {
+qemuBlockNamedNodeDataBitmapPtr bitmap;

+if (!(bitmap = 
qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData,
+ n, bitmapname)))
 continue;
-}

-if (chain_ended)
+if (found || bitmap->inconsistent || !bitmap->persistent || 
!bitmap->recording)
 return false;

-chain_started = true;
-
-if (bitmap->inconsistent)
-return false;
+found = true;
 }

-return chain_started;
+return found;
 }


-- 
2.26.2

[RFC PATCH 35/41] qemublocktest: Add 'snapshots' tests for backup bitmap handling

The 'snapshots' case has multiple layers so we need to make sure that
the bitmaps are merged with the appropriate temporary bitmaps formatted
from the allocation bitmap for any backing chain layer above.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c |  4 ++
 .../backupmerge/snapshots-deep-out.json   | 46 +++
 .../backupmerge/snapshots-flat-out.json   | 25 ++
 .../snapshots-intermediate-out.json   | 31 +
 4 files changed, 106 insertions(+)
 create mode 100644 tests/qemublocktestdata/backupmerge/snapshots-deep-out.json
 create mode 100644 tests/qemublocktestdata/backupmerge/snapshots-flat-out.json
 create mode 100644 
tests/qemublocktestdata/backupmerge/snapshots-intermediate-out.json

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 5cdc4d4be6..a04719b2df 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1271,6 +1271,10 @@ mymain(void)
 TEST_BACKUP_BITMAP_CALCULATE("basic-intermediate", bitmapSourceChain, "d", 
"basic");
 TEST_BACKUP_BITMAP_CALCULATE("basic-deep", bitmapSourceChain, "a", 
"basic");

+TEST_BACKUP_BITMAP_CALCULATE("snapshots-flat", bitmapSourceChain, 
"current", "snapshots");
+TEST_BACKUP_BITMAP_CALCULATE("snapshots-intermediate", bitmapSourceChain, 
"d", "snapshots");
+TEST_BACKUP_BITMAP_CALCULATE("snapshots-deep", bitmapSourceChain, "a", 
"snapshots");
+
 #define TEST_CHECKPOINT_DELETE(testname, delbmp, named) \
 do { \
 checkpointdeletedata.name = testname; \
diff --git a/tests/qemublocktestdata/backupmerge/snapshots-deep-out.json 
b/tests/qemublocktestdata/backupmerge/snapshots-deep-out.json
new file mode 100644
index 00..0cd4ec2b8a
--- /dev/null
+++ b/tests/qemublocktestdata/backupmerge/snapshots-deep-out.json
@@ -0,0 +1,46 @@
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "target_node",
+  "name": "target-bitmap-name",
+  "persistent": false,
+  "disabled": true,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "target_node",
+  "target": "target-bitmap-name",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "libvirt-tmp-allocation"
+},
+{
+  "node": "libvirt-2-format",
+  "name": "libvirt-tmp-allocation"
+},
+{
+  "node": "libvirt-3-format",
+  "name": "libvirt-tmp-allocation"
+},
+{
+  "node": "libvirt-4-format",
+  "name": "libvirt-tmp-allocation"
+},
+{
+  "node": "libvirt-5-format",
+  "name": "a"
+}
+  ]
+}
+  }
+]
+allocation bitmap:
+libvirt-4-format
+libvirt-3-format
+libvirt-2-format
+libvirt-1-format
diff --git a/tests/qemublocktestdata/backupmerge/snapshots-flat-out.json 
b/tests/qemublocktestdata/backupmerge/snapshots-flat-out.json
new file mode 100644
index 00..4637bbc377
--- /dev/null
+++ b/tests/qemublocktestdata/backupmerge/snapshots-flat-out.json
@@ -0,0 +1,25 @@
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "target_node",
+  "name": "target-bitmap-name",
+  "persistent": false,
+  "disabled": true,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "target_node",
+  "target": "target-bitmap-name",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "current"
+}
+  ]
+}
+  }
+]
diff --git 
a/tests/qemublocktestdata/backupmerge/snapshots-intermediate-out.json 
b/tests/qemublocktestdata/backupmerge/snapshots-intermediate-out.json
new file mode 100644
index 00..d6cf4e3a17
--- /dev/null
+++ b/tests/qemublocktestdata/backupmerge/snapshots-intermediate-out.json
@@ -0,0 +1,31 @@
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "target_node",
+  "name": "target-bitmap-name",
+  "persistent": false,
+  "disabled": true,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "target_node",
+  "target": "target-bitmap-name",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "libvirt-tmp-allocation"
+},
+{
+  "node": "libvirt-2-format",
+  "name": "d"
+}
+  ]
+}
+  }
+]
+allocation bitmap:
+libvirt-1-format
-- 
2.26.2

[RFC PATCH 36/41] qemu: Rewrite bitmap handling for block commit

Reuse qemuBlockGetBitmapMergeActions which allows to remove the ad-hoc
implementatio of bitmap merging for block commit. The new approach is
way simpler and more robust and also allows us to get rid of the
disabling of bitmaps done prior to the start as we actually do want to
update the bitmaps in the base.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_block.c | 203 +-
 src/qemu/qemu_block.h |  10 +-
 src/qemu/qemu_blockjob.c  |  24 ++-
 src/qemu/qemu_driver.c|  56 +
 tests/qemublocktest.c |  21 +-
 .../qemublocktestdata/bitmapblockcommit/empty |   1 -
 6 files changed, 47 insertions(+), 268 deletions(-)

diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
index 6eab9cb4e2..fd3fe4c354 100644
--- a/src/qemu/qemu_block.c
+++ b/src/qemu/qemu_block.c
@@ -3176,117 +3176,7 @@ qemuBlockBitmapsHandleBlockcopy(virStorageSourcePtr src,
 /**
  * @topsrc: virStorageSource representing 'top' of the job
  * @basesrc: virStorageSource representing 'base' of the job
- * @blockNamedNodeData: hash table containing data about bitmaps
- * @actions: filled with arguments for a 'transaction' command
- * @disabledBitmapsBase: filled with a list of bitmap names which must be 
disabled
- *
- * Prepares data for correctly handling bitmaps during the start of a commit
- * job. The bitmaps in the 'base' image must be disabled, so that the writes
- * done by the blockjob don't dirty the enabled bitmaps.
- *
- * @actions and @disabledBitmapsBase are untouched if no bitmaps need
- * to be disabled.
- */
-int
-qemuBlockBitmapsHandleCommitStart(virStorageSourcePtr topsrc,
-  virStorageSourcePtr basesrc,
-  virHashTablePtr blockNamedNodeData,
-  virJSONValuePtr *actions,
-  char ***disabledBitmapsBase)
-{
-g_autoptr(virJSONValue) act = virJSONValueNewArray();
-VIR_AUTOSTRINGLIST bitmaplist = NULL;
-size_t curbitmapstr = 0;
-qemuBlockNamedNodeDataPtr entry;
-bool disable_bitmaps = false;
-size_t i;
-
-if (!(entry = virHashLookup(blockNamedNodeData, basesrc->nodeformat)))
-return 0;
-
-bitmaplist = g_new0(char *, entry->nbitmaps + 1);
-
-for (i = 0; i < entry->nbitmaps; i++) {
-qemuBlockNamedNodeDataBitmapPtr bitmap = entry->bitmaps[i];
-
-if (!bitmap->recording || bitmap->inconsistent ||
-!qemuBlockBitmapChainIsValid(topsrc, bitmap->name, 
blockNamedNodeData))
-continue;
-
-disable_bitmaps = true;
-
-if (qemuMonitorTransactionBitmapDisable(act, basesrc->nodeformat,
-bitmap->name) < 0)
-return -1;
-
-bitmaplist[curbitmapstr++] = g_strdup(bitmap->name);
-}
-
-if (disable_bitmaps) {
-*actions = g_steal_pointer();
-*disabledBitmapsBase = g_steal_pointer();
-}
-
-return 0;
-}
-
-
-struct qemuBlockBitmapsHandleCommitData {
-bool skip;
-bool create;
-bool enable;
-const char *basenode;
-virJSONValuePtr merge;
-unsigned long long granularity;
-bool persistent;
-};
-
-
-static void
-qemuBlockBitmapsHandleCommitDataFree(void *opaque)
-{
-struct qemuBlockBitmapsHandleCommitData *data = opaque;
-
-virJSONValueFree(data->merge);
-g_free(data);
-}
-
-
-static int
-qemuBlockBitmapsHandleCommitFinishIterate(void *payload,
-  const void *entryname,
-  void *opaque)
-{
-struct qemuBlockBitmapsHandleCommitData *data = payload;
-const char *bitmapname = entryname;
-virJSONValuePtr actions = opaque;
-
-if (data->skip)
-return 0;
-
-if (data->create) {
-if (qemuMonitorTransactionBitmapAdd(actions, data->basenode, 
bitmapname,
-data->persistent, !data->enable,
-data->granularity) < 0)
-return -1;
-} else {
-if (data->enable &&
-qemuMonitorTransactionBitmapEnable(actions, data->basenode, 
bitmapname) < 0)
-return -1;
-}
-
-if (data->merge &&
-qemuMonitorTransactionBitmapMerge(actions, data->basenode, bitmapname,
-  >merge) < 0)
-return -1;
-
-return 0;
-}
-
-
-/**
- * @topsrc: virStorageSource representing 'top' of the job
- * @basesrc: virStorageSource representing 'base' of the job
+ * @active: commit job is an active layer block-commit
  * @blockNamedNodeData: hash table containing data about bitmaps
  * @actions: filled with arguments for a 'transaction' command
  * @disabledBitmapsBase: bitmap names which were disabled
@@ -3299,95 +3189,22 @@ qemuBlockBitmapsHandleCommitFinishIterate(void *payload,
 int

[RFC PATCH 34/41] qemublocktest: Add 'basic' tests for backup bitmap handling

The 'basic' case is just a single backing store layer containing the
bitmaps so we just copy the bitmaps over to the backup bitmap.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c |  4 +++
 .../backupmerge/basic-deep-out.json   | 25 +++
 .../backupmerge/basic-flat-out.json   | 25 +++
 .../backupmerge/basic-intermediate-out.json   | 25 +++
 4 files changed, 79 insertions(+)
 create mode 100644 tests/qemublocktestdata/backupmerge/basic-deep-out.json
 create mode 100644 tests/qemublocktestdata/backupmerge/basic-flat-out.json
 create mode 100644 
tests/qemublocktestdata/backupmerge/basic-intermediate-out.json

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 2488aa2200..5cdc4d4be6 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1267,6 +1267,10 @@ mymain(void)

 TEST_BACKUP_BITMAP_CALCULATE("empty", bitmapSourceChain, "a", "empty");

+TEST_BACKUP_BITMAP_CALCULATE("basic-flat", bitmapSourceChain, "current", 
"basic");
+TEST_BACKUP_BITMAP_CALCULATE("basic-intermediate", bitmapSourceChain, "d", 
"basic");
+TEST_BACKUP_BITMAP_CALCULATE("basic-deep", bitmapSourceChain, "a", 
"basic");
+
 #define TEST_CHECKPOINT_DELETE(testname, delbmp, named) \
 do { \
 checkpointdeletedata.name = testname; \
diff --git a/tests/qemublocktestdata/backupmerge/basic-deep-out.json 
b/tests/qemublocktestdata/backupmerge/basic-deep-out.json
new file mode 100644
index 00..ff77af789b
--- /dev/null
+++ b/tests/qemublocktestdata/backupmerge/basic-deep-out.json
@@ -0,0 +1,25 @@
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "target_node",
+  "name": "target-bitmap-name",
+  "persistent": false,
+  "disabled": true,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "target_node",
+  "target": "target-bitmap-name",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "a"
+}
+  ]
+}
+  }
+]
diff --git a/tests/qemublocktestdata/backupmerge/basic-flat-out.json 
b/tests/qemublocktestdata/backupmerge/basic-flat-out.json
new file mode 100644
index 00..4637bbc377
--- /dev/null
+++ b/tests/qemublocktestdata/backupmerge/basic-flat-out.json
@@ -0,0 +1,25 @@
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "target_node",
+  "name": "target-bitmap-name",
+  "persistent": false,
+  "disabled": true,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "target_node",
+  "target": "target-bitmap-name",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "current"
+}
+  ]
+}
+  }
+]
diff --git a/tests/qemublocktestdata/backupmerge/basic-intermediate-out.json 
b/tests/qemublocktestdata/backupmerge/basic-intermediate-out.json
new file mode 100644
index 00..f2f3b3f568
--- /dev/null
+++ b/tests/qemublocktestdata/backupmerge/basic-intermediate-out.json
@@ -0,0 +1,25 @@
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "target_node",
+  "name": "target-bitmap-name",
+  "persistent": false,
+  "disabled": true,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "target_node",
+  "target": "target-bitmap-name",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "d"
+}
+  ]
+}
+  }
+]
-- 
2.26.2

[RFC PATCH 37/41] qemublocktest: Add 'basic' tests for commit bitmap handling

In the 'basic case we have few bitmaps in only the top layer. Simulate
commit into the backing of the top layer and also 2 levels deep.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c |   4 +
 .../bitmapblockcommit/basic-1-2   | 145 ++
 .../bitmapblockcommit/basic-1-3   | 145 ++
 .../bitmapblockcommit/basic-2-3   |   1 +
 4 files changed, 295 insertions(+)
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/basic-1-2
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/basic-1-3
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/basic-2-3

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 5624c96d5f..d1148ec02e 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1365,6 +1365,10 @@ mymain(void)

 TEST_BITMAP_BLOCKCOMMIT("empty", 1, 2, "empty");

+TEST_BITMAP_BLOCKCOMMIT("basic-1-2", 1, 2, "basic");
+TEST_BITMAP_BLOCKCOMMIT("basic-1-3", 1, 3, "basic");
+TEST_BITMAP_BLOCKCOMMIT("basic-2-3", 2, 3, "basic");
+
  cleanup:
 qemuTestDriverFree();
 VIR_FREE(capslatest_x86_64);
diff --git a/tests/qemublocktestdata/bitmapblockcommit/basic-1-2 
b/tests/qemublocktestdata/bitmapblockcommit/basic-1-2
new file mode 100644
index 00..eeca383dec
--- /dev/null
+++ b/tests/qemublocktestdata/bitmapblockcommit/basic-1-2
@@ -0,0 +1,145 @@
+merge bitmpas:
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "libvirt-2-format",
+  "name": "current",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "libvirt-2-format",
+  "target": "current",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "current"
+},
+{
+  "node": "libvirt-2-format",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "libvirt-2-format",
+  "name": "d",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "libvirt-2-format",
+  "target": "d",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "d"
+},
+{
+  "node": "libvirt-2-format",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "libvirt-2-format",
+  "name": "c",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "libvirt-2-format",
+  "target": "c",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "c"
+},
+{
+  "node": "libvirt-2-format",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "libvirt-2-format",
+  "name": "b",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "libvirt-2-format",
+  "target": "b",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "b"
+},
+{
+  "node": "libvirt-2-format",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "libvirt-2-format",
+  "name": "a",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "libvirt-2-format",
+  "target": "a",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "a"
+},
+{
+  "node": "libvirt-2-format",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-remove",
+"data": {
+  "node": "libvirt-2-format",
+  "name": "libvirt-tmp-activewrite"
+}
+  }
+]
diff --git a/tests/qemublocktestdata/bitmapblockcommit/basic-1-3 
b/tests/qemublocktestdata/bitmapblockcommit/basic-1-3
new file mode 100644
index 00..3db21cb465
--- /dev/null
+++ b/tests/qemublocktestdata/bitmapblockcommit/basic-1-3
@@ -0,0 +1,145 @@
+merge bitmpas:
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "libvirt-3-format",
+  "name": "current",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node":

[RFC PATCH 15/41] qemublocktest: Extract printing of nodename list

There will be multiple places where we'll need to print nodenames from a
GSList of virStorageSource for testing purposes. Extract the code into a
function.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c | 27 ++-
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 28cbaeba38..0ed9b99bc4 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -647,6 +647,23 @@ testQemuDetectBitmaps(const void *opaque)
 }


+static void
+testQemuBitmapListPrint(const char *title,
+GSList *next,
+virBufferPtr buf)
+{
+if (!next)
+return;
+
+virBufferAsprintf(buf, "%s\n", title);
+
+for (; next; next = next->next) {
+virStorageSourcePtr src = next->data;
+virBufferAsprintf(buf, "%s\n", src->nodeformat);
+}
+}
+
+
 static virStorageSourcePtr
 testQemuBackupIncrementalBitmapCalculateGetFakeImage(size_t idx)
 {
@@ -829,7 +846,6 @@ testQemuCheckpointDeleteMerge(const void *opaque)
 g_autoptr(virHashTable) nodedata = NULL;
 g_autoptr(GSList) reopenimages = NULL;
 g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
-GSList *tmp;

 expectpath = g_strdup_printf("%s/%s%s-out.json", abs_srcdir,
  checkpointDeletePrefix, data->name);
@@ -858,14 +874,7 @@ testQemuCheckpointDeleteMerge(const void *opaque)
 virBufferAddLit(, "NULL\n");
 }

-if (reopenimages) {
-virBufferAddLit(, "reopen nodes:\n");
-
-for (tmp = reopenimages; tmp; tmp = tmp->next) {
-virStorageSourcePtr src = tmp->data;
-virBufferAsprintf(, "%s\n", src->nodeformat);
-}
-}
+testQemuBitmapListPrint("reopen nodes:", reopenimages, );

 actual = virBufferContentAndReset();

-- 
2.26.2

[RFC PATCH 27/41] qemu: monitor: Add support for 'block-dirty-bitmap-populate' transaction member

Add monitor code for driving the block-dirty-bitmap-populate blockjob.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_monitor.c  | 11 +++
 src/qemu/qemu_monitor.h  |  7 +++
 src/qemu/qemu_monitor_json.c | 18 ++
 src/qemu/qemu_monitor_json.h |  6 ++
 tests/qemumonitorjsontest.c  |  3 ++-
 5 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
index 3ec22b939f..623730154d 100644
--- a/src/qemu/qemu_monitor.c
+++ b/src/qemu/qemu_monitor.c
@@ -4617,3 +4617,14 @@ qemuMonitorTransactionBackup(virJSONValuePtr actions,
 return qemuMonitorJSONTransactionBackup(actions, device, jobname, target,
 bitmap, syncmode);
 }
+
+
+int
+qemuMonitorTransactionBitmapPopulate(virJSONValuePtr actions,
+ const char *nodename,
+ const char *bitmapname,
+ const char *jobname)
+{
+return qemuMonitorJSONTransactionBitmapPopulate(actions, nodename,
+bitmapname, jobname);
+}
diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
index 50a337715d..ccc946108a 100644
--- a/src/qemu/qemu_monitor.h
+++ b/src/qemu/qemu_monitor.h
@@ -1431,3 +1431,10 @@ qemuMonitorTransactionBackup(virJSONValuePtr actions,
  const char *target,
  const char *bitmap,
  qemuMonitorTransactionBackupSyncMode syncmode);
+
+
+int
+qemuMonitorTransactionBitmapPopulate(virJSONValuePtr actions,
+ const char *nodename,
+ const char *bitmapname,
+ const char *jobname);
diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c
index 3070c1e6b3..2ed3358252 100644
--- a/src/qemu/qemu_monitor_json.c
+++ b/src/qemu/qemu_monitor_json.c
@@ -9261,6 +9261,24 @@ 
qemuMonitorJSONTransactionBitmapMergeSourceAddBitmap(virJSONValuePtr sources,
 }


+int
+qemuMonitorJSONTransactionBitmapPopulate(virJSONValuePtr actions,
+ const char *nodename,
+ const char *bitmapname,
+ const char *jobname)
+{
+return qemuMonitorJSONTransactionAdd(actions,
+ "block-dirty-bitmap-populate",
+ "s:node", nodename,
+ "s:name", bitmapname,
+ "s:job-id", jobname,
+ "s:pattern", "allocation-top",
+ "T:auto-finalize", 
VIR_TRISTATE_BOOL_YES,
+ "T:auto-dismiss", 
VIR_TRISTATE_BOOL_NO,
+ NULL);
+}
+
+
 int
 qemuMonitorJSONTransactionSnapshotLegacy(virJSONValuePtr actions,
  const char *device,
diff --git a/src/qemu/qemu_monitor_json.h b/src/qemu/qemu_monitor_json.h
index 83c5e25ca5..7ae4857672 100644
--- a/src/qemu/qemu_monitor_json.h
+++ b/src/qemu/qemu_monitor_json.h
@@ -667,6 +667,12 @@ 
qemuMonitorJSONTransactionBitmapMergeSourceAddBitmap(virJSONValuePtr sources,
  const char *sourcenode,
  const char *sourcebitmap);

+int
+qemuMonitorJSONTransactionBitmapPopulate(virJSONValuePtr actions,
+ const char *nodename,
+ const char *bitmapname,
+ const char *jobname);
+
 int
 qemuMonitorJSONTransactionSnapshotLegacy(virJSONValuePtr actions,
  const char *device,
diff --git a/tests/qemumonitorjsontest.c b/tests/qemumonitorjsontest.c
index 6d5a1d5fe2..befd41fd25 100644
--- a/tests/qemumonitorjsontest.c
+++ b/tests/qemumonitorjsontest.c
@@ -2992,7 +2992,8 @@ testQemuMonitorJSONTransaction(const void *opaque)
 qemuMonitorTransactionBackup(actions, "dev9", "job9", "target9", 
"bitmap9",
  
QEMU_MONITOR_TRANSACTION_BACKUP_SYNC_MODE_INCREMENTAL) < 0 ||
 qemuMonitorTransactionBackup(actions, "devA", "jobA", "targetA", 
"bitmapA",
- 
QEMU_MONITOR_TRANSACTION_BACKUP_SYNC_MODE_FULL) < 0)
+ 
QEMU_MONITOR_TRANSACTION_BACKUP_SYNC_MODE_FULL) < 0 ||
+qemuMonitorJSONTransactionBitmapPopulate(actions, "nodeB", "bitmapB", 
"jobB") < 0)
 return -1;

 if (qemuMonitorTestAddItem(test, "transaction", "{\"return\":{}}") < 0)
-- 
2.26.2

[RFC PATCH 18/41] qemu: snapshot: Don't propagate bitmaps to upper layers

With the upcoming changes to use 'dirty-bitmap-populate' job, this will
no longer be required to do the backups, so we can stop creating bitmaps
which would be pointless.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_driver.c | 41 -
 1 file changed, 41 deletions(-)

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 88517ba6a7..5a3b3bb35b 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -15016,44 +15016,6 @@ 
qemuDomainSnapshotDiskCleanup(qemuDomainSnapshotDiskDataPtr data,
 }


-/**
- * qemuDomainSnapshotDiskBitmapsPropagate:
- *
- * This function propagates any active persistent bitmap present in the 
original
- * image into the new snapshot. This is necessary to keep tracking the changed
- * blocks in the active bitmaps as the backing file will become read-only.
- * We leave the original bitmap active as in cases when the overlay is
- * discarded (snapshot revert with abandoning the history) everything works as
- * expected.
- */
-static int
-qemuDomainSnapshotDiskBitmapsPropagate(qemuDomainSnapshotDiskDataPtr dd,
-   virJSONValuePtr actions,
-   virHashTablePtr blockNamedNodeData)
-{
-qemuBlockNamedNodeDataPtr entry;
-size_t i;
-
-if (!(entry = virHashLookup(blockNamedNodeData, 
dd->disk->src->nodeformat)))
-return 0;
-
-for (i = 0; i < entry->nbitmaps; i++) {
-qemuBlockNamedNodeDataBitmapPtr bitmap = entry->bitmaps[i];
-
-/* we don't care about temporary, inconsistent, or disabled bitmaps */
-if (!bitmap->persistent || !bitmap->recording || bitmap->inconsistent)
-continue;
-
-if (qemuMonitorTransactionBitmapAdd(actions, dd->src->nodeformat,
-bitmap->name, true, false,
-bitmap->granularity) < 0)
-return -1;
-}
-
-return 0;
-}
-
-
 static int
 qemuDomainSnapshotDiskPrepareOneBlockdev(virQEMUDriverPtr driver,
  virDomainObjPtr vm,
@@ -15200,9 +15162,6 @@ qemuDomainSnapshotDiskPrepareOne(virQEMUDriverPtr 
driver,
  blockNamedNodeData, 
asyncJob) < 0)
 return -1;

-if (qemuDomainSnapshotDiskBitmapsPropagate(dd, actions, 
blockNamedNodeData) < 0)
-return -1;
-
 if (qemuBlockSnapshotAddBlockdev(actions, dd->disk, dd->src) < 0)
 return -1;
 } else {
-- 
2.26.2

[RFC PATCH 38/41] qemublocktest: Add 'snapshots' tests for block commit bitmap handling

Simulate commit between all the combinations of layers in the
'snapshots' case to see whether the code merges the correct bitmaps with
the correct depth of temporary bitmaps.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c |  14 ++
 .../bitmapblockcommit/snapshots-1-2   |  37 +
 .../bitmapblockcommit/snapshots-1-3   |  70 +
 .../bitmapblockcommit/snapshots-1-4   | 141 ++
 .../bitmapblockcommit/snapshots-1-5   | 141 ++
 .../bitmapblockcommit/snapshots-2-3   |  26 
 .../bitmapblockcommit/snapshots-2-4   |  82 ++
 .../bitmapblockcommit/snapshots-2-5   |  82 ++
 .../bitmapblockcommit/snapshots-3-4   |  49 ++
 .../bitmapblockcommit/snapshots-3-5   |  49 ++
 .../bitmapblockcommit/snapshots-4-5   |   1 +
 11 files changed, 692 insertions(+)
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-1-2
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-1-3
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-1-4
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-1-5
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-2-3
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-2-4
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-2-5
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-3-4
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-3-5
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-4-5

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index d1148ec02e..8706f1a5ec 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1369,6 +1369,20 @@ mymain(void)
 TEST_BITMAP_BLOCKCOMMIT("basic-1-3", 1, 3, "basic");
 TEST_BITMAP_BLOCKCOMMIT("basic-2-3", 2, 3, "basic");

+TEST_BITMAP_BLOCKCOMMIT("snapshots-1-2", 1, 2, "snapshots");
+TEST_BITMAP_BLOCKCOMMIT("snapshots-1-3", 1, 3, "snapshots");
+TEST_BITMAP_BLOCKCOMMIT("snapshots-1-4", 1, 4, "snapshots");
+TEST_BITMAP_BLOCKCOMMIT("snapshots-1-5", 1, 5, "snapshots");
+
+TEST_BITMAP_BLOCKCOMMIT("snapshots-2-3", 2, 3, "snapshots");
+TEST_BITMAP_BLOCKCOMMIT("snapshots-2-4", 2, 4, "snapshots");
+TEST_BITMAP_BLOCKCOMMIT("snapshots-2-5", 2, 5, "snapshots");
+
+TEST_BITMAP_BLOCKCOMMIT("snapshots-3-4", 3, 4, "snapshots");
+TEST_BITMAP_BLOCKCOMMIT("snapshots-3-5", 3, 5, "snapshots");
+
+TEST_BITMAP_BLOCKCOMMIT("snapshots-4-5", 4, 5, "snapshots");
+
  cleanup:
 qemuTestDriverFree();
 VIR_FREE(capslatest_x86_64);
diff --git a/tests/qemublocktestdata/bitmapblockcommit/snapshots-1-2 
b/tests/qemublocktestdata/bitmapblockcommit/snapshots-1-2
new file mode 100644
index 00..70aa9608c7
--- /dev/null
+++ b/tests/qemublocktestdata/bitmapblockcommit/snapshots-1-2
@@ -0,0 +1,37 @@
+merge bitmpas:
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "libvirt-2-format",
+  "name": "current",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "libvirt-2-format",
+  "target": "current",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "current"
+},
+{
+  "node": "libvirt-2-format",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-remove",
+"data": {
+  "node": "libvirt-2-format",
+  "name": "libvirt-tmp-activewrite"
+}
+  }
+]
diff --git a/tests/qemublocktestdata/bitmapblockcommit/snapshots-1-3 
b/tests/qemublocktestdata/bitmapblockcommit/snapshots-1-3
new file mode 100644
index 00..a223ef998e
--- /dev/null
+++ b/tests/qemublocktestdata/bitmapblockcommit/snapshots-1-3
@@ -0,0 +1,70 @@
+merge bitmpas:
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "libvirt-3-format",
+  "name": "current",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "libvirt-3-format",
+  "target": "current",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "current"
+},
+{
+  "node": "libvirt-3-format",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "libvirt-3-format",
+  "name": "d",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "libvirt-3-format",
+  "target": "d",
+  "bitmaps": [
+{
+  "node":

[RFC PATCH 31/41] qemu: block: Add helper to add temporary block bitmaps from allocation maps

qemuBlockBitmapTemporaryAdd uses the 'block-dirty-bitmap-populate' qemu
blockjob to fill a new bitmap from the allocation layer. This is useful
to restore bitmaps for backing chain layers where a specific bitmap is
not necessary or the layer was created outside of libvirt.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_block.c| 190 +++
 src/qemu/qemu_block.h|  11 +++
 src/qemu/qemu_blockjob.c |   2 +-
 3 files changed, 202 insertions(+), 1 deletion(-)

diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
index f42fd200a3..69578e861c 100644
--- a/src/qemu/qemu_block.c
+++ b/src/qemu/qemu_block.c
@@ -3434,3 +3434,193 @@ qemuBlockUpdateRelativeBacking(virDomainObjPtr vm,

 return 0;
 }
+
+
+static void
+qemuBlockBitmapTemporaryRemoveDuplicates(GSList *images)
+{
+g_autoptr(virHashTable) duplicates = virHashNew(NULL);
+GSList *next;
+GSList *prev = NULL;
+
+for (next = images; next; next = next->next) {
+virStorageSourcePtr img = next->data;
+
+if (virHashHasEntry(duplicates, img->nodeformat)) {
+next = g_slist_delete_link(prev, next);
+continue;
+}
+
+virHashAddEntry(duplicates, img->nodeformat, NULL);
+
+prev = next;
+}
+}
+
+
+/**
+ * qemuBlockBitmapTemporaryAdd:
+ * @vm: domain object
+ * @blockNamedNodeData: hash table filled with qemuBlockNamedNodeData
+ * @images: a GSList of virStorageSources to add the temporary bitmaps for
+ * @asyncJob: qemu asynchronous job type
+ *
+ * Add temporary block dirty bitmaps populated from the allocation map of
+ * images for list of virStorageSources @images. The bitmaps added are called
+ * "libvirt-tmp-allocation" and are not made persistent. After this function
+ * returns @images is updated to the actual list of bitmaps which were added 
and
+ * qemuBlockBitmapTemporaryRemove can be used to undo the changes.
+ */
+int
+qemuBlockBitmapTemporaryAdd(virDomainObjPtr vm,
+virHashTablePtr blockNamedNodeData,
+GSList **images,
+qemuDomainAsyncJob asyncJob)
+{
+qemuDomainObjPrivatePtr priv = vm->privateData;
+g_autoptr(virJSONValue) actions = virJSONValueNewArray();
+bool failed = false;
+GSList *next;
+int rc;
+
+if (!*images)
+return 0;
+
+qemuBlockBitmapTemporaryRemoveDuplicates(*images);
+
+for (next = *images; next; next = next->next) {
+virStorageSourcePtr img = next->data;
+qemuBlockNamedNodeDataBitmapPtr tmpbitmap;
+qemuDomainStorageSourcePrivatePtr srcPriv;
+
+if ((tmpbitmap = 
qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData, img,
+   
"libvirt-tmp-allocation"))) {
+if (!tmpbitmap->recording || tmpbitmap->persistent || 
tmpbitmap->inconsistent) {
+virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+   _("internal bitmap in unexpected state"));
+return -1;
+}
+} else {
+if (qemuMonitorTransactionBitmapAdd(actions,
+img->nodeformat,
+"libvirt-tmp-allocation",
+false, false, 0) < 0)
+return -1;
+}
+
+if (!(srcPriv = qemuDomainStorageSourcePrivateFetch(img)))
+return -1;
+
+if (!(srcPriv->blockjob = qemuBlockJobNewPopulate(vm, img)))
+return -1;
+
+qemuBlockJobSyncBegin(srcPriv->blockjob);
+
+if (qemuMonitorTransactionBitmapPopulate(actions,
+ img->nodeformat,
+ "libvirt-tmp-allocation",
+ srcPriv->blockjob->name) < 0)
+return -1;
+}
+
+if (qemuDomainObjEnterMonitorAsync(priv->driver, vm, asyncJob) < 0)
+return -1;
+
+rc = qemuMonitorTransaction(priv->mon, );
+
+if (qemuDomainObjExitMonitor(priv->driver, vm) < 0 || rc < 0)
+return -1;
+
+while (true) {
+bool update = false;
+bool running = false;
+
+for (next = *images; next; next = next->next) {
+virStorageSourcePtr img = next->data;
+qemuDomainStorageSourcePrivatePtr srcPriv = 
QEMU_DOMAIN_STORAGE_SOURCE_PRIVATE(img);
+
+if (!srcPriv->blockjob)
+continue;
+
+if (srcPriv->blockjob->newstate != -1)
+update = true;
+
+qemuBlockJobUpdate(vm, srcPriv->blockjob, asyncJob);
+
+switch ((qemuBlockjobState) srcPriv->blockjob->state) {
+case QEMU_BLOCKJOB_STATE_NEW:
+case QEMU_BLOCKJOB_STATE_RUNNING:
+running = true;
+break;
+
+case QEMU_BLOCKJOB_STATE_FAILED:
+

[RFC PATCH 33/41] qemu: backup: Rewrite backup bitmap handling to the new bitmap semantics

Reuse qemuBlockGetBitmapMergeActions which allows to remove the ad-hoc
iplementatio of bitmap merging for backup. The new approach is simpler
and also more robust in case some of the bitmaps break as they remove
the dependancy on the whole chain of bitmaps working.

The new approach also allows backups if a snapshot is created outside of
libvirt.

Additionally the code is greatly simplified.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_backup.c| 239 +-
 src/qemu/qemu_backup.h|  13 +-
 tests/qemublocktest.c |  90 ++-
 .../backupmerge/empty-out.json|   4 +-
 4 files changed, 88 insertions(+), 258 deletions(-)

diff --git a/src/qemu/qemu_backup.c b/src/qemu/qemu_backup.c
index 5729aac858..242a75431b 100644
--- a/src/qemu/qemu_backup.c
+++ b/src/qemu/qemu_backup.c
@@ -170,199 +170,68 @@ qemuBackupDiskDataCleanup(virDomainObjPtr vm,
 }


-/**
- * qemuBackupBeginCollectIncrementalCheckpoints:
- * @vm: domain object
- * @incrFrom: name of checkpoint representing starting point of incremental 
backup
- *
- * Returns a NULL terminated list of pointers to checkpoint definitions in
- * chronological order starting from the 'current' checkpoint until reaching
- * @incrFrom.
- */
-static virDomainMomentDefPtr *
-qemuBackupBeginCollectIncrementalCheckpoints(virDomainObjPtr vm,
- const char *incrFrom)
-{
-virDomainMomentObjPtr n = virDomainCheckpointGetCurrent(vm->checkpoints);
-g_autofree virDomainMomentDefPtr *incr = NULL;
-size_t nincr = 0;
-
-while (n) {
-virDomainMomentDefPtr def = n->def;
-
-if (VIR_APPEND_ELEMENT_COPY(incr, nincr, def) < 0)
-return NULL;
-
-if (STREQ(def->name, incrFrom)) {
-def = NULL;
-if (VIR_APPEND_ELEMENT_COPY(incr, nincr, def) < 0)
-return NULL;
-
-return g_steal_pointer();
-}
-
-if (!n->def->parent_name)
-break;
-
-n = virDomainCheckpointFindByName(vm->checkpoints, 
n->def->parent_name);
-}
-
-virReportError(VIR_ERR_OPERATION_INVALID,
-   _("could not locate checkpoint '%s' for incremental 
backup"),
-   incrFrom);
-return NULL;
-}
-
-
-static int
-qemuBackupGetBitmapMergeRange(virStorageSourcePtr from,
-  const char *bitmapname,
-  virJSONValuePtr *actions,
-  virStorageSourcePtr *to,
-  const char *diskdst,
-  virHashTablePtr blockNamedNodeData)
+int
+qemuBackupDiskPrepareOneBitmapsChain(virStorageSourcePtr backingChain,
+ virStorageSourcePtr targetsrc,
+ const char *targetbitmap,
+ const char *incremental,
+ virJSONValuePtr actions,
+ GSList **allocationbitmapnodes,
+ virHashTablePtr blockNamedNodeData)
 {
-g_autoptr(virJSONValue) ret = virJSONValueNewArray();
-virStorageSourcePtr tmpsrc = NULL;
-virStorageSourcePtr n;
-bool foundbitmap = false;
-
-for (n = from; virStorageSourceIsBacking(n); n = n->backingStore) {
-qemuBlockNamedNodeDataBitmapPtr bitmap = NULL;
-
-if (!(bitmap = 
qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData,
- n,
- bitmapname)))
-break;
-
-foundbitmap = true;
-
-if (bitmap->inconsistent) {
-virReportError(VIR_ERR_INVALID_ARG,
-   _("bitmap '%s' for image '%s%u' is inconsistent"),
-   bitmap->name, diskdst, n->id);
-return -1;
-}
-
-if (qemuMonitorTransactionBitmapMergeSourceAddBitmap(ret,
- n->nodeformat,
- bitmapname) < 0)
-return -1;
-
-tmpsrc = n;
-}
+g_autoptr(virJSONValue) tmpactions = NULL;
+g_autoptr(GSList) tmpallocbitmapnodes = NULL;

-if (!foundbitmap) {
-virReportError(VIR_ERR_INVALID_ARG,
-   _("failed to find bitmap '%s' in image '%s%u'"),
-   bitmapname, diskdst, from->id);
+if (qemuBlockGetBitmapMergeActions(backingChain, NULL, targetsrc,
+   incremental, targetbitmap, NULL,
+   , ,
+   blockNamedNodeData) < 0)
 return -1;
-}

-*actions = g_steal_pointer();
-*to = tmpsrc;
-
-return 0;
-}
-
-
-virJSONValuePtr

[RFC PATCH 39/41] qemu: blockjob: Remove 'disabledBitmapsBase' field from commit job private data

New semantics of the bitmap handling don't need this. Remove the field
and all uses of it including the status XML.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_blockjob.c  | 44 ---
 src/qemu/qemu_blockjob.h  |  3 --
 src/qemu/qemu_domain.c| 22 --
 src/qemu/qemu_driver.c|  3 +-
 .../blockjob-blockdev-in.xml  |  4 --
 5 files changed, 1 insertion(+), 75 deletions(-)

diff --git a/src/qemu/qemu_blockjob.c b/src/qemu/qemu_blockjob.c
index ef22bc60ab..801d88a9fb 100644
--- a/src/qemu/qemu_blockjob.c
+++ b/src/qemu/qemu_blockjob.c
@@ -85,11 +85,6 @@ qemuBlockJobDataDisposeJobdata(qemuBlockJobDataPtr job)
 virObjectUnref(job->data.backup.store);
 g_free(job->data.backup.bitmap);
 }
-
-if (job->type == QEMU_BLOCKJOB_TYPE_COMMIT ||
-job->type == QEMU_BLOCKJOB_TYPE_ACTIVE_COMMIT) {
-virStringListFree(job->data.commit.disabledBitmapsBase);
-}
 }


@@ -291,7 +286,6 @@ qemuBlockJobDiskNewCommit(virDomainObjPtr vm,
   virStorageSourcePtr topparent,
   virStorageSourcePtr top,
   virStorageSourcePtr base,
-  char ***disabledBitmapsBase,
   bool delete_imgs,
   unsigned int jobflags)
 {
@@ -317,7 +311,6 @@ qemuBlockJobDiskNewCommit(virDomainObjPtr vm,
 job->data.commit.top = top;
 job->data.commit.base = base;
 job->data.commit.deleteCommittedImages = delete_imgs;
-job->data.commit.disabledBitmapsBase = 
g_steal_pointer(disabledBitmapsBase);
 job->jobflags = jobflags;

 if (qemuBlockJobRegister(job, vm, disk, true) < 0)
@@ -1405,40 +1398,6 @@ 
qemuBlockJobProcessEventFailedActiveCommit(virQEMUDriverPtr driver,
 }


-static void
-qemuBlockJobProcessEventFailedCommitCommon(virDomainObjPtr vm,
-   qemuBlockJobDataPtr job,
-   qemuDomainAsyncJob asyncJob)
-{
-qemuDomainObjPrivatePtr priv = vm->privateData;
-g_autoptr(virJSONValue) actions = virJSONValueNewArray();
-char **disabledBitmaps = job->data.commit.disabledBitmapsBase;
-
-if (!disabledBitmaps || !*disabledBitmaps)
-return;
-
-for (; *disabledBitmaps; disabledBitmaps++) {
-qemuMonitorTransactionBitmapEnable(actions,
-   job->data.commit.base->nodeformat,
-   *disabledBitmaps);
-}
-
-if (qemuBlockReopenReadWrite(vm, job->data.commit.base, asyncJob) < 0)
-return;
-
-if (qemuDomainObjEnterMonitorAsync(priv->driver, vm, asyncJob) < 0)
-return;
-
-qemuMonitorTransaction(priv->mon, );
-
-if (qemuDomainObjExitMonitor(priv->driver, vm) < 0)
-return;
-
-if (qemuBlockReopenReadOnly(vm, job->data.commit.base, asyncJob) < 0)
-return;
-}
-
-
 static void
 qemuBlockJobProcessEventConcludedCreate(virQEMUDriverPtr driver,
 virDomainObjPtr vm,
@@ -1578,8 +1537,6 @@ 
qemuBlockJobEventProcessConcludedTransition(qemuBlockJobDataPtr job,
 case QEMU_BLOCKJOB_TYPE_COMMIT:
 if (success)
 qemuBlockJobProcessEventCompletedCommit(driver, vm, job, asyncJob);
-else
-qemuBlockJobProcessEventFailedCommitCommon(vm, job, asyncJob);
 break;

 case QEMU_BLOCKJOB_TYPE_ACTIVE_COMMIT:
@@ -1587,7 +1544,6 @@ 
qemuBlockJobEventProcessConcludedTransition(qemuBlockJobDataPtr job,
 qemuBlockJobProcessEventCompletedActiveCommit(driver, vm, job, 
asyncJob);
 } else {
 qemuBlockJobProcessEventFailedActiveCommit(driver, vm, job);
-qemuBlockJobProcessEventFailedCommitCommon(vm, job, asyncJob);
 }
 break;

diff --git a/src/qemu/qemu_blockjob.h b/src/qemu/qemu_blockjob.h
index 0b9da7dd5b..89747b30c0 100644
--- a/src/qemu/qemu_blockjob.h
+++ b/src/qemu/qemu_blockjob.h
@@ -89,8 +89,6 @@ struct _qemuBlockJobCommitData {
 virStorageSourcePtr top;
 virStorageSourcePtr base;
 bool deleteCommittedImages;
-char **disabledBitmapsBase; /* a NULL-terminated list of bitmap names which
-   were disabled in @base for the commit job */
 };


@@ -196,7 +194,6 @@ qemuBlockJobDiskNewCommit(virDomainObjPtr vm,
   virStorageSourcePtr topparent,
   virStorageSourcePtr top,
   virStorageSourcePtr base,
-  char ***disabledBitmapsBase,
   bool delete_imgs,
   unsigned int jobflags);

diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index bceeaf0342..e8ab42db70 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -2634,7 +2634,6 @@ qemuDomainPrivateBlockJobFormatCommit(qemuBlockJobDataPtr

[RFC PATCH 32/41] qemu: block: Add universal helper for merging dirty bitmaps for all scenarios

Add a function which allows to merge bitmaps according to the new
semantics and will allow to replace all the specific ad-hoc functions
currently in use for 'backup', 'block commit', 'block copy' and will
also be usable in the future for 'block pull' and non-shared storage
migration.

The semantics are a bit quirky for the 'backup' case but these quirks
are documented and will prevent us from having two slightly different
algorithms.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_block.c | 151 ++
 src/qemu/qemu_block.h |  11 +++
 2 files changed, 162 insertions(+)

diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
index 69578e861c..6eab9cb4e2 100644
--- a/src/qemu/qemu_block.c
+++ b/src/qemu/qemu_block.c
@@ -2847,6 +2847,157 @@ qemuBlockGetNamedNodeData(virDomainObjPtr vm,
 }


+/**
+ * qemuBlockGetBitmapMergeActions:
+ * @topsrc: top of the chain to merge bitmaps in
+ * @basesrc: bottom of the chain to merge bitmaps in (NULL for full chain)
+ * @target: destination storage source of the merge (may be part of original 
chain)
+ * @bitmapname: name of bitmap to perform the merge (NULL for all bitmaps)
+ * @dstbitmapname: name of destination bitmap of the merge (see below for 
caveats)
+ * @writebitmapsrc: storage source corresponding to the node containing the 
write temporary bitmap
+ * @actions: returns actions for a 'transaction' QMP command for executing the 
merge
+ * @allocationbitmapnodes: list of storage sources which will require an 
allocation bitmap
+ * @blockNamedNodeData: hash table filled with qemuBlockNamedNodeData
+ *
+ * Calculate handling of dirty block bitmaps between @topsrc and @basesrc. If
+ * @basesrc is NULL the end of the chain is considered. @target is the 
destination
+ * storage source definition of the merge and may or may not be part of the
+ * merged chain.
+ *
+ * Specifically the merging algorithm ensures that each considered bitmap is
+ * merged with the appropriate allocation bitmaps so that it properly describes
+ * the state of dirty blocks when looked at from @topsrc based on the depth
+ * of the backing chain where the bitmap is placed.
+ *
+ * If @bitmapname is non-NULL only bitmaps with that name are handled, 
otherwise
+ * all bitmaps are considered.
+ *
+ * If @dstbitmap is non-NULL everything is merged into a bitmap with that name,
+ * otherwise each bitmap is merged into a bitmap with the same name into 
@target.
+ * Additionally if @dstbitmap is non-NULL the target bitmap is created as 
'inactive'
+ * and 'transient' as a special case for the backup operation.
+ *
+ * If @writebitmapsrc is non-NULL, the 'libvirt-tmp-activewrite' bitmap from
+ * given node is merged along with others. This bitmap corresponds to the 
writes
+ * which occured between an active layer job finished and the rest of the 
bitmap
+ * merging.
+ *
+ * If the bitmap is not valid somehow (see qemuBlockBitmapChainIsValid) given
+ * bitmap is silently skipped, so callers must ensure that given bitmap is 
valid
+ * if they care about it.
+ *
+ * The resulting 'transacion' QMP command actions are filled in and returned 
via
+ * @actions. In cases when the merging requires use of bitmap created from the
+ * allocation map of certain parts of the chain the @allocationbitmapnodes list
+ * is filled with pointers to the virStorageSource structure of the backing
+ * chain members where the allocation bitmap must be added prior to the merge.
+ *
+ * Note that @actions and @allocationbitmapnodes may be NULL if no merging is
+ * required.
+ */
+int
+qemuBlockGetBitmapMergeActions(virStorageSourcePtr topsrc,
+   virStorageSourcePtr basesrc,
+   virStorageSourcePtr target,
+   const char *bitmapname,
+   const char *dstbitmapname,
+   virStorageSourcePtr writebitmapsrc,
+   virJSONValuePtr *actions,
+   GSList **allocationbitmapnodes,
+   virHashTablePtr blockNamedNodeData)
+{
+g_autoptr(virJSONValue) act = virJSONValueNewArray();
+virStorageSourcePtr n;
+qemuBlockNamedNodeDataPtr entry;
+g_autoptr(virJSONValue) tmpbitmaps = virJSONValueNewArray();
+g_autoptr(GSList) tmpbitmapnodes = NULL;
+g_autoptr(GSList) tmpbitmapnodesused = NULL;
+size_t i;
+
+for (n = topsrc; virStorageSourceIsBacking(n) && n != basesrc; n = 
n->backingStore) {
+bool copytmpbitmapnodes = false;
+
+if (!(entry = virHashLookup(blockNamedNodeData, n->nodeformat)))
+continue;
+
+for (i = 0; i < entry->nbitmaps; i++) {
+qemuBlockNamedNodeDataBitmapPtr bitmap = entry->bitmaps[i];
+g_autoptr(virJSONValue) merge = NULL;
+const char *mergebitmapname = dstbitmapname;
+bool mergebitmappersistent = false;
+bool mergebitmapdisabled = true;
+
+

[RFC PATCH 29/41] qemu: blockjob: Introduce 'populate' blockjob

This is a helper blockjob which uses the 'block-dirty-bitmap-populate'
command to convert the allocation map of a qcow2 image into a dirty
bitmap.

Internally it's a blockjob so we need to treat it as such. We only care
about the results when there's a sync thread waiting for it, otherwise
we clean up the created bitmap.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_blockjob.c | 69 
 src/qemu/qemu_blockjob.h | 13 
 src/qemu/qemu_domain.c   | 13 
 src/qemu/qemu_driver.c   |  1 +
 4 files changed, 96 insertions(+)

diff --git a/src/qemu/qemu_blockjob.c b/src/qemu/qemu_blockjob.c
index 17dc08476b..79820c6ca8 100644
--- a/src/qemu/qemu_blockjob.c
+++ b/src/qemu/qemu_blockjob.c
@@ -69,6 +69,7 @@ VIR_ENUM_IMPL(qemuBlockjob,
   "backup",
   "",
   "create",
+  "populate",
   "broken");

 static virClassPtr qemuBlockJobDataClass;
@@ -356,6 +357,38 @@ qemuBlockJobNewCreate(virDomainObjPtr vm,
 }


+/**
+ * qemuBlockJobNewPopulate:
+ * @vm: domain object
+ * @src: storage source the populate job is running on
+ *
+ * Instantiate block job data for running a 'dirty-bitmap-populate' blockjob.
+ * Note that this job is expected to run as part of a single 'modify' qemu
+ * domain job as this job is not registered with the disk itself and @src must
+ * be valid for the whole time the job is running.
+ */
+qemuBlockJobDataPtr
+qemuBlockJobNewPopulate(virDomainObjPtr vm,
+   virStorageSourcePtr src)
+{
+g_autoptr(qemuBlockJobData) job = NULL;
+g_autofree char *jobname = NULL;
+const char *nodename = src->nodeformat;
+
+jobname = g_strdup_printf("bitmap-populate-%s", nodename);
+
+if (!(job = qemuBlockJobDataNew(QEMU_BLOCKJOB_TYPE_POPULATE, jobname)))
+return NULL;
+
+ job->data.populate.src = src;
+
+if (qemuBlockJobRegister(job, vm, NULL, true) < 0)
+return NULL;
+
+return g_steal_pointer();
+}
+
+
 qemuBlockJobDataPtr
 qemuBlockJobDiskNewCopy(virDomainObjPtr vm,
 virDomainDiskDefPtr disk,
@@ -1478,6 +1511,38 @@ qemuBlockJobProcessEventConcludedBackup(virQEMUDriverPtr 
driver,
 }


+static void
+qemuBlockJobProcessEventConcludedPopulate(virQEMUDriverPtr driver,
+  virDomainObjPtr vm,
+  qemuBlockJobDataPtr job,
+  qemuDomainAsyncJob asyncJob)
+{
+g_autoptr(virJSONValue) actions = NULL;
+
+/* On successful completion there must be a synchronous handler waiting for
+ * this job. If there isn't one we need to clean up the associated bitmap.
+ */
+if (job->newstate == QEMU_BLOCKJOB_STATE_COMPLETED &&
+job->synchronous)
+return;
+
+if (qemuMonitorTransactionBitmapRemove(actions,
+   job->data.populate.src->nodeformat,
+   "libvirt-tmp-bitmap") < 0)
+return;
+
+if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
+return;
+
+qemuMonitorTransaction(qemuDomainGetMonitor(vm), );
+
+if (qemuDomainObjExitMonitor(driver, vm) < 0)
+return;
+
+return;
+}
+
+
 static void
 qemuBlockJobEventProcessConcludedTransition(qemuBlockJobDataPtr job,
 virQEMUDriverPtr driver,
@@ -1527,6 +1592,10 @@ 
qemuBlockJobEventProcessConcludedTransition(qemuBlockJobDataPtr job,
 progressTotal);
 break;

+case QEMU_BLOCKJOB_TYPE_POPULATE:
+qemuBlockJobProcessEventConcludedPopulate(driver, vm, job, asyncJob);
+break;
+
 case QEMU_BLOCKJOB_TYPE_BROKEN:
 case QEMU_BLOCKJOB_TYPE_NONE:
 case QEMU_BLOCKJOB_TYPE_INTERNAL:
diff --git a/src/qemu/qemu_blockjob.h b/src/qemu/qemu_blockjob.h
index 19498b5bd8..0b9da7dd5b 100644
--- a/src/qemu/qemu_blockjob.h
+++ b/src/qemu/qemu_blockjob.h
@@ -64,6 +64,7 @@ typedef enum {
 /* Additional enum values local to qemu */
 QEMU_BLOCKJOB_TYPE_INTERNAL,
 QEMU_BLOCKJOB_TYPE_CREATE,
+QEMU_BLOCKJOB_TYPE_POPULATE,
 QEMU_BLOCKJOB_TYPE_BROKEN,
 QEMU_BLOCKJOB_TYPE_LAST
 } qemuBlockJobType;
@@ -119,6 +120,13 @@ struct _qemuBlockJobBackupData {
 };


+typedef struct _qemuBlockJobBitmapPopulateData qemuBlockJobBitmapPopulateData;
+typedef qemuBlockJobBitmapPopulateData *qemuBlockJobDataBitmapPopulatePtr;
+
+struct _qemuBlockJobBitmapPopulateData {
+virStorageSourcePtr src;
+};
+
 typedef struct _qemuBlockJobData qemuBlockJobData;
 typedef qemuBlockJobData *qemuBlockJobDataPtr;

@@ -140,6 +148,7 @@ struct _qemuBlockJobData {
 qemuBlockJobCreateData create;
 qemuBlockJobCopyData copy;
 qemuBlockJobBackupData backup;
+qemuBlockJobBitmapPopulateData populate;
 } data;

 int type; /* qemuBlockJobType */
@@ -197,6 +206,10 @@

[RFC PATCH 40/41] qemu: Rewrite bitmap handling for block copy

Reuse qemuBlockGetBitmapMergeActions which allows to remove the ad-hoc
implementatio of bitmap merging for block copy.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_block.c| 116 +++
 src/qemu/qemu_block.h|   3 +-
 src/qemu/qemu_blockjob.c |  45 +++
 src/qemu/qemu_driver.c   |  13 +++--
 tests/qemublocktest.c|  12 +++-
 5 files changed, 71 insertions(+), 118 deletions(-)

diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
index fd3fe4c354..d46a631671 100644
--- a/src/qemu/qemu_block.c
+++ b/src/qemu/qemu_block.c
@@ -3035,38 +3035,6 @@ qemuBlockBitmapChainIsValid(virStorageSourcePtr src,
 }


-struct qemuBlockBitmapsHandleBlockcopyConcatData {
-virHashTablePtr bitmaps_merge;
-virJSONValuePtr actions;
-const char *mirrornodeformat;
-bool has_bitmaps;
-};
-
-
-static int
-qemuBlockBitmapsHandleBlockcopyConcatActions(void *payload,
- const void *name,
- void *opaque)
-{
-struct qemuBlockBitmapsHandleBlockcopyConcatData *data = opaque;
-virJSONValuePtr createactions = payload;
-const char *bitmapname = name;
-g_autoptr(virJSONValue) mergebitmaps = virHashSteal(data->bitmaps_merge, 
bitmapname);
-
-data->has_bitmaps = true;
-
-virJSONValueArrayConcat(data->actions, createactions);
-
-if (qemuMonitorTransactionBitmapMerge(data->actions,
-  data->mirrornodeformat,
-  bitmapname,
-  ) < 0)
-return -1;
-
-return 0;
-}
-
-
 /**
  * qemuBlockBitmapsHandleBlockcopy:
  * @src: disk source
@@ -3087,88 +3055,18 @@ qemuBlockBitmapsHandleBlockcopy(virStorageSourcePtr src,
 virStorageSourcePtr mirror,
 virHashTablePtr blockNamedNodeData,
 bool shallow,
-virJSONValuePtr *actions)
+virJSONValuePtr *actions,
+GSList **allocationbitmapnodes)
 {
-g_autoptr(virHashTable) bitmaps = virHashNew(virJSONValueHashFree);
-g_autoptr(virHashTable) bitmaps_merge = virHashNew(virJSONValueHashFree);
-g_autoptr(virHashTable) bitmaps_skip = virHashNew(NULL);
-g_autoptr(virJSONValue) tmpactions = virJSONValueNewArray();
-qemuBlockNamedNodeDataPtr entry;
-virStorageSourcePtr n;
-size_t i;
-struct qemuBlockBitmapsHandleBlockcopyConcatData data = { .bitmaps_merge = 
bitmaps_merge,
-  .actions = 
tmpactions,
-  
.mirrornodeformat = mirror->nodeformat,
-  .has_bitmaps = 
false, };
-
-for (n = src; n; n = n->backingStore) {
-if (!(entry = virHashLookup(blockNamedNodeData, n->nodeformat)))
-continue;
-
-for (i = 0; i < entry->nbitmaps; i++) {
-qemuBlockNamedNodeDataBitmapPtr bitmap = entry->bitmaps[i];
-virJSONValuePtr bitmap_merge;
-
-if (virHashHasEntry(bitmaps_skip, bitmap->name))
-continue;
-
-if (!(bitmap_merge = virHashLookup(bitmaps_merge, bitmap->name))) {
-g_autoptr(virJSONValue) tmp = NULL;
-bool disabled = !bitmap->recording;
-
-/* disable any non top-layer bitmaps */
-if (n != src)
-disabled = true;
-
-if (!bitmap->persistent ||
-!(qemuBlockBitmapChainIsValid(n, bitmap->name,
-  blockNamedNodeData))) {
-ignore_value(virHashAddEntry(bitmaps_skip, bitmap->name, 
NULL));
-continue;
-}
-
-/* prepare the data for adding the bitmap to the mirror */
-tmp = virJSONValueNewArray();
-
-if (qemuMonitorTransactionBitmapAdd(tmp,
-mirror->nodeformat,
-bitmap->name,
-true,
-disabled,
-bitmap->granularity) < 0)
-return -1;
-
-if (virHashAddEntry(bitmaps, bitmap->name, tmp) < 0)
-return -1;
-
-tmp = NULL;
-
-/* prepare array for merging all the bitmaps from the original 
chain */
-tmp = virJSONValueNewArray();
+virStorageSourcePtr base = NULL;

-if (virHashAddEntry(bitmaps_merge, bitmap->name, tmp) < 0)
-return -1;
+if (shallow)
+base = src->backingStore;

-

[RFC PATCH 09/41] qemublocktest: Add 'empty' case for incremental backup test

Use the new test data when calculating incremental backup operations. As
incremental backup fails with no bitmap the test code is modified to
allow testing this case too.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c | 19 ++-
 .../backupmerge/empty-out.json|  1 +
 2 files changed, 11 insertions(+), 9 deletions(-)
 create mode 100644 tests/qemublocktestdata/backupmerge/empty-out.json

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 8d613d7cac..9e54c254e8 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -793,17 +793,16 @@ testQemuBackupIncrementalBitmapCalculate(const void 
*opaque)

 incremental = testQemuBackupGetIncremental(data->incremental);

-if (!(mergebitmaps = qemuBackupDiskPrepareOneBitmapsChain(incremental,
-  data->chain,
-  nodedata,
-  "testdisk"))) {
-VIR_TEST_VERBOSE("failed to calculate merged bitmaps");
-return -1;
+if ((mergebitmaps = qemuBackupDiskPrepareOneBitmapsChain(incremental,
+ data->chain,
+ nodedata,
+ "testdisk"))) {
+if (!(actual = virJSONValueToString(mergebitmaps, true)))
+return -1;
+} else {
+actual = g_strdup("NULL\n");
 }

-if (!(actual = virJSONValueToString(mergebitmaps, true)))
-return -1;
-
 return virTestCompareToFile(actual, expectpath);
 }

@@ -1312,6 +1311,8 @@ mymain(void)
 ret = -1; \
 } while (0)

+TEST_BACKUP_BITMAP_CALCULATE("empty", bitmapSourceChain, "a", "empty");
+
 TEST_BACKUP_BITMAP_CALCULATE("basic-flat", bitmapSourceChain, "current", 
"basic");
 TEST_BACKUP_BITMAP_CALCULATE("basic-intermediate", bitmapSourceChain, "d", 
"basic");
 TEST_BACKUP_BITMAP_CALCULATE("basic-deep", bitmapSourceChain, "a", 
"basic");
diff --git a/tests/qemublocktestdata/backupmerge/empty-out.json 
b/tests/qemublocktestdata/backupmerge/empty-out.json
new file mode 100644
index 00..7951defec1
--- /dev/null
+++ b/tests/qemublocktestdata/backupmerge/empty-out.json
@@ -0,0 +1 @@
+NULL
-- 
2.26.2

[RFC PATCH 19/41] qemublocktest: Replace 'snapshots' bitmap detection test case data

Use test data which conforms to the new semantics which changed in the
previous patch.

The test data was created by the same set of commands as originally in
commit 0b27b655b1bac480186ce80457113cd5dc34e6a1

Signed-off-by: Peter Krempa 
---
 tests/qemublocktestdata/bitmap/snapshots.json | 1254 ++---
 tests/qemublocktestdata/bitmap/snapshots.out  |6 +-
 2 files changed, 440 insertions(+), 820 deletions(-)

diff --git a/tests/qemublocktestdata/bitmap/snapshots.json 
b/tests/qemublocktestdata/bitmap/snapshots.json
index 87e77ad408..054269d9bc 100644
--- a/tests/qemublocktestdata/bitmap/snapshots.json
+++ b/tests/qemublocktestdata/bitmap/snapshots.json
@@ -1,836 +1,460 @@
 [
-{
-"iops_rd": 0,
-"detect_zeroes": "off",
-"image": {
-"backing-image": {
-"backing-image": {
-"backing-image": {
-"backing-image": {
-"virtual-size": 10485760,
-"filename": "/tmp/pull4.qcow2",
-"cluster-size": 65536,
-"format": "qcow2",
-"actual-size": 208896,
-"format-specific": {
-"type": "qcow2",
-"data": {
-"compat": "1.1",
-"lazy-refcounts": false,
-"bitmaps": [
-{
-"flags": [
-"auto"
-],
-"name": "a",
-"granularity": 65536
-}
-],
-"refcount-bits": 16,
-"corrupt": false
-}
-},
-"dirty-flag": false
-},
-"backing-filename-format": "qcow2",
-"virtual-size": 10485760,
-"filename": "/tmp/pull4.1575911522",
-"cluster-size": 65536,
-"format": "qcow2",
-"actual-size": 208896,
-"format-specific": {
-"type": "qcow2",
-"data": {
-"compat": "1.1",
-"lazy-refcounts": false,
-"bitmaps": [
-{
-"flags": [
-"auto"
-],
-"name": "a",
-"granularity": 65536
-}
-],
-"refcount-bits": 16,
-"corrupt": false
-}
-},
-"full-backing-filename": "/tmp/pull4.qcow2",
-"backing-filename": "/tmp/pull4.qcow2",
-"dirty-flag": false
-},
-"backing-filename-format": "qcow2",
-"virtual-size": 10485760,
-"filename": "/tmp/pull4.1575911527",
-"cluster-size": 65536,
-"format": "qcow2",
-"actual-size": 217088,
-"format-specific": {
-"type": "qcow2",
-"data": {
-"compat": "1.1",
-"lazy-refcounts": false,
-"bitmaps": [
-{
-"flags": [
-"auto"
-],
-"name": "c",
-"granularity": 65536
-},
-{
-"flags": [
-
-],
-"name": "b",
-"granularity": 65536
-},
-{
-"flags": [
-
-],
-"name": "a",
-"granularity": 65536
-}
-],
-"refcount-bits":

[RFC PATCH 25/41] qemublocktest: Re-introduce testing of checkpoint deletion

Exercise the now arguably simpler checkpoint deletion code on the
'basic', 'snapshots', and 'synthetic' test data sets.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c | 18 ++
 .../checkpointdelete/basic-current-out.json   |  9 +
 .../basic-intermediate1-out.json  |  9 +
 .../basic-intermediate2-out.json  |  9 +
 .../basic-intermediate3-out.json  |  9 +
 .../checkpointdelete/basic-noparent-out.json  |  9 +
 .../snapshots-current-out.json|  9 +
 .../snapshots-intermediate1-out.json  | 11 +++
 .../snapshots-intermediate2-out.json  | 11 +++
 .../snapshots-intermediate3-out.json  | 11 +++
 .../snapshots-noparent-out.json   | 11 +++
 .../synthetic-current-out.json|  9 +
 .../synthetic-intermediate1-out.json  | 11 +++
 .../synthetic-intermediate2-out.json  | 11 +++
 .../synthetic-intermediate3-out.json  | 19 +++
 .../synthetic-noparent-out.json   | 11 +++
 16 files changed, 177 insertions(+)
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/basic-current-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/basic-intermediate1-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/basic-intermediate2-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/basic-intermediate3-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/basic-noparent-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/snapshots-current-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/snapshots-intermediate1-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/snapshots-intermediate2-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/snapshots-intermediate3-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/snapshots-noparent-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/synthetic-current-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/synthetic-intermediate1-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/synthetic-intermediate2-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/synthetic-intermediate3-out.json
 create mode 100644 
tests/qemublocktestdata/checkpointdelete/synthetic-noparent-out.json

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index f7773f4a51..c10eabb6a0 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1330,6 +1330,24 @@ mymain(void)

 TEST_CHECKPOINT_DELETE("empty", "a", "empty");

+TEST_CHECKPOINT_DELETE("basic-noparent", "a", "basic");
+TEST_CHECKPOINT_DELETE("basic-intermediate1", "b", "basic");
+TEST_CHECKPOINT_DELETE("basic-intermediate2", "c", "basic");
+TEST_CHECKPOINT_DELETE("basic-intermediate3", "d", "basic");
+TEST_CHECKPOINT_DELETE("basic-current", "current", "basic");
+
+TEST_CHECKPOINT_DELETE("snapshots-noparent", "a", "snapshots");
+TEST_CHECKPOINT_DELETE("snapshots-intermediate1", "b", "snapshots");
+TEST_CHECKPOINT_DELETE("snapshots-intermediate2", "c", "snapshots");
+TEST_CHECKPOINT_DELETE("snapshots-intermediate3", "d", "snapshots");
+TEST_CHECKPOINT_DELETE("snapshots-current", "current", "snapshots");
+
+TEST_CHECKPOINT_DELETE("synthetic-noparent", "a", "synthetic");
+TEST_CHECKPOINT_DELETE("synthetic-intermediate1", "b", "synthetic");
+TEST_CHECKPOINT_DELETE("synthetic-intermediate2", "c", "synthetic");
+TEST_CHECKPOINT_DELETE("synthetic-intermediate3", "d", "synthetic");
+TEST_CHECKPOINT_DELETE("synthetic-current", "current", "synthetic");
+
 #define TEST_BITMAP_VALIDATE(testname, bitmap, rc) \
 do { \
 blockbitmapvalidatedata.name = testname; \
diff --git a/tests/qemublocktestdata/checkpointdelete/basic-current-out.json 
b/tests/qemublocktestdata/checkpointdelete/basic-current-out.json
new file mode 100644
index 00..6ed1b63b66
--- /dev/null
+++ b/tests/qemublocktestdata/checkpointdelete/basic-current-out.json
@@ -0,0 +1,9 @@
+[
+  {
+"type": "block-dirty-bitmap-remove",
+"data": {
+  "node": "libvirt-1-format",
+  "name": "current"
+}
+  }
+]
diff --git 
a/tests/qemublocktestdata/checkpointdelete/basic-intermediate1-out.json 
b/tests/qemublocktestdata/checkpointdelete/basic-intermediate1-out.json
new file mode 100644
index 00..e1dd4920cd
--- /dev/null
+++ b/tests/qemublocktestdata/checkpointdelete/basic-intermediate1-out.json
@@ -0,0 +1,9 @@
+[
+  {
+"type": "block-dirty-bitmap-remove",
+"data": {
+  "node": "libvirt-1-format",
+  "name": "b"
+}
+  }
+]
diff --git 
a/tests/qemublocktestdata/checkpointdelete/basic-intermediate2-out.json

[RFC PATCH 41/41] qemublocktest: Add test cases for handling bitmaps during block-copy

Test both 'basic' and 'snapshots' cases on shallow and deep copy modes.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c |   5 +
 .../bitmapblockcopy/basic-deep-out.json   | 144 ++
 .../bitmapblockcopy/basic-shallow-out.json| 144 ++
 .../bitmapblockcopy/snapshots-deep-out.json   | 185 ++
 .../snapshots-shallow-out.json|  36 
 5 files changed, 514 insertions(+)
 create mode 100644 tests/qemublocktestdata/bitmapblockcopy/basic-deep-out.json
 create mode 100644 
tests/qemublocktestdata/bitmapblockcopy/basic-shallow-out.json
 create mode 100644 
tests/qemublocktestdata/bitmapblockcopy/snapshots-deep-out.json
 create mode 100644 
tests/qemublocktestdata/bitmapblockcopy/snapshots-shallow-out.json

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 91796e96d8..8c3d1cfc47 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1357,6 +1357,11 @@ mymain(void)
 TEST_BITMAP_BLOCKCOPY("empty-shallow", true, "empty");
 TEST_BITMAP_BLOCKCOPY("empty-deep", false, "empty");

+TEST_BITMAP_BLOCKCOPY("basic-shallow", true, "basic");
+TEST_BITMAP_BLOCKCOPY("basic-deep", false, "basic");
+
+TEST_BITMAP_BLOCKCOPY("snapshots-shallow", true, "snapshots");
+TEST_BITMAP_BLOCKCOPY("snapshots-deep", false, "snapshots");

 #define TEST_BITMAP_BLOCKCOMMIT(testname, topimg, baseimg, ndf) \
 do {\
diff --git a/tests/qemublocktestdata/bitmapblockcopy/basic-deep-out.json 
b/tests/qemublocktestdata/bitmapblockcopy/basic-deep-out.json
new file mode 100644
index 00..95ee835d30
--- /dev/null
+++ b/tests/qemublocktestdata/bitmapblockcopy/basic-deep-out.json
@@ -0,0 +1,144 @@
+[
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "mirror-format-node",
+  "name": "current",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "mirror-format-node",
+  "target": "current",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "current"
+},
+{
+  "node": "mirror-format-node",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "mirror-format-node",
+  "name": "d",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "mirror-format-node",
+  "target": "d",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "d"
+},
+{
+  "node": "mirror-format-node",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "mirror-format-node",
+  "name": "c",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "mirror-format-node",
+  "target": "c",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "c"
+},
+{
+  "node": "mirror-format-node",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "mirror-format-node",
+  "name": "b",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "mirror-format-node",
+  "target": "b",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "b"
+},
+{
+  "node": "mirror-format-node",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-add",
+"data": {
+  "node": "mirror-format-node",
+  "name": "a",
+  "persistent": true,
+  "disabled": false,
+  "granularity": 65536
+}
+  },
+  {
+"type": "block-dirty-bitmap-merge",
+"data": {
+  "node": "mirror-format-node",
+  "target": "a",
+  "bitmaps": [
+{
+  "node": "libvirt-1-format",
+  "name": "a"
+},
+{
+  "node": "mirror-format-node",
+  "name": "libvirt-tmp-activewrite"
+}
+  ]
+}
+  },
+  {
+"type": "block-dirty-bitmap-remove",
+"data": {
+  "node": "mirror-format-node",
+  "name": "libvirt-tmp-activewrite"
+}
+  }
+]
diff --git a/tests/qemublocktestdata/bitmapblockcopy/basic-shallow-out.json 
b/tests/qemublocktestdata/bitmapblockcopy/basic-shallow-out.json
new file mode 100644
index 00..95ee835d30
--- /dev/null
+++

[RFC PATCH 23/41] qemu: checkpoint: Don't merge checkpoints during deletion

Now that we've switched to the simple handling, the first thing that can
be massively simplified is checkpoint deletion. We now need to only go
through the backing chain and find the appropriately named bitmaps and
delete them, no complex lookups or merging.

Note that compared to other functions this deletes the bitmap in all
layers compared to others where we expect only exactly 1 bitmap of a
name in the backing chain to prevent potential problems.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_checkpoint.c | 153 ++---
 src/qemu/qemu_checkpoint.h |   1 -
 tests/qemublocktest.c  |   7 +-
 3 files changed, 25 insertions(+), 136 deletions(-)

diff --git a/src/qemu/qemu_checkpoint.c b/src/qemu/qemu_checkpoint.c
index 9b4f3ad396..c24d97443c 100644
--- a/src/qemu/qemu_checkpoint.c
+++ b/src/qemu/qemu_checkpoint.c
@@ -105,140 +105,41 @@ qemuCheckpointWriteMetadata(virDomainObjPtr vm,
 }


-/**
- * qemuCheckpointFindActiveDiskInParent:
- * @vm: domain object
- * @from: starting moment object
- * @diskname: name (target) of the disk to find
- *
- * Find the first checkpoint starting from @from continuing through parents
- * of the checkpoint which describes disk @diskname. Return the pointer to the
- * definition of the disk.
- */
-static virDomainCheckpointDiskDef *
-qemuCheckpointFindActiveDiskInParent(virDomainObjPtr vm,
- virDomainMomentObjPtr from,
- const char *diskname)
-{
-virDomainMomentObjPtr parent = from;
-virDomainCheckpointDefPtr parentdef = NULL;
-size_t i;
-
-while (parent) {
-parentdef = virDomainCheckpointObjGetDef(parent);
-
-for (i = 0; i < parentdef->ndisks; i++) {
-virDomainCheckpointDiskDef *chkdisk = >disks[i];
-
-if (STRNEQ(chkdisk->name, diskname))
-continue;
-
-/* currently inspected checkpoint doesn't describe the disk,
- * continue into parent checkpoint */
-if (chkdisk->type != VIR_DOMAIN_CHECKPOINT_TYPE_BITMAP)
-break;
-
-return chkdisk;
-}
-
-parent = virDomainCheckpointFindByName(vm->checkpoints,
-   parentdef->parent.parent_name);
-}
-
-return NULL;
-}
-
-
 int
 qemuCheckpointDiscardDiskBitmaps(virStorageSourcePtr src,
  virHashTablePtr blockNamedNodeData,
  const char *delbitmap,
- const char *parentbitmap,
  virJSONValuePtr actions,
  const char *diskdst,
  GSList **reopenimages)
 {
-virStorageSourcePtr n = src;
+virStorageSourcePtr n;
+bool found = false;

 /* find the backing chain entry with bitmap named '@delbitmap' */
-while (n) {
-qemuBlockNamedNodeDataBitmapPtr tmp;
-
-if ((tmp = qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData,
- n, delbitmap))) {
-break;
-}
-
-n = n->backingStore;
-}
-
-if (!n) {
-virReportError(VIR_ERR_INTERNAL_ERROR,
-   _("bitmap '%s' not found in backing chain of '%s'"),
-   delbitmap, diskdst);
-return -1;
-}
-
-while (n) {
-qemuBlockNamedNodeDataBitmapPtr srcbitmap;
-
-if (!(srcbitmap = 
qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData,
-n, delbitmap)))
-break;
-
-/* For the actual checkpoint deletion we will merge any bitmap into the
- * bitmap of the parent checkpoint (@parentbitmap) or for any image
- * where the parent checkpoint bitmap is not present we must rename
- * the bitmap of the deleted checkpoint into the bitmap of the parent
- * checkpoint as qemu can't currently take the allocation map and turn
- * it into a bitmap and thus we wouldn't be able to do a backup. */
-if (parentbitmap) {
-qemuBlockNamedNodeDataBitmapPtr dstbitmap;
-g_autoptr(virJSONValue) arr = NULL;
-
-dstbitmap = 
qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData,
-  n, parentbitmap);
-
-if (dstbitmap) {
-if (srcbitmap->recording && !dstbitmap->recording) {
-if (qemuMonitorTransactionBitmapEnable(actions,
-   n->nodeformat,
-   dstbitmap->name) < 
0)
-return -1;
-}
-
-} else {
-if (qemuMonitorTransactionBitmapAdd(actions,
-n->nodeformat,
-

[RFC PATCH 28/41] qemuDomainStorageSourcePrivate: Add per-source private blockjob

We'll need to track multiple blockjobs for populating bitmaps per
storage source for internal use. Add a variable into the storage source
private data for this purpose.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_domain.c | 1 +
 src/qemu/qemu_domain.h | 4 
 2 files changed, 5 insertions(+)

diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index 3239ac1a52..0cd9cf8582 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -1230,6 +1230,7 @@ qemuDomainStorageSourcePrivateDispose(void *obj)
 g_clear_pointer(>secinfo, qemuDomainSecretInfoFree);
 g_clear_pointer(>encinfo, qemuDomainSecretInfoFree);
 g_clear_pointer(>httpcookie, qemuDomainSecretInfoFree);
+virObjectUnref(priv->blockjob);
 }


diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index 41d3f1561d..821832c986 100644
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -483,6 +483,10 @@ struct _qemuDomainStorageSourcePrivate {

 /* secure passthrough of the http cookie */
 qemuDomainSecretInfoPtr httpcookie;
+
+/* Optional per-source internal blockjob. Regular blockjobs are sill 
tracked
+ * at 'disk' level */
+qemuBlockJobDataPtr blockjob;
 };

 virObjectPtr qemuDomainStorageSourcePrivateNew(void);
-- 
2.26.2

[RFC PATCH 30/41] qemu: domain: Introduce helper for always fetching virStorageSource private data

Add a helper which will always return the storage source private data
even if it was not allocated before.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_domain.c | 10 ++
 src/qemu/qemu_domain.h |  2 ++
 2 files changed, 12 insertions(+)

diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index 80d1d14e4f..bceeaf0342 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -1234,6 +1234,16 @@ qemuDomainStorageSourcePrivateDispose(void *obj)
 }


+qemuDomainStorageSourcePrivatePtr
+qemuDomainStorageSourcePrivateFetch(virStorageSourcePtr src)
+{
+if (!src->privateData)
+src->privateData = qemuDomainStorageSourcePrivateNew();
+
+return QEMU_DOMAIN_STORAGE_SOURCE_PRIVATE(src);
+}
+
+
 static virClassPtr qemuDomainVcpuPrivateClass;
 static void qemuDomainVcpuPrivateDispose(void *obj);

diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index 821832c986..7b4e6bbd84 100644
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -490,6 +490,8 @@ struct _qemuDomainStorageSourcePrivate {
 };

 virObjectPtr qemuDomainStorageSourcePrivateNew(void);
+qemuDomainStorageSourcePrivatePtr
+qemuDomainStorageSourcePrivateFetch(virStorageSourcePtr src);

 typedef struct _qemuDomainVcpuPrivate qemuDomainVcpuPrivate;
 typedef qemuDomainVcpuPrivate *qemuDomainVcpuPrivatePtr;
-- 
2.26.2

[RFC PATCH 24/41] qemublocktest: Rename TEST_CHECKPOINT_DELETE_MERGE to TEST_CHECKPOINT_DELETE

Also rename the helper struct and function.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c | 14 +++---
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 2dc06fd420..f7773f4a51 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -826,7 +826,7 @@ testQemuBackupIncrementalBitmapCalculate(const void *opaque)

 static const char *checkpointDeletePrefix = 
"qemublocktestdata/checkpointdelete/";

-struct testQemuCheckpointDeleteMergeData {
+struct testQemuCheckpointDeleteData {
 const char *name;
 virStorageSourcePtr chain;
 const char *deletebitmap;
@@ -835,9 +835,9 @@ struct testQemuCheckpointDeleteMergeData {


 static int
-testQemuCheckpointDeleteMerge(const void *opaque)
+testQemuCheckpointDelete(const void *opaque)
 {
-const struct testQemuCheckpointDeleteMergeData *data = opaque;
+const struct testQemuCheckpointDeleteData *data = opaque;
 g_autofree char *actual = NULL;
 g_autofree char *expectpath = NULL;
 g_autoptr(virJSONValue) actions = NULL;
@@ -1037,7 +1037,7 @@ mymain(void)
 struct testJSONtoJSONData jsontojsondata;
 struct testQemuImageCreateData imagecreatedata;
 struct testQemuBackupIncrementalBitmapCalculateData backupbitmapcalcdata;
-struct testQemuCheckpointDeleteMergeData checkpointdeletedata;
+struct testQemuCheckpointDeleteData checkpointdeletedata;
 struct testQemuBlockBitmapValidateData blockbitmapvalidatedata;
 struct testQemuBlockBitmapBlockcopyData blockbitmapblockcopydata;
 struct testQemuBlockBitmapBlockcommitData blockbitmapblockcommitdata;
@@ -1317,18 +1317,18 @@ mymain(void)

 TEST_BACKUP_BITMAP_CALCULATE("empty", bitmapSourceChain, "a", "empty");

-#define TEST_CHECKPOINT_DELETE_MERGE(testname, delbmp, named) \
+#define TEST_CHECKPOINT_DELETE(testname, delbmp, named) \
 do { \
 checkpointdeletedata.name = testname; \
 checkpointdeletedata.chain = bitmapSourceChain; \
 checkpointdeletedata.deletebitmap = delbmp; \
 checkpointdeletedata.nodedatafile = named; \
 if (virTestRun("checkpoint delete " testname, \
-   testQemuCheckpointDeleteMerge, ) < 
0) \
+   testQemuCheckpointDelete, ) < 0) \
 ret = -1; \
 } while (0)

-TEST_CHECKPOINT_DELETE_MERGE("empty", "a", "empty");
+TEST_CHECKPOINT_DELETE("empty", "a", "empty");

 #define TEST_BITMAP_VALIDATE(testname, bitmap, rc) \
 do { \
-- 
2.26.2

[RFC PATCH 01/41] virErrorPreserveLast: Return the saved error object in addition to storing it

In some cases it's easier to use the returned value rather than passing
the variable pointer in via argument.

Signed-off-by: Peter Krempa 
---
 src/util/virerror.c | 18 --
 src/util/virerror.h |  2 +-
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/util/virerror.c b/src/util/virerror.c
index 774c36bca3..1277a0bdfc 100644
--- a/src/util/virerror.c
+++ b/src/util/virerror.c
@@ -417,22 +417,28 @@ virSaveLastError(void)
  * virErrorPreserveLast:
  * @saveerr: pointer to virErrorPtr for storing last error object
  *
- * Preserves the currently set last error (for the thread) into @saveerr so 
that
- * it can be restored via virErrorRestore(). @saveerr must be passed to
+ * Preserves the currently set last error (for the thread) into @saveerr (if
+ * non-NULL) and in the return value so that it can be restored via
+ * virErrorRestore(). @saveerr or the return value must be passed to
  * virErrorRestore()
+ *
+ * Returns the last error object or NULL if there is no error.
  */
-void
+virErrorPtr
 virErrorPreserveLast(virErrorPtr *saveerr)
 {
 int saved_errno = errno;
 virErrorPtr lasterr = virGetLastError();
-
-*saveerr = NULL;
+virErrorPtr copy = NULL;

 if (lasterr)
-*saveerr = virErrorCopyNew(lasterr);
+copy = virErrorCopyNew(lasterr);
+
+if (saveerr)
+*saveerr = copy;

 errno = saved_errno;
+return copy;
 }


diff --git a/src/util/virerror.h b/src/util/virerror.h
index 9d3e40d65a..42f2835a85 100644
--- a/src/util/virerror.h
+++ b/src/util/virerror.h
@@ -200,7 +200,7 @@ void virErrorSetErrnoFromLastError(void);

 bool virLastErrorIsSystemErrno(int errnum);

-void virErrorPreserveLast(virErrorPtr *saveerr);
+virErrorPtr virErrorPreserveLast(virErrorPtr *saveerr);
 void virErrorRestore(virErrorPtr *savederr);

 void virLastErrorPrefixMessage(const char *fmt, ...)
-- 
2.26.2

[RFC PATCH 06/41] conf: backup: Store incremental backup checkpoint name per-disk

In preparation to allow heterogenous backups store the 'incremental'
field per-disk and fill it by default from the per-backup field.

Having this will be important once we'll want to allow incremental
backup working while hotplugging a new disk.

Signed-off-by: Peter Krempa 
---
 src/conf/backup_conf.c | 8 
 src/conf/backup_conf.h | 1 +
 2 files changed, 9 insertions(+)

diff --git a/src/conf/backup_conf.c b/src/conf/backup_conf.c
index c0c6f25d10..92106d8aaa 100644
--- a/src/conf/backup_conf.c
+++ b/src/conf/backup_conf.c
@@ -72,6 +72,7 @@ virDomainBackupDefFree(virDomainBackupDefPtr def)
 virDomainBackupDiskDefPtr disk = def->disks + i;

 g_free(disk->name);
+g_free(disk->incremental);
 g_free(disk->exportname);
 g_free(disk->exportbitmap);
 virObjectUnref(disk->store);
@@ -505,5 +506,12 @@ virDomainBackupAlignDisks(virDomainBackupDefPtr def,
 }
 }

+for (i = 0; i < def->ndisks; i++) {
+virDomainBackupDiskDefPtr backupdisk = >disks[i];
+
+if (def->incremental && !backupdisk->incremental)
+backupdisk->incremental = g_strdup(def->incremental);
+}
+
 return 0;
 }
diff --git a/src/conf/backup_conf.h b/src/conf/backup_conf.h
index b5685317c5..172eb1cf1c 100644
--- a/src/conf/backup_conf.h
+++ b/src/conf/backup_conf.h
@@ -51,6 +51,7 @@ typedef virDomainBackupDiskDef *virDomainBackupDiskDefPtr;
 struct _virDomainBackupDiskDef {
 char *name; /* name matching the

[RFC PATCH 17/41] qemublocktest: Replace 'basic' bitmap detection test case data

Use test data which conforms to the new semantics which changed in the
previous patch.

The test data was created by the same set of commands as originally in
commit 9aac9d5bdab039a50de2d8c627b3a1f1578ed471

Signed-off-by: Peter Krempa 
---
 tests/qemublocktestdata/bitmap/basic.json | 229 +++---
 tests/qemublocktestdata/bitmap/basic.out  |   8 +-
 2 files changed, 119 insertions(+), 118 deletions(-)

diff --git a/tests/qemublocktestdata/bitmap/basic.json 
b/tests/qemublocktestdata/bitmap/basic.json
index 9d418b1a37..718106bf99 100644
--- a/tests/qemublocktestdata/bitmap/basic.json
+++ b/tests/qemublocktestdata/bitmap/basic.json
@@ -1,117 +1,118 @@
 [
-{
-"iops_rd": 0,
-"detect_zeroes": "off",
-"image": {
-"virtual-size": 10485760,
-"filename": "/tmp/pull4.qcow2",
-"cluster-size": 65536,
-"format": "qcow2",
-"actual-size": 200704,
-"format-specific": {
-"type": "qcow2",
-"data": {
-"compat": "1.1",
-"lazy-refcounts": false,
-"refcount-bits": 16,
-"corrupt": false
-}
-},
-"dirty-flag": false
-},
-"iops_wr": 0,
-"ro": false,
-"node-name": "libvirt-1-format",
-"backing_file_depth": 0,
-"drv": "qcow2",
-"iops": 0,
-"bps_wr": 0,
-"write_threshold": 0,
-"dirty-bitmaps": [
-{
-"name": "current",
-"recording": true,
-"persistent": true,
-"busy": false,
-"status": "active",
-"granularity": 65536,
-"count": 0
-},
-{
-"name": "d",
-"recording": false,
-"persistent": true,
-"busy": false,
-"status": "disabled",
-"granularity": 65536,
-"count": 0
-},
-{
-"name": "c",
-"recording": false,
-"persistent": true,
-"busy": false,
-"status": "disabled",
-"granularity": 65536,
-"count": 0
-},
-{
-"name": "b",
-"recording": false,
-"persistent": true,
-"busy": false,
-"status": "disabled",
-"granularity": 65536,
-"count": 0
-},
-{
-"name": "a",
-"recording": false,
-"persistent": true,
-"busy": false,
-"status": "disabled",
-"granularity": 65536,
-"count": 0
-}
-],
-"encrypted": false,
-"bps": 0,
-"bps_rd": 0,
-"cache": {
-"no-flush": false,
-"direct": false,
-"writeback": true
-},
-"file": "/tmp/pull4.qcow2",
-"encryption_key_missing": false
+  {
+"iops_rd": 0,
+"detect_zeroes": "off",
+"image": {
+  "virtual-size": 10485760,
+  "filename": "/tmp/bitmaps.qcow2",
+  "cluster-size": 65536,
+  "format": "qcow2",
+  "actual-size": 200704,
+  "format-specific": {
+"type": "qcow2",
+"data": {
+  "compat": "1.1",
+  "compression-type": "zlib",
+  "lazy-refcounts": false,
+  "refcount-bits": 16,
+  "corrupt": false
+}
+  },
+  "dirty-flag": false
 },
-{
-"iops_rd": 0,
-"detect_zeroes": "off",
-"image": {
-"virtual-size": 197120,
-"filename": "/tmp/pull4.qcow2",
-"format": "file",
-"actual-size": 200704,
-"dirty-flag": false
-},
-"iops_wr": 0,
-"ro": false,
-"node-name": "libvirt-1-storage",
-"backing_file_depth": 0,
-"drv": "file",
-"iops": 0,
-"bps_wr": 0,
-"write_threshold": 0,
-"encrypted": false,
-"bps": 0,
-"bps_rd": 0,
-"cache": {
-"no-flush": false,
-"direct": false,
-"writeback": true
-},
-"file": "/tmp/pull4.qcow2",
-"encryption_key_missing": false
-}
+"iops_wr": 0,
+"ro": false,
+"node-name": "libvirt-1-format",
+"backing_file_depth": 0,
+"drv": "qcow2",
+"iops": 0,
+"bps_wr": 0,
+"write_threshold": 0,
+"dirty-bitmaps": [
+  {
+"name": "current",
+"recording": true,
+"persistent": true,
+"busy": false,
+"status": "active",
+"granularity": 65536,
+"count": 0
+  },
+  {
+"name": "d",
+"recording": true,
+"persistent": true,
+

[RFC PATCH 13/41] qemublocktest: Disable testcases for all bitmap handling

Upcoming patches are going to rewrite and semantically modify how
bitmaps are handled during blockjobs. This is possible as incremental
backup is not yet fully enabled.

As the changes are going to be incompatible with any current test data
remove all test cases for bitmap handling during checkpoint deletion,
incremental backups, block commit, block copy, and bitmap validation
operations.

The tests will be gradually added back later after the code and
test-data is refactored.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c |  94 -
 .../backupmerge/basic-deep-out.json   |  22 ---
 .../backupmerge/basic-flat-out.json   |   6 -
 .../backupmerge/basic-intermediate-out.json   |  10 --
 .../backupmerge/snapshot-deep-out.json|  38 -
 .../backupmerge/snapshot-flat-out.json|   6 -
 .../snapshot-intermediate-out.json|  14 --
 .../bitmapblockcommit/basic-1-2   | 119 
 .../bitmapblockcommit/basic-1-3   | 119 
 .../bitmapblockcommit/basic-2-3   |   2 -
 .../bitmapblockcommit/snapshots-1-2   |  49 ---
 .../bitmapblockcommit/snapshots-1-3   |  76 --
 .../bitmapblockcommit/snapshots-1-4   | 126 -
 .../bitmapblockcommit/snapshots-1-5   | 130 -
 .../bitmapblockcommit/snapshots-2-3   |  49 ---
 .../bitmapblockcommit/snapshots-2-4   |  99 -
 .../bitmapblockcommit/snapshots-2-5   | 103 --
 .../bitmapblockcommit/snapshots-3-4   |  72 --
 .../bitmapblockcommit/snapshots-3-5   |  76 --
 .../bitmapblockcommit/snapshots-4-5   |  33 -
 .../snapshots-synthetic-broken-1-2|  57 
 .../snapshots-synthetic-broken-1-3| 112 ---
 .../snapshots-synthetic-broken-1-4| 119 
 .../snapshots-synthetic-broken-1-5| 119 
 .../snapshots-synthetic-broken-2-3|  89 
 .../snapshots-synthetic-broken-2-4|  96 -
 .../snapshots-synthetic-broken-2-5|  96 -
 .../snapshots-synthetic-broken-3-4|  27 
 .../snapshots-synthetic-broken-3-5|  27 
 .../snapshots-synthetic-broken-4-5|  20 ---
 .../bitmapblockcopy/basic-deep-out.json   | 117 ---
 .../bitmapblockcopy/basic-shallow-out.json| 117 ---
 .../bitmapblockcopy/snapshots-deep-out.json   | 133 --
 .../snapshots-shallow-out.json|  48 ---
 .../checkpointdelete/basic-current-out.json   |  29 
 .../basic-intermediate1-out.json  |  22 ---
 .../basic-intermediate2-out.json  |  22 ---
 .../basic-intermediate3-out.json  |  22 ---
 .../checkpointdelete/basic-noparent-out.json  |   9 --
 .../snapshots-current-out.json|  29 
 .../snapshots-intermediate1-out.json  |  24 
 .../snapshots-intermediate2-out.json  |  62 
 .../snapshots-intermediate3-out.json  |  61 
 .../snapshots-noparent-out.json   |  27 
 ...hots-synthetic-checkpoint-current-out.json |  29 
 ...ynthetic-checkpoint-intermediate1-out.json |  31 
 ...ynthetic-checkpoint-intermediate2-out.json |  34 -
 ...ynthetic-checkpoint-intermediate3-out.json |  61 
 ...ots-synthetic-checkpoint-noparent-out.json |  27 
 49 files changed, 2909 deletions(-)
 delete mode 100644 tests/qemublocktestdata/backupmerge/basic-deep-out.json
 delete mode 100644 tests/qemublocktestdata/backupmerge/basic-flat-out.json
 delete mode 100644 
tests/qemublocktestdata/backupmerge/basic-intermediate-out.json
 delete mode 100644 tests/qemublocktestdata/backupmerge/snapshot-deep-out.json
 delete mode 100644 tests/qemublocktestdata/backupmerge/snapshot-flat-out.json
 delete mode 100644 
tests/qemublocktestdata/backupmerge/snapshot-intermediate-out.json
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/basic-1-2
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/basic-1-3
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/basic-2-3
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-1-2
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-1-3
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-1-4
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-1-5
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-2-3
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-2-4
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-2-5
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-3-4
 delete mode 100644 tests/qemublocktestdata/bitmapblockcommit/snapshots-3-5
 delete mode 100644

[RFC PATCH 16/41] qemu: checkpoint: Don't chain bitmaps for checkpoints

Chaining bitmaps for checkpoints (disabling the active one and creating
a new) severely overcomplicated all operations in regards to bitmaps.

Specifically it requires us re-matching the on-disk state to the
internal metadata and in case of merging during block jobs it makes it
almost impossible to cover all corner cases.

Since the checkpoints and incremental backups were not yet enabled,
let's change the design to keep one bitmap per checkpoint. In case of
layered snapshots this will be filled in by using dirty-bitmap-populate.

Finally the main reason for this unnecessary complexity was the fear
that qemu's performance could degrade. In the end I think that
addressing the performance issue will be better done in qemu (e.g by
keeping an internal bitmap updated with changes and merging it
periodically back to the real bitmaps. QEMU writes out changes to disk
at shutdown so consistency is not a problem).

Removing the relationships between bitmaps frees us from complex
handling and also makes all the surrounding code more robust as one
broken bitmap doesn't necessarily invalidate whole chains of backups.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_checkpoint.c | 21 +
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/src/qemu/qemu_checkpoint.c b/src/qemu/qemu_checkpoint.c
index b71b4a7d14..9b4f3ad396 100644
--- a/src/qemu/qemu_checkpoint.c
+++ b/src/qemu/qemu_checkpoint.c
@@ -456,7 +456,6 @@ qemuCheckpointPrepare(virQEMUDriverPtr driver,
 static int
 qemuCheckpointAddActions(virDomainObjPtr vm,
  virJSONValuePtr actions,
- virDomainMomentObjPtr old_current,
  virDomainCheckpointDefPtr def)
 {
 size_t i;
@@ -464,7 +463,6 @@ qemuCheckpointAddActions(virDomainObjPtr vm,
 for (i = 0; i < def->ndisks; i++) {
 virDomainCheckpointDiskDef *chkdisk = >disks[i];
 virDomainDiskDefPtr domdisk = virDomainDiskByTarget(vm->def, 
chkdisk->name);
-virDomainCheckpointDiskDef *parentchkdisk = NULL;

 /* checkpoint definition validator mandates that the corresponding
  * domdisk should exist */
@@ -475,23 +473,6 @@ qemuCheckpointAddActions(virDomainObjPtr vm,
 if (qemuMonitorTransactionBitmapAdd(actions, domdisk->src->nodeformat,
 chkdisk->bitmap, true, false, 0) < 
0)
 return -1;
-
-/* We only want one active bitmap for a disk along the
- * checkpoint chain, then later differential backups will
- * merge the bitmaps (only one active) between the bounding
- * checkpoint and the leaf checkpoint.  If the same disks are
- * involved in each checkpoint, this search terminates in one
- * iteration; but it is also possible to have to search
- * further than the immediate parent to find another
- * checkpoint with a bitmap on the same disk.  */
-if ((parentchkdisk = qemuCheckpointFindActiveDiskInParent(vm, 
old_current,
-  
chkdisk->name))) {
-
-if (qemuMonitorTransactionBitmapDisable(actions,
-domdisk->src->nodeformat,
-parentchkdisk->bitmap) < 0)
-return -1;
-}
 }
 return 0;
 }
@@ -540,7 +521,7 @@ qemuCheckpointCreateCommon(virQEMUDriverPtr driver,

 tmpactions = virJSONValueNewArray();

-if (qemuCheckpointAddActions(vm, tmpactions, parent, *def) < 0)
+if (qemuCheckpointAddActions(vm, tmpactions, *def) < 0)
 return -1;

 if (!(*chk = virDomainCheckpointAssignDef(vm->checkpoints, *def)))
-- 
2.26.2

[RFC PATCH 08/41] qemublocktest: Add 'empty' test case for bitmaps

Add test data for an image without bitmaps.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c |  4 ++
 tests/qemublocktestdata/bitmap/empty.json | 70 +++
 tests/qemublocktestdata/bitmap/empty.out  |  1 +
 3 files changed, 75 insertions(+)
 create mode 100644 tests/qemublocktestdata/bitmap/empty.json
 create mode 100644 tests/qemublocktestdata/bitmap/empty.out

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index f00d2ff129..8d613d7cac 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1292,6 +1292,8 @@ mymain(void)
 ret = -1; \
 } while (0)

+TEST_BITMAP_DETECT("empty");
+
 TEST_BITMAP_DETECT("basic");
 TEST_BITMAP_DETECT("synthetic");
 TEST_BITMAP_DETECT("snapshots");
@@ -1360,6 +1362,8 @@ mymain(void)
 ret = -1; \
 } while (0)

+TEST_BITMAP_VALIDATE("empty", "a", false);
+
 TEST_BITMAP_VALIDATE("basic", "a", true);
 TEST_BITMAP_VALIDATE("basic", "b", true);
 TEST_BITMAP_VALIDATE("basic", "c", true);
diff --git a/tests/qemublocktestdata/bitmap/empty.json 
b/tests/qemublocktestdata/bitmap/empty.json
new file mode 100644
index 00..ec43b25f0d
--- /dev/null
+++ b/tests/qemublocktestdata/bitmap/empty.json
@@ -0,0 +1,70 @@
+[
+{
+"iops_rd": 0,
+"detect_zeroes": "off",
+"image": {
+"virtual-size": 10485760,
+"filename": "/tmp/pull4.qcow2",
+"cluster-size": 65536,
+"format": "qcow2",
+"actual-size": 200704,
+"format-specific": {
+"type": "qcow2",
+"data": {
+"compat": "1.1",
+"lazy-refcounts": false,
+"refcount-bits": 16,
+"corrupt": false
+}
+},
+"dirty-flag": false
+},
+"iops_wr": 0,
+"ro": false,
+"node-name": "libvirt-1-format",
+"backing_file_depth": 0,
+"drv": "qcow2",
+"iops": 0,
+"bps_wr": 0,
+"write_threshold": 0,
+"encrypted": false,
+"bps": 0,
+"bps_rd": 0,
+"cache": {
+"no-flush": false,
+"direct": false,
+"writeback": true
+},
+"file": "/tmp/pull4.qcow2",
+"encryption_key_missing": false
+},
+{
+"iops_rd": 0,
+"detect_zeroes": "off",
+"image": {
+"virtual-size": 197120,
+"filename": "/tmp/pull4.qcow2",
+"format": "file",
+"actual-size": 200704,
+"dirty-flag": false
+},
+"iops_wr": 0,
+"ro": false,
+"node-name": "libvirt-1-storage",
+"backing_file_depth": 0,
+"drv": "file",
+"iops": 0,
+"bps_wr": 0,
+"write_threshold": 0,
+"encrypted": false,
+"bps": 0,
+"bps_rd": 0,
+"cache": {
+"no-flush": false,
+"direct": false,
+"writeback": true
+},
+"file": "/tmp/pull4.qcow2",
+"encryption_key_missing": false
+}
+]
diff --git a/tests/qemublocktestdata/bitmap/empty.out 
b/tests/qemublocktestdata/bitmap/empty.out
new file mode 100644
index 00..3787cbd354
--- /dev/null
+++ b/tests/qemublocktestdata/bitmap/empty.out
@@ -0,0 +1 @@
+libvirt-1-format:
-- 
2.26.2

[RFC PATCH 03/41] qemuBackupDiskDataCleanup: Use VIR_ERROR_AUTO_PRESERVE_LAST

Example usage of the new macro.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_backup.c | 5 +
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/qemu/qemu_backup.c b/src/qemu/qemu_backup.c
index b317b841cd..ac7fa5def3 100644
--- a/src/qemu/qemu_backup.c
+++ b/src/qemu/qemu_backup.c
@@ -157,19 +157,16 @@ qemuBackupDiskDataCleanup(virDomainObjPtr vm,
   struct qemuBackupDiskData *dd,
   size_t ndd)
 {
-virErrorPtr orig_err;
+VIR_ERROR_AUTOPRESERVE_LAST;
 size_t i;

 if (!dd)
 return;

-virErrorPreserveLast(_err);
-
 for (i = 0; i < ndd; i++)
 qemuBackupDiskDataCleanupOne(vm, dd + i);

 g_free(dd);
-virErrorRestore(_err);
 }


-- 
2.26.2

[RFC PATCH 05/41] qemu: backup: Fix backup of disk skipped in an intermediate checkpoint

If a disk is not captured by one of the intermediate checkpoints the
code would fail, but we can easily calculate the bitmaps to merge
correctly by skipping over checkpoints which don't describe the disk.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_backup.c | 24 
 tests/qemublocktest.c  |  6 ++
 2 files changed, 30 insertions(+)

diff --git a/src/qemu/qemu_backup.c b/src/qemu/qemu_backup.c
index 0a4c08e01e..cb1df9ffae 100644
--- a/src/qemu/qemu_backup.c
+++ b/src/qemu/qemu_backup.c
@@ -237,6 +237,30 @@ qemuBackupDiskPrepareOneBitmapsChain(virDomainMomentDefPtr 
*incremental,
 for (incridx = 0; incremental[incridx]; incridx++) {
 g_autoptr(virJSONValue) tmp = virJSONValueNewArray();
 virStorageSourcePtr tmpsrc = NULL;
+virDomainCheckpointDefPtr chkdef = (virDomainCheckpointDefPtr) 
incremental[incridx];
+bool checkpoint_has_disk = false;
+size_t i;
+
+for (i = 0; i < chkdef->ndisks; i++) {
+if (STRNEQ_NULLABLE(diskdst, chkdef->disks[i].name))
+continue;
+
+if (chkdef->disks[i].type == VIR_DOMAIN_CHECKPOINT_TYPE_BITMAP)
+checkpoint_has_disk = true;
+
+break;
+}
+
+if (!checkpoint_has_disk) {
+if (!incremental[incridx + 1]) {
+virReportError(VIR_ERR_INVALID_ARG,
+   _("disk '%s' not found in checkpoint '%s'"),
+   diskdst, incremental[incridx]->name);
+return NULL;
+}
+
+continue;
+}

 if (qemuBackupGetBitmapMergeRange(n, incremental[incridx]->name,
   , , diskdst,
diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 0cdedb9ad4..f00d2ff129 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -727,6 +727,12 @@ testQemuBackupGetIncrementalMoment(const char *name)
 if (!(checkpoint = virDomainCheckpointDefNew()))
 abort();

+checkpoint->disks = g_new0(virDomainCheckpointDiskDef, 1);
+checkpoint->ndisks = 1;
+
+checkpoint->disks[0].name = g_strdup("testdisk");
+checkpoint->disks[0].type = VIR_DOMAIN_CHECKPOINT_TYPE_BITMAP;
+
 checkpoint->parent.name = g_strdup(name);

 return (virDomainMomentDefPtr) checkpoint;
-- 
2.26.2

[RFC PATCH 02/41] util: error: Introduce VIR_ERROR_AUTOPRESERVE_LAST macro

The macro is used to automatically preserve the last error object on
cleanup paths which might override. Note that the 'ignore_value' is
required to silence clang's unused variable checker.

Signed-off-by: Peter Krempa 
---
 src/util/virerror.h | 10 ++
 1 file changed, 10 insertions(+)

diff --git a/src/util/virerror.h b/src/util/virerror.h
index 42f2835a85..ce0b48b3ad 100644
--- a/src/util/virerror.h
+++ b/src/util/virerror.h
@@ -207,3 +207,13 @@ void virLastErrorPrefixMessage(const char *fmt, ...)
 G_GNUC_PRINTF(1, 2);

 G_DEFINE_AUTOPTR_CLEANUP_FUNC(virError, virFreeError);
+
+/**
+ * VIR_ERROR_AUTO_PRESERVE_LAST:
+ *
+ * This macro ensures that the last error object is restored to the state it 
was
+ * at the point where the macro was expanded when the scope ends.
+ */
+#define VIR_ERROR_AUTOPRESERVE_LAST \
+__attribute__((cleanup(virErrorRestore))) virErrorPtr orig_err_preserve = 
virErrorPreserveLast(NULL); \
+ignore_value(_err_preserve)
-- 
2.26.2

[RFC PATCH 22/41] qemublocktest: Add new 'synthetic' bitmap detection and validation test case

Based on the 'snapshots' example with manual tweaks to introduce
inactive, transient, inconsistent and duplicate bitmaps in various parts
of the chain to excercise detection and new validation code.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c |  11 +
 tests/qemublocktestdata/bitmap/synthetic.json | 506 ++
 tests/qemublocktestdata/bitmap/synthetic.out  |  15 +
 3 files changed, 532 insertions(+)
 create mode 100644 tests/qemublocktestdata/bitmap/synthetic.json
 create mode 100644 tests/qemublocktestdata/bitmap/synthetic.out

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 6d1bf3f250..63112eea24 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1303,6 +1303,7 @@ mymain(void)

 TEST_BITMAP_DETECT("basic");
 TEST_BITMAP_DETECT("snapshots");
+TEST_BITMAP_DETECT("synthetic");

 #define TEST_BACKUP_BITMAP_CALCULATE(testname, source, incrbackup, named) \
 do { \
@@ -1358,6 +1359,16 @@ mymain(void)
 TEST_BITMAP_VALIDATE("snapshots", "d", true);
 TEST_BITMAP_VALIDATE("snapshots", "current", true);

+TEST_BITMAP_VALIDATE("synthetic", "a", true);
+TEST_BITMAP_VALIDATE("synthetic", "b", false);
+TEST_BITMAP_VALIDATE("synthetic", "c", false);
+TEST_BITMAP_VALIDATE("synthetic", "d", false);
+TEST_BITMAP_VALIDATE("synthetic", "current", false);
+TEST_BITMAP_VALIDATE("synthetic", "top-ok", true);
+TEST_BITMAP_VALIDATE("synthetic", "top-inactive", false);
+TEST_BITMAP_VALIDATE("synthetic", "top-transient", false);
+TEST_BITMAP_VALIDATE("synthetic", "top-inactive-transient", false);
+
 #define TEST_BITMAP_BLOCKCOPY(testname, shllw, ndf) \
 do { \
 blockbitmapblockcopydata.name = testname; \
diff --git a/tests/qemublocktestdata/bitmap/synthetic.json 
b/tests/qemublocktestdata/bitmap/synthetic.json
new file mode 100644
index 00..3712c8e5fc
--- /dev/null
+++ b/tests/qemublocktestdata/bitmap/synthetic.json
@@ -0,0 +1,506 @@
+[
+  {
+"iops_rd": 0,
+"detect_zeroes": "off",
+"image": {
+  "backing-filename-format": "qcow2",
+  "virtual-size": 10485760,
+  "filename": "/tmp/bitmaps.1590749073",
+  "cluster-size": 65536,
+  "format": "qcow2",
+  "actual-size": 208896,
+  "format-specific": {
+"type": "qcow2",
+"data": {
+  "compat": "1.1",
+  "compression-type": "zlib",
+  "lazy-refcounts": false,
+  "bitmaps": [
+{
+  "flags": [
+"in-use",
+"auto"
+  ],
+  "name": "current",
+  "granularity": 65536
+}
+  ],
+  "refcount-bits": 16,
+  "corrupt": false
+}
+  },
+  "full-backing-filename": "/tmp/bitmaps.1590749012",
+  "backing-filename": "/tmp/bitmaps.1590749012",
+  "dirty-flag": false
+},
+"iops_wr": 0,
+"ro": false,
+"node-name": "libvirt-1-format",
+"backing_file_depth": 0,
+"drv": "qcow2",
+"iops": 0,
+"bps_wr": 0,
+"write_threshold": 0,
+"backing_file": "/tmp/bitmaps.1590749012",
+"dirty-bitmaps": [
+  {
+"name": "current",
+"recording": true,
+"persistent": true,
+"busy": false,
+"status": "active",
+"inconsistent": true,
+"granularity": 65536,
+"count": 0
+  },
+  {
+"name": "top-ok",
+"recording": true,
+"persistent": true,
+"busy": false,
+"status": "active",
+"granularity": 65536,
+"count": 0
+  },
+  {
+"name": "top-inactive",
+"recording": false,
+"persistent": true,
+"busy": false,
+"status": "active",
+"granularity": 65536,
+"count": 0
+  },
+  {
+"name": "top-transient",
+"recording": true,
+"persistent": false,
+"busy": false,
+"status": "active",
+"granularity": 65536,
+"count": 0
+  },
+  {
+"name": "top-transient-inactive",
+"recording": false,
+"persistent": false,
+"busy": false,
+"status": "active",
+"granularity": 65536,
+"count": 0
+  }
+],
+"encrypted": false,
+"bps": 0,
+"bps_rd": 0,
+"cache": {
+  "no-flush": false,
+  "direct": false,
+  "writeback": true
+},
+"file": "/tmp/bitmaps.1590749073",
+"encryption_key_missing": false
+  },
+  {
+"iops_rd": 0,
+"detect_zeroes": "off",
+"image": {
+  "virtual-size": 328192,
+  "filename": "/tmp/bitmaps.1590749073",
+  "format": "file",
+  "actual-size": 208896,
+  "dirty-flag": false
+},
+"iops_wr": 0,
+"ro": false,
+"node-name": "libvirt-1-storage",
+"backing_file_depth": 0,
+"drv": "file",
+"iops": 0,
+"bps_wr": 0,
+"write_threshold": 0,
+"encrypted": false,
+"bps": 0,
+

[RFC PATCH 21/41] qemublocktest: Re-add bitmap validation for 'basic' and 'snapshots' cases

Now that we've updated both the test data and the validator to new
semantics we can start testing again.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c | 12 
 1 file changed, 12 insertions(+)

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 0ed9b99bc4..6d1bf3f250 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1346,6 +1346,18 @@ mymain(void)

 TEST_BITMAP_VALIDATE("empty", "a", false);

+TEST_BITMAP_VALIDATE("basic", "a", true);
+TEST_BITMAP_VALIDATE("basic", "b", true);
+TEST_BITMAP_VALIDATE("basic", "c", true);
+TEST_BITMAP_VALIDATE("basic", "d", true);
+TEST_BITMAP_VALIDATE("basic", "current", true);
+
+TEST_BITMAP_VALIDATE("snapshots", "a", true);
+TEST_BITMAP_VALIDATE("snapshots", "b", true);
+TEST_BITMAP_VALIDATE("snapshots", "c", true);
+TEST_BITMAP_VALIDATE("snapshots", "d", true);
+TEST_BITMAP_VALIDATE("snapshots", "current", true);
+
 #define TEST_BITMAP_BLOCKCOPY(testname, shllw, ndf) \
 do { \
 blockbitmapblockcopydata.name = testname; \
-- 
2.26.2

[RFC PATCH 14/41] qemublocktest: Delete 'synthetic' bitmap test cases

They will be replaced by a different set which will test scenarios
relevant for the new semantics.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c |   3 -
 .../bitmap/snapshots-synthetic-broken.json| 837 --
 .../bitmap/snapshots-synthetic-broken.out |  14 -
 .../snapshots-synthetic-checkpoint.json   | 827 -
 .../bitmap/snapshots-synthetic-checkpoint.out |  13 -
 tests/qemublocktestdata/bitmap/synthetic.json | 118 ---
 tests/qemublocktestdata/bitmap/synthetic.out  |   6 -
 7 files changed, 1818 deletions(-)
 delete mode 100644 
tests/qemublocktestdata/bitmap/snapshots-synthetic-broken.json
 delete mode 100644 
tests/qemublocktestdata/bitmap/snapshots-synthetic-broken.out
 delete mode 100644 
tests/qemublocktestdata/bitmap/snapshots-synthetic-checkpoint.json
 delete mode 100644 
tests/qemublocktestdata/bitmap/snapshots-synthetic-checkpoint.out
 delete mode 100644 tests/qemublocktestdata/bitmap/synthetic.json
 delete mode 100644 tests/qemublocktestdata/bitmap/synthetic.out

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index d2f105486e..28cbaeba38 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1293,10 +1293,7 @@ mymain(void)
 TEST_BITMAP_DETECT("empty");

 TEST_BITMAP_DETECT("basic");
-TEST_BITMAP_DETECT("synthetic");
 TEST_BITMAP_DETECT("snapshots");
-TEST_BITMAP_DETECT("snapshots-synthetic-checkpoint");
-TEST_BITMAP_DETECT("snapshots-synthetic-broken");

 #define TEST_BACKUP_BITMAP_CALCULATE(testname, source, incrbackup, named) \
 do { \
diff --git a/tests/qemublocktestdata/bitmap/snapshots-synthetic-broken.json 
b/tests/qemublocktestdata/bitmap/snapshots-synthetic-broken.json
deleted file mode 100644
index 8cf14d4baa..00
--- a/tests/qemublocktestdata/bitmap/snapshots-synthetic-broken.json
+++ /dev/null
@@ -1,837 +0,0 @@
-[
-{
-"iops_rd": 0,
-"detect_zeroes": "off",
-"image": {
-"backing-image": {
-"backing-image": {
-"backing-image": {
-"backing-image": {
-"virtual-size": 10485760,
-"filename": "/tmp/pull4.qcow2",
-"cluster-size": 65536,
-"format": "qcow2",
-"actual-size": 208896,
-"format-specific": {
-"type": "qcow2",
-"data": {
-"compat": "1.1",
-"lazy-refcounts": false,
-"bitmaps": [
-{
-"flags": [
-"auto"
-],
-"name": "a",
-"granularity": 65536
-}
-],
-"refcount-bits": 16,
-"corrupt": false
-}
-},
-"dirty-flag": false
-},
-"backing-filename-format": "qcow2",
-"virtual-size": 10485760,
-"filename": "/tmp/pull4.1575911522",
-"cluster-size": 65536,
-"format": "qcow2",
-"actual-size": 208896,
-"format-specific": {
-"type": "qcow2",
-"data": {
-"compat": "1.1",
-"lazy-refcounts": false,
-"bitmaps": [
-{
-"flags": [
-"auto"
-],
-"name": "a",
-"granularity": 65536
-}
-],
-"refcount-bits": 16,
-"corrupt": false
-}
-},
-"full-backing-filename": "/tmp/pull4.qcow2",
-"backing-filename": "/tmp/pull4.qcow2",
-"dirty-flag": false
-},
-"backing-filename-format": "qcow2",
-"virtual-size": 10485760,
-"filename": "/tmp/pull4.1575911527",
-"cluster-size": 65536,
-"format": "qcow2",
-

[RFC PATCH 04/41] qemu: backup: Split up code traversing checkpoint list looking for bitmaps

The algorithm is getting quite complex. Split out the lookup of range of
backing chain storage sources and bitmaps contained in them which
correspond to one checkpoint.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_backup.c | 111 ++---
 1 file changed, 61 insertions(+), 50 deletions(-)

diff --git a/src/qemu/qemu_backup.c b/src/qemu/qemu_backup.c
index ac7fa5def3..0a4c08e01e 100644
--- a/src/qemu/qemu_backup.c
+++ b/src/qemu/qemu_backup.c
@@ -170,72 +170,83 @@ qemuBackupDiskDataCleanup(virDomainObjPtr vm,
 }


+static int
+qemuBackupGetBitmapMergeRange(virStorageSourcePtr from,
+  const char *bitmapname,
+  virJSONValuePtr *actions,
+  virStorageSourcePtr *to,
+  const char *diskdst,
+  virHashTablePtr blockNamedNodeData)
+{
+g_autoptr(virJSONValue) ret = virJSONValueNewArray();
+virStorageSourcePtr tmpsrc = NULL;
+virStorageSourcePtr n;
+bool foundbitmap = false;
+
+for (n = from; virStorageSourceIsBacking(n); n = n->backingStore) {
+qemuBlockNamedNodeDataBitmapPtr bitmap = NULL;
+
+if (!(bitmap = 
qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData,
+ n,
+ bitmapname)))
+break;
+
+foundbitmap = true;
+
+if (bitmap->inconsistent) {
+virReportError(VIR_ERR_INVALID_ARG,
+   _("bitmap '%s' for image '%s%u' is inconsistent"),
+   bitmap->name, diskdst, n->id);
+return -1;
+}
+
+if (qemuMonitorTransactionBitmapMergeSourceAddBitmap(ret,
+ n->nodeformat,
+ bitmapname) < 0)
+return -1;
+
+tmpsrc = n;
+}
+
+if (!foundbitmap) {
+virReportError(VIR_ERR_INVALID_ARG,
+   _("failed to find bitmap '%s' in image '%s%u'"),
+   bitmapname, diskdst, from->id);
+return -1;
+}
+
+*actions = g_steal_pointer();
+*to = tmpsrc;
+
+return 0;
+}
+
+
 virJSONValuePtr
 qemuBackupDiskPrepareOneBitmapsChain(virDomainMomentDefPtr *incremental,
  virStorageSourcePtr backingChain,
  virHashTablePtr blockNamedNodeData,
  const char *diskdst)
 {
-qemuBlockNamedNodeDataBitmapPtr bitmap;
 g_autoptr(virJSONValue) ret = NULL;
 size_t incridx = 0;
+virStorageSourcePtr n = backingChain;

 ret = virJSONValueNewArray();

-if (!(bitmap = qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData,
- backingChain,
- 
incremental[0]->name))) {
-virReportError(VIR_ERR_INVALID_ARG,
-   _("failed to find bitmap '%s' in image '%s%u'"),
-   incremental[0]->name, diskdst, backingChain->id);
-return NULL;
-}
+for (incridx = 0; incremental[incridx]; incridx++) {
+g_autoptr(virJSONValue) tmp = virJSONValueNewArray();
+virStorageSourcePtr tmpsrc = NULL;

-while (bitmap) {
-if (bitmap->inconsistent) {
-virReportError(VIR_ERR_INVALID_ARG,
-   _("bitmap '%s' for image '%s%u' is inconsistent"),
-   bitmap->name, diskdst, backingChain->id);
+if (qemuBackupGetBitmapMergeRange(n, incremental[incridx]->name,
+  , , diskdst,
+  blockNamedNodeData) < 0)
 return NULL;
-}

-if (qemuMonitorTransactionBitmapMergeSourceAddBitmap(ret,
- 
backingChain->nodeformat,
- bitmap->name) < 0)
+if (virJSONValueArrayConcat(ret, tmp) < 0)
 return NULL;

-if (backingChain->backingStore &&
-(bitmap = qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData,
-
backingChain->backingStore,
-
incremental[incridx]->name))) {
-backingChain = backingChain->backingStore;
-continue;
-}
-
-if (incremental[incridx + 1]) {
-if ((bitmap = 
qemuBlockNamedNodeDataGetBitmapByName(blockNamedNodeData,
-backingChain,
-
incremental[incridx + 1]->name))) {
-incridx++;
-

[RFC PATCH 00/41] qemu: Rewrite checkpoint code for 'block-dirty-bitmap-populate'

Use 'block-dirty-bitmap-populate' and change how we create bitmaps
corresponding to checkpoints to simplify the code and also properly
integrate with backing chain images created outside of libvirt.

This patchset changes how we approach checkpoints by keeping one bitmap
per checkpoint and disk and not propagating the bitmaps into overlays on
snapshots. This massively simplifies the code handling all the
operations during blockjobs and backups.

While the change isn't compatible with checkpoints created previously,
we didn't yet enable the support for checkpoints/backups.

Note that 'block-dirty-bitmap-populate' is _not_ in finished state in
qemu yet, so I'm posting this as RFC and as reference for qemu
developers of the usefulnes of it.


The changes can be fetched by:

  git fetch https://gitlab.com/pipo.sk/libvirt.git checkpoint-bitmap-populate

Note that the above branch also contains a commit enabling incremental
backup for simpler testing.

I've pushed the appropriate qemu patches for convenience here:

  git fetch https://gitlab.com/pipo.sk/qemu.git block-dirty-bitmap-populate

Peter Krempa (41):
  virErrorPreserveLast: Return the saved error object in addition to
storing it
  util: error: Introduce VIR_ERROR_AUTOPRESERVE_LAST macro
  qemuBackupDiskDataCleanup: Use VIR_ERROR_AUTO_PRESERVE_LAST
  qemu: backup: Split up code traversing checkpoint list looking for
bitmaps
  qemu: backup: Fix backup of disk skipped in an intermediate checkpoint
  conf: backup: Store incremental backup checkpoint name per-disk
  qemu: backup: Move fetching of checkpoint list for incremental backup
  qemublocktest: Add 'empty' test case for bitmaps
  qemublocktest: Add 'empty' case for incremental backup test
  qemublocktest: Add 'empty' case for checkpoint deletion
  qemublocktest: Add 'empty' case for blockcopy bitmap handling test
  qemublocktest: Add 'empty' case for checkpoint bitmap handling
  qemublocktest: Disable testcases for all bitmap handling
  qemublocktest: Delete 'synthetic' bitmap test cases
  qemublocktest: Extract printing of nodename list
  qemu: checkpoint: Don't chain bitmaps for checkpoints
  qemublocktest: Replace 'basic' bitmap detection test case data
  qemu: snapshot: Don't propagate bitmaps to upper layers
  qemublocktest: Replace 'snapshots' bitmap detection test case data
  qemuBlockBitmapChainIsValid: Adjust to new semantics of bitmaps
  qemublocktest: Re-add bitmap validation for 'basic' and 'snapshots'
cases
  qemublocktest: Add new 'synthetic' bitmap detection and validation
test case
  qemu: checkpoint: Don't merge checkpoints during deletion
  qemublocktest: Rename TEST_CHECKPOINT_DELETE_MERGE to
TEST_CHECKPOINT_DELETE
  qemublocktest: Re-introduce testing of checkpoint deletion
  qemucapabilitiestest: Bump qemu-5.1 caps on x86_64 for
'dirty-bitmap-populate'
  qemu: monitor: Add support for 'block-dirty-bitmap-populate'
transaction member
  qemuDomainStorageSourcePrivate: Add per-source private blockjob
  qemu: blockjob: Introduce 'populate' blockjob
  qemu: domain: Introduce helper for always fetching virStorageSource
private data
  qemu: block: Add helper to add temporary block bitmaps from allocation
maps
  qemu: block: Add universal helper for merging dirty bitmaps for all
scenarios
  qemu: backup: Rewrite backup bitmap handling to the new bitmap
semantics
  qemublocktest: Add 'basic' tests for backup bitmap handling
  qemublocktest: Add 'snapshots' tests for backup bitmap handling
  qemu: Rewrite bitmap handling for block commit
  qemublocktest: Add 'basic' tests for commit bitmap handling
  qemublocktest: Add 'snapshots' tests for block commit bitmap handling
  qemu: blockjob: Remove 'disabledBitmapsBase' field from commit job
private data
  qemu: Rewrite bitmap handling for block copy
  qemublocktest: Add test cases for handling bitmaps during block-copy

 src/conf/backup_conf.c|8 +
 src/conf/backup_conf.h|1 +
 src/qemu/qemu_backup.c|  221 +-
 src/qemu/qemu_backup.h|   13 +-
 src/qemu/qemu_block.c |  674 ++--
 src/qemu/qemu_block.h |   35 +-
 src/qemu/qemu_blockjob.c  |  182 +-
 src/qemu/qemu_blockjob.h  |   16 +-
 src/qemu/qemu_checkpoint.c|  174 +-
 src/qemu/qemu_checkpoint.h|1 -
 src/qemu/qemu_domain.c|   46 +-
 src/qemu/qemu_domain.h|6 +
 src/qemu/qemu_driver.c|  112 +-
 src/qemu/qemu_monitor.c   |   11 +
 src/qemu/qemu_monitor.h   |7 +
 src/qemu/qemu_monitor_json.c  |   18 +
 src/qemu/qemu_monitor_json.h  |6 +
 src/util/virerror.c   |   18 +-
 src/util/virerror.h   |   12 +-

[RFC PATCH 11/41] qemublocktest: Add 'empty' case for blockcopy bitmap handling test

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c  | 3 +++
 tests/qemublocktestdata/bitmapblockcopy/empty-deep-out.json| 0
 tests/qemublocktestdata/bitmapblockcopy/empty-shallow-out.json | 0
 3 files changed, 3 insertions(+)
 create mode 100644 tests/qemublocktestdata/bitmapblockcopy/empty-deep-out.json
 create mode 100644 
tests/qemublocktestdata/bitmapblockcopy/empty-shallow-out.json

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 1821c227d5..2ddfab72eb 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1408,6 +1408,9 @@ mymain(void)
 ret = -1; \
 } while (0)

+TEST_BITMAP_BLOCKCOPY("empty-shallow", true, "empty");
+TEST_BITMAP_BLOCKCOPY("empty-deep", false, "empty");
+
 TEST_BITMAP_BLOCKCOPY("basic-shallow", true, "basic");
 TEST_BITMAP_BLOCKCOPY("basic-deep", false, "basic");

diff --git a/tests/qemublocktestdata/bitmapblockcopy/empty-deep-out.json 
b/tests/qemublocktestdata/bitmapblockcopy/empty-deep-out.json
new file mode 100644
index 00..e69de29bb2
diff --git a/tests/qemublocktestdata/bitmapblockcopy/empty-shallow-out.json 
b/tests/qemublocktestdata/bitmapblockcopy/empty-shallow-out.json
new file mode 100644
index 00..e69de29bb2
-- 
2.26.2

[RFC PATCH 12/41] qemublocktest: Add 'empty' case for checkpoint bitmap handling

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c   | 2 ++
 tests/qemublocktestdata/bitmapblockcommit/empty | 2 ++
 2 files changed, 4 insertions(+)
 create mode 100644 tests/qemublocktestdata/bitmapblockcommit/empty

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 2ddfab72eb..d66e169fd8 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -1430,6 +1430,8 @@ mymain(void)
 ret = -1; \
 } while (0)

+TEST_BITMAP_BLOCKCOMMIT("empty", 1, 2, "empty");
+
 TEST_BITMAP_BLOCKCOMMIT("basic-1-2", 1, 2, "basic");
 TEST_BITMAP_BLOCKCOMMIT("basic-1-3", 1, 3, "basic");
 TEST_BITMAP_BLOCKCOMMIT("basic-2-3", 2, 3, "basic");
diff --git a/tests/qemublocktestdata/bitmapblockcommit/empty 
b/tests/qemublocktestdata/bitmapblockcommit/empty
new file mode 100644
index 00..bfc58f994e
--- /dev/null
+++ b/tests/qemublocktestdata/bitmapblockcommit/empty
@@ -0,0 +1,2 @@
+pre job bitmap disable:
+merge bitmpas:
-- 
2.26.2

[RFC PATCH 10/41] qemublocktest: Add 'empty' case for checkpoint deletion

Use the new test data for checkpoint deletion testing. This test also
requires modification of the internals to allow checking for test
failure.

Signed-off-by: Peter Krempa 
---
 tests/qemublocktest.c   | 13 +++--
 .../checkpointdelete/empty-out.json |  1 +
 2 files changed, 8 insertions(+), 6 deletions(-)
 create mode 100644 tests/qemublocktestdata/checkpointdelete/empty-out.json

diff --git a/tests/qemublocktest.c b/tests/qemublocktest.c
index 9e54c254e8..1821c227d5 100644
--- a/tests/qemublocktest.c
+++ b/tests/qemublocktest.c
@@ -851,14 +851,13 @@ testQemuCheckpointDeleteMerge(const void *opaque)
  data->parentbitmap,
  actions,
  "testdisk",
- ) < 0) {
-VIR_TEST_VERBOSE("failed to generate checkpoint delete transaction\n");
-return -1;
+ ) >= 0) {
+if (virJSONValueToBuffer(actions, , true) < 0)
+return -1;
+} else {
+virBufferAddLit(, "NULL\n");
 }

-if (virJSONValueToBuffer(actions, , true) < 0)
-return -1;
-
 if (reopenimages) {
 virBufferAddLit(, "reopen nodes:\n");

@@ -1333,6 +1332,8 @@ mymain(void)
 ret = -1; \
 } while (0)

+TEST_CHECKPOINT_DELETE_MERGE("empty", "a", NULL, "empty");
+
 TEST_CHECKPOINT_DELETE_MERGE("basic-noparent", "a", NULL, "basic");
 TEST_CHECKPOINT_DELETE_MERGE("basic-intermediate1", "b", "a", "basic");
 TEST_CHECKPOINT_DELETE_MERGE("basic-intermediate2", "c", "b", "basic");
diff --git a/tests/qemublocktestdata/checkpointdelete/empty-out.json 
b/tests/qemublocktestdata/checkpointdelete/empty-out.json
new file mode 100644
index 00..7951defec1
--- /dev/null
+++ b/tests/qemublocktestdata/checkpointdelete/empty-out.json
@@ -0,0 +1 @@
+NULL
-- 
2.26.2

[RFC PATCH 07/41] qemu: backup: Move fetching of checkpoint list for incremental backup

Fetch the checkpoint list for every disk specifically based on the new
per-disk 'incremental' field.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_backup.c | 108 -
 1 file changed, 52 insertions(+), 56 deletions(-)

diff --git a/src/qemu/qemu_backup.c b/src/qemu/qemu_backup.c
index cb1df9ffae..5729aac858 100644
--- a/src/qemu/qemu_backup.c
+++ b/src/qemu/qemu_backup.c
@@ -170,6 +170,50 @@ qemuBackupDiskDataCleanup(virDomainObjPtr vm,
 }


+/**
+ * qemuBackupBeginCollectIncrementalCheckpoints:
+ * @vm: domain object
+ * @incrFrom: name of checkpoint representing starting point of incremental 
backup
+ *
+ * Returns a NULL terminated list of pointers to checkpoint definitions in
+ * chronological order starting from the 'current' checkpoint until reaching
+ * @incrFrom.
+ */
+static virDomainMomentDefPtr *
+qemuBackupBeginCollectIncrementalCheckpoints(virDomainObjPtr vm,
+ const char *incrFrom)
+{
+virDomainMomentObjPtr n = virDomainCheckpointGetCurrent(vm->checkpoints);
+g_autofree virDomainMomentDefPtr *incr = NULL;
+size_t nincr = 0;
+
+while (n) {
+virDomainMomentDefPtr def = n->def;
+
+if (VIR_APPEND_ELEMENT_COPY(incr, nincr, def) < 0)
+return NULL;
+
+if (STREQ(def->name, incrFrom)) {
+def = NULL;
+if (VIR_APPEND_ELEMENT_COPY(incr, nincr, def) < 0)
+return NULL;
+
+return g_steal_pointer();
+}
+
+if (!n->def->parent_name)
+break;
+
+n = virDomainCheckpointFindByName(vm->checkpoints, 
n->def->parent_name);
+}
+
+virReportError(VIR_ERR_OPERATION_INVALID,
+   _("could not locate checkpoint '%s' for incremental 
backup"),
+   incrFrom);
+return NULL;
+}
+
+
 static int
 qemuBackupGetBitmapMergeRange(virStorageSourcePtr from,
   const char *bitmapname,
@@ -331,11 +375,11 @@ qemuBackupDiskPrepareDataOne(virDomainObjPtr vm,
  struct qemuBackupDiskData *dd,
  virJSONValuePtr actions,
  bool pull,
- virDomainMomentDefPtr *incremental,
  virHashTablePtr blockNamedNodeData,
  virQEMUDriverConfigPtr cfg)
 {
 qemuDomainObjPrivatePtr priv = vm->privateData;
+g_autofree virDomainMomentDefPtr *incremental = NULL;

 /* set data structure */
 dd->backupdisk = backupdisk;
@@ -360,7 +404,7 @@ qemuBackupDiskPrepareDataOne(virDomainObjPtr vm,
  * pull mode:
  *   both: original disk
  */
-if (pull || (incremental && dd->store->format >= 
VIR_STORAGE_FILE_BACKING)) {
+if (pull || (dd->backupdisk->incremental && dd->store->format >= 
VIR_STORAGE_FILE_BACKING)) {
 dd->backingStore = dd->domdisk->src;
 } else {
 dd->backingStore = dd->terminator = virStorageSourceNew();
@@ -372,7 +416,10 @@ qemuBackupDiskPrepareDataOne(virDomainObjPtr vm,
 if (qemuDomainPrepareStorageSourceBlockdev(NULL, dd->store, priv, cfg) < 0)
 return -1;

-if (incremental) {
+if (dd->backupdisk->incremental) {
+if (!(incremental = qemuBackupBeginCollectIncrementalCheckpoints(vm, 
dd->backupdisk->incremental)))
+return -1;
+
 if (dd->backupdisk->exportbitmap)
 dd->incrementalBitmap = g_strdup(dd->backupdisk->exportbitmap);
 else
@@ -441,7 +488,6 @@ qemuBackupDiskPrepareDataOnePull(virJSONValuePtr actions,
 static ssize_t
 qemuBackupDiskPrepareData(virDomainObjPtr vm,
   virDomainBackupDefPtr def,
-  virDomainMomentDefPtr *incremental,
   virHashTablePtr blockNamedNodeData,
   virJSONValuePtr actions,
   virQEMUDriverConfigPtr cfg,
@@ -464,8 +510,7 @@ qemuBackupDiskPrepareData(virDomainObjPtr vm,
 ndisks++;

 if (qemuBackupDiskPrepareDataOne(vm, backupdisk, dd, actions, pull,
- incremental, blockNamedNodeData,
- cfg) < 0)
+ blockNamedNodeData, cfg) < 0)
 goto error;

 if (pull) {
@@ -622,50 +667,6 @@ qemuBackupBeginPullExportDisks(virDomainObjPtr vm,
 }


-/**
- * qemuBackupBeginCollectIncrementalCheckpoints:
- * @vm: domain object
- * @incrFrom: name of checkpoint representing starting point of incremental 
backup
- *
- * Returns a NULL terminated list of pointers to checkpoint definitions in
- * chronological order starting from the 'current' checkpoint until reaching
- * @incrFrom.
- */
-static virDomainMomentDefPtr *
-qemuBackupBeginCollectIncrementalCheckpoints(virDomainObjPtr vm,
- const char *incrFrom)
-{
-virDomainMomentObjPtr n =

Re: [PATCH v2 4/8] qemu: Validate firmware blob configuration

On Tue, Jun 09, 2020 at 04:44:46PM +0200, Michal Privoznik wrote:
> On 6/9/20 11:52 AM, Daniel P. Berrangé wrote:
> > On Thu, Jun 04, 2020 at 08:44:05PM +0200, Michal Privoznik wrote:
> > > There are recommendations and limitations to the name of the
> > > config blobs we need to follow [1].
> > > 
> > > Firstly, we don't want users to change any value only add new
> > > blobs. This means, that the name must have "opt/" prefix and at
> > > the same time must not begin with "opt/ovmf" nor "opt/org.qemu"
> > > as these are reserved for OVMF or QEMU respectively.
> > > 
> > > Secondly, there is a limit (FW_CFG_MAX_FILE_PATH in qemu.git) of
> > > 56 characters for filename.
> > 
> > Ewww, that is horrible. I'm have inclined to say we should leave the
> > limit unchecked in libvirt, and file a BZ against QEMU. It should be
> > using g_strdup_printf() with filenames and not allocating on the
> > stack. We already see peoiple exceeding the 100 charater limit of UNIX
> > sockets, so a 56 character limit is going to be trivially exceeded
> > without even trying hard.
> 
> Ah, I got it wrong when reading the documentation. The limit is the name of
> the entry, not file name associated. For instance the following works:
> 
> -fw_cfg 
> name=opt/com.example/blah,file=/tmp/some_very_long_path_that_is_more_than_fifty_six_characters_long_to_see_what_happens_if_I_do_that
> 
> but if @name would be too long QEMU would fail:
> 
> qemu-system-x86_64: -fw_cfg
> name=opt/com.example/some_very_long_...,file=...: name too long (max. 55
> char)
> 
> And this comes from kernel's implementation of qemu_fw_cfg. However, the
> name can be up to PAGE_SIZE long (minus 2 for terminating \n and NUL),
> according to sysfs documentation (Documentation/filesystems/sysfs.rst:241):
> 
> - The buffer will always be PAGE_SIZE bytes in length. On i386, this
>   is 4096.
> 
> Given this new realization, I think I'll just remove the check and not fill
> bug. I don't think we need longer names, do we?

Yep, fine with me.
 
Regards,
Daniel
-- 
|: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o-https://fstop138.berrange.com :|
|: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|

Re: [PATCH v2 4/8] qemu: Validate firmware blob configuration

2020-06-09 Thread Michal Privoznik

On 6/9/20 11:52 AM, Daniel P. Berrangé wrote:

On Thu, Jun 04, 2020 at 08:44:05PM +0200, Michal Privoznik wrote:

There are recommendations and limitations to the name of the
config blobs we need to follow [1].

Firstly, we don't want users to change any value only add new
blobs. This means, that the name must have "opt/" prefix and at
the same time must not begin with "opt/ovmf" nor "opt/org.qemu"
as these are reserved for OVMF or QEMU respectively.

Secondly, there is a limit (FW_CFG_MAX_FILE_PATH in qemu.git) of
56 characters for filename.

Ewww, that is horrible. I'm have inclined to say we should leave the
limit unchecked in libvirt, and file a BZ against QEMU. It should be
using g_strdup_printf() with filenames and not allocating on the
stack. We already see peoiple exceeding the 100 charater limit of UNIX
sockets, so a 56 character limit is going to be trivially exceeded
without even trying hard.

Ah, I got it wrong when reading the documentation. The limit is the name
of the entry, not file name associated. For instance the following works:

-fw_cfg
name=opt/com.example/blah,file=/tmp/some_very_long_path_that_is_more_than_fifty_six_characters_long_to_see_what_happens_if_I_do_that

but if @name would be too long QEMU would fail:

qemu-system-x86_64: -fw_cfg
name=opt/com.example/some_very_long_...,file=...: name too long (max. 55
char)

And this comes from kernel's implementation of qemu_fw_cfg. However, the
name can be up to PAGE_SIZE long (minus 2 for terminating \n and NUL),
according to sysfs documentation (Documentation/filesystems/sysfs.rst:241):

- The buffer will always be PAGE_SIZE bytes in length. On i386, this
is 4096.

Given this new realization, I think I'll just remove the check and not
fill bug. I don't think we need longer names, do we?

Michal

Re: [libvirt PATCH v2 0/2] ci: Enable Cirrus CI integration

2020-06-09 Thread Erik Skultety

On Tue, Jun 09, 2020 at 12:31:42PM +0200, Andrea Bolognani wrote:
> Changes from [v1]:
>
>   * skip Cirrus CI jobs if the necessary variables are not set.
>
> [v1] https://www.redhat.com/archives/libvir-list/2020-June/msg00249.html

Reviewed-by: Erik Skultety 

but I'd wait a little longer if DanPB doesn't have more comments on the job
definition.

[PATCH v5] numa: forbid '-numa node, mem' for 5.1 and newer machine types

2020-06-09 Thread Igor Mammedov

Deprecation period is run out and it's a time to flip the switch
introduced by cd5ff8333a.  Disable legacy option for new machine
types (since 5.1) and amend documentation.

'-numa node,memdev' shall be used instead of disabled option
with new machine types.

Signed-off-by: Igor Mammedov 
Reviewed-by: Michal Privoznik 
Reviewed-by: Michael S. Tsirkin 
Reviewed-by: Greg Kurz 
---
v1:
 - rebased on top of current master
 - move compat mode from 4.2 to 5.0
v2:
 - move deprecation text to recently removed section
v3:
 - increase title line length for (deprecated.rst)
 '``-numa node,mem=``\ *size* (removed in 5.1)'
v4:
 - use error_append_hint() for suggesting valid CLI
v5:
 - add "\n" at the end of error_append_hint()
 - fix gramar/spelling in moved deprecation text

CC: peter.mayd...@linaro.org
CC: ehabk...@redhat.com
CC: marcel.apfelb...@gmail.com
CC: m...@redhat.com
CC: pbonz...@redhat.com
CC: r...@twiddle.net
CC: da...@gibson.dropbear.id.au
CC: libvir-list@redhat.com
CC: qemu-...@nongnu.org
CC: qemu-...@nongnu.org
CC: ebl...@redhat.com
CC: gr...@kaod.org
---
 docs/system/deprecated.rst | 37 -
 hw/arm/virt.c  |  2 +-
 hw/core/numa.c |  7 +++
 hw/i386/pc.c   |  1 -
 hw/i386/pc_piix.c  |  1 +
 hw/i386/pc_q35.c   |  1 +
 hw/ppc/spapr.c |  2 +-
 qemu-options.hx|  9 +
 8 files changed, 36 insertions(+), 24 deletions(-)

diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index 544ece0a45..72666ac764 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -101,23 +101,6 @@ error in the future.
 The ``-realtime mlock=on|off`` argument has been replaced by the
 ``-overcommit mem-lock=on|off`` argument.
 
-``-numa node,mem=``\ *size* (since 4.1)
-'''
-
-The parameter ``mem`` of ``-numa node`` is used to assign a part of
-guest RAM to a NUMA node. But when using it, it's impossible to manage 
specified
-RAM chunk on the host side (like bind it to a host node, setting bind policy, 
...),
-so guest end-ups with the fake NUMA configuration with suboptiomal performance.
-However since 2014 there is an alternative way to assign RAM to a NUMA node
-using parameter ``memdev``, which does the same as ``mem`` and adds
-means to actualy manage node RAM on the host side. Use parameter ``memdev``
-with *memory-backend-ram* backend as an replacement for parameter ``mem``
-to achieve the same fake NUMA effect or a properly configured
-*memory-backend-file* backend to actually benefit from NUMA configuration.
-In future new machine versions will not accept the option but it will still
-work with old machine types. User can check QAPI schema to see if the legacy
-option is supported by looking at MachineInfo::numa-mem-supported property.
-
 ``-numa`` node (without memory specified) (since 4.1)
 '
 
@@ -516,3 +499,23 @@ long starting at 1MiB, the old command::
 can be rewritten as::
 
   qemu-nbd -t --image-opts 
driver=raw,offset=1M,size=100M,file.driver=qcow2,file.file.driver=file,file.file.filename=file.qcow2
+
+Command line options
+
+
+``-numa node,mem=``\ *size* (removed in 5.1)
+
+
+The parameter ``mem`` of ``-numa node`` was used to assign a part of
+guest RAM to a NUMA node. But when using it, it's impossible to manage a 
specified
+RAM chunk on the host side (like bind it to a host node, setting bind policy, 
...),
+so the guest ends up with the fake NUMA configuration with suboptiomal 
performance.
+However since 2014 there is an alternative way to assign RAM to a NUMA node
+using parameter ``memdev``, which does the same as ``mem`` and adds
+means to actually manage node RAM on the host side. Use parameter ``memdev``
+with *memory-backend-ram* backend as replacement for parameter ``mem``
+to achieve the same fake NUMA effect or a properly configured
+*memory-backend-file* backend to actually benefit from NUMA configuration.
+New machine versions (since 5.1) will not accept the option but it will still
+work with old machine types. User can check the QAPI schema to see if the 
legacy
+option is supported by looking at MachineInfo::numa-mem-supported property.
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 37462a6f78..063d4703f7 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -2262,7 +2262,6 @@ static void virt_machine_class_init(ObjectClass *oc, void 
*data)
 hc->pre_plug = virt_machine_device_pre_plug_cb;
 hc->plug = virt_machine_device_plug_cb;
 hc->unplug_request = virt_machine_device_unplug_request_cb;
-mc->numa_mem_supported = true;
 mc->nvdimm_supported = true;
 mc->auto_enable_numa_with_memhp = true;
 mc->default_ram_id = "mach-virt.ram";
@@ -2375,6 +2374,7 @@ DEFINE_VIRT_MACHINE_AS_LATEST(5, 1)
 static void virt_machine_5_0_options(MachineClass *mc)
 {

Re: [libvirt-tck PATCH] gitlab: add CONTRIBUTING.rst file to indicate use of merge requests

On Mon, Jun 08, 2020 at 11:22:51AM +0100, Daniel P. Berrangé wrote:
> On Mon, Jun 08, 2020 at 12:10:53PM +0200, Erik Skultety wrote:
> > On Mon, Jun 08, 2020 at 10:02:07AM +0100, Daniel P. Berrangé wrote:
> > > On Thu, Jun 04, 2020 at 03:23:05PM +0200, Erik Skultety wrote:
> > > > With the introduction of automated CI pipelines, we are now ready to
> > > > switch to using merge requests for the project. With this switch we no
> > > > longer wish to have patches sent to the mailing list, and thus the
> > > > git-publish config is removed.
> > >
> > > We don't actually have any CI running for TCK yet.
> > 
> > True, bad wording for the commit message, but TCK is not a high-traffic 
> > libvirt
> > project and since I already kind of started the trend I don't see a reason 
> > why
> > we couldn't make it official, I'll reword the commit message if that is only
> > what's needed.
> 
> Or just wait and I'll send a CI patch later today

This is merged now, so for this patch

Reviewed-by: Daniel P. Berrangé 


Regards,
Daniel
-- 
|: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o-https://fstop138.berrange.com :|
|: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|

Re: [libvirt-tck PATCH] gitlab: add CI jobs for validating build across platforms

On Mon, 2020-06-08 at 15:54 +0100, Daniel P. Berrangé wrote:
> The general goal is that the CI jobs are self-contained and don't
> depend on the build artifacts from the libvirt repo. We also want
> to avoid having an explicit dependency on the libvirt-ci repo, or
> on the Quay.io service. Contributors to the PHP module need to be

s/PHP/TCK/

> Neither CentOS nor Ubuntu have the required perl deps to run the
> TCK so those platforms are currently skipped.

You should also mention openSUSE here, since you are excluding that
platform as well for what I assume are the same reasons.

> +.container_job_template: _job_definition
> +  image: docker:stable
> +  stage: containers
> +  services:
> +- docker:dind
> +  before_script:
> +- export TAG="$CI_REGISTRY_IMAGE/ci-$NAME:latest"
> +- export COMMON_TAG="$CI_REGISTRY/libvirt/libvirt-perl/ci-$NAME:latest"

s/perl/tck/


With these nits fixed,

  Reviewed-by: Andrea Bolognani 

-- 
Andrea Bolognani / Red Hat / Virtualization

Re: [libvirt PATCH 0/9] qemu: Add support for -cpu host, migratable=on|off

2020-06-09 Thread Michal Privoznik


On 6/5/20 8:31 PM, Jiri Denemark wrote:


Jiri Denemark (9):
   conf: Use g_auto* in virCPUDefParseXML
   qemu: Probe for .migratable property of a CPU
   qemu: Probe for migrtability support in CPU expansion
   qemu: Avoid probing unsupported migratable CPU expansion
   conf: Introduce migratable attribute for the  element
   conf: Advertise migratable attribute for CPU in domcaps
   qemu: Advertise migratable attribute for CPU in domcaps
   qemu: Fill default value in //cpu/@migratable attribute
   qemu: Pass migratable=on|off property for -cpu host

  docs/formatdomain.html.in |  14 +-
  docs/formatdomaincaps.html.in |  13 +-
  docs/schemas/domaincaps.rng   |   3 +
  docs/schemas/domaincommon.rng |   5 +
  src/conf/cpu_conf.c   | 157 +-
  src/conf/cpu_conf.h   |   1 +
  src/conf/domain_capabilities.c|  13 +-
  src/conf/domain_capabilities.h|   1 +
  src/qemu/qemu_capabilities.c  |  28 +++-
  src/qemu/qemu_capabilities.h  |   4 +
  src/qemu/qemu_command.c   |  15 ++
  src/qemu/qemu_domain.c|  14 +-
  .../domaincapsdata/qemu_1.5.3-q35.x86_64.xml  |   6 +-
  tests/domaincapsdata/qemu_1.5.3.x86_64.xml|   6 +-
  .../domaincapsdata/qemu_1.6.0-q35.x86_64.xml  |   6 +-
  tests/domaincapsdata/qemu_1.6.0.x86_64.xml|   6 +-
  .../domaincapsdata/qemu_1.7.0-q35.x86_64.xml  |   6 +-
  tests/domaincapsdata/qemu_1.7.0.x86_64.xml|   6 +-
  .../domaincapsdata/qemu_2.1.1-q35.x86_64.xml  |   6 +-
  tests/domaincapsdata/qemu_2.1.1.x86_64.xml|   6 +-
  .../domaincapsdata/qemu_2.10.0-q35.x86_64.xml |   6 +-
  .../qemu_2.10.0-virt.aarch64.xml  |   6 +-
  tests/domaincapsdata/qemu_2.10.0.aarch64.xml  |   6 +-
  tests/domaincapsdata/qemu_2.10.0.ppc64.xml|   6 +-
  tests/domaincapsdata/qemu_2.10.0.s390x.xml|   6 +-
  tests/domaincapsdata/qemu_2.10.0.x86_64.xml   |   6 +-
  .../domaincapsdata/qemu_2.11.0-q35.x86_64.xml |   6 +-
  tests/domaincapsdata/qemu_2.11.0.s390x.xml|   6 +-
  tests/domaincapsdata/qemu_2.11.0.x86_64.xml   |   6 +-
  .../domaincapsdata/qemu_2.12.0-q35.x86_64.xml |   7 +-
  .../qemu_2.12.0-virt.aarch64.xml  |   6 +-
  tests/domaincapsdata/qemu_2.12.0.aarch64.xml  |   6 +-
  tests/domaincapsdata/qemu_2.12.0.ppc64.xml|   6 +-
  tests/domaincapsdata/qemu_2.12.0.s390x.xml|   6 +-
  tests/domaincapsdata/qemu_2.12.0.x86_64.xml   |   7 +-
  .../domaincapsdata/qemu_2.4.0-q35.x86_64.xml  |   6 +-
  tests/domaincapsdata/qemu_2.4.0.x86_64.xml|   6 +-
  .../domaincapsdata/qemu_2.5.0-q35.x86_64.xml  |   6 +-
  tests/domaincapsdata/qemu_2.5.0.x86_64.xml|   6 +-
  .../domaincapsdata/qemu_2.6.0-q35.x86_64.xml  |   6 +-
  .../qemu_2.6.0-virt.aarch64.xml   |   6 +-
  tests/domaincapsdata/qemu_2.6.0.aarch64.xml   |   6 +-
  tests/domaincapsdata/qemu_2.6.0.ppc64.xml |   6 +-
  tests/domaincapsdata/qemu_2.6.0.x86_64.xml|   6 +-
  .../domaincapsdata/qemu_2.7.0-q35.x86_64.xml  |   6 +-
  tests/domaincapsdata/qemu_2.7.0.s390x.xml |   6 +-
  tests/domaincapsdata/qemu_2.7.0.x86_64.xml|   6 +-
  .../domaincapsdata/qemu_2.8.0-q35.x86_64.xml  |   6 +-
  tests/domaincapsdata/qemu_2.8.0.s390x.xml |   6 +-
  tests/domaincapsdata/qemu_2.8.0.x86_64.xml|   6 +-
  .../domaincapsdata/qemu_2.9.0-q35.x86_64.xml  |   6 +-
  tests/domaincapsdata/qemu_2.9.0.ppc64.xml |   6 +-
  tests/domaincapsdata/qemu_2.9.0.s390x.xml |   6 +-
  tests/domaincapsdata/qemu_2.9.0.x86_64.xml|   6 +-
  .Reviewed-by: Michal Privoznik 

../domaincapsdata/qemu_3.0.0-q35.x86_64.xml  |   7 +-

  tests/domaincapsdata/qemu_3.0.0.ppc64.xml |   6 +-
  tests/domaincapsdata/qemu_3.0.0.s390x.xml |   6 +-
  tests/domaincapsdata/qemu_3.0.0.x86_64.xml|   7 +-
  .../domaincapsdata/qemu_3.1.0-q35.x86_64.xml  |   7 +-
  tests/domaincapsdata/qemu_3.1.0.ppc64.xml |   6 +-
  tests/domaincapsdata/qemu_3.1.0.x86_64.xml|   7 +-
  .../domaincapsdata/qemu_4.0.0-q35.x86_64.xml  |   7 +-
  .../qemu_4.0.0-virt.aarch64.xml   |   6 +-
  tests/domaincapsdata/qemu_4.0.0.aarch64.xml   |   6 +-
  tests/domaincapsdata/qemu_4.0.0.ppc64.xml |   6 +-
  tests/domaincapsdata/qemu_4.0.0.s390x.xml |   6 +-
  tests/domaincapsdata/qemu_4.0.0.x86_64.xml|   7 +-
  .../domaincapsdata/qemu_4.1.0-q35.x86_64.xml  |   7 +-
  tests/domaincapsdata/qemu_4.1.0.x86_64.xml|   7 +-
  .../domaincapsdata/qemu_4.2.0-q35.x86_64.xml  |   7 +-
  .../qemu_4.2.0-virt.aarch64.xml   |   6 +-
  tests/domaincapsdata/qemu_4.2.0.aarch64.xml   |   6 +-
  tests/domaincapsdata/qemu_4.2.0.ppc64.xml |   6 +-
  tests/domaincapsdata/qemu_4.2.0.s390x.xml |   6 +-
  tests/domaincapsdata/qemu_4.2.0.x86_64.xml|   7 +-
  .../domaincapsdata/qemu_5.0.0-q35.x86_64.xml  |   7 +-
  .../qemu_5.0.0-virt.aarch64.xml   |   6 +-

[libvirt PATCH v2 1/2] ci: Enable Cirrus CI integration

We use cirrus-run to trigger Cirrus CI jobs from GitLab CI jobs,
making it possible to extend our platform coverage to include
FreeBSD without having to maintain our own runners; additionally,
we'll be able to ditch Travis CI and, since results for Cirrus CI
jobs are reflected back to the GitLab CI jobs that triggered them,
we will be able to get all information from a single dashboard.

The FreeBSD and macOS job definitions can be improved further: for
example, we will want to enable caching to speed up builds, and
ultimately we should figure out a way to generate at least part of
them, notably the list of packages to be installed, using lcitool.
All of that will happen in later patches: for now, this is good
enough to start using Cirrus CI.

Signed-off-by: Andrea Bolognani 
---
 .gitlab-ci.yml  | 38 +++
 ci/README.rst   | 59 ++
 ci/cirrus/freebsd-11.yml.j2 | 73 +
 ci/cirrus/freebsd-12.yml.j2 | 73 +
 ci/cirrus/macos-1015.yml.j2 | 38 +++
 5 files changed, 281 insertions(+)
 create mode 100644 ci/README.rst
 create mode 100644 ci/cirrus/freebsd-11.yml.j2
 create mode 100644 ci/cirrus/freebsd-12.yml.j2
 create mode 100644 ci/cirrus/macos-1015.yml.j2

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7113a1283c..d6effee842 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -38,6 +38,29 @@ stages:
 - master
 - /^ci-full-.*$/
 
+# Jobs that we delegate to Cirrus CI because they require an operating
+# system other than Linux. These jobs will only run if the required
+# setup has been performed on the GitLab account (see ci/README.rst).
+.cirrus_build_default_job_template: _build_default_job_definition
+  stage: native_build
+  image: registry.gitlab.com/libvirt/libvirt-ci/cirrus-run:master
+  script:
+- cirrus-run ci/cirrus/$NAME.yml.j2
+  only:
+variables:
+  - $CIRRUS_GITHUB_REPO
+  - $CIRRUS_API_TOKEN
+
+.cirrus_build_extra_job_template: _build_extra_job_definition
+  <<: *cirrus_build_default_job_definition
+  only:
+variables:
+  - $CIRRUS_GITHUB_REPO
+  - $CIRRUS_API_TOKEN
+refs:
+  - master
+  - /^ci-full-.*$/
+
 
 # Default cross build jobs that are always run
 .cross_build_default_job_template: _build_default_job_definition
@@ -109,6 +132,21 @@ x64-ubuntu-2004:
   <<: *native_build_default_job_definition
   image: quay.io/libvirt/buildenv-libvirt-ubuntu-2004:latest
 
+x64-freebsd-11-build:
+  <<: *cirrus_build_extra_job_definition
+  variables:
+NAME: freebsd-11
+
+x64-freebsd-12-build:
+  <<: *cirrus_build_default_job_definition
+  variables:
+NAME: freebsd-12
+
+x64-macos-1015-build:
+  <<: *cirrus_build_default_job_definition
+  variables:
+NAME: macos-1015
+
 
 # Cross compiled build jobs
 
diff --git a/ci/README.rst b/ci/README.rst
new file mode 100644
index 00..033db1c847
--- /dev/null
+++ b/ci/README.rst
@@ -0,0 +1,59 @@
+==
+CI for libvirt
+==
+
+This document provides some information related to the CI capabilities for the
+libvirt project.
+
+
+Cirrus CI integration
+=
+
+libvirt currently supports three non-Linux operating systems: Windows, FreeBSD
+and macOS. Windows cross-builds can be prepared on Linux by using `MinGW`_, but
+for both FreeBSD and macOS we need to use the actual operating system, and
+unfortunately GitLab shared runners are currently not available for either.
+
+To work around this limitation, we take advantage of `Cirrus CI`_'s free
+offering: more specifically, we use the `cirrus-run`_ script to trigger Cirrus
+CI jobs from GitLab CI jobs so that the workaround is almost entirely
+transparent to users and there's no need to constantly check two separate CI
+dashboards.
+
+There is, however, some one-time setup required. If you want FreeBSD and macOS
+builds to happen when you push to your GitLab repository, you need to
+
+* set up a GitHub repository for the project, eg. ``yourusername/libvirt``.
+  This repository needs to exist for cirrus-run to work, but it doesn't need to
+  be kept up to date, so you can create it and then forget about it;
+
+* enable the `Cirrus CI GitHub app`_  for your GitHub account;
+
+* sign up for Cirrus CI. It's enough to log into the website using your GitHub
+  account;
+
+* grab an API token from the `Cirrus CI settings`_ page;
+
+* in the *CI/CD / Variables* section of the settings page for your GitLab
+  repository, create two new variables:
+
+  * ``CIRRUS_GITHUB_REPO``, containing the name of the GitHub repository
+created earlier, eg. ``yourusername/libvirt``;
+
+  * ``CIRRUS_API_TOKEN``, containing the Cirrus CI API token generated earlier.
+This variable **must** be marked as *Masked*, because anyone with knowledge
+of it can impersonate you as far as Cirrus CI is concerned.
+
+  Neither of these variables should be marked as *Protected*, because in
+

[libvirt PATCH v2 2/2] ci: Remove Travis CI

Since we now use Cirrus CI for macOS jobs, we no longer need to
keep Travis CI around.

Signed-off-by: Andrea Bolognani 
---
 .travis.yml | 58 -
 1 file changed, 58 deletions(-)
 delete mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 7c8a114054..00
--- a/.travis.yml
+++ /dev/null
@@ -1,58 +0,0 @@
-language: c
-compiler: clang
-os: osx
-
-branches:
-  except:
-- /^.*-maint$/
-
-addons:
-  homebrew:
-update: true
-packages:
-  - ccache
-  - rpcgen
-  - xz
-  - yajl
-  - glib
-  - docutils
-  - gnutls
-
-matrix:
-  include:
-- osx_image: xcode11.3
-- osx_image: xcode10.3
-
-env:
-  global:
-- 
PATH="/usr/local/opt/gettext/bin:/usr/local/opt/ccache/libexec:/usr/local/opt/rpcgen/bin:$PATH"
-- PKG_CONFIG_PATH="/usr/local/opt/libxml2/lib/pkgconfig"
-
-before_script:
-  # Hack to blow away py2
-  - brew link --overwrite python
-
-script:
-  # We can't run 'distcheck' or 'syntax-check' because they fail on
-  # macOS, but doing 'install' and 'dist' gives us some useful coverage
-  - mkdir build && cd build
-  - ../autogen.sh --prefix=$(pwd)/install-root && make -j3 && make -j3 install 
&& make -j3 dist
-
-git:
-  submodules: true
-
-notifications:
-  irc:
-# The channel name "irc.oftc.net#virt" is encrypted against libvirt/libvirt
-# to prevent IRC notifications from github forks. This was created using:
-# $ travis encrypt -r "libvirt/libvirt" "irc.oftc.net#virt"
-channels:
-  - secure: 
"hUPdkLxX7nh75+clpnk4U0XLExLfV9DFKSvQSAUtf5JtDNMslj7AeOCf2wcbkNsEhkiF557odTAnov1s5m1w/yaa56zbjFAh5agzqRKya3QjqsrvlBKw/WuN+l82iMNLLeebTgCPAXrbAbGWH8YmYssp/7+eMsnKaVh84EQQNbMCHlLg6ovE26Fs18mZ6J5RC3OPa1vbv+xkdCHvGg/Oyp4K8bpU7RYyimA56jdxI/OfdTH9HxntHYSzykR7hDbyzZhdIlAUyRKReQVjcV5+R8fdDL/1imyGA/88KTztMeKXpZ5Rf+Ss3vYLZb6qsLLegCZ4AU/q0vvbWxjpZGJZoeyrVpfBTZdYGIzmLTMl9GYXXa/gDwFlbvRDiPDG4TIy6GlMUROinj7KRKEHu1fWRYu012ife5OjidxcwrTnz21vYaCv3AKWPpMPxwIzQPkY1hex9uLLX6z+TrAxxDLF+7UzRT9w2RLFBkLYlj2aDVrLAVb/ynRsxDz5CGzC61FSQVft2e308SkGjdn8YxvguCuXv+N70Fu1cvFyh5XYeHb4fbBRo0Ctzaec78leHlQvRGWKJxXDXRkE2lvvBc7YbBNSAYh7Fs8Y+zY7l7rMxvXdrt3nuaNQhe74V3yhxPDAld66qmAn9TYMmaZW2f5/KKKILLbCa0t2MxiAc6L2OI8="
-on_success: change
-on_failure: always
-  email:
-# The list name 'libvirt...@redhat.com" is encrypted against 
libvirt/libvirt
-# to prevent IRC notifications from github forks. This was created using:
-# $ travis encrypt -r "libvirt/libvirt" "libvirt...@redhat.com"
-recipients:
-  - secure: 
"QcU9eP96P0RlDNzVRZl/4sxyydPStGzECrpgJhr2IPB/7pHk23yaBrmUsq9S830tB+jwLGma1IscNB8uf7Sf7WY+cYIpfR8v030OffWnaipo/Gcs0dpnlfURWHjOFQI3RJzGEihsqvbwUFOwsM+3IDyO3qdWaiT6cN2Tj9ROlwYCySSX5YWzLyX7arBZ4lp8ESs7ohQaEwp2cegnMP2oGPJJe4SebvlCDjHZbjkU5aEradwUWnRQDJZWTKknpNLArVFxN2/ixp6f/MGY4DmkHoDweio6mHIPN5zTs5Jt32aiX6wDBa+bBa4v8TCRqzhYkQ63ZZhNV8bY5Uf9ufTdyvt96yIANyakd85b1QpMdAX76IyJi1l0/Uub6DTQZAcq3vK7iPjGeTVSpyoXrqTfGy4JxMjqDoocpWvv8ALX1wrYI/HfN2R2Aepw9jModTimOsebYhJ1yMhSt8qnh5AQNftGKL2JBKoA1LWdU2YJ5fO1bGjKNiVEkGFQTPYFWrYCUY5JcT+s5WCzNeMNm8s9na8liYhGl3WtS3rPr5M8bof+BMsBhG2hQ0loduc94x2GkvyhQZUgRbqrwNR+y4hn+rWFC3hBzzyiAULs43vY/PJ+eBdKEf3VAc0MkhQ8GgXGSA61fR6aXYonroI/WnBVItwDmUnnMfSziZXxk09GLl4="
-- 
2.25.4

[libvirt PATCH v2 0/2] ci: Enable Cirrus CI integration

Changes from [v1]:

  * skip Cirrus CI jobs if the necessary variables are not set.

[v1] https://www.redhat.com/archives/libvir-list/2020-June/msg00249.html

Andrea Bolognani (2):
  ci: Enable Cirrus CI integration
  ci: Remove Travis CI

 .gitlab-ci.yml  | 38 +++
 .travis.yml | 58 -
 ci/README.rst   | 59 ++
 ci/cirrus/freebsd-11.yml.j2 | 73 +
 ci/cirrus/freebsd-12.yml.j2 | 73 +
 ci/cirrus/macos-1015.yml.j2 | 38 +++
 6 files changed, 281 insertions(+), 58 deletions(-)
 delete mode 100644 .travis.yml
 create mode 100644 ci/README.rst
 create mode 100644 ci/cirrus/freebsd-11.yml.j2
 create mode 100644 ci/cirrus/freebsd-12.yml.j2
 create mode 100644 ci/cirrus/macos-1015.yml.j2

-- 
2.25.4

Re: [libvirt PATCH 0/2] ci: Enable Cirrus CI integration

On Tue, Jun 09, 2020 at 11:53:41AM +0200, Andrea Bolognani wrote:
> On Mon, 2020-06-08 at 19:07 +0200, Andrea Bolognani wrote:
> > On Mon, 2020-06-08 at 17:34 +0100, Daniel P. Berrangé wrote:
> > > On Mon, Jun 08, 2020 at 05:42:31PM +0200, Andrea Bolognani wrote:
> > > > To see this in action, check out the
> > > > 
> > > >   https://gitlab.com/abologna/libvirt/-/pipelines/153963978
> > > > 
> > > > pipeline, and in particular the
> > > > 
> > > >   https://gitlab.com/abologna/libvirt/-/jobs/585678408
> > > >   https://gitlab.com/abologna/libvirt/-/jobs/585678413
> > > 
> > > Ideally the actual build output from Cirrus CI would be pulled into
> > > the build output of the GitLab CI job. That way email notifications
> > > from failed builds will include the actual compile failure details.
> > > This also avoids having to cut+paste non-clickable links to the
> > > Cirrus CI job.
> > 
> > I agree that this would be desirable.
> > 
> > Earlier today, while reviewing the code for cirrus-run, I found a
> > reference to a CIRRUS_LOG_FILE environment variable that should
> > allow us to do just that, and was intending to try it out. If that
> > doesn't work out, we should still be able to fetch the full job
> > output from Cirrus CI after the fact.
> 
> I tried setting CIRRUS_LOG_FILE and it doesn't seem to work the way
> we want. Getting the taks log directly from Cirrus CI is definitely
> possible - you can do so from the Web UI already - but obtaining the
> correct URL requires calling the GraphQL API and I need some more
> time to figure out how that works. Possibly make some changes to
> cirrus-run as well.
> 
> Since it's already possible, although not very convenient, to get to
> the full log, I think this should not block the series and it can
> come as a follow-up instead. I'll post a v2 that addresses the other
> issue you pointed out.

Yep, that's fine


Regards,
Daniel
-- 
|: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o-https://fstop138.berrange.com :|
|: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|

Re: [PATCH v2 3/8] conf: Add firmware blob configuration