[Libguestfs] [PATCH v3 1/2] common: extract UTF-8 conversion function

[Cédric Bosdonnat]

libxml2-utils.c local_string_to_utf8() function could easily be reused
in other places. This commit extracts it with a new parameter to allow
giving the encoding of the input string and publishes it in
guestfs-utils.h as guestfs_int_string_to_utf8()
---
 common/utils/guestfs-utils.h | 11 +++
 common/utils/libxml2-utils.c | 69 +---
 common/utils/utils.c | 64 
 3 files changed, 76 insertions(+), 68 deletions(-)

diff --git a/common/utils/guestfs-utils.h b/common/utils/guestfs-utils.h
index 90e7c3dd9..86da693bc 100644
--- a/common/utils/guestfs-utils.h
+++ b/common/utils/guestfs-utils.h
@@ -33,6 +33,7 @@
 #define GUESTFS_UTILS_H_
 
 #include 
+#include 
 
 #include "guestfs-internal-all.h"
 #include "cleanups.h"
@@ -70,6 +71,16 @@ extern int guestfs_int_is_fifo (int64_t mode);
 extern int guestfs_int_is_lnk (int64_t mode);
 extern int guestfs_int_is_sock (int64_t mode);
 extern char *guestfs_int_full_path (const char *dir, const char *name);
+extern char *guestfs_int_string_to_utf8 (/* const */ char *input, const char 
*encoding);
+
+/* Would be const, but the interface to iconv is not const-correct on
+ * all platforms.  The input string is not touched.
+ */
+static inline char *
+guestfs_int_local_string_to_utf8 (/* const */ char *input)
+{
+  return guestfs_int_string_to_utf8 (input, nl_langinfo (CODESET));
+}
 
 /* Not all language bindings know how to deal with Pointer arguments.
  * Those that don't will use this macro which complains noisily and
diff --git a/common/utils/libxml2-utils.c b/common/utils/libxml2-utils.c
index 8a05aa5b1..a71db30dd 100644
--- a/common/utils/libxml2-utils.c
+++ b/common/utils/libxml2-utils.c
@@ -30,8 +30,6 @@
 #include 
 #include 
 #include 
-#include 
-#include 
 
 #include 
 
@@ -42,8 +40,6 @@
 #include "guestfs-utils.h"
 #include "libxml2-utils.h"
 
-static char *local_string_to_utf8 (/* const */ char *input);
-
 /**
  * This is a wrapper around C.  That function cannot
  * handle spaces and some non-ASCII characters found in URIs.  This
@@ -73,7 +69,7 @@ guestfs_int_parse_nonstandard_uri (const char *arg)
   xmlURIPtr ret;
 
   /* Convert the string to UTF-8. */
-  uri = local_string_to_utf8 ((char *) arg);
+  uri = guestfs_int_local_string_to_utf8 ((char *) arg);
   if (uri == NULL)
 return NULL;
 
@@ -113,66 +109,3 @@ guestfs_int_parse_nonstandard_uri (const char *arg)
 
   return ret;
 }
-
-/* Would be const, but the interface to iconv is not const-correct on
- * all platforms.  The input string is not touched.
- */
-static char *
-local_string_to_utf8 (/* const */ char *input)
-{
-  iconv_t ic;
-  size_t len, inlen, outlen, outalloc, r, prev;
-  int err;
-  char *out, *inp, *outp;
-
-  /* Convert from input locale to UTF-8. */
-  ic = iconv_open ("UTF-8", nl_langinfo (CODESET));
-  if (ic == (iconv_t) -1)
-return NULL;
-
-  len = strlen (input);
-  outalloc = len;   /* Initial guess. */
-
- again:
-  inlen = len;
-  outlen = outalloc;
-  out = malloc (outlen + 1);
-  if (out == NULL) {
-err = errno;
-iconv_close (ic);
-errno = err;
-return NULL;
-  }
-  inp = input;
-  outp = out;
-
-  r = iconv (ic, (ICONV_CONST char **) , , , );
-  if (r == (size_t) -1) {
-if (errno == E2BIG) {
-  err = errno;
-  prev = outalloc;
-  /* Try again with a larger output buffer. */
-  free (out);
-  outalloc *= 2;
-  if (outalloc < prev) {
-iconv_close (ic);
-errno = err;
-return NULL;
-  }
-  goto again;
-}
-else {
-  /* Else some other conversion failure, eg. EILSEQ, EINVAL. */
-  err = errno;
-  iconv_close (ic);
-  free (out);
-  errno = err;
-  return NULL;
-}
-  }
-
-  *outp = '\0';
-  iconv_close (ic);
-
-  return out;
-}
diff --git a/common/utils/utils.c b/common/utils/utils.c
index 22af62b0f..faef7c089 100644
--- a/common/utils/utils.c
+++ b/common/utils/utils.c
@@ -35,6 +35,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /* NB: MUST NOT require linking to gnulib, because that will break the
  * Python 'sdist' which includes a copy of this file.  It's OK to
@@ -733,3 +734,66 @@ guestfs_int_full_path (const char *dir, const char *name)
 
   return path;
 }
+
+/* Would be const, but the interface to iconv is not const-correct on
+ * all platforms.  The input string is not touched.
+ */
+char *
+guestfs_int_string_to_utf8 (/* const */ char *input, const char *encoding)
+{
+  iconv_t ic;
+  size_t len, inlen, outlen, outalloc, r, prev;
+  int err;
+  char *out, *inp, *outp;
+
+  /* Convert from input encoding to UTF-8. */
+  ic = iconv_open ("UTF-8", encoding);
+  if (ic == (iconv_t) -1)
+return NULL;
+
+  len = strlen (input);
+  outalloc = len;   /* Initial guess. */
+
+ again:
+  inlen = len;
+  outlen = outalloc;
+  out = malloc (outlen + 1);
+  if (out == NULL) {
+err = errno;
+iconv_close (ic);
+errno = err;
+return 

[Libguestfs] [PATCH v3 2/2] inspector: rpm summary and description may not be utf-8

[Cédric Bosdonnat]

The application inspection code assumes the data in the RPM database
are encoded in UTF-8. However this is not always the case.

As a basic workaround, try to parse the string to UTF-8 and if that
fails, try converting it from latin-1.
---
 inspector/expected-fedora.img.xml |  4 
 lib/inspect-apps.c| 30 +++
 test-data/phony-guests/fedora-packages.db.txt |  4 ++--
 3 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/inspector/expected-fedora.img.xml 
b/inspector/expected-fedora.img.xml
index df6060a73..c29f9770e 100644
--- a/inspector/expected-fedora.img.xml
+++ b/inspector/expected-fedora.img.xml
@@ -34,12 +34,16 @@
 1.0
 1.fc14
 x86_64
+summary with ö
+description with ö
   
   
 test2
 2.0
 2.fc14
 x86_64
+summary with ö
+description with ö
   
   
 test3
diff --git a/lib/inspect-apps.c b/lib/inspect-apps.c
index f0cf16b38..fdea85188 100644
--- a/lib/inspect-apps.c
+++ b/lib/inspect-apps.c
@@ -22,6 +22,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifdef HAVE_ENDIAN_H
 #include 
@@ -43,6 +44,7 @@
 #include "guestfs.h"
 #include "guestfs-internal.h"
 #include "guestfs-internal-actions.h"
+#include "guestfs-utils.h"
 #include "structs-cleanups.h"
 
 #ifdef DB_DUMP
@@ -251,7 +253,7 @@ get_rpm_header_tag (guestfs_h *g, const unsigned char 
*header_start,
   /* This function parses the RPM header structure to pull out various
* tag strings (version, release, arch, etc.).  For more detail on the
* header format, see:
-   * 
http://www.rpm.org/max-rpm/s1-rpm-file-format-rpm-file-format.html#S2-RPM-FILE-FORMAT-HEADER
+   * http://rpm.org/devel_doc/file_format.html#24-header-format
*/
 
   /* The minimum header size that makes sense here is 24 bytes.  Four
@@ -301,6 +303,20 @@ struct read_package_data {
   struct guestfs_application2_list *apps;
 };
 
+static char *
+to_utf8 (guestfs_h *g, char *input)
+{
+  char *out = NULL;
+
+  out = guestfs_int_string_to_utf8 (input, "UTF-8");
+  if (!out) {
+out = guestfs_int_string_to_utf8 (input, "ISO-8859-1");
+perrorf (g, "Not an UTF-8 or latin-1 string: '%s'", input);
+  }
+
+  return out;
+}
+
 static int
 read_package (guestfs_h *g,
   const unsigned char *key, size_t keylen,
@@ -311,7 +327,7 @@ read_package (guestfs_h *g,
   struct rpm_name nkey, *entry;
   CLEANUP_FREE char *version = NULL, *release = NULL,
 *epoch_str = NULL, *arch = NULL, *url = NULL, *summary = NULL,
-*description = NULL;
+*description = NULL, *summary_raw = NULL, *description_raw = NULL;
   int32_t epoch;
 
   /* This function reads one (key, value) pair from the Packages
@@ -342,8 +358,14 @@ read_package (guestfs_h *g,
   epoch_str = get_rpm_header_tag (g, value, valuelen, RPMTAG_EPOCH, 'i');
   arch = get_rpm_header_tag (g, value, valuelen, RPMTAG_ARCH, 's');
   url = get_rpm_header_tag (g, value, valuelen, RPMTAG_URL, 's');
-  summary = get_rpm_header_tag (g, value, valuelen, RPMTAG_SUMMARY, 's');
-  description = get_rpm_header_tag (g, value, valuelen, RPMTAG_DESCRIPTION, 
's');
+  summary_raw = get_rpm_header_tag (g, value, valuelen, RPMTAG_SUMMARY, 's');
+  description_raw = get_rpm_header_tag (g, value, valuelen, 
RPMTAG_DESCRIPTION, 's');
+
+  /* Try (not too hard) to get UTF-8 */
+  if (summary_raw)
+summary = to_utf8 (g, summary_raw);
+  if (description_raw)
+description = to_utf8 (g, description_raw);
 
   /* The epoch is stored as big-endian integer. */
   if (epoch_str)
diff --git a/test-data/phony-guests/fedora-packages.db.txt 
b/test-data/phony-guests/fedora-packages.db.txt
index f16a5aa76..927d6eb5f 100644
--- a/test-data/phony-guests/fedora-packages.db.txt
+++ b/test-data/phony-guests/fedora-packages.db.txt
@@ -5,9 +5,9 @@ h_nelem=3
 db_pagesize=4096
 HEADER=END
  \01\00\00\00
- 
\00\00\00\03\00\00\00\11\00\00\03\e9\00\00\00\00\00\00\00\00\00\00\00\00\00\00\03\ea\00\00\00\00\00\00\00\04\00\00\00\00\00\00\03\fe\00\00\00\00\00\00\00\0b\00\00\00\001.0\001.fc14\00x86_64\00
+ 
\00\00\00\05\00\00\00\33\00\00\03\e9\00\00\00\00\00\00\00\00\00\00\00\00\00\00\03\ea\00\00\00\00\00\00\00\04\00\00\00\00\00\00\03\fe\00\00\00\00\00\00\00\0b\00\00\00\00\00\00\03\ec\00\00\00\00\00\00\00\12\00\00\00\00\00\00\03\ed\00\00\00\00\00\00\00\21\00\00\00\001.0\001.fc14\00x86_64\00summary
 with \f6\00description with \f6\00
  \02\00\00\00
- 
\00\00\00\03\00\00\00\11\00\00\03\e9\00\00\00\00\00\00\00\00\00\00\00\00\00\00\03\ea\00\00\00\00\00\00\00\04\00\00\00\00\00\00\03\fe\00\00\00\00\00\00\00\0b\00\00\00\002.0\002.fc14\00x86_64\00
+ 
\00\00\00\05\00\00\00\35\00\00\03\e9\00\00\00\00\00\00\00\00\00\00\00\00\00\00\03\ea\00\00\00\00\00\00\00\04\00\00\00\00\00\00\03\fe\00\00\00\00\00\00\00\0b\00\00\00\00\00\00\03\ec\00\00\00\00\00\00\00\12\00\00\00\00\00\00\03\ed\00\00\00\00\00\00\00\22\00\00\00\002.0\002.fc14\00x86_64\00summary
 with \c3\b6\00description with 

[Libguestfs] [PATCH v3 0/2] inspect: basic UTF-8 encoding for rpm

[Cédric Bosdonnat]

Diff to v2:
  * inlined local_string_to_utf8

Cédric Bosdonnat (2):
  common: extract UTF-8 conversion function
  inspector: rpm summary and description may not be utf-8

 common/utils/guestfs-utils.h  | 11 +
 common/utils/libxml2-utils.c  | 69 +--
 common/utils/utils.c  | 64 +
 inspector/expected-fedora.img.xml |  4 ++
 lib/inspect-apps.c| 30 ++--
 test-data/phony-guests/fedora-packages.db.txt |  4 +-
 6 files changed, 108 insertions(+), 74 deletions(-)

-- 
2.16.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v2 1/2] common: extract UTF-8 conversion function

[Cédric Bosdonnat]

libxml2-utils.c local_string_to_utf8() function could easily be reused
in other places. This commit extracts it with a new parameter to allow
giving the encoding of the input string and publishes it in
guestfs-utils.h as guestfs_int_string_to_utf8()
---
 common/utils/guestfs-utils.h |  1 +
 common/utils/libxml2-utils.c | 57 +--
 common/utils/utils.c | 64 
 3 files changed, 66 insertions(+), 56 deletions(-)

diff --git a/common/utils/guestfs-utils.h b/common/utils/guestfs-utils.h
index 90e7c3dd9..68e704a2c 100644
--- a/common/utils/guestfs-utils.h
+++ b/common/utils/guestfs-utils.h
@@ -70,6 +70,7 @@ extern int guestfs_int_is_fifo (int64_t mode);
 extern int guestfs_int_is_lnk (int64_t mode);
 extern int guestfs_int_is_sock (int64_t mode);
 extern char *guestfs_int_full_path (const char *dir, const char *name);
+extern char * guestfs_int_string_to_utf8 (/* const */ char *input, const char 
*encoding);
 
 /* Not all language bindings know how to deal with Pointer arguments.
  * Those that don't will use this macro which complains noisily and
diff --git a/common/utils/libxml2-utils.c b/common/utils/libxml2-utils.c
index 8a05aa5b1..f83bb177e 100644
--- a/common/utils/libxml2-utils.c
+++ b/common/utils/libxml2-utils.c
@@ -31,7 +31,6 @@
 #include 
 #include 
 #include 
-#include 
 
 #include 
 
@@ -120,59 +119,5 @@ guestfs_int_parse_nonstandard_uri (const char *arg)
 static char *
 local_string_to_utf8 (/* const */ char *input)
 {
-  iconv_t ic;
-  size_t len, inlen, outlen, outalloc, r, prev;
-  int err;
-  char *out, *inp, *outp;
-
-  /* Convert from input locale to UTF-8. */
-  ic = iconv_open ("UTF-8", nl_langinfo (CODESET));
-  if (ic == (iconv_t) -1)
-return NULL;
-
-  len = strlen (input);
-  outalloc = len;   /* Initial guess. */
-
- again:
-  inlen = len;
-  outlen = outalloc;
-  out = malloc (outlen + 1);
-  if (out == NULL) {
-err = errno;
-iconv_close (ic);
-errno = err;
-return NULL;
-  }
-  inp = input;
-  outp = out;
-
-  r = iconv (ic, (ICONV_CONST char **) , , , );
-  if (r == (size_t) -1) {
-if (errno == E2BIG) {
-  err = errno;
-  prev = outalloc;
-  /* Try again with a larger output buffer. */
-  free (out);
-  outalloc *= 2;
-  if (outalloc < prev) {
-iconv_close (ic);
-errno = err;
-return NULL;
-  }
-  goto again;
-}
-else {
-  /* Else some other conversion failure, eg. EILSEQ, EINVAL. */
-  err = errno;
-  iconv_close (ic);
-  free (out);
-  errno = err;
-  return NULL;
-}
-  }
-
-  *outp = '\0';
-  iconv_close (ic);
-
-  return out;
+  return guestfs_int_string_to_utf8 (input, nl_langinfo (CODESET));
 }
diff --git a/common/utils/utils.c b/common/utils/utils.c
index 22af62b0f..faef7c089 100644
--- a/common/utils/utils.c
+++ b/common/utils/utils.c
@@ -35,6 +35,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /* NB: MUST NOT require linking to gnulib, because that will break the
  * Python 'sdist' which includes a copy of this file.  It's OK to
@@ -733,3 +734,66 @@ guestfs_int_full_path (const char *dir, const char *name)
 
   return path;
 }
+
+/* Would be const, but the interface to iconv is not const-correct on
+ * all platforms.  The input string is not touched.
+ */
+char *
+guestfs_int_string_to_utf8 (/* const */ char *input, const char *encoding)
+{
+  iconv_t ic;
+  size_t len, inlen, outlen, outalloc, r, prev;
+  int err;
+  char *out, *inp, *outp;
+
+  /* Convert from input encoding to UTF-8. */
+  ic = iconv_open ("UTF-8", encoding);
+  if (ic == (iconv_t) -1)
+return NULL;
+
+  len = strlen (input);
+  outalloc = len;   /* Initial guess. */
+
+ again:
+  inlen = len;
+  outlen = outalloc;
+  out = malloc (outlen + 1);
+  if (out == NULL) {
+err = errno;
+iconv_close (ic);
+errno = err;
+return NULL;
+  }
+  inp = input;
+  outp = out;
+
+  r = iconv (ic, (ICONV_CONST char **) , , , );
+  if (r == (size_t) -1) {
+if (errno == E2BIG) {
+  err = errno;
+  prev = outalloc;
+  /* Try again with a larger output buffer. */
+  free (out);
+  outalloc *= 2;
+  if (outalloc < prev) {
+iconv_close (ic);
+errno = err;
+return NULL;
+  }
+  goto again;
+}
+else {
+  /* Else some other conversion failure, eg. EILSEQ, EINVAL. */
+  err = errno;
+  iconv_close (ic);
+  free (out);
+  errno = err;
+  return NULL;
+}
+  }
+
+  *outp = '\0';
+  iconv_close (ic);
+
+  return out;
+}
-- 
2.16.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v2 2/2] inspector: rpm summary and description may not be utf-8

[Cédric Bosdonnat]

The application inspection code assumes the data in the RPM database
are encoded in UTF-8. However this is not always the case.

As a basic workaround, try to parse the string to UTF-8 and if that
fails, try converting it from latin-1.
---
 inspector/expected-fedora.img.xml |  4 
 lib/inspect-apps.c| 30 +++
 test-data/phony-guests/fedora-packages.db.txt |  4 ++--
 3 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/inspector/expected-fedora.img.xml 
b/inspector/expected-fedora.img.xml
index 8d40e8cb7..ffefce177 100644
--- a/inspector/expected-fedora.img.xml
+++ b/inspector/expected-fedora.img.xml
@@ -33,12 +33,16 @@
 1.0
 1.fc14
 x86_64
+summary with ö
+description with ö
   
   
 test2
 2.0
 2.fc14
 x86_64
+summary with ö
+description with ö
   
   
 test3
diff --git a/lib/inspect-apps.c b/lib/inspect-apps.c
index f0cf16b38..fdea85188 100644
--- a/lib/inspect-apps.c
+++ b/lib/inspect-apps.c
@@ -22,6 +22,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifdef HAVE_ENDIAN_H
 #include 
@@ -43,6 +44,7 @@
 #include "guestfs.h"
 #include "guestfs-internal.h"
 #include "guestfs-internal-actions.h"
+#include "guestfs-utils.h"
 #include "structs-cleanups.h"
 
 #ifdef DB_DUMP
@@ -251,7 +253,7 @@ get_rpm_header_tag (guestfs_h *g, const unsigned char 
*header_start,
   /* This function parses the RPM header structure to pull out various
* tag strings (version, release, arch, etc.).  For more detail on the
* header format, see:
-   * 
http://www.rpm.org/max-rpm/s1-rpm-file-format-rpm-file-format.html#S2-RPM-FILE-FORMAT-HEADER
+   * http://rpm.org/devel_doc/file_format.html#24-header-format
*/
 
   /* The minimum header size that makes sense here is 24 bytes.  Four
@@ -301,6 +303,20 @@ struct read_package_data {
   struct guestfs_application2_list *apps;
 };
 
+static char *
+to_utf8 (guestfs_h *g, char *input)
+{
+  char *out = NULL;
+
+  out = guestfs_int_string_to_utf8 (input, "UTF-8");
+  if (!out) {
+out = guestfs_int_string_to_utf8 (input, "ISO-8859-1");
+perrorf (g, "Not an UTF-8 or latin-1 string: '%s'", input);
+  }
+
+  return out;
+}
+
 static int
 read_package (guestfs_h *g,
   const unsigned char *key, size_t keylen,
@@ -311,7 +327,7 @@ read_package (guestfs_h *g,
   struct rpm_name nkey, *entry;
   CLEANUP_FREE char *version = NULL, *release = NULL,
 *epoch_str = NULL, *arch = NULL, *url = NULL, *summary = NULL,
-*description = NULL;
+*description = NULL, *summary_raw = NULL, *description_raw = NULL;
   int32_t epoch;
 
   /* This function reads one (key, value) pair from the Packages
@@ -342,8 +358,14 @@ read_package (guestfs_h *g,
   epoch_str = get_rpm_header_tag (g, value, valuelen, RPMTAG_EPOCH, 'i');
   arch = get_rpm_header_tag (g, value, valuelen, RPMTAG_ARCH, 's');
   url = get_rpm_header_tag (g, value, valuelen, RPMTAG_URL, 's');
-  summary = get_rpm_header_tag (g, value, valuelen, RPMTAG_SUMMARY, 's');
-  description = get_rpm_header_tag (g, value, valuelen, RPMTAG_DESCRIPTION, 
's');
+  summary_raw = get_rpm_header_tag (g, value, valuelen, RPMTAG_SUMMARY, 's');
+  description_raw = get_rpm_header_tag (g, value, valuelen, 
RPMTAG_DESCRIPTION, 's');
+
+  /* Try (not too hard) to get UTF-8 */
+  if (summary_raw)
+summary = to_utf8 (g, summary_raw);
+  if (description_raw)
+description = to_utf8 (g, description_raw);
 
   /* The epoch is stored as big-endian integer. */
   if (epoch_str)
diff --git a/test-data/phony-guests/fedora-packages.db.txt 
b/test-data/phony-guests/fedora-packages.db.txt
index f16a5aa76..927d6eb5f 100644
--- a/test-data/phony-guests/fedora-packages.db.txt
+++ b/test-data/phony-guests/fedora-packages.db.txt
@@ -5,9 +5,9 @@ h_nelem=3
 db_pagesize=4096
 HEADER=END
  \01\00\00\00
- 
\00\00\00\03\00\00\00\11\00\00\03\e9\00\00\00\00\00\00\00\00\00\00\00\00\00\00\03\ea\00\00\00\00\00\00\00\04\00\00\00\00\00\00\03\fe\00\00\00\00\00\00\00\0b\00\00\00\001.0\001.fc14\00x86_64\00
+ 
\00\00\00\05\00\00\00\33\00\00\03\e9\00\00\00\00\00\00\00\00\00\00\00\00\00\00\03\ea\00\00\00\00\00\00\00\04\00\00\00\00\00\00\03\fe\00\00\00\00\00\00\00\0b\00\00\00\00\00\00\03\ec\00\00\00\00\00\00\00\12\00\00\00\00\00\00\03\ed\00\00\00\00\00\00\00\21\00\00\00\001.0\001.fc14\00x86_64\00summary
 with \f6\00description with \f6\00
  \02\00\00\00
- 
\00\00\00\03\00\00\00\11\00\00\03\e9\00\00\00\00\00\00\00\00\00\00\00\00\00\00\03\ea\00\00\00\00\00\00\00\04\00\00\00\00\00\00\03\fe\00\00\00\00\00\00\00\0b\00\00\00\002.0\002.fc14\00x86_64\00
+ 
\00\00\00\05\00\00\00\35\00\00\03\e9\00\00\00\00\00\00\00\00\00\00\00\00\00\00\03\ea\00\00\00\00\00\00\00\04\00\00\00\00\00\00\03\fe\00\00\00\00\00\00\00\0b\00\00\00\00\00\00\03\ec\00\00\00\00\00\00\00\12\00\00\00\00\00\00\03\ed\00\00\00\00\00\00\00\22\00\00\00\002.0\002.fc14\00x86_64\00summary
 with \c3\b6\00description with 

[Libguestfs] [PATCH v2 0/2] inspect: basic UTF-8 encoding for rpm

[Cédric Bosdonnat]

This needs Richard's patch:
https://www.redhat.com/archives/libguestfs/2018-February/msg00099.html

Diff to v1:
  * factorized the UTF-8 conversion functions
  * small style fixes

Cédric Bosdonnat (2):
  common: extract UTF-8 conversion function
  inspector: rpm summary and description may not be utf-8

 common/utils/guestfs-utils.h  |  1 +
 common/utils/libxml2-utils.c  | 57 +---
 common/utils/utils.c  | 64 +++
 inspector/expected-fedora.img.xml |  4 ++
 lib/inspect-apps.c| 30 +++--
 test-data/phony-guests/fedora-packages.db.txt |  4 +-
 6 files changed, 98 insertions(+), 62 deletions(-)

-- 
2.16.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH] inspector: rpm summary and description may not be utf-8

[Cédric Bosdonnat]

The application inspection code assumes the data in the RPM database
are encoded in UTF-8. However this is not always the case.

As a basic workaround, try to parse the string to UTF-8 and if that
fails, try converting it from latin-1.
---
 inspector/expected-fedora.img.xml |  4 ++
 lib/inspect-apps.c| 75 +--
 test-data/phony-guests/fedora-packages.db.txt |  4 +-
 3 files changed, 77 insertions(+), 6 deletions(-)

diff --git a/inspector/expected-fedora.img.xml 
b/inspector/expected-fedora.img.xml
index 8d40e8cb7..ffefce177 100644
--- a/inspector/expected-fedora.img.xml
+++ b/inspector/expected-fedora.img.xml
@@ -33,12 +33,16 @@
 1.0
 1.fc14
 x86_64
+summary with ö
+description with ö
   
   
 test2
 2.0
 2.fc14
 x86_64
+summary with ö
+description with ö
   
   
 test3
diff --git a/lib/inspect-apps.c b/lib/inspect-apps.c
index f0cf16b38..5adfabfe6 100644
--- a/lib/inspect-apps.c
+++ b/lib/inspect-apps.c
@@ -22,6 +22,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifdef HAVE_ENDIAN_H
 #include 
@@ -251,7 +252,7 @@ get_rpm_header_tag (guestfs_h *g, const unsigned char 
*header_start,
   /* This function parses the RPM header structure to pull out various
* tag strings (version, release, arch, etc.).  For more detail on the
* header format, see:
-   * 
http://www.rpm.org/max-rpm/s1-rpm-file-format-rpm-file-format.html#S2-RPM-FILE-FORMAT-HEADER
+   * http://rpm.org/devel_doc/file_format.html#24-header-format
*/
 
   /* The minimum header size that makes sense here is 24 bytes.  Four
@@ -301,6 +302,66 @@ struct read_package_data {
   struct guestfs_application2_list *apps;
 };
 
+static char *
+to_utf8 (guestfs_h *g, char *input)
+{
+  iconv_t cd_utf8_utf8 = (iconv_t)(-1);
+  iconv_t cd_utf8_latin1 = (iconv_t)(-1);
+  size_t in_left, out_left, res;
+  char *in_ptr;
+  char *out_ptr;
+  char *output = NULL;
+  char *result = NULL;
+
+  cd_utf8_utf8 = iconv_open("UTF-8", "UTF-8");
+  if (cd_utf8_utf8 == (iconv_t)(-1)) {
+perrorf(g, "No iconv UTF-8 encoding");
+goto cleanup;
+  }
+
+  in_ptr = input;
+  in_left = strlen(input) + 1;
+  out_left = in_left * 4;
+  output = safe_malloc(g, out_left);
+  out_ptr = output;
+
+  res = iconv(cd_utf8_utf8, _ptr, _left, _ptr, _left);
+  if (res == (size_t)(-1)) {
+if (errno == E2BIG) {
+  perrorf(g, "iconv: '%s', buffer length: %lu", input, strlen(input) * 4);
+  goto cleanup;
+}
+
+/* Try latin-1 encoding */
+cd_utf8_latin1 = iconv_open("UTF-8", "ISO-8859-1");
+if (cd_utf8_latin1 == (iconv_t)(-1)) {
+  perrorf(g, "No iconv ISO-8859-1 encoding");
+  goto cleanup;
+}
+
+in_ptr = input;
+in_left = strlen(input) + 1;
+out_left = in_left * 4;
+out_ptr = output;
+
+res = iconv(cd_utf8_latin1, _ptr, _left, _ptr, _left);
+if (res == (size_t)(-1)) {
+  perrorf(g, "Failed to parse latin-1: '%s'", input);
+  goto cleanup;
+}
+  }
+
+  result = output;
+
+ cleanup:
+   iconv_close(cd_utf8_utf8);
+   iconv_close(cd_utf8_latin1);
+   if (!result)
+ free(output);
+
+   return result;
+}
+
 static int
 read_package (guestfs_h *g,
   const unsigned char *key, size_t keylen,
@@ -311,7 +372,7 @@ read_package (guestfs_h *g,
   struct rpm_name nkey, *entry;
   CLEANUP_FREE char *version = NULL, *release = NULL,
 *epoch_str = NULL, *arch = NULL, *url = NULL, *summary = NULL,
-*description = NULL;
+*description = NULL, *summary_raw = NULL, *description_raw = NULL;
   int32_t epoch;
 
   /* This function reads one (key, value) pair from the Packages
@@ -342,8 +403,14 @@ read_package (guestfs_h *g,
   epoch_str = get_rpm_header_tag (g, value, valuelen, RPMTAG_EPOCH, 'i');
   arch = get_rpm_header_tag (g, value, valuelen, RPMTAG_ARCH, 's');
   url = get_rpm_header_tag (g, value, valuelen, RPMTAG_URL, 's');
-  summary = get_rpm_header_tag (g, value, valuelen, RPMTAG_SUMMARY, 's');
-  description = get_rpm_header_tag (g, value, valuelen, RPMTAG_DESCRIPTION, 
's');
+  summary_raw = get_rpm_header_tag (g, value, valuelen, RPMTAG_SUMMARY, 's');
+  description_raw = get_rpm_header_tag (g, value, valuelen, 
RPMTAG_DESCRIPTION, 's');
+
+  /* Try (not too hard) to get UTF-8 */
+  if (summary_raw)
+summary = to_utf8(g, summary_raw);
+  if (description_raw)
+description = to_utf8(g, description_raw);
 
   /* The epoch is stored as big-endian integer. */
   if (epoch_str)
diff --git a/test-data/phony-guests/fedora-packages.db.txt 
b/test-data/phony-guests/fedora-packages.db.txt
index f16a5aa76..927d6eb5f 100644
--- a/test-data/phony-guests/fedora-packages.db.txt
+++ b/test-data/phony-guests/fedora-packages.db.txt
@@ -5,9 +5,9 @@ h_nelem=3
 db_pagesize=4096
 HEADER=END
  \01\00\00\00
- 

[Libguestfs] [PATCH v3 1/3] daemon: make sgdisk_info_extract_uuid_field more generic

[Cédric Bosdonnat]

Rename the sgdisk_info_extract_uuid_field to
sgdisk_info_extract_field in order to reuse it for other field types.
Just like its C ancestor, it now needs an extractor function to be
passed as parameter.
---
 daemon/parted.ml | 22 --
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/daemon/parted.ml b/daemon/parted.ml
index d6638867a..6fe803613 100644
--- a/daemon/parted.ml
+++ b/daemon/parted.ml
@@ -124,12 +124,11 @@ let part_get_parttype device =
   | _ ->
  failwithf "%s: cannot parse the output of parted" device
 
-let rec part_get_gpt_type device partnum =
-  sgdisk_info_extract_uuid_field device partnum "Partition GUID code"
-and part_get_gpt_guid device partnum =
-  sgdisk_info_extract_uuid_field device partnum "Partition unique GUID"
+let extract_guid value =
+  (* The value contains only valid GUID characters. *)
+  String.sub value 0 (String.span value "-0123456789ABCDEF")
 
-and sgdisk_info_extract_uuid_field device partnum field =
+let sgdisk_info_extract_field device partnum field extractor =
   if partnum <= 0 then failwith "partition number must be >= 1";
 
   udev_settle ();
@@ -167,13 +166,16 @@ and sgdisk_info_extract_uuid_field device partnum field =
(* Skip any whitespace after the colon. *)
let value = String.triml value in
 
-   (* Extract the UUID. *)
-   extract_uuid value
+   (* Extract the value. *)
+   extractor value
 
 | _ :: lines -> loop lines
   in
   loop lines
 
-and extract_uuid value =
-  (* The value contains only valid GUID characters. *)
-  String.sub value 0 (String.span value "-0123456789ABCDEF")
+let rec part_get_gpt_type device partnum =
+  sgdisk_info_extract_field device partnum "Partition GUID code"
+extract_guid
+and part_get_gpt_guid device partnum =
+  sgdisk_info_extract_field device partnum "Partition unique GUID"
+extract_guid
-- 
2.15.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v3 0/3] copy GPT attributes

[Cédric Bosdonnat]

Hi all,

Here is v3 of the series, taking Richard's comments in account.


Cédric Bosdonnat (3):
  daemon: make sgdisk_info_extract_uuid_field more generic
  New APIs: part_set_gpt_attributes and part_get_gpt_attributes
  resize: copy GPT partition flags

 daemon/parted.ml  | 45 +++--
 daemon/parted.mli |  2 ++
 generator/actions_core.ml | 37 +
 generator/proc_nr.ml  |  2 ++
 lib/MAX_PROC_NR   |  2 +-
 resize/resize.ml  | 15 ---
 6 files changed, 89 insertions(+), 14 deletions(-)

-- 
2.15.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v3 3/3] resize: copy GPT partition flags

[Cédric Bosdonnat]

In some cases, the first stage bootloader needs the 'Legacy BIOS
bootable' flag to be set on the partition. This change copies all
flags (including this one) for each partition of the old disk to the
new one to avoid ending up with non-bootable disks.
---
 resize/resize.ml | 15 ---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/resize/resize.ml b/resize/resize.ml
index 880fa98cb..1a21e4dff 100644
--- a/resize/resize.ml
+++ b/resize/resize.ml
@@ -50,6 +50,7 @@ type partition = {
   p_type : partition_content;(* Content type and content size. *)
   p_label : string option;   (* Label/name. *)
   p_guid : string option;(* Partition GUID (GPT only). *)
+  p_attributes : int64 option;   (* Partition attributes bit mask (GPT only). 
*)
 
   (* What we're going to do: *)
   mutable p_operation : partition_operation;
@@ -493,6 +494,12 @@ read the man page virt-resize(1).
   let label =
 try Some (g#part_get_name "/dev/sda" part_num)
 with G.Error _ -> None in
+  let attributes =
+match parttype with
+| MBR -> None
+| GPT ->
+  try Some (g#part_get_gpt_attributes "/dev/sda" part_num)
+  with G.Error _ -> None in
   let guid =
 match parttype with
 | MBR -> None
@@ -502,7 +509,7 @@ read the man page virt-resize(1).
 
   { p_name = name; p_part = part;
 p_bootable = bootable; p_id = id; p_type = typ;
-p_label = label; p_guid = guid;
+p_label = label; p_guid = guid; p_attributes = attributes;
 p_operation = OpCopy; p_target_partnum = 0;
 p_target_start = 0L; p_target_end = 0L }
   ) parts in
@@ -1150,6 +1157,7 @@ read the man page virt-resize(1).
  part_size = 0L };
   p_bootable = false; p_id = No_ID; p_type = ContentUnknown;
   p_label = None; p_guid = None;
+  p_attributes = None;
 
   (* Target information is meaningful. *)
   p_operation = OpIgnore;
@@ -1191,12 +1199,13 @@ read the man page virt-resize(1).
* is changed from primary to extended.  Thus we need to set the
* MBR ID before doing the copy so sfdisk doesn't corrupt things.
*)
-  let set_partition_bootable_and_id p =
+  let set_partition_attributes p =
   if p.p_bootable then
 g#part_set_bootable "/dev/sdb" p.p_target_partnum true;
 
   Option.may (g#part_set_name "/dev/sdb" p.p_target_partnum) p.p_label;
   Option.may (g#part_set_gpt_guid "/dev/sdb" p.p_target_partnum) p.p_guid;
+  Option.may (g#part_set_gpt_attributes "/dev/sdb" p.p_target_partnum) 
p.p_attributes;
 
   match parttype, p.p_id with
   | GPT, GPT_Type gpt_type ->
@@ -1205,7 +1214,7 @@ read the man page virt-resize(1).
 g#part_set_mbr_id "/dev/sdb" p.p_target_partnum mbr_id
   | GPT, (No_ID|MBR_ID _) | MBR, (No_ID|GPT_Type _) -> ()
   in
-  List.iter set_partition_bootable_and_id partitions;
+  List.iter set_partition_attributes partitions;
 
   (* Copy over the data. *)
   let copy_partition p =
-- 
2.15.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v2 0/3] copying gpt attributes

[Cédric Bosdonnat]

Hi all,

Here is the latest version of the series addressing Pino's comments.

Cédric Bosdonnat (3):
  daemon: make sgdisk_info_extract_uuid_field more generic
  New APIs: part_set_gpt_attributes and part_get_gpt_attributes
  resize: copy GPT partition flags

 daemon/parted.ml  | 45 +++--
 daemon/parted.mli |  3 +++
 generator/actions_core.ml | 37 +
 generator/proc_nr.ml  |  2 ++
 lib/MAX_PROC_NR   |  2 +-
 resize/resize.ml  | 15 ---
 6 files changed, 90 insertions(+), 14 deletions(-)

-- 
2.15.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v2 1/3] daemon: make sgdisk_info_extract_uuid_field more generic

[Cédric Bosdonnat]

Rename the sgdisk_info_extract_uuid_field to
sgdisk_info_extract_field in order to reuse it for other field types.
Just like its C ancestor, it now needs an extractor function to be
passed as parameter.
---
 daemon/parted.ml | 22 --
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/daemon/parted.ml b/daemon/parted.ml
index d6638867a..6fe803613 100644
--- a/daemon/parted.ml
+++ b/daemon/parted.ml
@@ -124,12 +124,11 @@ let part_get_parttype device =
   | _ ->
  failwithf "%s: cannot parse the output of parted" device
 
-let rec part_get_gpt_type device partnum =
-  sgdisk_info_extract_uuid_field device partnum "Partition GUID code"
-and part_get_gpt_guid device partnum =
-  sgdisk_info_extract_uuid_field device partnum "Partition unique GUID"
+let extract_guid value =
+  (* The value contains only valid GUID characters. *)
+  String.sub value 0 (String.span value "-0123456789ABCDEF")
 
-and sgdisk_info_extract_uuid_field device partnum field =
+let sgdisk_info_extract_field device partnum field extractor =
   if partnum <= 0 then failwith "partition number must be >= 1";
 
   udev_settle ();
@@ -167,13 +166,16 @@ and sgdisk_info_extract_uuid_field device partnum field =
(* Skip any whitespace after the colon. *)
let value = String.triml value in
 
-   (* Extract the UUID. *)
-   extract_uuid value
+   (* Extract the value. *)
+   extractor value
 
 | _ :: lines -> loop lines
   in
   loop lines
 
-and extract_uuid value =
-  (* The value contains only valid GUID characters. *)
-  String.sub value 0 (String.span value "-0123456789ABCDEF")
+let rec part_get_gpt_type device partnum =
+  sgdisk_info_extract_field device partnum "Partition GUID code"
+extract_guid
+and part_get_gpt_guid device partnum =
+  sgdisk_info_extract_field device partnum "Partition unique GUID"
+extract_guid
-- 
2.15.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v2 2/3] New APIs: part_set_gpt_attributes and part_get_gpt_attributes

[Cédric Bosdonnat]

Allow reading and setting the GPT partition attribute flags.
---
 daemon/parted.ml  | 23 +++
 daemon/parted.mli |  3 +++
 generator/actions_core.ml | 37 +
 generator/proc_nr.ml  |  2 ++
 lib/MAX_PROC_NR   |  2 +-
 5 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/daemon/parted.ml b/daemon/parted.ml
index 6fe803613..e3ab823bd 100644
--- a/daemon/parted.ml
+++ b/daemon/parted.ml
@@ -124,10 +124,30 @@ let part_get_parttype device =
   | _ ->
  failwithf "%s: cannot parse the output of parted" device
 
+let part_set_gpt_attributes device partnum attributes =
+  if partnum <= 0 then failwith "partition number must be >= 1";
+
+  udev_settle ();
+
+  let hex = Printf.sprintf "%LX" attributes in
+  let arg = string_of_int partnum ^ ":=:" ^ hex in
+  let r, _, err =
+commandr ~fold_stdout_on_stderr:true
+ "sgdisk" [ device; "-A"; arg ] in
+  if r <> 0 then
+failwithf "sgdisk: %s" err;
+
+  udev_settle ()
+
 let extract_guid value =
   (* The value contains only valid GUID characters. *)
   String.sub value 0 (String.span value "-0123456789ABCDEF")
 
+let extract_hex value =
+  (* The value contains only valid numeric characters. *)
+  let str = String.sub value 0 (String.span value "0123456789ABCDEF") in
+  Int64.of_string ("0x" ^ str)
+
 let sgdisk_info_extract_field device partnum field extractor =
   if partnum <= 0 then failwith "partition number must be >= 1";
 
@@ -179,3 +199,6 @@ let rec part_get_gpt_type device partnum =
 and part_get_gpt_guid device partnum =
   sgdisk_info_extract_field device partnum "Partition unique GUID"
 extract_guid
+and part_get_gpt_attributes device partnum =
+  sgdisk_info_extract_field device partnum "Attribute flags"
+extract_hex
diff --git a/daemon/parted.mli b/daemon/parted.mli
index cbcb7b503..9f57bbac7 100644
--- a/daemon/parted.mli
+++ b/daemon/parted.mli
@@ -30,3 +30,6 @@ val part_get_parttype : string -> string
 
 val part_get_gpt_type : string -> int -> string
 val part_get_gpt_guid : string -> int -> string
+val part_get_gpt_attributes : string -> int -> int64
+
+val part_set_gpt_attributes : string -> int -> int64 -> unit
diff --git a/generator/actions_core.ml b/generator/actions_core.ml
index 02759a6b7..78eee61dd 100644
--- a/generator/actions_core.ml
+++ b/generator/actions_core.ml
@@ -8265,6 +8265,43 @@ Return the type GUID of numbered GPT partition 
C. For MBR partitions,
 return an appropriate GUID corresponding to the MBR type. Behaviour is 
undefined
 for other partition types." };
 
+  { defaults with
+name = "part_set_gpt_attributes"; added = (1, 21, 1);
+style = RErr, [String (Device, "device"); Int "partnum"; Int64 
"attributes"], [];
+impl = OCaml "Parted.part_set_gpt_attributes";
+optional = Some "gdisk";
+tests = [
+  InitGPT, Always, TestResult (
+[["part_set_gpt_attributes"; "/dev/sda"; "1";
+  "4"];
+ ["part_get_gpt_attributes"; "/dev/sda"; "1"]],
+"4"), [];
+];
+shortdesc = "set the attribute flags of a GPT partition";
+longdesc = "\
+Set the attribute flags of numbered GPT partition C to C. 
Return an
+error if the partition table of C isn't GPT.
+
+See L
+for a useful list of partition attributes." };
+
+  { defaults with
+name = "part_get_gpt_attributes"; added = (1, 21, 1);
+style = RInt64 "attributes", [String (Device, "device"); Int "partnum"], 
[];
+impl = OCaml "Parted.part_get_gpt_attributes";
+optional = Some "gdisk";
+tests = [
+  InitGPT, Always, TestResult (
+[["part_set_gpt_attributes"; "/dev/sda"; "1";
+  "0"];
+ ["part_get_gpt_attributes"; "/dev/sda"; "1"]],
+"4"), [];
+];
+shortdesc = "get the attribute flags of a GPT partition";
+longdesc = "\
+Return the attribute flags of numbered GPT partition C as a 
hexadecimal bit mask.
+For MBR partitions, return all flags set to 0." };
+
   { defaults with
 name = "rename"; added = (1, 21, 5);
 style = RErr, [String (Pathname, "oldpath"); String (Pathname, 
"newpath")], [];
diff --git a/generator/proc_nr.ml b/generator/proc_nr.ml
index 3e393da73..9e16ab14a 100644
--- a/generator/proc_nr.ml
+++ b/generator/proc_nr.ml
@@ -510,6 +510,8 @@ let proc_nr = [
 500, "inspect_get_mountpoints";
 501, "inspect_get_filesystems";
 502, "inspect_get_drive_mappings";
+503, "part_set_gpt_attributes";
+504, "part_get_gpt_attributes";
 ]
 
 (* End of list.  If adding a new entry, add it at the end of the list
diff --git a/lib/MAX_PROC_NR b/lib/MAX_PROC_NR
index cc5027eed..3091e8eea 100644
--- a/lib/MAX_PROC_NR
+++ b/lib/MAX_PROC_NR
@@ -1 +1 @@
-502
+504
-- 
2.15.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH 1/3] daemon: make sgdisk_info_extract_uuid_field more generic

[Cédric Bosdonnat]

Rename the sgdisk_info_extract_uuid_field to
sgdisk_info_extract_field in order to reuse it for other field types.
To avoid possible confusion, the list of valid characters used to
extract the value is added to the function parameters.
---
 daemon/parted.ml | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/daemon/parted.ml b/daemon/parted.ml
index d6638867a..cf1a54a08 100644
--- a/daemon/parted.ml
+++ b/daemon/parted.ml
@@ -124,12 +124,16 @@ let part_get_parttype device =
   | _ ->
  failwithf "%s: cannot parse the output of parted" device
 
+let hex_chars = "0123456789ABCDEF"
+
 let rec part_get_gpt_type device partnum =
-  sgdisk_info_extract_uuid_field device partnum "Partition GUID code"
+  sgdisk_info_extract_field device partnum "Partition GUID code"
+("-" ^ hex_chars)
 and part_get_gpt_guid device partnum =
-  sgdisk_info_extract_uuid_field device partnum "Partition unique GUID"
+  sgdisk_info_extract_field device partnum "Partition unique GUID"
+("-" ^ hex_chars)
 
-and sgdisk_info_extract_uuid_field device partnum field =
+and sgdisk_info_extract_field device partnum field chars =
   if partnum <= 0 then failwith "partition number must be >= 1";
 
   udev_settle ();
@@ -167,13 +171,13 @@ and sgdisk_info_extract_uuid_field device partnum field =
(* Skip any whitespace after the colon. *)
let value = String.triml value in
 
-   (* Extract the UUID. *)
-   extract_uuid value
+   (* Extract the value. *)
+   extract_string chars value
 
 | _ :: lines -> loop lines
   in
   loop lines
 
-and extract_uuid value =
+and extract_string chars value =
   (* The value contains only valid GUID characters. *)
-  String.sub value 0 (String.span value "-0123456789ABCDEF")
+  String.sub value 0 (String.span value chars)
-- 
2.15.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH 3/3] resize: copy GPT partition flags

[Cédric Bosdonnat]

In some cases, the first stage bootloader needs the 'Legacy BIOS
bootable' flag to be set on the partition. This change copies all
flags (including this one) for each partition of the old disk to the
new one to avoid ending up with non-bootable disks.
---
 resize/resize.ml | 15 ---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/resize/resize.ml b/resize/resize.ml
index 880fa98cb..4328d162e 100644
--- a/resize/resize.ml
+++ b/resize/resize.ml
@@ -50,6 +50,7 @@ type partition = {
   p_type : partition_content;(* Content type and content size. *)
   p_label : string option;   (* Label/name. *)
   p_guid : string option;(* Partition GUID (GPT only). *)
+  p_attributes : string option;  (* Partition attributes hex bit mask (GPT 
only). *)
 
   (* What we're going to do: *)
   mutable p_operation : partition_operation;
@@ -493,6 +494,12 @@ read the man page virt-resize(1).
   let label =
 try Some (g#part_get_name "/dev/sda" part_num)
 with G.Error _ -> None in
+  let attributes =
+match parttype with
+| MBR -> None
+| GPT ->
+  try Some (g#part_get_gpt_attributes "/dev/sda" part_num)
+  with G.Error _ -> None in
   let guid =
 match parttype with
 | MBR -> None
@@ -502,7 +509,7 @@ read the man page virt-resize(1).
 
   { p_name = name; p_part = part;
 p_bootable = bootable; p_id = id; p_type = typ;
-p_label = label; p_guid = guid;
+p_label = label; p_guid = guid; p_attributes = attributes;
 p_operation = OpCopy; p_target_partnum = 0;
 p_target_start = 0L; p_target_end = 0L }
   ) parts in
@@ -1150,6 +1157,7 @@ read the man page virt-resize(1).
  part_size = 0L };
   p_bootable = false; p_id = No_ID; p_type = ContentUnknown;
   p_label = None; p_guid = None;
+  p_attributes = None;
 
   (* Target information is meaningful. *)
   p_operation = OpIgnore;
@@ -1191,12 +1199,13 @@ read the man page virt-resize(1).
* is changed from primary to extended.  Thus we need to set the
* MBR ID before doing the copy so sfdisk doesn't corrupt things.
*)
-  let set_partition_bootable_and_id p =
+  let set_partition_bootable_attributes_and_id p =
   if p.p_bootable then
 g#part_set_bootable "/dev/sdb" p.p_target_partnum true;
 
   Option.may (g#part_set_name "/dev/sdb" p.p_target_partnum) p.p_label;
   Option.may (g#part_set_gpt_guid "/dev/sdb" p.p_target_partnum) p.p_guid;
+  Option.may (g#part_set_gpt_attributes "/dev/sdb" p.p_target_partnum) 
p.p_attributes;
 
   match parttype, p.p_id with
   | GPT, GPT_Type gpt_type ->
@@ -1205,7 +1214,7 @@ read the man page virt-resize(1).
 g#part_set_mbr_id "/dev/sdb" p.p_target_partnum mbr_id
   | GPT, (No_ID|MBR_ID _) | MBR, (No_ID|GPT_Type _) -> ()
   in
-  List.iter set_partition_bootable_and_id partitions;
+  List.iter set_partition_bootable_attributes_and_id partitions;
 
   (* Copy over the data. *)
   let copy_partition p =
-- 
2.15.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH 2/3] New APIs: part_set_gpt_attributes and part_get_gpt_attributes

[Cédric Bosdonnat]

Allow reading and setting the GPT partition attribute flags.
---
 daemon/parted.ml  | 18 +-
 daemon/parted.mli |  3 +++
 generator/actions_core.ml | 40 
 generator/proc_nr.ml  |  2 ++
 lib/MAX_PROC_NR   |  2 +-
 5 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/daemon/parted.ml b/daemon/parted.ml
index cf1a54a08..5f553c2da 100644
--- a/daemon/parted.ml
+++ b/daemon/parted.ml
@@ -124,7 +124,19 @@ let part_get_parttype device =
   | _ ->
  failwithf "%s: cannot parse the output of parted" device
 
-let hex_chars = "0123456789ABCDEF"
+let part_set_gpt_attributes device partnum attributes =
+  if partnum <= 0 then failwith "partition number must be >= 1";
+
+  udev_settle ();
+
+  let arg = string_of_int partnum ^ ":=:" ^ attributes in
+  let r, _, err =
+commandr ~fold_stdout_on_stderr:true
+ "sgdisk" [ device; "-A"; arg ] in
+  if r <> 0 then
+failwithf "sgdisk: %s" err;
+
+  udev_settle ()
 
 let rec part_get_gpt_type device partnum =
   sgdisk_info_extract_field device partnum "Partition GUID code"
@@ -132,6 +144,8 @@ let rec part_get_gpt_type device partnum =
 and part_get_gpt_guid device partnum =
   sgdisk_info_extract_field device partnum "Partition unique GUID"
 ("-" ^ hex_chars)
+and part_get_gpt_attributes device partnum =
+  sgdisk_info_extract_field device partnum "Attribute flags" hex_chars
 
 and sgdisk_info_extract_field device partnum field chars =
   if partnum <= 0 then failwith "partition number must be >= 1";
@@ -178,6 +192,8 @@ and sgdisk_info_extract_field device partnum field chars =
   in
   loop lines
 
+and hex_chars = "0123456789ABCDEF"
+
 and extract_string chars value =
   (* The value contains only valid GUID characters. *)
   String.sub value 0 (String.span value chars)
diff --git a/daemon/parted.mli b/daemon/parted.mli
index cbcb7b503..300adfa75 100644
--- a/daemon/parted.mli
+++ b/daemon/parted.mli
@@ -30,3 +30,6 @@ val part_get_parttype : string -> string
 
 val part_get_gpt_type : string -> int -> string
 val part_get_gpt_guid : string -> int -> string
+val part_get_gpt_attributes : string -> int -> string
+
+val part_set_gpt_attributes : string -> int -> string -> unit
diff --git a/generator/actions_core.ml b/generator/actions_core.ml
index 02759a6b7..786953d0c 100644
--- a/generator/actions_core.ml
+++ b/generator/actions_core.ml
@@ -8265,6 +8265,46 @@ Return the type GUID of numbered GPT partition 
C. For MBR partitions,
 return an appropriate GUID corresponding to the MBR type. Behaviour is 
undefined
 for other partition types." };
 
+  { defaults with
+name = "part_set_gpt_attributes"; added = (1, 21, 1);
+style = RErr, [String (Device, "device"); Int "partnum"; String 
(PlainString, "attributes")], [];
+impl = OCaml "Parted.part_set_gpt_attributes";
+optional = Some "gdisk";
+tests = [
+  InitGPT, Always, TestLastFail (
+[["part_set_gpt_attributes"; "/dev/sda"; "1"; "foo"]]), [];
+  InitGPT, Always, TestResultString (
+[["part_set_gpt_attributes"; "/dev/sda"; "1";
+  "0004"];
+ ["part_get_gpt_attributes"; "/dev/sda"; "1"]],
+"0004"), [];
+];
+shortdesc = "set the attribute flags of a GPT partition";
+longdesc = "\
+Set the attribute flags of numbered GPT partition C to C. 
Return an
+error if the partition table of C isn't GPT, or if C is 
not a
+valid hexadecimal value.
+
+See L
+for a useful list of partition attributes." };
+
+  { defaults with
+name = "part_get_gpt_attributes"; added = (1, 21, 1);
+style = RString (RPlainString, "attributes"), [String (Device, "device"); 
Int "partnum"], [];
+impl = OCaml "Parted.part_get_gpt_attributes";
+optional = Some "gdisk";
+tests = [
+  InitGPT, Always, TestResultString (
+[["part_set_gpt_attributes"; "/dev/sda"; "1";
+  ""];
+ ["part_get_gpt_attributes"; "/dev/sda"; "1"]],
+"0004"), [];
+];
+shortdesc = "get the attribute flags of a GPT partition";
+longdesc = "\
+Return the attribute flags of numbered GPT partition C as a 
hexadecimal bit mask.
+For MBR partitions, return all flags set to 0." };
+
   { defaults with
 name = "rename"; added = (1, 21, 5);
 style = RErr, [String (Pathname, "oldpath"); String (Pathname, 
"newpath")], [];
diff --git a/generator/proc_nr.ml b/generator/proc_nr.ml
index 3e393da73..9e16ab14a 100644
--- a/generator/proc_nr.ml
+++ b/generator/proc_nr.ml
@@ -510,6 +510,8 @@ let proc_nr = [
 500, "inspect_get_mountpoints";
 501, "inspect_get_filesystems";
 502, "inspect_get_drive_mappings";
+503, "part_set_gpt_attributes";
+504, "part_get_gpt_attributes";
 ]
 
 (* End of list.  If adding a new entry, add it at the end of the list
diff --git a/lib/MAX_PROC_NR b/lib/MAX_PROC_NR

[Libguestfs] [PATCH 0/3] Handle GPT attribute flags

[Cédric Bosdonnat]

Hi all,

Here is the series fixing the bug I mentioned on IRC regarding the GPT
attribute flags to copy to the new disk in a virt-resize.

Cédric Bosdonnat (3):
  daemon: make sgdisk_info_extract_uuid_field more generic
  New APIs: part_set_gpt_attributes and part_get_gpt_attributes
  resize: copy GPT partition flags

 daemon/parted.ml  | 34 +++---
 daemon/parted.mli |  3 +++
 generator/actions_core.ml | 40 
 generator/proc_nr.ml  |  2 ++
 lib/MAX_PROC_NR   |  2 +-
 resize/resize.ml  | 15 ---
 6 files changed, 85 insertions(+), 11 deletions(-)

-- 
2.15.1

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v13 0/3] virt-builder-repository

[Cédric Bosdonnat]

Hey there,

Here is an update of the series. The changes:

 * Incorporate Richard's comments. Left out the with_openfile one
   since that leads to a double close.

 * Change the ask option return type to string (removing the option)
   since if the use doesn't input anything we're using the default,
   and the default is now a mandatory parameter.

 * Make sure there are items in the lvs list before getting the
   first one.

Cédric Bosdonnat (3):
  builder: change arch type to distinguish guesses
  builder: add a template parameter to get_index
  New tool: virt-builder-repository

 .gitignore  |   4 +
 builder/Makefile.am |  87 -
 builder/builder.ml  |   6 +-
 builder/cache.ml|   6 +-
 builder/cache.mli   |   6 +-
 builder/downloader.mli  |   2 +-
 builder/index.ml|   9 +-
 builder/index.mli   |   8 +-
 builder/index_parser.ml |  55 ++-
 builder/index_parser.mli|   5 +-
 builder/list_entries.ml |   6 +-
 builder/repository_main.ml  | 595 
 builder/simplestreams_parser.ml |   2 +-
 builder/test-docs.sh|   2 +
 builder/test-virt-builder-repository.sh | 100 ++
 builder/utils.ml|   4 +
 builder/utils.mli   |   3 +
 builder/virt-builder-repository.pod | 213 
 builder/virt-builder.pod|   4 +
 fish/guestfish.pod  |   1 +
 installcheck.sh.in  |   1 +
 lib/guestfs.pod |   1 +
 22 files changed, 1091 insertions(+), 29 deletions(-)
 create mode 100644 builder/repository_main.ml
 create mode 100755 builder/test-virt-builder-repository.sh
 create mode 100644 builder/virt-builder-repository.pod

-- 
2.15.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v13 3/3] New tool: virt-builder-repository

[Cédric Bosdonnat]

thing wrong happens when running the tool, the repository is
+left untouched.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--help>
+
+Display help.
+
+=item B<--gpg> GPG
+
+Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary.  You can
+also use this to add gpg parameters, for example to specify an
+alternate home directory:
+
+ virt-builder-repository --gpg "gpg --homedir /tmp" [...]
+
+This can also be used to avoid gpg asking for the key passphrase:
+
+ virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...]
+
+=item B<-K> KEYID
+
+=item B<--gpg-key> KEYID
+
+Specify the GPG key to be used to sign the repository index file.
+If not provided, the index will left unsigned. C is used to
+identify the GPG key to use. This value is passed to gpg’s
+I<--default-key> option and can thus be an email address or a
+fingerprint.
+
+B: by default, virt-builder-repository searches for the key
+in the user’s GPG keyring.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data. Default values are computed from the disk
+image.
+
+When prompted for data, inputting C<-> corresponds to leaving the
+value empty. This can be used to avoid setting the default computed value.
+
+=item B<--keep-index>
+
+When using a GPG key, don’t remove the unsigned index.
+
+=item B<--no-compression>
+
+Don’t compress the template images.
+
+=item B<--machine-readable>
+
+This option is used to make the output more machine friendly
+when being parsed by other programs.  See
+L below.
+
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages.  This is the default
+when the output is a tty.  If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don’t print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 MACHINE READABLE OUTPUT
+
+The I<--machine-readable> option can be used to make the output more
+machine friendly, which is useful when calling virt-builder-repository from
+other programs, GUIs etc.
+
+Use the option on its own to query the capabilities of the
+virt-builder-repository binary.  Typical output looks like this:
+
+ $ virt-builder-repository --machine-readable
+ virt-builder-repository
+
+A list of features is printed, one per line, and the program exits
+with status 0.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdon...@suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod
index 74bf7bb11..1ed18a7c7 100644
--- a/builder/virt-builder.pod
+++ b/builder/virt-builder.pod
@@ -1319,6 +1319,9 @@ digital signature):
 The part in square brackets is the C, which is the same
 string that is used on the virt-builder command line to build that OS.
 
+The index file creation and signature can be eased with the
+L<virt-builder-repository(1)> tool.
+
 After preparing the C file in the correct format, clearsign it
 using the following command:
 
@@ -1875,6 +1878,7 @@ error.
 L<guestfs(3)>,
 L<guestfish(1)>,
 L<guestmount(1)>,
+L<virt-builder-repository(1)>,
 L<virt-copy-out(1)>,
 L<virt-customize(1)>,
 L<virt-get-kernel(1)>,
diff --git a/fish/guestfish.pod b/fish/guestfish.pod
index 11f2bbeb5..c37189bd8 100644
--- a/fish/guestfish.pod
+++ b/fish/guestfish.pod
@@ -1617,6 +1617,7 @@ L<guestfs(3)>,
 L<http://libguestfs.org/>,
 L<virt-alignment-scan(1)>,
 L<virt-builder(1)>,
+L<virt-builder-repository(1)>,
 L<virt-cat(1)>,
 L<virt-copy-in(1)>,
 L<virt-copy-out(1)>,
diff --git a/installcheck.sh.in b/installcheck.sh.in
index 6b05ab812..a4829cda6 100644
--- a/installcheck.sh.in
+++ b/installcheck.sh.in
@@ -46,6 +46,7 @@ cp @bindir@/guestfish fish/
 cp @bindir@/guestmountfuse/
 cp @bindir@/virt-alignment-scan   align/
 cp @bindir@/virt-builder  builder/
+cp @bindir@/virt-builder-repository builder/
 cp @bindir@/virt-cat  cat/
 cp @bindir@/virt-copy-in  fish/
 cp @bindir@/virt-copy-out fish/
diff --git a/lib/guestfs.pod b/lib/guestfs.pod
index 8d31f3200..55467e92e 100644
--- a/lib/guestfs.pod
+++ b/lib/guestfs.pod
@@ -3417,6 +3417,7 @@ L<guestfish(1)>,
 L<guestmount(1)>,
 L<virt-alignment-scan(1)>,
 L<virt-builder(1)>,
+L<virt-builder-repository(1)>,
 L<virt-cat(1)>,
 L<virt-copy-in(1)>,
 L<virt-copy-out(1)>,
-- 
2.15.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v13 2/3] builder: add a template parameter to get_index

[Cédric Bosdonnat]

get_index now gets a new template parameter. Setting it to true will
make the index parsing less picky about missing important data. This
can be used to parse a partial index file.
---
 builder/index_parser.ml  | 51 
 builder/index_parser.mli |  5 -
 2 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index d79c807e4..b6e721a00 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -25,7 +25,7 @@ open Utils
 open Printf
 open Unix
 
-let get_index ~downloader ~sigchecker { Sources.uri; proxy } =
+let get_index ~downloader ~sigchecker ?(template = false) { Sources.uri; proxy 
} =
   let corrupt_file () =
 error (f_"The index file downloaded from ‘%s’ is corrupt.\nYou need to ask 
the supplier of this file to fix it and upload a fixed version.") uri
   in
@@ -99,8 +99,23 @@ let get_index ~downloader ~sigchecker { Sources.uri; proxy } 
=
   let arch =
 try Index.Arch (List.assoc ("arch", None) fields)
 with Not_found ->
-  eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
-corrupt_file () in
+  if template then
+let g = open_guestfs ~identifier:"template" () in
+g#add_drive_ro file_uri;
+g#launch ();
+let roots = g#inspect_os () in
+let nroots = Array.length roots in
+if nroots <> 1 then (
+  eprintf (f_"%s: no ‘arch’ entry for %s and failed to guess 
it\n") prog n;
+  corrupt_file ()
+);
+let inspected_arch = g#inspect_get_arch (Array.get roots 0) in
+g#close();
+Index.GuessedArch inspected_arch
+  else (
+eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let signature_uri =
 try Some (make_absolute_uri (List.assoc ("sig", None) fields))
 with Not_found -> None in
@@ -112,21 +127,41 @@ let get_index ~downloader ~sigchecker { Sources.uri; 
proxy } =
   let revision =
 try Rev_int (int_of_string (List.assoc ("revision", None) fields))
 with
-| Not_found -> Rev_int 1
+| Not_found -> if template then Rev_int 0 else Rev_int 1
 | Failure _ ->
   eprintf (f_"%s: cannot parse ‘revision’ field for ‘%s’\n") prog 
n;
   corrupt_file () in
   let format =
 try Some (List.assoc ("format", None) fields) with Not_found -> 
None in
   let size =
+let get_image_size filepath =
+  (* If a compressed image manages to reach this code, qemu-img 
just
+ returns a virtual-size equal to actual-size *)
+  match detect_file_type filepath with
+  | `Unknown ->
+let infos = Utils.get_image_infos filepath in
+Yajl.object_get_number "virtual-size" infos
+  | `XZ | `GZip | `Tar | ` Zip ->
+eprintf (f_"%s: cannot determine the virtal size of %s due to 
compression")
+prog filepath;
+corrupt_file () in
+  
 try Int64.of_string (List.assoc ("size", None) fields)
 with
 | Not_found ->
-  eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file ()
+  if template then
+get_image_size file_uri
+  else (
+eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  )
 | Failure _ ->
-  eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file () in
+  if template then
+get_image_size file_uri
+  else (
+eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let compressed_size =
 try Some (Int64.of_string (List.assoc ("compressed_size", None) 
fields))
 with
diff --git a/builder/index_parser.mli b/builder/index_parser.mli
index f77ae9376..dc6b0b407 100644
--- a/builder/index_parser.mli
+++ b/builder/index_parser.mli
@@ -16,7 +16,10 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
Sources.source -> Index.index
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
?template:bool -> Sources.source -> Index.index
+(** [get_index download sigchecker template source] will parse the source
+index file into an index entry list. If the template flag is set to
+true, the parser will be less picky about missing values. *)
 
 val write_entry : out_channel -> 

[Libguestfs] [PATCH v13 1/3] builder: change arch type to distinguish guesses

[Cédric Bosdonnat]

Change Index.arch to the type (Arch of string | GuessedArch of string).

In a future commit, the index parser will allow arch not to be set
for some cases. Thus arch value will be guessed by inspecting the
image. However we need to distinguish between a set value and a guessed
one. Using this new type will help it:

match arch with
| Arch s-> (* This is a set value *)
| GuessedArch s -> (* This is a guessed value *)
---
 builder/builder.ml  | 6 +++---
 builder/cache.ml| 6 --
 builder/cache.mli   | 6 +++---
 builder/downloader.mli  | 2 +-
 builder/index.ml| 9 +++--
 builder/index.mli   | 8 +++-
 builder/index_parser.ml | 4 ++--
 builder/list_entries.ml | 6 +++---
 builder/simplestreams_parser.ml | 2 +-
 9 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index 3f7c79bc9..41c0a4ccc 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -94,7 +94,7 @@ let selected_cli_item cmdline index =
   let item =
 try List.find (
   fun (name, { Index.arch = a }) ->
-name = arg && cmdline.arch = normalize_arch a
+name = arg && cmdline.arch = normalize_arch (Index.string_of_arch a)
 ) index
 with Not_found ->
   error (f_"cannot find os-version ‘%s’ with architecture ‘%s’.\nUse 
--list to list available guest types.")
@@ -252,7 +252,7 @@ let main () =
 List.iter (
   fun (name,
{ Index.revision; file_uri; proxy }) ->
-let template = name, cmdline.arch, revision in
+let template = name, Index.Arch cmdline.arch, revision in
 message (f_"Downloading: %s") file_uri;
 let progress_bar = not (quiet ()) in
 ignore (Downloader.download downloader ~template ~progress_bar
@@ -300,7 +300,7 @@ let main () =
   let template =
 let template, delete_on_exit =
   let { Index.revision; file_uri; proxy } = entry in
-  let template = arg, cmdline.arch, revision in
+  let template = arg, Index.Arch cmdline.arch, revision in
   message (f_"Downloading: %s") file_uri;
   let progress_bar = not (quiet ()) in
   Downloader.download downloader ~template ~progress_bar ~proxy
diff --git a/builder/cache.ml b/builder/cache.ml
index dbd222fda..d2693b943 100644
--- a/builder/cache.ml
+++ b/builder/cache.ml
@@ -41,7 +41,9 @@ let create ~directory =
   }
 
 let cache_of_name t name arch revision =
-  t.directory // sprintf "%s.%s.%s" name arch (string_of_revision revision)
+  t.directory // sprintf "%s.%s.%s" name
+(Index.string_of_arch arch)
+(string_of_revision revision)
 
 let is_cached t name arch revision =
   let filename = cache_of_name t name arch revision in
@@ -54,6 +56,6 @@ let print_item_status t ~header l =
   List.iter (
 fun (name, arch, revision) ->
   let cached = is_cached t name arch revision in
-  printf "%-24s %-10s %s\n" name arch
+  printf "%-24s %-10s %s\n" name (Index.string_of_arch arch)
 (if cached then s_"cached" else (*s_*)"no")
   ) l
diff --git a/builder/cache.mli b/builder/cache.mli
index f27fc235b..f88cbdf2f 100644
--- a/builder/cache.mli
+++ b/builder/cache.mli
@@ -27,16 +27,16 @@ type t
 val create : directory:string -> t
 (** Create the abstract type. *)
 
-val cache_of_name : t -> string -> string -> Utils.revision -> string
+val cache_of_name : t -> string -> Index.arch -> Utils.revision -> string
 (** [cache_of_name t name arch revision] return the filename
 of the cached file.  (Note: It doesn't check if the filename
 exists, this is just a simple string transformation). *)
 
-val is_cached : t -> string -> string -> Utils.revision -> bool
+val is_cached : t -> string -> Index.arch -> Utils.revision -> bool
 (** [is_cached t name arch revision] return whether the file with
 specified name, architecture and revision is cached. *)
 
-val print_item_status : t -> header:bool -> (string * string * Utils.revision) 
list -> unit
+val print_item_status : t -> header:bool -> (string * Index.arch * 
Utils.revision) list -> unit
 (** [print_item_status t header items] print the status in the cache
 of the specified items (which are tuples of name, architecture,
 and revision).
diff --git a/builder/downloader.mli b/builder/downloader.mli
index 7f39f7e36..a199440cb 100644
--- a/builder/downloader.mli
+++ b/builder/downloader.mli
@@ -27,7 +27,7 @@ type t
 val create : curl:string -> tmpdir:string -> cache:Cache.t option -> t
 (** Create the abstract type. *)
 
-val download : t -> ?template:(string*string*Utils.revision) -> 
?progress_bar:bool -> ?proxy:Curl.proxy -> uri -> (filename * bool)
+val download : t -> ?template:string * Index.arch * Utils.revision -> 
?progress_bar:bool -> ?proxy:Curl.proxy -> uri -> filename * bool
 (** Download the URI, returning the downloaded 

[Libguestfs] [PATCH v12 3/3] New tool: virt-builder-repository

[Cédric Bosdonnat]

ndex file>.
+
+=head2 Update images in an existing repository
+
+In this use case, an new image or a new revision of an existing image
+needs to be added to the repository. Place the corresponding image
+template files in the repository folder.
+
+To update the revision of an image, the file needs to have the same
+name than the existing one (without the C extension).
+
+As in the repository creation use case, a minimal fragment can be
+added to the index file for the automated mode. This can be done
+on the signed index even if it may sound a strange idea: the index
+will be signed again by the tool.
+
+To remove an image from the repository, just remove the corresponding
+image file before running virt-builder-repository.
+
+Then running the following command will complete and update the index
+file:
+
+ virt-builder-repository --gpg-key "j...@hacker.org" -i /path/to/folder
+
+virt-builder-repository works in a temporary folder inside the repository
+one. If anything wrong happens when running the tool, the repository is
+left untouched.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--help>
+
+Display help.
+
+=item B<--gpg> GPG
+
+Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary.  You can
+also use this to add gpg parameters, for example to specify an
+alternate home directory:
+
+ virt-builder-repository --gpg "gpg --homedir /tmp" [...]
+
+This can also be used to avoid gpg asking for the key passphrase:
+
+ virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...]
+
+=item B<-K> KEYID
+
+=item B<--gpg-key> KEYID
+
+Specify the GPG key to be used to sign the repository index file.
+If not provided, the index will left unsigned. C is used to
+identify the GPG key to use. This value is passed to gpg’s
+I<--default-key> option and can thus be an email address or a
+fingerprint.
+
+B: by default, virt-builder-repository searches for the key
+in the user’s GPG keyring.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data. Default values are computed from the disk
+image.
+
+When prompted for data, inputting C<-> corresponds to leaving the
+value empty. This can be used to avoid setting the default computed value.
+
+=item B<--keep-index>
+
+When using a GPG key, don’t remove the unsigned index.
+
+=item B<--no-compression>
+
+Don’t compress the template images.
+
+=item B<--machine-readable>
+
+This option is used to make the output more machine friendly
+when being parsed by other programs.  See
+L below.
+
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages.  This is the default
+when the output is a tty.  If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don’t print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 MACHINE READABLE OUTPUT
+
+The I<--machine-readable> option can be used to make the output more
+machine friendly, which is useful when calling virt-builder-repository from
+other programs, GUIs etc.
+
+Use the option on its own to query the capabilities of the
+virt-builder-repository binary.  Typical output looks like this:
+
+ $ virt-builder-repository --machine-readable
+ virt-builder-repository
+
+A list of features is printed, one per line, and the program exits
+with status 0.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdon...@suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod
index 74bf7bb11..1ed18a7c7 100644
--- a/builder/virt-builder.pod
+++ b/builder/virt-builder.pod
@@ -1319,6 +1319,9 @@ digital signature):
 The part in square brackets is the C, which is the same
 string that is used on the virt-builder command line to build that OS.
 
+The index file creation and signature can be eased with the
+L<virt-builder-repository(1)> tool.
+
 After preparing the C file in the correct format, clearsign it
 using the following command:
 
@@ -1875,6 +1878,7 @@ error.
 L<guestfs(3)>,
 L<guestfish(1)>,
 L<guestmount(1)>,
+L<virt-builder-repository(1)>,
 L<virt-copy-out(1)>,
 L<virt-customize(1)>,
 L<virt-get-kernel(1)>,
diff --git a/fish/guestfish.pod b/fish/guestfish.pod
index 11f2bbeb5..c37189bd8 100644
--- a/fish/guestfish.pod
+++ b/fish/guestfish.pod
@@ -1617,6 +1617,7 @@ L<guestfs(3)>,
 L<http://libguestfs.org/>,
 L<virt-alignment-scan(1)>,
 L<virt-builder(1)>,
+L<virt-builder-repository(1)>,
 L<virt-cat(1)>,
 L<virt-copy-in(1)>,
 L<virt-copy-out(1)>,
diff --git a/installcheck.sh.in b/installcheck.sh.in
index 6b05ab812..a4829cda6 100644
--- a/installcheck.sh.in
+++ b/installcheck.sh.in
@@ -46,6 +46,7 @@ cp @bindir@/guestfish fish/
 cp @bindir@/guestmountfuse/
 cp @bindir@/virt-alignment-scan   align/
 cp @bindir@/virt-builder  builder/
+cp @bindir@/virt-builder-repository builder/
 cp @bindir@/virt-cat  cat/
 cp @bindir@/virt-copy-in  fish/
 cp @bindir@/virt-copy-out fish/
diff --git a/lib/guestfs.pod b/lib/guestfs.pod
index 8d31f3200..55467e92e 100644
--- a/lib/guestfs.pod
+++ b/lib/guestfs.pod
@@ -3417,6 +3417,7 @@ L<guestfish(1)>,
 L<guestmount(1)>,
 L<virt-alignment-scan(1)>,
 L<virt-builder(1)>,
+L<virt-builder-repository(1)>,
 L<virt-cat(1)>,
 L<virt-copy-in(1)>,
 L<virt-copy-out(1)>,
-- 
2.14.3

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v12 1/3] builder: change arch type to distinguish guesses

[Cédric Bosdonnat]

Change Index.arch to the type (Arch of string | GuessedArch of string).

In a future commit, the index parser will allow arch not to be set
for some cases. Thus arch value will be guessed by inspecting the
image. However we need to distinguish between a set value and a guessed
one. Using this new type will help it:

match arch with
| Arch s-> (* This is a set value *)
| GuessedArch s -> (* This is a guessed value *)
---
 builder/builder.ml  |  8 +---
 builder/cache.ml|  8 
 builder/cache.mli   |  6 +++---
 builder/downloader.mli  |  2 +-
 builder/index.ml| 10 --
 builder/index.mli   |  5 -
 builder/index_parser.ml |  6 --
 builder/list_entries.ml | 13 ++---
 builder/simplestreams_parser.ml |  2 +-
 9 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index 3f7c79bc9..8a950cd8f 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -94,7 +94,9 @@ let selected_cli_item cmdline index =
   let item =
 try List.find (
   fun (name, { Index.arch = a }) ->
-name = arg && cmdline.arch = normalize_arch a
+match a with
+| Index.Arch a
+| Index.GuessedArch a -> name = arg && cmdline.arch = normalize_arch a
 ) index
 with Not_found ->
   error (f_"cannot find os-version ‘%s’ with architecture ‘%s’.\nUse 
--list to list available guest types.")
@@ -252,7 +254,7 @@ let main () =
 List.iter (
   fun (name,
{ Index.revision; file_uri; proxy }) ->
-let template = name, cmdline.arch, revision in
+let template = name, (Index.Arch cmdline.arch), revision in
 message (f_"Downloading: %s") file_uri;
 let progress_bar = not (quiet ()) in
 ignore (Downloader.download downloader ~template ~progress_bar
@@ -300,7 +302,7 @@ let main () =
   let template =
 let template, delete_on_exit =
   let { Index.revision; file_uri; proxy } = entry in
-  let template = arg, cmdline.arch, revision in
+  let template = arg, (Index.Arch cmdline.arch), revision in
   message (f_"Downloading: %s") file_uri;
   let progress_bar = not (quiet ()) in
   Downloader.download downloader ~template ~progress_bar ~proxy
diff --git a/builder/cache.ml b/builder/cache.ml
index dbd222fda..e313a8bcf 100644
--- a/builder/cache.ml
+++ b/builder/cache.ml
@@ -41,6 +41,10 @@ let create ~directory =
   }
 
 let cache_of_name t name arch revision =
+  let arch =
+match arch with
+| Index.Arch arch
+| Index.GuessedArch arch -> arch in
   t.directory // sprintf "%s.%s.%s" name arch (string_of_revision revision)
 
 let is_cached t name arch revision =
@@ -54,6 +58,10 @@ let print_item_status t ~header l =
   List.iter (
 fun (name, arch, revision) ->
   let cached = is_cached t name arch revision in
+  let arch =
+match arch with
+| Index.Arch arch
+| Index.GuessedArch arch -> arch in
   printf "%-24s %-10s %s\n" name arch
 (if cached then s_"cached" else (*s_*)"no")
   ) l
diff --git a/builder/cache.mli b/builder/cache.mli
index f27fc235b..f88cbdf2f 100644
--- a/builder/cache.mli
+++ b/builder/cache.mli
@@ -27,16 +27,16 @@ type t
 val create : directory:string -> t
 (** Create the abstract type. *)
 
-val cache_of_name : t -> string -> string -> Utils.revision -> string
+val cache_of_name : t -> string -> Index.arch -> Utils.revision -> string
 (** [cache_of_name t name arch revision] return the filename
 of the cached file.  (Note: It doesn't check if the filename
 exists, this is just a simple string transformation). *)
 
-val is_cached : t -> string -> string -> Utils.revision -> bool
+val is_cached : t -> string -> Index.arch -> Utils.revision -> bool
 (** [is_cached t name arch revision] return whether the file with
 specified name, architecture and revision is cached. *)
 
-val print_item_status : t -> header:bool -> (string * string * Utils.revision) 
list -> unit
+val print_item_status : t -> header:bool -> (string * Index.arch * 
Utils.revision) list -> unit
 (** [print_item_status t header items] print the status in the cache
 of the specified items (which are tuples of name, architecture,
 and revision).
diff --git a/builder/downloader.mli b/builder/downloader.mli
index 7f39f7e36..e2dd49f27 100644
--- a/builder/downloader.mli
+++ b/builder/downloader.mli
@@ -27,7 +27,7 @@ type t
 val create : curl:string -> tmpdir:string -> cache:Cache.t option -> t
 (** Create the abstract type. *)
 
-val download : t -> ?template:(string*string*Utils.revision) -> 
?progress_bar:bool -> ?proxy:Curl.proxy -> uri -> (filename * bool)
+val download : t -> ?template:(string*Index.arch*Utils.revision) -> 
?progress_bar:bool -> ?proxy:Curl.proxy -> uri -> (filename * bool)
 (** Download the URI, returning the downloaded filename and a

[Libguestfs] [PATCH v12 2/3] builder: add a template parameter to get_index

[Cédric Bosdonnat]

get_index now gets a new template parameter. Setting it to true will
make the index parsing less picky about missing important data. This
can be used to parse a partial index file.
---
 builder/index_parser.ml  | 44 
 builder/index_parser.mli |  5 -
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index a4d1e466e..bf1c6a557 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -25,7 +25,7 @@ open Utils
 open Printf
 open Unix
 
-let get_index ~downloader ~sigchecker { Sources.uri; proxy } =
+let get_index ~downloader ~sigchecker ?(template = false) { Sources.uri; proxy 
} =
   let corrupt_file () =
 error (f_"The index file downloaded from ‘%s’ is corrupt.\nYou need to ask 
the supplier of this file to fix it and upload a fixed version.") uri
   in
@@ -99,8 +99,23 @@ let get_index ~downloader ~sigchecker { Sources.uri; proxy } 
=
   let arch =
 try Index.Arch (List.assoc ("arch", None) fields)
 with Not_found ->
-  eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
-corrupt_file () in
+  if template then
+let g = new Guestfs.guestfs () in
+g#add_drive_ro file_uri;
+g#launch ();
+let roots = g#inspect_os () in
+let nroots = Array.length roots in
+if nroots <> 1 then (
+  eprintf (f_"%s: no ‘arch’ entry for %s and failed to guess 
it\n") prog n;
+  corrupt_file ()
+);
+let inspected_arch = g#inspect_get_arch (Array.get roots 0) in
+g#close();
+Index.GuessedArch inspected_arch
+  else (
+eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let signature_uri =
 try Some (make_absolute_uri (List.assoc ("sig", None) fields))
 with Not_found -> None in
@@ -112,21 +127,34 @@ let get_index ~downloader ~sigchecker { Sources.uri; 
proxy } =
   let revision =
 try Rev_int (int_of_string (List.assoc ("revision", None) fields))
 with
-| Not_found -> Rev_int 1
+| Not_found -> if template then Rev_int 0 else Rev_int 1
 | Failure _ ->
   eprintf (f_"%s: cannot parse ‘revision’ field for ‘%s’\n") prog 
n;
   corrupt_file () in
   let format =
 try Some (List.assoc ("format", None) fields) with Not_found -> 
None in
   let size =
+let get_image_size filepath =
+  (* If a compressed image manages to reach this code, qemu-img 
just
+ returns a virtual-size equal to actual-size *)
+  let infos = Utils.get_image_infos filepath in
+  Yajl.object_get_number "virtual-size" infos in
 try Int64.of_string (List.assoc ("size", None) fields)
 with
 | Not_found ->
-  eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file ()
+  if template then
+get_image_size file_uri
+  else (
+eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  )
 | Failure _ ->
-  eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file () in
+  if template then
+get_image_size file_uri
+  else (
+eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let compressed_size =
 try Some (Int64.of_string (List.assoc ("compressed_size", None) 
fields))
 with
diff --git a/builder/index_parser.mli b/builder/index_parser.mli
index f77ae9376..dc6b0b407 100644
--- a/builder/index_parser.mli
+++ b/builder/index_parser.mli
@@ -16,7 +16,10 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
Sources.source -> Index.index
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
?template:bool -> Sources.source -> Index.index
+(** [get_index download sigchecker template source] will parse the source
+index file into an index entry list. If the template flag is set to
+true, the parser will be less picky about missing values. *)
 
 val write_entry : out_channel -> (string * Index.entry) -> unit
 (** [write_entry chan entry] writes the index entry to the chan output
-- 
2.14.3

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v12 0/3] virt-builder-repository tool

[Cédric Bosdonnat]

Hi there!

Here is the latest version of the series including Richard's comments.
I also reworked the repository_main.ml code to avoid setting an
empty entry if not found.

Cédric Bosdonnat (3):
  builder: change arch type to distinguish guesses
  builder: add a template parameter to get_index
  New tool: virt-builder-repository

 .gitignore  |   4 +
 builder/Makefile.am |  87 -
 builder/builder.ml  |   8 +-
 builder/cache.ml|   8 +
 builder/cache.mli   |   6 +-
 builder/downloader.mli  |   2 +-
 builder/index.ml|  10 +-
 builder/index.mli   |   5 +-
 builder/index_parser.ml |  50 ++-
 builder/index_parser.mli|   5 +-
 builder/list_entries.ml |  13 +-
 builder/repository_main.ml  | 621 
 builder/simplestreams_parser.ml |   2 +-
 builder/test-docs.sh|   2 +
 builder/test-virt-builder-repository.sh |  98 +
 builder/utils.ml|   4 +
 builder/utils.mli   |   3 +
 builder/virt-builder-repository.pod | 213 +++
 builder/virt-builder.pod|   4 +
 fish/guestfish.pod  |   1 +
 installcheck.sh.in  |   1 +
 lib/guestfs.pod |   1 +
 22 files changed, 1121 insertions(+), 27 deletions(-)
 create mode 100644 builder/repository_main.ml
 create mode 100755 builder/test-virt-builder-repository.sh
 create mode 100644 builder/virt-builder-repository.pod

-- 
2.14.3

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v11 3/8] builder: change arch type to (string, string option) maybe.

[Cédric Bosdonnat]

In a future commit, the index parser will allow arch not to be set
for some cases. In such cases, it will be guessed by inspecting the
image, but we need to distinguish between a set value and a guessed
one. Using the '(string, string option) maybe' type will help it:

match arch with
| Either s -> (* This is a set value *)
| Or Some s -> (* This is a guessed value *)
| Or None -> (* No value and no guess *)
---
 builder/builder.ml  |  9 ++---
 builder/cache.ml| 10 ++
 builder/cache.mli   |  6 +++---
 builder/downloader.mli  |  2 +-
 builder/index.ml| 13 +++--
 builder/index.mli   |  7 ++-
 builder/index_parser.ml |  2 +-
 builder/list_entries.ml | 16 +---
 builder/simplestreams_parser.ml |  2 +-
 9 files changed, 52 insertions(+), 15 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index 3f7c79bc9..519cdbc79 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -94,7 +94,10 @@ let selected_cli_item cmdline index =
   let item =
 try List.find (
   fun (name, { Index.arch = a }) ->
-name = arg && cmdline.arch = normalize_arch a
+match a with
+| Either a
+| Or Some a -> name = arg && cmdline.arch = normalize_arch a
+| Or None -> false
 ) index
 with Not_found ->
   error (f_"cannot find os-version ‘%s’ with architecture ‘%s’.\nUse 
--list to list available guest types.")
@@ -252,7 +255,7 @@ let main () =
 List.iter (
   fun (name,
{ Index.revision; file_uri; proxy }) ->
-let template = name, cmdline.arch, revision in
+let template = name, (Either cmdline.arch), revision in
 message (f_"Downloading: %s") file_uri;
 let progress_bar = not (quiet ()) in
 ignore (Downloader.download downloader ~template ~progress_bar
@@ -300,7 +303,7 @@ let main () =
   let template =
 let template, delete_on_exit =
   let { Index.revision; file_uri; proxy } = entry in
-  let template = arg, cmdline.arch, revision in
+  let template = arg, (Either cmdline.arch), revision in
   message (f_"Downloading: %s") file_uri;
   let progress_bar = not (quiet ()) in
   Downloader.download downloader ~template ~progress_bar ~proxy
diff --git a/builder/cache.ml b/builder/cache.ml
index dbd222fda..c4a6b0578 100644
--- a/builder/cache.ml
+++ b/builder/cache.ml
@@ -41,6 +41,11 @@ let create ~directory =
   }
 
 let cache_of_name t name arch revision =
+  let arch =
+match arch with
+| Either arch
+| Or Some arch -> arch
+| Or None -> "" in
   t.directory // sprintf "%s.%s.%s" name arch (string_of_revision revision)
 
 let is_cached t name arch revision =
@@ -54,6 +59,11 @@ let print_item_status t ~header l =
   List.iter (
 fun (name, arch, revision) ->
   let cached = is_cached t name arch revision in
+  let arch =
+match arch with
+| Either arch
+| Or Some arch -> arch
+| Or None -> "" in
   printf "%-24s %-10s %s\n" name arch
 (if cached then s_"cached" else (*s_*)"no")
   ) l
diff --git a/builder/cache.mli b/builder/cache.mli
index f27fc235b..f88cbdf2f 100644
--- a/builder/cache.mli
+++ b/builder/cache.mli
@@ -27,16 +27,16 @@ type t
 val create : directory:string -> t
 (** Create the abstract type. *)
 
-val cache_of_name : t -> string -> string -> Utils.revision -> string
+val cache_of_name : t -> string -> Index.arch -> Utils.revision -> string
 (** [cache_of_name t name arch revision] return the filename
 of the cached file.  (Note: It doesn't check if the filename
 exists, this is just a simple string transformation). *)
 
-val is_cached : t -> string -> string -> Utils.revision -> bool
+val is_cached : t -> string -> Index.arch -> Utils.revision -> bool
 (** [is_cached t name arch revision] return whether the file with
 specified name, architecture and revision is cached. *)
 
-val print_item_status : t -> header:bool -> (string * string * Utils.revision) 
list -> unit
+val print_item_status : t -> header:bool -> (string * Index.arch * 
Utils.revision) list -> unit
 (** [print_item_status t header items] print the status in the cache
 of the specified items (which are tuples of name, architecture,
 and revision).
diff --git a/builder/downloader.mli b/builder/downloader.mli
index 7f39f7e36..e2dd49f27 100644
--- a/builder/downloader.mli
+++ b/builder/downloader.mli
@@ -27,7 +27,7 @@ type t
 val create : curl:string -> tmpdir:string -> cache:Cache.t option -> t
 (** Create the abstract type. *)
 
-val download : t -> ?template:(string*string*Utils.revision) -> 
?progress_bar:bool -> ?proxy:Curl.proxy -> uri -> (filename * bool)
+val download : t -> ?template:(string*Index.arch*Utils.revision) -> 
?progress_bar:bool -> ?proxy:Curl.proxy -> uri -> (filename * bool)
 (** Download the URI, returning the 

[Libguestfs] [PATCH v11 7/8] mllib: add XPath helper xpath_get_nodes

[Cédric Bosdonnat]

This function will allow more OCaml-ish processing of XPath queries
with multiple results.
---
 common/mltools/xpath_helpers.ml  |  9 +++
 common/mltools/xpath_helpers.mli |  4 +++
 v2v/output_libvirt.ml| 11 ++--
 v2v/test-harness/v2v_test_harness.ml | 51 +++-
 4 files changed, 30 insertions(+), 45 deletions(-)

diff --git a/common/mltools/xpath_helpers.ml b/common/mltools/xpath_helpers.ml
index 3afee8b21..d2bfd3fb9 100644
--- a/common/mltools/xpath_helpers.ml
+++ b/common/mltools/xpath_helpers.ml
@@ -40,3 +40,12 @@ let xpath_eval parsefn xpathctx expr =
 let xpath_string = xpath_eval identity
 let xpath_int = xpath_eval int_of_string
 let xpath_int64 = xpath_eval Int64.of_string
+
+let xpath_get_nodes xpathctx expr =
+  let obj = Xml.xpath_eval_expression xpathctx expr in
+  let nodes = ref [] in
+  for i = 0 to Xml.xpathobj_nr_nodes obj - 1 do
+let node = Xml.xpathobj_node obj i in
+push_front node nodes
+  done;
+  List.rev !nodes
diff --git a/common/mltools/xpath_helpers.mli b/common/mltools/xpath_helpers.mli
index 3a8190b05..3a2607aeb 100644
--- a/common/mltools/xpath_helpers.mli
+++ b/common/mltools/xpath_helpers.mli
@@ -25,3 +25,7 @@ val xpath_int : Xml.xpathctx -> string -> int option
 val xpath_int64 : Xml.xpathctx -> string -> int64 option
 (** Parse an xpath expression and return a string/int.  Returns
 [Some v], or [None] if the expression doesn't match. *)
+
+val xpath_get_nodes : Xml.xpathctx -> string -> Xml.node list
+(** Parse an XPath expression and return a list with the matching
+XML nodes. *)
diff --git a/v2v/output_libvirt.ml b/v2v/output_libvirt.ml
index 02b4d54ff..729f8b67a 100644
--- a/v2v/output_libvirt.ml
+++ b/v2v/output_libvirt.ml
@@ -55,15 +55,8 @@ let target_features_of_capabilities_doc doc arch =
 Xml.xpathctx_set_current_context xpathctx node;
 
 (* Get guest/features/* nodes. *)
-let obj = Xml.xpath_eval_expression xpathctx "features/*" in
-
-let features = ref [] in
-for i = 0 to Xml.xpathobj_nr_nodes obj - 1 do
-  let feature_node = Xml.xpathobj_node obj i in
-  let feature_name = Xml.node_name feature_node in
-  push_front feature_name features
-done;
-!features
+let features = xpath_get_nodes xpathctx "features/*" in
+List.map Xml.node_name features
   )
 
 class output_libvirt oc output_pool = object
diff --git a/v2v/test-harness/v2v_test_harness.ml 
b/v2v/test-harness/v2v_test_harness.ml
index ae0033dde..79e97a4b2 100644
--- a/v2v/test-harness/v2v_test_harness.ml
+++ b/v2v/test-harness/v2v_test_harness.ml
@@ -25,6 +25,7 @@ open Printf
 
 open Std_utils
 open Tools_utils
+open Xpath_helpers
 
 type test_plan = {
   guest_clock : float option;
@@ -90,29 +91,18 @@ let run ~test ?input_disk ?input_xml ?(test_plan = 
default_plan) () =
 g, root
   in
 
-  let nodes_of_xpathobj doc xpathobj =
-let nodes = ref [] in
-for i = 0 to Xml.xpathobj_nr_nodes xpathobj - 1 do
-  push_front (Xml.xpathobj_node xpathobj i) nodes
-done;
-List.rev !nodes
-  in
-
   let test_boot boot_disk boot_xml_doc =
 (* Modify boot XML (in memory). *)
 let xpathctx = Xml.xpath_new_context boot_xml_doc in
 
 (* Change  to something unique. *)
 let domname = "tmpv2v-" ^ test in
-let xpath = Xml.xpath_eval_expression xpathctx "/domain/name" in
-let nodes = nodes_of_xpathobj boot_xml_doc xpath in
+let nodes = xpath_get_nodes xpathctx "/domain/name" in
 List.iter (fun node -> Xml.node_set_content node domname) nodes;
 
 (* Limit the RAM used by the guest to 2GB. *)
-let xpath = Xml.xpath_eval_expression xpathctx "/domain/memory" in
-let nodes = nodes_of_xpathobj boot_xml_doc xpath in
-let xpath = Xml.xpath_eval_expression xpathctx "/domain/currentMemory" in
-let nodes = nodes @ nodes_of_xpathobj boot_xml_doc xpath in
+let nodes = xpath_get_nodes xpathctx "/domain/memory" in
+let nodes = nodes @ xpath_get_nodes xpathctx "/domain/currentMemory" in
 List.iter (
   fun node ->
 let i = int_of_string (Xml.node_as_string node) in
@@ -127,8 +117,7 @@ let run ~test ?input_disk ?input_xml ?(test_plan = 
default_plan) () =
 let adjustment = t -. time () in
 assert (adjustment <= 0.);
 let adjustment = int_of_float adjustment in
-let xpath = Xml.xpath_eval_expression xpathctx "/domain/clock" in
-let nodes = nodes_of_xpathobj boot_xml_doc xpath in
+let nodes = xpath_get_nodes xpathctx "/domain/clock" in
 let clock_node =
   match nodes with
   | [] ->
@@ -147,8 +136,7 @@ let run ~test ?input_disk ?input_xml ?(test_plan = 
default_plan) () =
 );
 
 (* Remove all devices except for a whitelist. *)
-let xpath = Xml.xpath_eval_expression xpathctx "/domain/devices/*" in
-let nodes = nodes_of_xpathobj boot_xml_doc xpath in
+let nodes = xpath_get_nodes xpathctx "/domain/devices/*" in
 List.iter (
   

[Libguestfs] [PATCH v11 4/8] builder: add Utils.get_image_infos function

[Cédric Bosdonnat]

This helper function calls qemu-img info on an image file and
returns the output as a JSON Yajl tree.

This function will be used in future commits.
---
 builder/Makefile.am | 2 +-
 builder/utils.ml| 6 ++
 builder/utils.mli   | 4 
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/builder/Makefile.am b/builder/Makefile.am
index 4a2f639c3..88392d327 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -61,12 +61,12 @@ SOURCES_MLI = \
yajl.mli
 
 SOURCES_ML = \
+   yajl.ml \
utils.ml \
pxzcat.ml \
setlocale.ml \
index.ml \
ini_reader.ml \
-   yajl.ml \
paths.ml \
languages.ml \
cache.ml \
diff --git a/builder/utils.ml b/builder/utils.ml
index acb6c2f4b..9fceee282 100644
--- a/builder/utils.ml
+++ b/builder/utils.ml
@@ -33,3 +33,9 @@ and revision =
 let string_of_revision = function
   | Rev_int n -> string_of_int n
   | Rev_string s -> s
+
+let get_image_infos filepath =
+  let qemuimg_cmd = "qemu-img info --output json " ^ (Std_utils.quote 
filepath) in
+  let lines = external_command qemuimg_cmd in
+  let line = String.concat "\n" lines in
+  Yajl.yajl_tree_parse line
diff --git a/builder/utils.mli b/builder/utils.mli
index 45385f713..4acde9f36 100644
--- a/builder/utils.mli
+++ b/builder/utils.mli
@@ -28,3 +28,7 @@ and revision =
 
 val string_of_revision : revision -> string
 (** Convert a {!revision} into a string. *)
+
+val get_image_infos : string -> Yajl.yajl_val 
+(** [get_image_infos path] Run qemu-img info on the image pointed at
+path as YAJL tree. *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v11 1/8] Ignore builder/*.out and *.img files

[Cédric Bosdonnat]

These ignores are covering test-console-ubuntu-12.04 test data.
---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index 36a193054..54dd5c6d0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -87,12 +87,14 @@ Makefile.in
 /build-aux/test-driver
 /build-aux/ylwrap
 /builder/.depend
+/builder/*.img
 /builder/index-parse.c
 /builder/index-parse.h
 /builder/index-scan.c
 /builder/libguestfs.conf
 /builder/opensuse.conf
 /builder/oUnit-*
+/builder/*.out
 /builder/*.qcow2
 /builder/stamp-virt-builder.pod
 /builder/stamp-virt-index-validate.pod
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v11 5/8] builder: add a template parameter to get_index

[Cédric Bosdonnat]

get_index now gets a new template parameter. Setting it to true will
make the index parsing less picky about missing important data. This
can be used to parse a partial index file.
---
 builder/index_parser.ml  | 46 ++
 builder/index_parser.mli |  5 -
 2 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index c715ccac7..7f64d0d98 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -25,7 +25,7 @@ open Utils
 open Printf
 open Unix
 
-let get_index ~downloader ~sigchecker { Sources.uri; proxy } =
+let get_index ~downloader ~sigchecker ?(template = false) { Sources.uri; proxy 
} =
   let corrupt_file () =
 error (f_"The index file downloaded from ‘%s’ is corrupt.\nYou need to ask 
the supplier of this file to fix it and upload a fixed version.") uri
   in
@@ -99,8 +99,25 @@ let get_index ~downloader ~sigchecker { Sources.uri; proxy } 
=
   let arch =
 try Either (List.assoc ("arch", None) fields)
 with Not_found ->
-  eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
-corrupt_file () in
+  if template then
+try
+  let g = new Guestfs.guestfs () in
+  g#add_drive_ro file_uri;
+  g#launch ();
+  let roots = g#inspect_os () in
+  let nroots = Array.length roots in
+  if nroots <> 1 then (
+eprintf (f_"%s: no ‘arch’ entry for %s and failed to guess 
it\n") prog n;
+corrupt_file ()
+  );
+  let inspected_arch = g#inspect_get_arch (Array.get roots 0) 
in
+  g#close();
+  Or (Some inspected_arch)
+with exn -> Or None
+  else (
+eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let signature_uri =
 try Some (make_absolute_uri (List.assoc ("sig", None) fields))
 with Not_found -> None in
@@ -112,21 +129,34 @@ let get_index ~downloader ~sigchecker { Sources.uri; 
proxy } =
   let revision =
 try Rev_int (int_of_string (List.assoc ("revision", None) fields))
 with
-| Not_found -> Rev_int 1
+| Not_found -> if template then Rev_int 0 else Rev_int 1
 | Failure _ ->
   eprintf (f_"%s: cannot parse ‘revision’ field for ‘%s’\n") prog 
n;
   corrupt_file () in
   let format =
 try Some (List.assoc ("format", None) fields) with Not_found -> 
None in
   let size =
+let get_image_size filepath =
+  (* If a compressed image manages to reach this code, qemu-img 
just
+ returns a virtual-size equal to actual-size *)
+  let infos = Utils.get_image_infos filepath in
+  Yajl.object_get_number "virtual-size" infos in
 try Int64.of_string (List.assoc ("size", None) fields)
 with
 | Not_found ->
-  eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file ()
+  if template then
+get_image_size file_uri
+  else (
+eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  )
 | Failure _ ->
-  eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file () in
+  if template then
+get_image_size file_uri
+  else (
+eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let compressed_size =
 try Some (Int64.of_string (List.assoc ("compressed_size", None) 
fields))
 with
diff --git a/builder/index_parser.mli b/builder/index_parser.mli
index b8d8ddf3d..324f4fc5a 100644
--- a/builder/index_parser.mli
+++ b/builder/index_parser.mli
@@ -16,4 +16,7 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
Sources.source -> Index.index
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
?template:bool -> Sources.source -> Index.index
+(** [get_index download sigchecker template source] will parse the source
+index file into an index entry list. If the template flag is set to
+true, the parser will be less picky about missing values. *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v11 8/8] New tool: virt-builder-repository

[Cédric Bosdonnat]

r the automated mode. This can be done
+on the signed index even if it may sound a strange idea: the index
+will be signed again by the tool.
+
+To remove an image from the repository, just remove the corresponding
+image file before running virt-builder-repository.
+
+Then running the following command will complete and update the index
+file:
+
+ virt-builder-repository --gpg-key "j...@hacker.org" -i /path/to/folder
+
+virt-builder-repository works in a temporary folder inside the repository
+one. If anything wrong happens when running the tool, the repository is
+left untouched.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--help>
+
+Display help.
+
+=item B<--gpg> GPG
+
+Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary.  You can
+also use this to add gpg parameters, for example to specify an
+alternate home directory:
+
+ virt-builder-repository --gpg "gpg --homedir /tmp" [...]
+
+This can also be used to avoid gpg asking for the key passphrase:
+
+ virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...]
+
+=item B<-K> KEYID
+
+=item B<--gpg-key> KEYID
+
+Specify the GPG key to be used to sign the repository index file.
+If not provided, the index will left unsigned. C is used to
+identify the GPG key to use. This value is passed to gpg’s
+I<--default-key> option and can thus be an email address or a
+fingerprint.
+
+B: by default, virt-builder-repository searches for the key
+in the user’s GPG keyring.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data. Default values are computed from the disk
+image.
+
+When prompted for data, inputting C<-> corresponds to leaving the
+value empty. This can be used to avoid setting the default computed value.
+
+=item B<--keep-index>
+
+When using a GPG key, don’t remove the unsigned index.
+
+=item B<--no-compression>
+
+Don’t compress the template images.
+
+=item B<--machine-readable>
+
+This option is used to make the output more machine friendly
+when being parsed by other programs.  See
+L below.
+
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages.  This is the default
+when the output is a tty.  If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don’t print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 MACHINE READABLE OUTPUT
+
+The I<--machine-readable> option can be used to make the output more
+machine friendly, which is useful when calling virt-builder-repository from
+other programs, GUIs etc.
+
+Use the option on its own to query the capabilities of the
+virt-builder-repository binary.  Typical output looks like this:
+
+ $ virt-builder-repository --machine-readable
+ virt-builder-repository
+
+A list of features is printed, one per line, and the program exits
+with status 0.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdon...@suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod
index 74bf7bb11..1ed18a7c7 100644
--- a/builder/virt-builder.pod
+++ b/builder/virt-builder.pod
@@ -1319,6 +1319,9 @@ digital signature):
 The part in square brackets is the C, which is the same
 string that is used on the virt-builder command line to build that OS.
 
+The index file creation and signature can be eased with the
+L<virt-builder-repository(1)> tool.
+
 After preparing the C file in the correct format, clearsign it
 using the following command:
 
@@ -1875,6 +1878,7 @@ error.
 L<guestfs(3)>,
 L<guestfish(1)>,
 L<guestmount(1)>,
+L<virt-builder-repository(1)>,
 L<virt-copy-out(1)>,
 L<virt-customize(1)>,
 L<virt-get-kernel(1)>,
diff --git a/fish/guestfish.pod b/fish/guestfish.pod
index 11f2bbeb5..c37189bd8 100644
--- a/fish/guestfish.pod
+++ b/fish/guestfish.pod
@@ -1617,6 +1617,7 @@ L<guestfs(3)>,
 L<http://libguestfs.org/>,
 L<virt-alignment-scan(1)>,
 L<virt-builder(1)>,
+L<virt-builder-repository(1)>,
 L<virt-cat(1)>,
 L<virt-copy-in(1)>,
 L<virt-copy-out(1)>,
diff --git a/installcheck.sh.in b/installcheck.sh.in
index 6b05ab812..a4829cda6 100644
--- a/installcheck.sh.in
+++ b/installcheck.sh.in
@@ -46,6 +46,7 @@ cp @bindir@/guestfish fish/
 cp @bindir@/guestmountfuse/
 cp @bindir@/virt-alignment-scan   align/
 cp @bindir@/virt-builder  builder/
+cp @bindir@/virt-builder-repository builder/
 cp @bindir@/virt-cat  cat/
 cp @bindir@/virt-copy-in  fish/
 cp @bindir@/virt-copy-out fish/
diff --git a/lib/guestfs.pod b/lib/guestfs.pod
index 8d31f3200..55467e92e 100644
--- a/lib/guestfs.pod
+++ b/lib/guestfs.pod
@@ -3417,6 +3417,7 @@ L<guestfish(1)>,
 L<guestmount(1)>,
 L<virt-alignment-scan(1)>,
 L<virt-builder(1)>,
+L<virt-builder-repository(1)>,
 L<virt-cat(1)>,
 L<virt-copy-in(1)>,
 L<virt-copy-out(1)>,
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v11 6/8] builder: add Index.write_entry function

[Cédric Bosdonnat]

Add a function to properly write virt-builder source index entries.
Note that this function is very similar to Index.print_entry that is
meant for debugging purposes.
---
 .gitignore|   1 +
 builder/Makefile.am   |  36 +++-
 builder/index.mli |   3 +
 builder/index_parser.ml   |  46 +++
 builder/index_parser.mli  |   4 ++
 builder/index_parser_tests.ml | 130 ++
 6 files changed, 218 insertions(+), 2 deletions(-)
 create mode 100644 builder/index_parser_tests.ml

diff --git a/.gitignore b/.gitignore
index 59bf52f2b..30165d59e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -108,6 +108,7 @@ Makefile.in
 /builder/virt-index-validate
 /builder/virt-index-validate.1
 /builder/*.xz
+/builder/index_parser_tests
 /builder/yajl_tests
 /cat/stamp-virt-*.pod
 /cat/virt-cat
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 88392d327..e4a347a09 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -239,13 +239,36 @@ yajl_tests_BOBJECTS = \
yajl_tests.cmo
 yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx)
 
+index_parser_tests_SOURCES = \
+   index-scan.c \
+   index-struct.c \
+   index-parser-c.c \
+   index-parse.c
+index_parser_tests_CPPFLAGS = $(virt_builder_CPPFLAGS)
+index_parser_tests_BOBJECTS = \
+   utils.cmo \
+   cache.cmo \
+   downloader.cmo \
+   sigchecker.cmo \
+   index.cmo \
+   ini_reader.cmo \
+   index_parser.cmo \
+   index_parser_tests.cmo
+index_parser_tests_XOBJECTS = $(index_parser_tests_BOBJECTS:.cmo=.cmx)
+
 # Can't call the following as _OBJECTS because automake gets confused.
 if HAVE_OCAMLOPT
 yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS)
 yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_XOBJECTS)
+index_parser_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 else
 yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS)
 yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_BOBJECTS)
+index_parser_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 endif
 
 yajl_tests_DEPENDENCIES = \
@@ -261,6 +284,15 @@ yajl_tests_LINK = \
  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
  $(yajl_tests_THEOBJECTS) -o $@
 
+index_parser_tests_DEPENDENCIES = \
+   $(index_parser_tests_THEOBJECTS) \
+   ../common/mltools/mltools.$(MLARCHIVE) \
+   $(top_srcdir)/ocaml-link.sh
+index_parser_tests_LINK = \
+   $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
+ $(index_parser_tests_THEOBJECTS) -o $@
+
 TESTS = \
test-docs.sh \
test-virt-builder-list.sh \
@@ -274,8 +306,8 @@ if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
 endif ENABLE_APPLIANCE
 if HAVE_OCAML_PKG_OUNIT
-check_PROGRAMS += yajl_tests
-TESTS += yajl_tests
+check_PROGRAMS += yajl_tests index_parser_tests
+TESTS += yajl_tests index_parser_tests
 endif
 
 check-valgrind:
diff --git a/builder/index.mli b/builder/index.mli
index 43d5485fb..3ed633ddc 100644
--- a/builder/index.mli
+++ b/builder/index.mli
@@ -44,3 +44,6 @@ and entry = {
 }
 
 val print_entry : out_channel -> (string * entry) -> unit
+(** Debugging helper function dumping an index entry to a stream.
+To write entries for non-debugging purpose, use the
+[Index_parser.write_entry] function. *)
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index 7f64d0d98..0fc3ecc06 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -256,3 +256,49 @@ let get_index ~downloader ~sigchecker ?(template = false) 
{ Sources.uri; proxy }
   in
 
   get_index ()
+
+let write_entry chan (name, { Index.printable_name; file_uri; arch; osinfo;
+  signature_uri; checksums; revision; format; size;
+  compressed_size; expand; lvexpand; notes;
+  aliases; hidden}) =
+  let fp fs = fprintf chan fs in
+  fp "[%s]\n" name;
+  Option.may (fp "name=%s\n") printable_name;
+  Option.may (fp "osinfo=%s\n") osinfo;
+  fp "file=%s\n" file_uri;
+  match arch with
+  | Either arch
+  | Or Some arch -> fp "arch=%s\n" arch
+  | Or None -> ();
+  Option.may (fp "sig=%s\n") signature_uri;
+  (match checksums with
+  | None -> ()
+  | Some checksums ->
+List.iter (
+  fun c ->
+fp "checksum[%s]=%s\n"
+  (Checksums.string_of_csum_t c) (Checksums.string_of_csum c)
+) checksums
+  );
+  fp "revision=%s\n" (string_of_revision revision);
+  Option.may (fp "format=%s\n") format;
+  fp "size=%Ld\n" size;
+  Option.may (fp "compressed_size=%Ld\n") compressed_size;
+  Option.may (fp "expand=%s\n") expand;
+  Option.may (fp "lvexpand=%s\n") lvexpand;
+
+  let format_notes notes =
+

[Libguestfs] [PATCH v11 2/8] builder: add simple OCaml osinfo-db reader

[Cédric Bosdonnat]

From: Pino Toscano 

Add a simple OCaml-based implementation of reader of the osinfo-db:
the only interface is an iterator that invokes an user-supplied
function with each XML file found.

This implementation behaves like the current C implementation, and
still supports the old libosinfo db.
---
 .gitignore  |  1 +
 builder/Makefile.am |  4 +++
 builder/osinfo.ml   | 76 +
 builder/osinfo.mli  | 22 
 4 files changed, 103 insertions(+)
 create mode 100644 builder/osinfo.ml
 create mode 100644 builder/osinfo.mli

diff --git a/.gitignore b/.gitignore
index 54dd5c6d0..59bf52f2b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -93,6 +93,7 @@ Makefile.in
 /builder/index-scan.c
 /builder/libguestfs.conf
 /builder/opensuse.conf
+/builder/osinfo_config.ml
 /builder/oUnit-*
 /builder/*.out
 /builder/*.qcow2
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 7aa97e31d..4a2f639c3 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -213,6 +213,10 @@ CLEANFILES += *.qcow2 *.xz
 
 check_DATA = $(disk_images)
 
+osinfo_config.ml: Makefile
+   echo 'let libosinfo_db_path = "$(datadir)/libosinfo/db"' > $@-t
+   mv $@-t $@
+
 fedora.qcow2: ../test-data/phony-guests/fedora.img
rm -f $@ $@-t
qemu-img convert -f raw -O qcow2 $< $@-t
diff --git a/builder/osinfo.ml b/builder/osinfo.ml
new file mode 100644
index 0..9d1b0169e
--- /dev/null
+++ b/builder/osinfo.ml
@@ -0,0 +1,76 @@
+(* virt-builder
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open Tools_utils
+open Osinfo_config
+
+let rec fold fn base =
+  let locations =
+(* (1) Try the shared osinfo directory, using either the
+ * $OSINFO_SYSTEM_DIR envvar or its default value.
+ *)
+let dir =
+  try Sys.getenv "OSINFO_SYSTEM_DIR"
+  with Not_found -> "/usr/share/osinfo" in
+((dir // "os"), read_osinfo_db_three_levels) ::
+
+  (* (2) Try the libosinfo directory, using the newer three-directory
+   * layout ($LIBOSINFO_DB_PATH / "os" / $group-ID / [file.xml]).
+   *)
+  let path = Osinfo_config.libosinfo_db_path // "os" in
+  (path, read_osinfo_db_three_levels) ::
+
+(* (3) Try the libosinfo directory, using the old flat directory
+ * layout ($LIBOSINFO_DB_PATH / "oses" / [file.xml]).
+ *)
+let path = Osinfo_config.libosinfo_db_path // "oses" in
+(path, read_osinfo_db_flat) :: [] in
+
+
+  let files =
+List.flatten (
+  filter_map (
+  fun (path, f) ->
+if is_directory path then Some (f path)
+(* This is not an error: RHBZ#948324. *)
+else None
+  ) locations
+  ) in
+
+  List.fold_left fn base files
+
+and read_osinfo_db_three_levels path =
+  debug "osinfo: loading 3-level-directories database from %s" path;
+  let entries = Array.to_list (Sys.readdir path) in
+  let entries = List.map ((//) path) entries in
+  (* Iterate only on directories. *)
+  let entries = List.filter is_directory entries in
+  List.flatten (List.map read_osinfo_db_directory entries)
+
+and read_osinfo_db_flat path =
+  debug "osinfo: loading flat database from %s" path;
+  read_osinfo_db_directory path
+
+and read_osinfo_db_directory path =
+  let entries = Sys.readdir path in
+  let entries = Array.to_list entries in
+  let entries = List.filter (fun x -> Filename.check_suffix x ".xml") entries 
in
+  let entries = List.map ((//) path) entries in
+  let entries = List.filter is_regular_file entries in
+  entries
diff --git a/builder/osinfo.mli b/builder/osinfo.mli
new file mode 100644
index 0..fa179509d
--- /dev/null
+++ b/builder/osinfo.mli
@@ -0,0 +1,22 @@
+(* virt-builder
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU 

[Libguestfs] [PATCH v11 2/6] builder: rename docs test script

[Cédric Bosdonnat]

Rename test-virt-builder-docs.sh into test-docs.sh to include test
for another tool's documentation.
---
 builder/Makefile.am | 4 ++--
 builder/{test-virt-builder-docs.sh => test-docs.sh} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (100%)

diff --git a/builder/Makefile.am b/builder/Makefile.am
index e315bc785..4a2f639c3 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -28,7 +28,7 @@ EXTRA_DIST = \
test-simplestreams/streams/v1/index.json \
test-simplestreams/streams/v1/net.cirros-cloud_released_download.json \
test-virt-builder.sh \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-builder-list-simplestreams.sh \
test-virt-builder-planner.sh \
@@ -262,7 +262,7 @@ yajl_tests_LINK = \
  $(yajl_tests_THEOBJECTS) -o $@
 
 TESTS = \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-index-validate.sh \
$(SLOW_TESTS)
diff --git a/builder/test-virt-builder-docs.sh b/builder/test-docs.sh
similarity index 100%
rename from builder/test-virt-builder-docs.sh
rename to builder/test-docs.sh
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v11 6/6] New tool: virt-builder-repository

[Cédric Bosdonnat]

mage, the file needs to have the same
+name than the existing one (without the C extension).
+
+As in the repository creation use case, a minimal fragment can be
+added to the index file for the automated mode. This can be done
+on the signed index even if it may sound a strange idea: the index
+will be signed again by the tool.
+
+To remove an image from the repository, just remove the corresponding
+image file before running virt-builder-repository.
+
+Then running the following command will complete and update the index
+file:
+
+ virt-builder-repository --gpg-key "j...@hacker.org" -i /path/to/folder
+
+virt-builder-repository works in a temporary folder inside the repository
+one. If anything wrong happens when running the tool, the repository is
+left untouched.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--help>
+
+Display help.
+
+=item B<--gpg> GPG
+
+Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary.  You can
+also use this to add gpg parameters, for example to specify an
+alternate home directory:
+
+ virt-builder-repository --gpg "gpg --homedir /tmp" [...]
+
+This can also be used to avoid gpg asking for the key passphrase:
+
+ virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...]
+
+=item B<-K> KEYID
+
+=item B<--gpg-key> KEYID
+
+Specify the GPG key to be used to sign the repository index file.
+If not provided, the index will left unsigned. C is used to
+identify the GPG key to use. This value is passed to gpg’s
+I<--default-key> option and can thus be an email address or a
+fingerprint.
+
+B: by default, virt-builder-repository searches for the key
+in the user’s GPG keyring.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data. Default values are computed from the disk
+image.
+
+When prompted for data, inputting C<-> corresponds to leaving the
+value empty. This can be used to avoid setting the default computed value.
+
+=item B<--keep-index>
+
+When using a GPG key, don’t remove the unsigned index.
+
+=item B<--no-compression>
+
+Don’t compress the template images.
+
+=item B<--machine-readable>
+
+This option is used to make the output more machine friendly
+when being parsed by other programs.  See
+L below.
+
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages.  This is the default
+when the output is a tty.  If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don’t print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 MACHINE READABLE OUTPUT
+
+The I<--machine-readable> option can be used to make the output more
+machine friendly, which is useful when calling virt-builder-repository from
+other programs, GUIs etc.
+
+Use the option on its own to query the capabilities of the
+virt-builder-repository binary.  Typical output looks like this:
+
+ $ virt-builder-repository --machine-readable
+ virt-builder-repository
+
+A list of features is printed, one per line, and the program exits
+with status 0.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdon...@suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v11 3/6] builder: add a template parameter to get_index

[Cédric Bosdonnat]

get_index now gets a new template parameter. Setting it to true will
make the index parsing less picky about missing important data. This
can be used to parse a partial index file.
---
 builder/builder.ml   |  2 +-
 builder/index_parser.ml  | 26 ++
 builder/index_parser.mli |  5 -
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index 3d0dbe7a8..a19eb2d7b 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -208,7 +208,7 @@ let main () =
   ~tmpdir in
   match source.Sources.format with
   | Sources.FormatNative ->
-Index_parser.get_index ~downloader ~sigchecker source
+Index_parser.get_index ~downloader ~sigchecker ~template:false 
source
   | Sources.FormatSimpleStreams ->
 Simplestreams_parser.get_index ~downloader ~sigchecker source
   ) sources
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index d6a4e2e86..6f611a7f5 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -25,7 +25,7 @@ open Utils
 open Printf
 open Unix
 
-let get_index ~downloader ~sigchecker { Sources.uri; proxy } =
+let get_index ~downloader ~sigchecker ~template { Sources.uri; proxy } =
   let corrupt_file () =
 error (f_"The index file downloaded from ‘%s’ is corrupt.\nYou need to ask 
the supplier of this file to fix it and upload a fixed version.") uri
   in
@@ -99,8 +99,10 @@ let get_index ~downloader ~sigchecker { Sources.uri; proxy } 
=
   let arch =
 try List.assoc ("arch", None) fields
 with Not_found ->
-  eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
-corrupt_file () in
+  if template then "" else (
+eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let signature_uri =
 try Some (make_absolute_uri (List.assoc ("sig", None) fields))
 with Not_found -> None in
@@ -112,7 +114,7 @@ let get_index ~downloader ~sigchecker { Sources.uri; proxy 
} =
   let revision =
 try Rev_int (int_of_string (List.assoc ("revision", None) fields))
 with
-| Not_found -> Rev_int 1
+| Not_found -> if template then Rev_int 0 else Rev_int 1
 | Failure _ ->
   eprintf (f_"%s: cannot parse ‘revision’ field for ‘%s’\n") prog 
n;
   corrupt_file () in
@@ -122,11 +124,19 @@ let get_index ~downloader ~sigchecker { Sources.uri; 
proxy } =
 try Int64.of_string (List.assoc ("size", None) fields)
 with
 | Not_found ->
-  eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file ()
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  )
 | Failure _ ->
-  eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file () in
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let compressed_size =
 try Some (Int64.of_string (List.assoc ("compressed_size", None) 
fields))
 with
diff --git a/builder/index_parser.mli b/builder/index_parser.mli
index b8d8ddf3d..a93e20825 100644
--- a/builder/index_parser.mli
+++ b/builder/index_parser.mli
@@ -16,4 +16,7 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
Sources.source -> Index.index
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
template:bool -> Sources.source -> Index.index
+(** [get_index download sigchecker template source] will parse the source
+index file into an index entry list. If the template flag is set to
+true, the parser will be less picky about missing values. *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v11 5/6] mllib: add XPath helper xpath_get_nodes()

[Cédric Bosdonnat]

This function will allow more OCaml-ish processing of XPath queries
with multiple results.
---
 common/mltools/xpath_helpers.ml  | 9 +
 common/mltools/xpath_helpers.mli | 4 
 2 files changed, 13 insertions(+)

diff --git a/common/mltools/xpath_helpers.ml b/common/mltools/xpath_helpers.ml
index 05fad89a4..a79733486 100644
--- a/common/mltools/xpath_helpers.ml
+++ b/common/mltools/xpath_helpers.ml
@@ -52,3 +52,12 @@ let xpath_eval_default parsefn xpath expr default =
 let xpath_string_default = xpath_eval_default identity
 let xpath_int_default = xpath_eval_default int_of_string
 let xpath_int64_default = xpath_eval_default Int64.of_string
+
+let xpath_get_nodes xpathctx expr =
+  let obj = Xml.xpath_eval_expression xpathctx expr in
+  let nodes = ref [] in
+  for i = 0 to Xml.xpathobj_nr_nodes obj - 1 do
+let node = Xml.xpathobj_node obj i in
+push_back nodes node
+  done;
+  !nodes
diff --git a/common/mltools/xpath_helpers.mli b/common/mltools/xpath_helpers.mli
index 7434ba645..83c770281 100644
--- a/common/mltools/xpath_helpers.mli
+++ b/common/mltools/xpath_helpers.mli
@@ -31,3 +31,7 @@ val xpath_int_default : Xml.xpathctx -> string -> int -> int
 val xpath_int64_default : Xml.xpathctx -> string -> int64 -> int64
 (** Parse an xpath expression and return a string/int; if the expression
 doesn't match, return the default. *)
+
+val xpath_get_nodes : Xml.xpathctx -> string -> Xml.node list
+(** Parse an XPath expression and return a list with the matching
+XML nodes. *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v11 0/6] virt-builder-repository

[Cédric Bosdonnat]

Hi there,

This is an update of the series. Just to rebase it on top of
Rich's latest changes.

Cédric Bosdonnat (5):
  builder: rename docs test script
  builder: add a template parameter to get_index
  builder: add Index.write_entry function
  mllib: add XPath helper xpath_get_nodes()
  New tool: virt-builder-repository

Pino Toscano (1):
  builder: add simple OCaml osinfo-db reader

 .gitignore |   5 +
 builder/Makefile.am| 128 -
 builder/builder.ml |   2 +-
 builder/index.mli  |   3 +
 builder/index_parser.ml|  80 ++-
 builder/index_parser.mli   |   9 +-
 builder/index_parser_tests.ml  | 129 +
 builder/osinfo.ml  |  80 +++
 builder/osinfo.mli |  22 +
 builder/repository_main.ml | 597 +
 .../{test-virt-builder-docs.sh => test-docs.sh}|   2 +
 builder/virt-builder-repository.pod| 213 
 common/mltools/xpath_helpers.ml|   9 +
 common/mltools/xpath_helpers.mli   |   4 +
 14 files changed, 1268 insertions(+), 15 deletions(-)
 create mode 100644 builder/index_parser_tests.ml
 create mode 100644 builder/osinfo.ml
 create mode 100644 builder/osinfo.mli
 create mode 100644 builder/repository_main.ml
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (93%)
 create mode 100644 builder/virt-builder-repository.pod

-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v11 1/6] builder: add simple OCaml osinfo-db reader

[Cédric Bosdonnat]

From: Pino Toscano 

Add a simple OCaml-based implementation of reader of the osinfo-db:
the only interface is an iterator that invokes an user-supplied
function with each XML file found.

This implementation behaves like the current C implementation, and
still supports the old libosinfo db.
---
 .gitignore  |  1 +
 builder/Makefile.am |  4 +++
 builder/osinfo.ml   | 80 +
 builder/osinfo.mli  | 22 +++
 4 files changed, 107 insertions(+)
 create mode 100644 builder/osinfo.ml
 create mode 100644 builder/osinfo.mli

diff --git a/.gitignore b/.gitignore
index 36a193054..75e69edbd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -92,6 +92,7 @@ Makefile.in
 /builder/index-scan.c
 /builder/libguestfs.conf
 /builder/opensuse.conf
+/builder/osinfo_config.ml
 /builder/oUnit-*
 /builder/*.qcow2
 /builder/stamp-virt-builder.pod
diff --git a/builder/Makefile.am b/builder/Makefile.am
index f3becd51d..e315bc785 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -213,6 +213,10 @@ CLEANFILES += *.qcow2 *.xz
 
 check_DATA = $(disk_images)
 
+osinfo_config.ml: Makefile
+   echo 'let libosinfo_db_path = "$(datadir)/libosinfo/db"' > $@-t
+   mv $@-t $@
+
 fedora.qcow2: ../test-data/phony-guests/fedora.img
rm -f $@ $@-t
qemu-img convert -f raw -O qcow2 $< $@-t
diff --git a/builder/osinfo.ml b/builder/osinfo.ml
new file mode 100644
index 0..bfafdbde9
--- /dev/null
+++ b/builder/osinfo.ml
@@ -0,0 +1,80 @@
+(* virt-builder
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open Tools_utils
+open Osinfo_config
+
+let rec iterate_db fn =
+  let locations = ref [] in
+
+  (* (1) Try the shared osinfo directory, using either the
+   * $OSINFO_SYSTEM_DIR envvar or its default value.
+   *)
+  let () =
+let dir =
+  try Sys.getenv "OSINFO_SYSTEM_DIR"
+  with Not_found -> "/usr/share/osinfo" in
+push_back locations ((dir // "os"), read_osinfo_db_three_levels)
+  in
+
+  (* (2) Try the libosinfo directory, using the newer three-directory
+   * layout ($LIBOSINFO_DB_PATH / "os" / $group-ID / [file.xml]).
+   *)
+  let () =
+let path = Osinfo_config.libosinfo_db_path // "os" in
+push_back locations (path, read_osinfo_db_three_levels)
+  in
+
+  (* (3) Try the libosinfo directory, using the old flat directory
+   * layout ($LIBOSINFO_DB_PATH / "oses" / [file.xml]).
+   *)
+  let () =
+let path = Osinfo_config.libosinfo_db_path // "oses" in
+push_back locations (path, read_osinfo_db_flat)
+  in
+
+  let rec loop = function
+| (path, f) :: paths ->
+  if is_directory path then f fn path
+  (* This is not an error: RHBZ#948324. *)
+  else loop paths
+| [] -> ()
+  in
+
+  loop !locations
+
+and read_osinfo_db_three_levels fn path =
+  debug "osinfo: loading 3-level-directories database from %s" path;
+  let entries = Array.to_list (Sys.readdir path) in
+  let entries = List.map ((//) path) entries in
+  (* Iterate only on directories. *)
+  let entries = List.filter is_directory entries in
+  List.iter (read_osinfo_db_directory fn) entries
+
+and read_osinfo_db_flat fn path =
+  debug "osinfo: loading flat database from %s" path;
+  read_osinfo_db_directory fn path
+
+and read_osinfo_db_directory fn path =
+  let entries = Array.to_list (Sys.readdir path) in
+  let entries = List.filter (fun x -> Filename.check_suffix x ".xml") entries 
in
+  let entries = List.map ((//) path) entries in
+  let entries = List.filter is_regular_file entries in
+  List.iter fn entries
+
diff --git a/builder/osinfo.mli b/builder/osinfo.mli
new file mode 100644
index 0..949d776a9
--- /dev/null
+++ b/builder/osinfo.mli
@@ -0,0 +1,22 @@
+(* virt-builder
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * 

[Libguestfs] [PATCH v11 4/6] builder: add Index.write_entry function

[Cédric Bosdonnat]

Add a function to properly write virt-builder source index entries.
Note that this function is very similar to Index.print_entry that is
meant for debugging purposes.
---
 .gitignore|   1 +
 builder/Makefile.am   |  36 +++-
 builder/index.mli |   3 +
 builder/index_parser.ml   |  54 ++
 builder/index_parser.mli  |   4 ++
 builder/index_parser_tests.ml | 129 ++
 6 files changed, 225 insertions(+), 2 deletions(-)
 create mode 100644 builder/index_parser_tests.ml

diff --git a/.gitignore b/.gitignore
index 75e69edbd..c68bc9088 100644
--- a/.gitignore
+++ b/.gitignore
@@ -106,6 +106,7 @@ Makefile.in
 /builder/virt-index-validate
 /builder/virt-index-validate.1
 /builder/*.xz
+/builder/index_parser_tests
 /builder/yajl_tests
 /cat/stamp-virt-*.pod
 /cat/virt-cat
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 4a2f639c3..fa049be4d 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -239,13 +239,36 @@ yajl_tests_BOBJECTS = \
yajl_tests.cmo
 yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx)
 
+index_parser_tests_SOURCES = \
+   index-scan.c \
+   index-struct.c \
+   index-parser-c.c \
+   index-parse.c
+index_parser_tests_CPPFLAGS = $(virt_builder_CPPFLAGS)
+index_parser_tests_BOBJECTS = \
+   utils.cmo \
+   cache.cmo \
+   downloader.cmo \
+   sigchecker.cmo \
+   index.cmo \
+   ini_reader.cmo \
+   index_parser.cmo \
+   index_parser_tests.cmo
+index_parser_tests_XOBJECTS = $(index_parser_tests_BOBJECTS:.cmo=.cmx)
+
 # Can't call the following as _OBJECTS because automake gets confused.
 if HAVE_OCAMLOPT
 yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS)
 yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_XOBJECTS)
+index_parser_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 else
 yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS)
 yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_BOBJECTS)
+index_parser_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 endif
 
 yajl_tests_DEPENDENCIES = \
@@ -261,6 +284,15 @@ yajl_tests_LINK = \
  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
  $(yajl_tests_THEOBJECTS) -o $@
 
+index_parser_tests_DEPENDENCIES = \
+   $(index_parser_tests_THEOBJECTS) \
+   ../mllib/mllib.$(MLARCHIVE) \
+   $(top_srcdir)/ocaml-link.sh
+index_parser_tests_LINK = \
+   $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
+ $(index_parser_tests_THEOBJECTS) -o $@
+
 TESTS = \
test-docs.sh \
test-virt-builder-list.sh \
@@ -274,8 +306,8 @@ if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
 endif ENABLE_APPLIANCE
 if HAVE_OCAML_PKG_OUNIT
-check_PROGRAMS += yajl_tests
-TESTS += yajl_tests
+check_PROGRAMS += yajl_tests index_parser_tests
+TESTS += yajl_tests index_parser_tests
 endif
 
 check-valgrind:
diff --git a/builder/index.mli b/builder/index.mli
index ff5ec4a35..6202d636e 100644
--- a/builder/index.mli
+++ b/builder/index.mli
@@ -39,3 +39,6 @@ and entry = {
 }
 
 val print_entry : out_channel -> (string * entry) -> unit
+(** Debugging helper function dumping an index entry to a stream.
+To write entries for non-debugging purpose, use the
+[Index_parser.write_entry] function. *)
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index 6f611a7f5..4405eca12 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -236,3 +236,57 @@ let get_index ~downloader ~sigchecker ~template { 
Sources.uri; proxy } =
   in
 
   get_index ()
+
+let write_entry chan (name, { Index.printable_name = printable_name;
+  file_uri = file_uri;
+  arch = arch;
+  osinfo = osinfo;
+  signature_uri = signature_uri;
+  checksums = checksums;
+  revision = revision;
+  format = format;
+  size = size;
+  compressed_size = compressed_size;
+  expand = expand;
+  lvexpand = lvexpand;
+  notes = notes;
+  aliases = aliases;
+  hidden = hidden }) =
+  let fp fs = fprintf chan fs in
+  fp "[%s]\n" name;
+  may (fp "name=%s\n") printable_name;
+  may (fp "osinfo=%s\n") osinfo;
+  fp "file=%s\n" file_uri;
+  fp "arch=%s\n" arch;
+  may (fp "sig=%s\n") signature_uri;
+  (match checksums with
+  | None -> ()
+  | Some checksums ->
+List.iter (
+  fun c ->
+fp 

[Libguestfs] [PATCH v10 0/6] virt-builder-repository

[Cédric Bosdonnat]

Hi all,

Diff to v9 includes the changes requested by Pino.

Cédric Bosdonnat (5):
  builder: rename docs test script
  builder: add a template parameter to get_index
  builder: add Index.write_entry function
  mllib: add XPath helper xpath_get_nodes()
  New tool: virt-builder-repository

Pino Toscano (1):
  builder: add simple OCaml osinfo-db reader

 .gitignore |   5 +
 builder/Makefile.am| 128 -
 builder/builder.ml |   2 +-
 builder/index.mli  |   3 +
 builder/index_parser.ml|  80 ++-
 builder/index_parser.mli   |   9 +-
 builder/index_parser_tests.ml  | 129 +
 builder/osinfo.ml  |  80 +++
 builder/osinfo.mli |  22 +
 builder/repository_main.ml | 597 +
 .../{test-virt-builder-docs.sh => test-docs.sh}|   2 +
 builder/virt-builder-repository.pod| 213 
 mllib/xpath_helpers.ml |   9 +
 mllib/xpath_helpers.mli|   4 +
 14 files changed, 1268 insertions(+), 15 deletions(-)
 create mode 100644 builder/index_parser_tests.ml
 create mode 100644 builder/osinfo.ml
 create mode 100644 builder/osinfo.mli
 create mode 100644 builder/repository_main.ml
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (93%)
 create mode 100644 builder/virt-builder-repository.pod

-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v10 1/6] builder: add simple OCaml osinfo-db reader

[Cédric Bosdonnat]

From: Pino Toscano 

Add a simple OCaml-based implementation of reader of the osinfo-db:
the only interface is an iterator that invokes an user-supplied
function with each XML file found.

This implementation behaves like the current C implementation, and
still supports the old libosinfo db.
---
 .gitignore  |  1 +
 builder/Makefile.am |  4 +++
 builder/osinfo.ml   | 80 +
 builder/osinfo.mli  | 22 +++
 4 files changed, 107 insertions(+)
 create mode 100644 builder/osinfo.ml
 create mode 100644 builder/osinfo.mli

diff --git a/.gitignore b/.gitignore
index 302aa2d81..faf6068fa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -92,6 +92,7 @@ Makefile.in
 /builder/index-scan.c
 /builder/libguestfs.conf
 /builder/opensuse.conf
+/builder/osinfo_config.ml
 /builder/oUnit-*
 /builder/*.qcow2
 /builder/stamp-virt-builder.pod
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 64cb20ade..76e6d4f3a 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -207,6 +207,10 @@ CLEANFILES += *.qcow2 *.xz
 
 check_DATA = $(disk_images)
 
+osinfo_config.ml: Makefile
+   echo 'let libosinfo_db_path = "$(datadir)/libosinfo/db"' > $@-t
+   mv $@-t $@
+
 fedora.qcow2: ../test-data/phony-guests/fedora.img
rm -f $@ $@-t
qemu-img convert -f raw -O qcow2 $< $@-t
diff --git a/builder/osinfo.ml b/builder/osinfo.ml
new file mode 100644
index 0..6c0361100
--- /dev/null
+++ b/builder/osinfo.ml
@@ -0,0 +1,80 @@
+(* virt-builder
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open Common_utils
+open Osinfo_config
+
+let rec iterate_db fn =
+  let locations = ref [] in
+
+  (* (1) Try the shared osinfo directory, using either the
+   * $OSINFO_SYSTEM_DIR envvar or its default value.
+   *)
+  let () =
+let dir =
+  try Sys.getenv "OSINFO_SYSTEM_DIR"
+  with Not_found -> "/usr/share/osinfo" in
+push_back locations ((dir // "os"), read_osinfo_db_three_levels)
+  in
+
+  (* (2) Try the libosinfo directory, using the newer three-directory
+   * layout ($LIBOSINFO_DB_PATH / "os" / $group-ID / [file.xml]).
+   *)
+  let () =
+let path = Osinfo_config.libosinfo_db_path // "os" in
+push_back locations (path, read_osinfo_db_three_levels)
+  in
+
+  (* (3) Try the libosinfo directory, using the old flat directory
+   * layout ($LIBOSINFO_DB_PATH / "oses" / [file.xml]).
+   *)
+  let () =
+let path = Osinfo_config.libosinfo_db_path // "oses" in
+push_back locations (path, read_osinfo_db_flat)
+  in
+
+  let rec loop = function
+| (path, f) :: paths ->
+  if is_directory path then f fn path
+  (* This is not an error: RHBZ#948324. *)
+  else loop paths
+| [] -> ()
+  in
+
+  loop !locations
+
+and read_osinfo_db_three_levels fn path =
+  debug "osinfo: loading 3-level-directories database from %s" path;
+  let entries = Array.to_list (Sys.readdir path) in
+  let entries = List.map ((//) path) entries in
+  (* Iterate only on directories. *)
+  let entries = List.filter is_directory entries in
+  List.iter (read_osinfo_db_directory fn) entries
+
+and read_osinfo_db_flat fn path =
+  debug "osinfo: loading flat database from %s" path;
+  read_osinfo_db_directory fn path
+
+and read_osinfo_db_directory fn path =
+  let entries = Array.to_list (Sys.readdir path) in
+  let entries = List.filter (fun x -> Filename.check_suffix x ".xml") entries 
in
+  let entries = List.map ((//) path) entries in
+  let entries = List.filter is_regular_file entries in
+  List.iter fn entries
+
diff --git a/builder/osinfo.mli b/builder/osinfo.mli
new file mode 100644
index 0..949d776a9
--- /dev/null
+++ b/builder/osinfo.mli
@@ -0,0 +1,22 @@
+(* virt-builder
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * 

[Libguestfs] [PATCH v10 5/6] mllib: add XPath helper xpath_get_nodes()

[Cédric Bosdonnat]

This function will allow more OCaml-ish processing of XPath queries
with multiple results.
---
 mllib/xpath_helpers.ml  | 9 +
 mllib/xpath_helpers.mli | 4 
 2 files changed, 13 insertions(+)

diff --git a/mllib/xpath_helpers.ml b/mllib/xpath_helpers.ml
index e6185bf3d..eb655e1fe 100644
--- a/mllib/xpath_helpers.ml
+++ b/mllib/xpath_helpers.ml
@@ -52,3 +52,12 @@ let xpath_eval_default parsefn xpath expr default =
 let xpath_string_default = xpath_eval_default identity
 let xpath_int_default = xpath_eval_default int_of_string
 let xpath_int64_default = xpath_eval_default Int64.of_string
+
+let xpath_get_nodes xpathctx expr =
+  let obj = Xml.xpath_eval_expression xpathctx expr in
+  let nodes = ref [] in
+  for i = 0 to Xml.xpathobj_nr_nodes obj - 1 do
+let node = Xml.xpathobj_node obj i in
+push_back nodes node
+  done;
+  !nodes
diff --git a/mllib/xpath_helpers.mli b/mllib/xpath_helpers.mli
index 7434ba645..83c770281 100644
--- a/mllib/xpath_helpers.mli
+++ b/mllib/xpath_helpers.mli
@@ -31,3 +31,7 @@ val xpath_int_default : Xml.xpathctx -> string -> int -> int
 val xpath_int64_default : Xml.xpathctx -> string -> int64 -> int64
 (** Parse an xpath expression and return a string/int; if the expression
 doesn't match, return the default. *)
+
+val xpath_get_nodes : Xml.xpathctx -> string -> Xml.node list
+(** Parse an XPath expression and return a list with the matching
+XML nodes. *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v10 6/6] New tool: virt-builder-repository

[Cédric Bosdonnat]

move the corresponding
+image file before running virt-builder-repository.
+
+Then running the following command will complete and update the index
+file:
+
+ virt-builder-repository --gpg-key "j...@hacker.org" -i /path/to/folder
+
+virt-builder-repository works in a temporary folder inside the repository
+one. If anything wrong happens when running the tool, the repository is
+left untouched.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--help>
+
+Display help.
+
+=item B<--gpg> GPG
+
+Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary.  You can
+also use this to add gpg parameters, for example to specify an
+alternate home directory:
+
+ virt-builder-repository --gpg "gpg --homedir /tmp" [...]
+
+This can also be used to avoid gpg asking for the key passphrase:
+
+ virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...]
+
+=item B<-K> KEYID
+
+=item B<--gpg-key> KEYID
+
+Specify the GPG key to be used to sign the repository index file.
+If not provided, the index will left unsigned. C is used to
+identify the GPG key to use. This value is passed to gpg’s
+I<--default-key> option and can thus be an email address or a
+fingerprint.
+
+B: by default, virt-builder-repository searches for the key
+in the user’s GPG keyring.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data. Default values are computed from the disk
+image.
+
+When prompted for data, inputting C<-> corresponds to leaving the
+value empty. This can be used to avoid setting the default computed value.
+
+=item B<--keep-index>
+
+When using a GPG key, don’t remove the unsigned index.
+
+=item B<--no-compression>
+
+Don’t compress the template images.
+
+=item B<--machine-readable>
+
+This option is used to make the output more machine friendly
+when being parsed by other programs.  See
+L below.
+
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages.  This is the default
+when the output is a tty.  If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don’t print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 MACHINE READABLE OUTPUT
+
+The I<--machine-readable> option can be used to make the output more
+machine friendly, which is useful when calling virt-builder-repository from
+other programs, GUIs etc.
+
+Use the option on its own to query the capabilities of the
+virt-builder-repository binary.  Typical output looks like this:
+
+ $ virt-builder-repository --machine-readable
+ virt-builder-repository
+
+A list of features is printed, one per line, and the program exits
+with status 0.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdon...@suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v10 3/6] builder: add a template parameter to get_index

[Cédric Bosdonnat]

get_index now gets a new template parameter. Setting it to true will
make the index parsing less picky about missing important data. This
can be used to parse a partial index file.
---
 builder/builder.ml   |  2 +-
 builder/index_parser.ml  | 26 ++
 builder/index_parser.mli |  5 -
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index 3c1f04c77..0bb145466 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -208,7 +208,7 @@ let main () =
   ~tmpdir in
   match source.Sources.format with
   | Sources.FormatNative ->
-Index_parser.get_index ~downloader ~sigchecker source
+Index_parser.get_index ~downloader ~sigchecker ~template:false 
source
   | Sources.FormatSimpleStreams ->
 Simplestreams_parser.get_index ~downloader ~sigchecker source
   ) sources
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index fb546831f..02c124df3 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -25,7 +25,7 @@ open Utils
 open Printf
 open Unix
 
-let get_index ~downloader ~sigchecker
+let get_index ~downloader ~sigchecker ~template
   { Sources.uri = uri; proxy = proxy } =
   let corrupt_file () =
 error (f_"The index file downloaded from ‘%s’ is corrupt.\nYou need to ask 
the supplier of this file to fix it and upload a fixed version.") uri
@@ -100,8 +100,10 @@ let get_index ~downloader ~sigchecker
   let arch =
 try List.assoc ("arch", None) fields
 with Not_found ->
-  eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
-corrupt_file () in
+  if template then "" else (
+eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let signature_uri =
 try Some (make_absolute_uri (List.assoc ("sig", None) fields))
 with Not_found -> None in
@@ -113,7 +115,7 @@ let get_index ~downloader ~sigchecker
   let revision =
 try Rev_int (int_of_string (List.assoc ("revision", None) fields))
 with
-| Not_found -> Rev_int 1
+| Not_found -> if template then Rev_int 0 else Rev_int 1
 | Failure _ ->
   eprintf (f_"%s: cannot parse ‘revision’ field for ‘%s’\n") prog 
n;
   corrupt_file () in
@@ -123,11 +125,19 @@ let get_index ~downloader ~sigchecker
 try Int64.of_string (List.assoc ("size", None) fields)
 with
 | Not_found ->
-  eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file ()
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  )
 | Failure _ ->
-  eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file () in
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let compressed_size =
 try Some (Int64.of_string (List.assoc ("compressed_size", None) 
fields))
 with
diff --git a/builder/index_parser.mli b/builder/index_parser.mli
index b8d8ddf3d..a93e20825 100644
--- a/builder/index_parser.mli
+++ b/builder/index_parser.mli
@@ -16,4 +16,7 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
Sources.source -> Index.index
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
template:bool -> Sources.source -> Index.index
+(** [get_index download sigchecker template source] will parse the source
+index file into an index entry list. If the template flag is set to
+true, the parser will be less picky about missing values. *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v10 2/6] builder: rename docs test script

[Cédric Bosdonnat]

Rename test-virt-builder-docs.sh into test-docs.sh to include test
for another tool's documentation.
---
 builder/Makefile.am | 4 ++--
 builder/{test-virt-builder-docs.sh => test-docs.sh} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (100%)

diff --git a/builder/Makefile.am b/builder/Makefile.am
index 76e6d4f3a..38f4c6968 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -28,7 +28,7 @@ EXTRA_DIST = \
test-simplestreams/streams/v1/index.json \
test-simplestreams/streams/v1/net.cirros-cloud_released_download.json \
test-virt-builder.sh \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-builder-list-simplestreams.sh \
test-virt-builder-planner.sh \
@@ -256,7 +256,7 @@ yajl_tests_LINK = \
  $(yajl_tests_THEOBJECTS) -o $@
 
 TESTS = \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-index-validate.sh \
$(SLOW_TESTS)
diff --git a/builder/test-virt-builder-docs.sh b/builder/test-docs.sh
similarity index 100%
rename from builder/test-virt-builder-docs.sh
rename to builder/test-docs.sh
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v10 4/6] builder: add Index.write_entry function

[Cédric Bosdonnat]

Add a function to properly write virt-builder source index entries.
Note that this function is very similar to Index.print_entry that is
meant for debugging purposes.
---
 .gitignore|   1 +
 builder/Makefile.am   |  36 +++-
 builder/index.mli |   3 +
 builder/index_parser.ml   |  54 ++
 builder/index_parser.mli  |   4 ++
 builder/index_parser_tests.ml | 129 ++
 6 files changed, 225 insertions(+), 2 deletions(-)
 create mode 100644 builder/index_parser_tests.ml

diff --git a/.gitignore b/.gitignore
index faf6068fa..9ee28181e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -106,6 +106,7 @@ Makefile.in
 /builder/virt-index-validate
 /builder/virt-index-validate.1
 /builder/*.xz
+/builder/index_parser_tests
 /builder/yajl_tests
 /cat/stamp-virt-*.pod
 /cat/virt-cat
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 38f4c6968..2af44ecd3 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -233,13 +233,36 @@ yajl_tests_BOBJECTS = \
yajl_tests.cmo
 yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx)
 
+index_parser_tests_SOURCES = \
+   index-scan.c \
+   index-struct.c \
+   index-parser-c.c \
+   index-parse.c
+index_parser_tests_CPPFLAGS = $(virt_builder_CPPFLAGS)
+index_parser_tests_BOBJECTS = \
+   utils.cmo \
+   cache.cmo \
+   downloader.cmo \
+   sigchecker.cmo \
+   index.cmo \
+   ini_reader.cmo \
+   index_parser.cmo \
+   index_parser_tests.cmo
+index_parser_tests_XOBJECTS = $(index_parser_tests_BOBJECTS:.cmo=.cmx)
+
 # Can't call the following as _OBJECTS because automake gets confused.
 if HAVE_OCAMLOPT
 yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS)
 yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_XOBJECTS)
+index_parser_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 else
 yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS)
 yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_BOBJECTS)
+index_parser_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 endif
 
 yajl_tests_DEPENDENCIES = \
@@ -255,6 +278,15 @@ yajl_tests_LINK = \
  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
  $(yajl_tests_THEOBJECTS) -o $@
 
+index_parser_tests_DEPENDENCIES = \
+   $(index_parser_tests_THEOBJECTS) \
+   ../mllib/mllib.$(MLARCHIVE) \
+   $(top_srcdir)/ocaml-link.sh
+index_parser_tests_LINK = \
+   $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
+ $(index_parser_tests_THEOBJECTS) -o $@
+
 TESTS = \
test-docs.sh \
test-virt-builder-list.sh \
@@ -268,8 +300,8 @@ if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
 endif ENABLE_APPLIANCE
 if HAVE_OCAML_PKG_OUNIT
-check_PROGRAMS += yajl_tests
-TESTS += yajl_tests
+check_PROGRAMS += yajl_tests index_parser_tests
+TESTS += yajl_tests index_parser_tests
 endif
 
 check-valgrind:
diff --git a/builder/index.mli b/builder/index.mli
index ff5ec4a35..6202d636e 100644
--- a/builder/index.mli
+++ b/builder/index.mli
@@ -39,3 +39,6 @@ and entry = {
 }
 
 val print_entry : out_channel -> (string * entry) -> unit
+(** Debugging helper function dumping an index entry to a stream.
+To write entries for non-debugging purpose, use the
+[Index_parser.write_entry] function. *)
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index 02c124df3..3987cc385 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -237,3 +237,57 @@ let get_index ~downloader ~sigchecker ~template
   in
 
   get_index ()
+
+let write_entry chan (name, { Index.printable_name = printable_name;
+  file_uri = file_uri;
+  arch = arch;
+  osinfo = osinfo;
+  signature_uri = signature_uri;
+  checksums = checksums;
+  revision = revision;
+  format = format;
+  size = size;
+  compressed_size = compressed_size;
+  expand = expand;
+  lvexpand = lvexpand;
+  notes = notes;
+  aliases = aliases;
+  hidden = hidden }) =
+  let fp fs = fprintf chan fs in
+  fp "[%s]\n" name;
+  may (fp "name=%s\n") printable_name;
+  may (fp "osinfo=%s\n") osinfo;
+  fp "file=%s\n" file_uri;
+  fp "arch=%s\n" arch;
+  may (fp "sig=%s\n") signature_uri;
+  (match checksums with
+  | None -> ()
+  | Some checksums ->
+List.iter (
+  fun c ->
+fp "checksum[%s]=%s\n"
+  

[Libguestfs] [PATCH v9 1/7] ocaml osinfo database iterator

[Cédric Bosdonnat]

From: Pino Toscano 

The C osinfo database parser has been deprecated, reimplement the base
of it in ocaml for virt-builder-repository to use. This provides an
Osinfo.iterate_db() function traversing the files of the osinfo database
and calling a function on each of them.
---
 .gitignore  |  1 +
 builder/Makefile.am |  4 +++
 builder/osinfo.ml   | 80 +
 builder/osinfo.mli  | 22 +++
 4 files changed, 107 insertions(+)
 create mode 100644 builder/osinfo.ml
 create mode 100644 builder/osinfo.mli

diff --git a/.gitignore b/.gitignore
index 302aa2d81..faf6068fa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -92,6 +92,7 @@ Makefile.in
 /builder/index-scan.c
 /builder/libguestfs.conf
 /builder/opensuse.conf
+/builder/osinfo_config.ml
 /builder/oUnit-*
 /builder/*.qcow2
 /builder/stamp-virt-builder.pod
diff --git a/builder/Makefile.am b/builder/Makefile.am
index e1c7bd016..cb3e0a055 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -208,6 +208,10 @@ CLEANFILES += *.qcow2 *.xz
 
 check_DATA = $(disk_images)
 
+osinfo_config.ml: Makefile
+   echo 'let libosinfo_db_path = "$(datadir)/libosinfo/db"' > $@-t
+   mv $@-t $@
+
 fedora.qcow2: ../test-data/phony-guests/fedora.img
rm -f $@ $@-t
qemu-img convert -f raw -O qcow2 $< $@-t
diff --git a/builder/osinfo.ml b/builder/osinfo.ml
new file mode 100644
index 0..6c0361100
--- /dev/null
+++ b/builder/osinfo.ml
@@ -0,0 +1,80 @@
+(* virt-builder
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open Common_utils
+open Osinfo_config
+
+let rec iterate_db fn =
+  let locations = ref [] in
+
+  (* (1) Try the shared osinfo directory, using either the
+   * $OSINFO_SYSTEM_DIR envvar or its default value.
+   *)
+  let () =
+let dir =
+  try Sys.getenv "OSINFO_SYSTEM_DIR"
+  with Not_found -> "/usr/share/osinfo" in
+push_back locations ((dir // "os"), read_osinfo_db_three_levels)
+  in
+
+  (* (2) Try the libosinfo directory, using the newer three-directory
+   * layout ($LIBOSINFO_DB_PATH / "os" / $group-ID / [file.xml]).
+   *)
+  let () =
+let path = Osinfo_config.libosinfo_db_path // "os" in
+push_back locations (path, read_osinfo_db_three_levels)
+  in
+
+  (* (3) Try the libosinfo directory, using the old flat directory
+   * layout ($LIBOSINFO_DB_PATH / "oses" / [file.xml]).
+   *)
+  let () =
+let path = Osinfo_config.libosinfo_db_path // "oses" in
+push_back locations (path, read_osinfo_db_flat)
+  in
+
+  let rec loop = function
+| (path, f) :: paths ->
+  if is_directory path then f fn path
+  (* This is not an error: RHBZ#948324. *)
+  else loop paths
+| [] -> ()
+  in
+
+  loop !locations
+
+and read_osinfo_db_three_levels fn path =
+  debug "osinfo: loading 3-level-directories database from %s" path;
+  let entries = Array.to_list (Sys.readdir path) in
+  let entries = List.map ((//) path) entries in
+  (* Iterate only on directories. *)
+  let entries = List.filter is_directory entries in
+  List.iter (read_osinfo_db_directory fn) entries
+
+and read_osinfo_db_flat fn path =
+  debug "osinfo: loading flat database from %s" path;
+  read_osinfo_db_directory fn path
+
+and read_osinfo_db_directory fn path =
+  let entries = Array.to_list (Sys.readdir path) in
+  let entries = List.filter (fun x -> Filename.check_suffix x ".xml") entries 
in
+  let entries = List.map ((//) path) entries in
+  let entries = List.filter is_regular_file entries in
+  List.iter fn entries
+
diff --git a/builder/osinfo.mli b/builder/osinfo.mli
new file mode 100644
index 0..949d776a9
--- /dev/null
+++ b/builder/osinfo.mli
@@ -0,0 +1,22 @@
+(* virt-builder
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public 

[Libguestfs] [PATCH v9 2/7] builder: rename docs test script

[Cédric Bosdonnat]

Rename test-virt-builder-docs.sh into test-docs.sh to include test
for another tool's documentation.
---
 builder/Makefile.am | 4 ++--
 builder/{test-virt-builder-docs.sh => test-docs.sh} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (100%)

diff --git a/builder/Makefile.am b/builder/Makefile.am
index cb3e0a055..cd653dcd3 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -28,7 +28,7 @@ EXTRA_DIST = \
test-simplestreams/streams/v1/index.json \
test-simplestreams/streams/v1/net.cirros-cloud_released_download.json \
test-virt-builder.sh \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-builder-list-simplestreams.sh \
test-virt-builder-planner.sh \
@@ -257,7 +257,7 @@ yajl_tests_LINK = \
  $(yajl_tests_THEOBJECTS) -o $@
 
 TESTS = \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-index-validate.sh \
$(SLOW_TESTS)
diff --git a/builder/test-virt-builder-docs.sh b/builder/test-docs.sh
similarity index 100%
rename from builder/test-virt-builder-docs.sh
rename to builder/test-docs.sh
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v9 5/7] mllib: add XPath helper xpath_get_nodes()

[Cédric Bosdonnat]

This function will allow more OCaml-ish processing of XPath queries
with multiple results.
---
 mllib/xpath_helpers.ml  | 9 +
 mllib/xpath_helpers.mli | 4 
 2 files changed, 13 insertions(+)

diff --git a/mllib/xpath_helpers.ml b/mllib/xpath_helpers.ml
index e6185bf3d..eb655e1fe 100644
--- a/mllib/xpath_helpers.ml
+++ b/mllib/xpath_helpers.ml
@@ -52,3 +52,12 @@ let xpath_eval_default parsefn xpath expr default =
 let xpath_string_default = xpath_eval_default identity
 let xpath_int_default = xpath_eval_default int_of_string
 let xpath_int64_default = xpath_eval_default Int64.of_string
+
+let xpath_get_nodes xpathctx expr =
+  let obj = Xml.xpath_eval_expression xpathctx expr in
+  let nodes = ref [] in
+  for i = 0 to Xml.xpathobj_nr_nodes obj - 1 do
+let node = Xml.xpathobj_node obj i in
+push_back nodes node
+  done;
+  !nodes
diff --git a/mllib/xpath_helpers.mli b/mllib/xpath_helpers.mli
index 7434ba645..83c770281 100644
--- a/mllib/xpath_helpers.mli
+++ b/mllib/xpath_helpers.mli
@@ -31,3 +31,7 @@ val xpath_int_default : Xml.xpathctx -> string -> int -> int
 val xpath_int64_default : Xml.xpathctx -> string -> int64 -> int64
 (** Parse an xpath expression and return a string/int; if the expression
 doesn't match, return the default. *)
+
+val xpath_get_nodes : Xml.xpathctx -> string -> Xml.node list
+(** Parse an XPath expression and return a list with the matching
+XML nodes. *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v9 3/7] builder: add a template parameter to get_index

[Cédric Bosdonnat]

get_index now gets a new template parameter. Setting it to true will
make the index parsing less picky about missing important data. This
can be used to parse a partial index file.
---
 builder/builder.ml   |  2 +-
 builder/index_parser.ml  | 26 ++
 builder/index_parser.mli |  4 +++-
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index 3c1f04c77..0bb145466 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -208,7 +208,7 @@ let main () =
   ~tmpdir in
   match source.Sources.format with
   | Sources.FormatNative ->
-Index_parser.get_index ~downloader ~sigchecker source
+Index_parser.get_index ~downloader ~sigchecker ~template:false 
source
   | Sources.FormatSimpleStreams ->
 Simplestreams_parser.get_index ~downloader ~sigchecker source
   ) sources
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index fb546831f..02c124df3 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -25,7 +25,7 @@ open Utils
 open Printf
 open Unix
 
-let get_index ~downloader ~sigchecker
+let get_index ~downloader ~sigchecker ~template
   { Sources.uri = uri; proxy = proxy } =
   let corrupt_file () =
 error (f_"The index file downloaded from ‘%s’ is corrupt.\nYou need to ask 
the supplier of this file to fix it and upload a fixed version.") uri
@@ -100,8 +100,10 @@ let get_index ~downloader ~sigchecker
   let arch =
 try List.assoc ("arch", None) fields
 with Not_found ->
-  eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
-corrupt_file () in
+  if template then "" else (
+eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let signature_uri =
 try Some (make_absolute_uri (List.assoc ("sig", None) fields))
 with Not_found -> None in
@@ -113,7 +115,7 @@ let get_index ~downloader ~sigchecker
   let revision =
 try Rev_int (int_of_string (List.assoc ("revision", None) fields))
 with
-| Not_found -> Rev_int 1
+| Not_found -> if template then Rev_int 0 else Rev_int 1
 | Failure _ ->
   eprintf (f_"%s: cannot parse ‘revision’ field for ‘%s’\n") prog 
n;
   corrupt_file () in
@@ -123,11 +125,19 @@ let get_index ~downloader ~sigchecker
 try Int64.of_string (List.assoc ("size", None) fields)
 with
 | Not_found ->
-  eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file ()
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  )
 | Failure _ ->
-  eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file () in
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let compressed_size =
 try Some (Int64.of_string (List.assoc ("compressed_size", None) 
fields))
 with
diff --git a/builder/index_parser.mli b/builder/index_parser.mli
index b8d8ddf3d..aa5f84730 100644
--- a/builder/index_parser.mli
+++ b/builder/index_parser.mli
@@ -16,4 +16,6 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
Sources.source -> Index.index
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
template:bool -> Sources.source -> Index.index
+(** [get_index download sigchecker source] will parse the source index file
+ into an index entry list. *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v9 4/7] builder: add Index.write_entry function

[Cédric Bosdonnat]

Add a function to properly write virt-builder source index entries.
Note that this function is very similar to Index.print_entry that is
meant for debugging purposes.
---
 .gitignore|   1 +
 builder/Makefile.am   |  36 +++-
 builder/index.mli |   3 +
 builder/index_parser.ml   |  54 ++
 builder/index_parser.mli  |   4 ++
 builder/index_parser_tests.ml | 129 ++
 6 files changed, 225 insertions(+), 2 deletions(-)
 create mode 100644 builder/index_parser_tests.ml

diff --git a/.gitignore b/.gitignore
index faf6068fa..9ee28181e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -106,6 +106,7 @@ Makefile.in
 /builder/virt-index-validate
 /builder/virt-index-validate.1
 /builder/*.xz
+/builder/index_parser_tests
 /builder/yajl_tests
 /cat/stamp-virt-*.pod
 /cat/virt-cat
diff --git a/builder/Makefile.am b/builder/Makefile.am
index cd653dcd3..59de42a57 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -234,13 +234,36 @@ yajl_tests_BOBJECTS = \
yajl_tests.cmo
 yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx)
 
+index_parser_tests_SOURCES = \
+   index-scan.c \
+   index-struct.c \
+   index-parser-c.c \
+   index-parse.c
+index_parser_tests_CPPFLAGS = $(virt_builder_CPPFLAGS)
+index_parser_tests_BOBJECTS = \
+   utils.cmo \
+   cache.cmo \
+   downloader.cmo \
+   sigchecker.cmo \
+   index.cmo \
+   ini_reader.cmo \
+   index_parser.cmo \
+   index_parser_tests.cmo
+index_parser_tests_XOBJECTS = $(index_parser_tests_BOBJECTS:.cmo=.cmx)
+
 # Can't call the following as _OBJECTS because automake gets confused.
 if HAVE_OCAMLOPT
 yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS)
 yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_XOBJECTS)
+index_parser_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 else
 yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS)
 yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_BOBJECTS)
+index_parser_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 endif
 
 yajl_tests_DEPENDENCIES = \
@@ -256,6 +279,15 @@ yajl_tests_LINK = \
  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
  $(yajl_tests_THEOBJECTS) -o $@
 
+index_parser_tests_DEPENDENCIES = \
+   $(index_parser_tests_THEOBJECTS) \
+   ../mllib/mllib.$(MLARCHIVE) \
+   $(top_srcdir)/ocaml-link.sh
+index_parser_tests_LINK = \
+   $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
+ $(index_parser_tests_THEOBJECTS) -o $@
+
 TESTS = \
test-docs.sh \
test-virt-builder-list.sh \
@@ -269,8 +301,8 @@ if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
 endif ENABLE_APPLIANCE
 if HAVE_OCAML_PKG_OUNIT
-check_PROGRAMS += yajl_tests
-TESTS += yajl_tests
+check_PROGRAMS += yajl_tests index_parser_tests
+TESTS += yajl_tests index_parser_tests
 endif
 
 check-valgrind:
diff --git a/builder/index.mli b/builder/index.mli
index ff5ec4a35..6202d636e 100644
--- a/builder/index.mli
+++ b/builder/index.mli
@@ -39,3 +39,6 @@ and entry = {
 }
 
 val print_entry : out_channel -> (string * entry) -> unit
+(** Debugging helper function dumping an index entry to a stream.
+To write entries for non-debugging purpose, use the
+[Index_parser.write_entry] function. *)
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index 02c124df3..3987cc385 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -237,3 +237,57 @@ let get_index ~downloader ~sigchecker ~template
   in
 
   get_index ()
+
+let write_entry chan (name, { Index.printable_name = printable_name;
+  file_uri = file_uri;
+  arch = arch;
+  osinfo = osinfo;
+  signature_uri = signature_uri;
+  checksums = checksums;
+  revision = revision;
+  format = format;
+  size = size;
+  compressed_size = compressed_size;
+  expand = expand;
+  lvexpand = lvexpand;
+  notes = notes;
+  aliases = aliases;
+  hidden = hidden }) =
+  let fp fs = fprintf chan fs in
+  fp "[%s]\n" name;
+  may (fp "name=%s\n") printable_name;
+  may (fp "osinfo=%s\n") osinfo;
+  fp "file=%s\n" file_uri;
+  fp "arch=%s\n" arch;
+  may (fp "sig=%s\n") signature_uri;
+  (match checksums with
+  | None -> ()
+  | Some checksums ->
+List.iter (
+  fun c ->
+fp "checksum[%s]=%s\n"
+  

[Libguestfs] [PATCH v9 7/7] New tool: virt-builder-repository

[Cédric Bosdonnat]

rt-builder-repository works in a temporary folder inside the repository
+one. If anything wrong happens when running the tool, the repository is
+left untouched.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--help>
+
+Display help.
+
+=item B<--gpg> GPG
+
+Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary.  You can
+also use this to add gpg parameters, for example to specify an
+alternate home directory:
+
+ virt-builder-repository --gpg "gpg --homedir /tmp" [...]
+
+This can also be used to avoid gpg asking for the key passphrase:
+
+ virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...]
+
+=item B<-K> KEYID
+
+=item B<--gpg-key> KEYID
+
+Specify the GPG key to be used to sign the repository index file.
+If not provided, the index will left unsigned. C is used to
+identify the GPG key to use. This value is passed to gpg's
+I<--default-key> option and can thus be an email address or a
+fingerprint.
+
+B: by default, virt-builder-repository searches for the key
+in the user's GPG keyring.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data. Default values are computed from the disk
+image.
+
+When prompted for data, inputting C<-> corresponds to leaving the
+value empty. This can be used to avoid setting the default computed value.
+
+=item B<--keep-index>
+
+When using a GPG key, don't remove the unsigned index.
+
+=item B<--no-compression>
+
+Don't compress the template images.
+
+=item B<--machine-readable>
+
+This option is used to make the output more machine friendly
+when being parsed by other programs.  See
+L below.
+
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages.  This is the default
+when the output is a tty.  If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don't print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 MACHINE READABLE OUTPUT
+
+The I<--machine-readable> option can be used to make the output more
+machine friendly, which is useful when calling virt-builder-repository from
+other programs, GUIs etc.
+
+Use the option on its own to query the capabilities of the
+virt-builder-repository binary.  Typical output looks like this:
+
+ $ virt-builder-repository --machine-readable
+ virt-builder-repository
+
+A list of features is printed, one per line, and the program exits
+with status 0.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdon...@suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v9 6/7] builder: remove useless fish dependency

[Cédric Bosdonnat]

---
 builder/Makefile.am | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/builder/Makefile.am b/builder/Makefile.am
index 59de42a57..2af44ecd3 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -101,8 +101,7 @@ virt_builder_CPPFLAGS = \
-I$(shell $(OCAMLC) -where) \
-I$(top_srcdir)/gnulib/lib \
-I$(top_srcdir)/common/utils \
-   -I$(top_srcdir)/lib \
-   -I$(top_srcdir)/fish
+   -I$(top_srcdir)/lib
 virt_builder_CFLAGS = \
-pthread \
$(WARN_CFLAGS) $(WERROR_CFLAGS) \
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v8 2/7] builder: rename docs test script

[Cédric Bosdonnat]

Rename test-virt-builder-docs.sh into test-docs.sh to include test
for another tool's documentation.
---
 builder/Makefile.am | 4 ++--
 builder/{test-virt-builder-docs.sh => test-docs.sh} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (100%)

diff --git a/builder/Makefile.am b/builder/Makefile.am
index cb3e0a055..cd653dcd3 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -28,7 +28,7 @@ EXTRA_DIST = \
test-simplestreams/streams/v1/index.json \
test-simplestreams/streams/v1/net.cirros-cloud_released_download.json \
test-virt-builder.sh \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-builder-list-simplestreams.sh \
test-virt-builder-planner.sh \
@@ -257,7 +257,7 @@ yajl_tests_LINK = \
  $(yajl_tests_THEOBJECTS) -o $@
 
 TESTS = \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-index-validate.sh \
$(SLOW_TESTS)
diff --git a/builder/test-virt-builder-docs.sh b/builder/test-docs.sh
similarity index 100%
rename from builder/test-virt-builder-docs.sh
rename to builder/test-docs.sh
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v8 6/7] mllib: add XPath helper xpath_get_nodes()

[Cédric Bosdonnat]

This function will allow more OCaml-ish processing of XPath queries
with multiple results.
---
 mllib/xpath_helpers.ml  | 9 +
 mllib/xpath_helpers.mli | 4 
 2 files changed, 13 insertions(+)

diff --git a/mllib/xpath_helpers.ml b/mllib/xpath_helpers.ml
index e6185bf3d..eb655e1fe 100644
--- a/mllib/xpath_helpers.ml
+++ b/mllib/xpath_helpers.ml
@@ -52,3 +52,12 @@ let xpath_eval_default parsefn xpath expr default =
 let xpath_string_default = xpath_eval_default identity
 let xpath_int_default = xpath_eval_default int_of_string
 let xpath_int64_default = xpath_eval_default Int64.of_string
+
+let xpath_get_nodes xpathctx expr =
+  let obj = Xml.xpath_eval_expression xpathctx expr in
+  let nodes = ref [] in
+  for i = 0 to Xml.xpathobj_nr_nodes obj - 1 do
+let node = Xml.xpathobj_node obj i in
+push_back nodes node
+  done;
+  !nodes
diff --git a/mllib/xpath_helpers.mli b/mllib/xpath_helpers.mli
index 7434ba645..83c770281 100644
--- a/mllib/xpath_helpers.mli
+++ b/mllib/xpath_helpers.mli
@@ -31,3 +31,7 @@ val xpath_int_default : Xml.xpathctx -> string -> int -> int
 val xpath_int64_default : Xml.xpathctx -> string -> int64 -> int64
 (** Parse an xpath expression and return a string/int; if the expression
 doesn't match, return the default. *)
+
+val xpath_get_nodes : Xml.xpathctx -> string -> Xml.node list
+(** Parse an XPath expression and return a list with the matching
+XML nodes. *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v8 3/7] builder: add a template parameter to get_index

[Cédric Bosdonnat]

get_index now gets a new template parameter. Setting it to true will
make the index parsing less picky about missing important data. This
can be used to parse a partial index file.
---
 builder/builder.ml   |  2 +-
 builder/index_parser.ml  | 26 ++
 builder/index_parser.mli |  4 +++-
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index 3c1f04c77..0bb145466 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -208,7 +208,7 @@ let main () =
   ~tmpdir in
   match source.Sources.format with
   | Sources.FormatNative ->
-Index_parser.get_index ~downloader ~sigchecker source
+Index_parser.get_index ~downloader ~sigchecker ~template:false 
source
   | Sources.FormatSimpleStreams ->
 Simplestreams_parser.get_index ~downloader ~sigchecker source
   ) sources
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index fb546831f..02c124df3 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -25,7 +25,7 @@ open Utils
 open Printf
 open Unix
 
-let get_index ~downloader ~sigchecker
+let get_index ~downloader ~sigchecker ~template
   { Sources.uri = uri; proxy = proxy } =
   let corrupt_file () =
 error (f_"The index file downloaded from ‘%s’ is corrupt.\nYou need to ask 
the supplier of this file to fix it and upload a fixed version.") uri
@@ -100,8 +100,10 @@ let get_index ~downloader ~sigchecker
   let arch =
 try List.assoc ("arch", None) fields
 with Not_found ->
-  eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
-corrupt_file () in
+  if template then "" else (
+eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let signature_uri =
 try Some (make_absolute_uri (List.assoc ("sig", None) fields))
 with Not_found -> None in
@@ -113,7 +115,7 @@ let get_index ~downloader ~sigchecker
   let revision =
 try Rev_int (int_of_string (List.assoc ("revision", None) fields))
 with
-| Not_found -> Rev_int 1
+| Not_found -> if template then Rev_int 0 else Rev_int 1
 | Failure _ ->
   eprintf (f_"%s: cannot parse ‘revision’ field for ‘%s’\n") prog 
n;
   corrupt_file () in
@@ -123,11 +125,19 @@ let get_index ~downloader ~sigchecker
 try Int64.of_string (List.assoc ("size", None) fields)
 with
 | Not_found ->
-  eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file ()
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  )
 | Failure _ ->
-  eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file () in
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let compressed_size =
 try Some (Int64.of_string (List.assoc ("compressed_size", None) 
fields))
 with
diff --git a/builder/index_parser.mli b/builder/index_parser.mli
index b8d8ddf3d..aa5f84730 100644
--- a/builder/index_parser.mli
+++ b/builder/index_parser.mli
@@ -16,4 +16,6 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
Sources.source -> Index.index
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
template:bool -> Sources.source -> Index.index
+(** [get_index download sigchecker source] will parse the source index file
+ into an index entry list. *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v8 5/7] mllib: add do_mv helper function to Common_utils

[Cédric Bosdonnat]

---
 mllib/common_utils.ml  | 6 ++
 mllib/common_utils.mli | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/mllib/common_utils.ml b/mllib/common_utils.ml
index 597128967..1126f233b 100644
--- a/mllib/common_utils.ml
+++ b/mllib/common_utils.ml
@@ -561,3 +561,9 @@ let inspect_decrypt g =
* function.
*)
   c_inspect_decrypt g#ocaml_handle (Guestfs.c_pointer g#ocaml_handle)
+
+let do_mv src dest =
+  let cmd = [ "mv"; src; dest ] in
+  let r = run_command cmd in
+  if r <> 0 then
+error (f_"moving file '%s' to '%s' failed") src dest
diff --git a/mllib/common_utils.mli b/mllib/common_utils.mli
index fa4d15054..2c9d30a99 100644
--- a/mllib/common_utils.mli
+++ b/mllib/common_utils.mli
@@ -170,3 +170,6 @@ val inspect_decrypt : Guestfs.guestfs -> unit
 (** Simple implementation of decryption: look for any [crypto_LUKS]
 partitions and decrypt them, then rescan for VGs.  This only works
 for Fedora whole-disk encryption. *)
+
+val do_mv : string -> string -> unit
+(** Run the mv command, and exit with an error if it failed *)
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v8 7/7] Add a virt-builder-repository tool

[Cédric Bosdonnat]

on use case, a minimal fragment can be
+added to the index file for the automated mode. This can be done
+on the signed index even if it may sound a strange idea: the index
+will be signed again by the tool.
+
+To remove an image from the repository, just remove the corresponding
+compressed image file before running virt-builder-repository.
+
+Then running the following command will complete and update the index
+file:
+
+ virt-builder-repository --gpg-key "j...@hacker.org" -i /path/to/folder
+
+virt-builder-repository works in a temporary folder inside the repository
+one. If anything wrong happens when running the tool, the repository is
+left untouched.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--help>
+
+Display help.
+
+=item B<--gpg> GPG
+
+Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary.  You can
+also use this to add gpg parameters, for example to specify an
+alternate home directory:
+
+ virt-builder-repository --gpg "gpg --homedir /tmp" [...]
+
+This can also be used to avoid gpg asking for the key passphrase:
+
+ virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...]
+
+=item B<-K> KEYID
+
+=item B<--gpg-key> KEYID
+
+Specify the GPG key to be used to sign the repository index file.
+If not provided, the index will left unsigned. C is used to
+identify the GPG key to use. This value is passed to gpg's
+C<--default-key> option and can thus be an email address or a
+fingerprint.
+
+B: by default, virt-builder-repository searches for the key
+in the user's GPG keyring.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data. Default values are computed from the disk
+image.
+
+When prompted for data, inputting C<-> corresponds to leaving the
+value empty. This can be used to avoid setting the default computed value.
+
+=item B<--keep-index>
+
+When using a GPG key, don't remove the unsigned index.
+
+=item B<--no-compression>
+
+Don't compress the template images.
+
+=item B<--machine-readable>
+
+This option is used to make the output more machine friendly
+when being parsed by other programs.  See
+L below.
+
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages.  This is the default
+when the output is a tty.  If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don't print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 MACHINE READABLE OUTPUT
+
+The I<--machine-readable> option can be used to make the output more
+machine friendly, which is useful when calling virt-builder-repository from
+other programs, GUIs etc.
+
+Use the option on its own to query the capabilities of the
+virt-builder-repository binary.  Typical output looks like this:
+
+ $ virt-builder-repository --machine-readable
+ virt-builder-repository
+
+A list of features is printed, one per line, and the program exits
+with status 0.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdon...@suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v8 0/7] virt-builder-repository tool

[Cédric Bosdonnat]

Hi all,

Here is the latest iteration on the virt-builder-repository
series. Diffs to previous version are: fixing things mentioned
by Pino, integrate Pino's osinfo ocaml iterator and adding a
check of the mime type to filter potential image files.

Cédric Bosdonnat (6):
  builder: rename docs test script
  builder: add a template parameter to get_index
  builder: add Index.write_entry function
  mllib: add do_mv helper function to Common_utils
  mllib: add XPath helper xpath_get_nodes()
  Add a virt-builder-repository tool

Pino Toscano (1):
  ocaml osinfo database iterator

 .gitignore |   5 +
 builder/Makefile.am| 133 -
 builder/builder.ml |   2 +-
 builder/index.mli  |   3 +
 builder/index_parser.ml|  80 ++-
 builder/index_parser.mli   |   8 +-
 builder/index_parser_tests.ml  | 129 +
 builder/osinfo.ml  |  80 +++
 builder/osinfo.mli |  22 +
 builder/repository_main.ml | 590 +
 .../{test-virt-builder-docs.sh => test-docs.sh}|   2 +
 builder/virt-builder-repository.pod| 213 
 mllib/common_utils.ml  |   6 +
 mllib/common_utils.mli |   3 +
 mllib/xpath_helpers.ml |   9 +
 mllib/xpath_helpers.mli|   4 +
 16 files changed, 1272 insertions(+), 17 deletions(-)
 create mode 100644 builder/index_parser_tests.ml
 create mode 100644 builder/osinfo.ml
 create mode 100644 builder/osinfo.mli
 create mode 100644 builder/repository_main.ml
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (93%)
 create mode 100644 builder/virt-builder-repository.pod

-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [supermin][PATCH v2] os-release: use ID_LIKE as a fallback for SUSE detection

[Cédric Bosdonnat]

SUSE distros all have in common suse in ID_LIKE field. The ID field
could be varying and is even set to 'Dummy' when building the packages.
If the usual values for openSUSE/SLE can't be found in ID, try with
ID_LIKE.
---
 Diff with v1:
   * Use Utils.string_split rather than too recent String.split_on_char

 src/os_release.ml  | 10 +-
 src/os_release.mli |  7 +++
 src/ph_rpm.ml  |  1 +
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/os_release.ml b/src/os_release.ml
index b2de259..abf2bea 100644
--- a/src/os_release.ml
+++ b/src/os_release.ml
@@ -29,6 +29,7 @@ let split sep str =
 
 type os_release = {
   id : string;
+  id_like : string list;
 }
 
 let data = ref None
@@ -52,6 +53,7 @@ and parse () =
 let lines = List.filter (fun s -> s.[0] <> '#') lines in
 
 let id = ref "" in
+let id_like = ref [] in
 
 List.iter (
   fun line ->
@@ -65,10 +67,11 @@ and parse () =
   else value in
 match field with
 | "ID" -> id := value
+| "ID_LIKE" -> id_like := string_split " " value
 | _ -> ()
 ) lines;
 
-Some { id = !id; }
+Some { id = !id; id_like = !id_like }
   ) else
 None
 
@@ -76,3 +79,8 @@ let get_id () =
   match get_data () with
   | None -> ""
   | Some d -> d.id
+
+let get_id_like () =
+  match get_data () with
+  | None -> []
+  | Some d -> d.id_like
diff --git a/src/os_release.mli b/src/os_release.mli
index 2ae349b..e9f2993 100644
--- a/src/os_release.mli
+++ b/src/os_release.mli
@@ -24,3 +24,10 @@ val get_id : unit -> string
 
 An empty string is returned if the file does not exist or cannot
 be read. *)
+
+val get_id_like : unit -> string list
+(** Get the value of the "ID_LIKE" field from the /etc/os-release file
+on the current system.
+
+An empty list is returned if the file does not exist, cannot
+be read or the ID_LIKE field is not defined. *)
diff --git a/src/ph_rpm.ml b/src/ph_rpm.ml
index b0a5eb2..fd87822 100644
--- a/src/ph_rpm.ml
+++ b/src/ph_rpm.ml
@@ -41,6 +41,7 @@ let opensuse_detect () =
   Config.rpm <> "no" && Config.rpm2cpio <> "no" && rpm_is_available () &&
 Config.zypper <> "no" &&
 (List.mem (Os_release.get_id ()) [ "opensuse"; "sled"; "sles" ] ||
+ List.mem "suse" (Os_release.get_id_like ()) ||
  try (stat "/etc/SuSE-release").st_kind = S_REG with Unix_error _ -> false)
 
 let mageia_detect () =
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [supermin][PATCH] os-release: use ID_LIKE as a fallback for SUSE detection

[Cédric Bosdonnat]

SUSE distros all have in common suse in ID_LIKE field. The ID field
could be varying and is even set to 'Dummy' when building the packages.
If the usual values for openSUSE/SLE can't be found in ID, try with
ID_LIKE.
---
 src/os_release.ml  | 10 +-
 src/os_release.mli |  7 +++
 src/ph_rpm.ml  |  1 +
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/os_release.ml b/src/os_release.ml
index b2de259..daa8b07 100644
--- a/src/os_release.ml
+++ b/src/os_release.ml
@@ -29,6 +29,7 @@ let split sep str =
 
 type os_release = {
   id : string;
+  id_like : string list;
 }
 
 let data = ref None
@@ -52,6 +53,7 @@ and parse () =
 let lines = List.filter (fun s -> s.[0] <> '#') lines in
 
 let id = ref "" in
+let id_like = ref [] in
 
 List.iter (
   fun line ->
@@ -65,10 +67,11 @@ and parse () =
   else value in
 match field with
 | "ID" -> id := value
+| "ID_LIKE" -> id_like := String.split_on_char ' ' value
 | _ -> ()
 ) lines;
 
-Some { id = !id; }
+Some { id = !id; id_like = !id_like }
   ) else
 None
 
@@ -76,3 +79,8 @@ let get_id () =
   match get_data () with
   | None -> ""
   | Some d -> d.id
+
+let get_id_like () =
+  match get_data () with
+  | None -> []
+  | Some d -> d.id_like
diff --git a/src/os_release.mli b/src/os_release.mli
index 2ae349b..e9f2993 100644
--- a/src/os_release.mli
+++ b/src/os_release.mli
@@ -24,3 +24,10 @@ val get_id : unit -> string
 
 An empty string is returned if the file does not exist or cannot
 be read. *)
+
+val get_id_like : unit -> string list
+(** Get the value of the "ID_LIKE" field from the /etc/os-release file
+on the current system.
+
+An empty list is returned if the file does not exist, cannot
+be read or the ID_LIKE field is not defined. *)
diff --git a/src/ph_rpm.ml b/src/ph_rpm.ml
index b0a5eb2..fd87822 100644
--- a/src/ph_rpm.ml
+++ b/src/ph_rpm.ml
@@ -41,6 +41,7 @@ let opensuse_detect () =
   Config.rpm <> "no" && Config.rpm2cpio <> "no" && rpm_is_available () &&
 Config.zypper <> "no" &&
 (List.mem (Os_release.get_id ()) [ "opensuse"; "sled"; "sles" ] ||
+ List.mem "suse" (Os_release.get_id_like ()) ||
  try (stat "/etc/SuSE-release").st_kind = S_REG with Unix_error _ -> false)
 
 let mageia_detect () =
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v2] appliance: use ID_LIKE as a fallback for SUSE distro detection

[Cédric Bosdonnat]

All SUSE distros have a ID_LIKE=suse, including the fake one used
for building that has a ID=Dummy value. Without reading ID_LIKE
on SUSE distros, the generated appliance packagelist is not correct.

This fix reads ID_LIKE as a fallback if ID contains nothing.
---
 m4/guestfs_appliance.m4 | 10 +-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/m4/guestfs_appliance.m4 b/m4/guestfs_appliance.m4
index fbba3373f..788afbd36 100644
--- a/m4/guestfs_appliance.m4
+++ b/m4/guestfs_appliance.m4
@@ -99,8 +99,16 @@ if test -f /etc/os-release; then
 DISTRO="`. /etc/os-release && echo $ID | tr '@<:@:lower:@:>@' 
'@<:@:upper:@:>@'`"
 AS_CASE([$DISTRO],
 [FEDORA | RHEL | CENTOS],[DISTRO=REDHAT],
-[OPENSUSE | SLED | SLES],[DISTRO=SUSE],
+[OPENSUSE | SLED | SLES | SUSE],[DISTRO=SUSE],
 [ARCH],[DISTRO=ARCHLINUX])
+dnl All SUSE-based distros have ID_LIKE containing 'suse', check for it if
+dnl ID wasn't helpful.
+if test -z "$DISTRO"; then
+DISTRO_LIKE="`. /etc/os-release && echo $ID_LIKE`"
+if echo $DISTRO_LIKE | tr " " "\n" | grep -i "^SUSE$"; then
+DISTRO=SUSE
+fi
+fi
 elif test -f /etc/debian_version; then
 DISTRO=DEBIAN
 if grep -q 'DISTRIB_ID=Ubuntu' /etc/lsb-release 2>_MESSAGE_LOG_FD; then
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH] appliance: read ID_LIKE from os-release as a fallback

[Cédric Bosdonnat]

In the appliance used to build the packages for openSUSE, os-release
is super minimal and only had ID_LIKE=suse. The code setting the
DISTRO variable only searches for ID variable so far, resulting in
invalid packagelist on openSUSE.

This fix reads ID_LIKE as a fallback if ID contains nothing.
---
 m4/guestfs_appliance.m4 | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/m4/guestfs_appliance.m4 b/m4/guestfs_appliance.m4
index fbba3373f..ce45256bc 100644
--- a/m4/guestfs_appliance.m4
+++ b/m4/guestfs_appliance.m4
@@ -97,9 +97,15 @@ AC_MSG_CHECKING([which Linux distro for package names])
 if test -f /etc/os-release; then
 ( . /etc/os-release && echo $ID | tr '@<:@:lower:@:>@' '@<:@:upper:@:>@' ) 
>_MESSAGE_LOG_FD
 DISTRO="`. /etc/os-release && echo $ID | tr '@<:@:lower:@:>@' 
'@<:@:upper:@:>@'`"
+dnl when building SUSE-family packages, the OBS appliance has no ID in 
os-release,
+dnl only ID_LIKE set to suse. Read ID_LIKE as a fallback if no ID is found.
+if test -z "$DISTRO"; then
+( . /etc/os-release && echo $ID_LIKE | tr '@<:@:lower:@:>@' 
'@<:@:upper:@:>@' ) >_MESSAGE_LOG_FD
+DISTRO="`. /etc/os-release && echo $ID_LIKE | tr '@<:@:lower:@:>@' 
'@<:@:upper:@:>@'`"
+fi
 AS_CASE([$DISTRO],
 [FEDORA | RHEL | CENTOS],[DISTRO=REDHAT],
-[OPENSUSE | SLED | SLES],[DISTRO=SUSE],
+[OPENSUSE | SLED | SLES | SUSE],[DISTRO=SUSE],
 [ARCH],[DISTRO=ARCHLINUX])
 elif test -f /etc/debian_version; then
 DISTRO=DEBIAN
-- 
2.13.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v7 5/9] builder: add a template parameter to get_index

[Cédric Bosdonnat]

get_index now gets a new template parameter. Setting it to true will
make the index parsing less picky about missing important data. This
can be used to parse a partial index file.
---
 builder/builder.ml   |  2 +-
 builder/index_parser.ml  | 26 ++
 builder/index_parser.mli |  4 +++-
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index b0a48ea89..99cd488b2 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -207,7 +207,7 @@ let main () =
   ~tmpdir in
   match source.Sources.format with
   | Sources.FormatNative ->
-Index_parser.get_index ~downloader ~sigchecker source
+Index_parser.get_index ~downloader ~sigchecker ~template:false 
source
   | Sources.FormatSimpleStreams ->
 Simplestreams_parser.get_index ~downloader ~sigchecker source
   ) sources
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index 468805cf8..c70909e44 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -24,7 +24,7 @@ open Utils
 open Printf
 open Unix
 
-let get_index ~downloader ~sigchecker
+let get_index ~downloader ~sigchecker ~template
   { Sources.uri = uri; proxy = proxy } =
   let corrupt_file () =
 error (f_"The index file downloaded from ‘%s’ is corrupt.\nYou need to ask 
the supplier of this file to fix it and upload a fixed version.") uri
@@ -99,8 +99,10 @@ let get_index ~downloader ~sigchecker
   let arch =
 try List.assoc ("arch", None) fields
 with Not_found ->
-  eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
-corrupt_file () in
+  if template then "" else (
+eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let signature_uri =
 try Some (make_absolute_uri (List.assoc ("sig", None) fields))
 with Not_found -> None in
@@ -112,7 +114,7 @@ let get_index ~downloader ~sigchecker
   let revision =
 try Rev_int (int_of_string (List.assoc ("revision", None) fields))
 with
-| Not_found -> Rev_int 1
+| Not_found -> if template then Rev_int 0 else Rev_int 1
 | Failure _ ->
   eprintf (f_"%s: cannot parse ‘revision’ field for ‘%s’\n") prog 
n;
   corrupt_file () in
@@ -122,11 +124,19 @@ let get_index ~downloader ~sigchecker
 try Int64.of_string (List.assoc ("size", None) fields)
 with
 | Not_found ->
-  eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file ()
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  )
 | Failure _ ->
-  eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file () in
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let compressed_size =
 try Some (Int64.of_string (List.assoc ("compressed_size", None) 
fields))
 with
diff --git a/builder/index_parser.mli b/builder/index_parser.mli
index b8d8ddf3d..aa5f84730 100644
--- a/builder/index_parser.mli
+++ b/builder/index_parser.mli
@@ -16,4 +16,6 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
Sources.source -> Index.index
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
template:bool -> Sources.source -> Index.index
+(** [get_index download sigchecker source] will parse the source index file
+ into an index entry list. *)
-- 
2.12.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v7 9/9] Add a virt-builder-repository tool

[Cédric Bosdonnat]

pg(1)> (GNU Privacy Guard) binary.  You can
+also use this to add gpg parameters, for example to specify an
+alternate home directory:
+
+ virt-builder-repository --gpg "gpg --homedir /tmp" [...]
+
+This can also be used to avoid gpg asking for the key passphrase:
+
+ virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...]
+
+=item B<-K> KEYID
+
+=item B<--gpg-key> KEYID
+
+Specify the GPG key to be used to sign the repository index file.
+If not provided, the index will left unsigned. C is used to
+identify the GPG key to use. This value is passed to gpg's
+C<--default-key> option and can thus be an email address or a
+fingerprint.
+
+B: by default, virt-builder-repository searches for the key
+in the user's GPG keyring.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data. Default values are computed from the disk
+image.
+
+When prompted for data, inputting C<-> corresponds to leaving the
+value empty. This can be used to avoid setting the default computed value.
+
+=item B<--keep-index>
+
+When using a GPG key, don't remove the unsigned index.
+
+=item B<--no-compression>
+
+Don't compress the template images.
+
+=item B<--machine-readable>
+
+This option is used to make the output more machine friendly
+when being parsed by other programs.  See
+L below.
+
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages.  This is the default
+when the output is a tty.  If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don't print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 MACHINE READABLE OUTPUT
+
+The I<--machine-readable> option can be used to make the output more
+machine friendly, which is useful when calling virt-builder-repository from
+other programs, GUIs etc.
+
+Use the option on its own to query the capabilities of the
+virt-builder-repository binary.  Typical output looks like this:
+
+ $ virt-builder-repository --machine-readable
+ virt-builder-repository
+
+A list of features is printed, one per line, and the program exits
+with status 0.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdon...@suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
-- 
2.12.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v7 1/9] lib/osinfo.c: Extract xml processing into a callback

[Cédric Bosdonnat]

In order to further reuse the osinfo database parsing in OCAML, this
commit extracts the XML processing for the distro ISOs and places it
into a newly created callback.

This will later help other code to traverse the osinfo DB files and
let them extract what they need from them.
---
 lib/osinfo.c | 80 +---
 1 file changed, 39 insertions(+), 41 deletions(-)

diff --git a/lib/osinfo.c b/lib/osinfo.c
index ea2a7659a..3514585c7 100644
--- a/lib/osinfo.c
+++ b/lib/osinfo.c
@@ -52,6 +52,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -71,10 +72,11 @@ gl_lock_define_initialized (static, osinfo_db_lock);
 static ssize_t osinfo_db_size = 0; /* 0 = unread, -1 = error, >= 1 = #records 
*/
 static struct osinfo *osinfo_db = NULL;
 
-static int read_osinfo_db (guestfs_h *g);
-static void free_osinfo_db_entry (struct osinfo *);
-
 #define XMLSTREQ(a,b) (xmlStrEqual((a),(b)) == 1)
+typedef int (*read_osinfo_db_callback) (guestfs_h *g, const char *path, void 
*opaque);
+
+static int read_osinfo_db (guestfs_h *g, read_osinfo_db_callback callback, 
void *opaque);
+static void free_osinfo_db_entry (struct osinfo *);
 
 /* Given one or more fields from the header of a CD/DVD/ISO, look up
  * the media in the libosinfo database and return our best guess for
@@ -87,14 +89,24 @@ static void free_osinfo_db_entry (struct osinfo *);
  */
 int
 guestfs_int_osinfo_map (guestfs_h *g, const struct guestfs_isoinfo *isoinfo,
-   const struct osinfo **osinfo_ret)
+const struct osinfo **osinfo_ret)
 {
   size_t i;
 
   /* We only need to lock the database when reading it for the first time. */
   gl_lock_lock (osinfo_db_lock);
   if (osinfo_db_size == 0) {
-if (read_osinfo_db (g) == -1) {
+if (read_osinfo_db (g, read_osinfo_db_xml, NULL) == -1) {
+  /* Fatal error: free any database entries which have been read, and
+   * mark the database as having a permanent error.
+   */
+  if (osinfo_db_size > 0) {
+for (i = 0; i < (size_t) osinfo_db_size; ++i)
+  free_osinfo_db_entry (_db[i]);
+  }
+  free (osinfo_db);
+  osinfo_db = NULL;
+  osinfo_db_size = -1;
   gl_lock_unlock (osinfo_db_lock);
   return -1;
 }
@@ -156,19 +168,16 @@ guestfs_int_osinfo_map (guestfs_h *g, const struct 
guestfs_isoinfo *isoinfo,
  * Try to use the shared osinfo database layout (and location) first:
  * https://gitlab.com/libosinfo/libosinfo/blob/master/docs/database-layout.txt
  */
-static int read_osinfo_db_xml (guestfs_h *g, const char *filename);
-
-static int read_osinfo_db_flat (guestfs_h *g, const char *directory);
-static int read_osinfo_db_three_levels (guestfs_h *g, const char *directory);
-static int read_osinfo_db_directory (guestfs_h *g, const char *directory);
+static int read_osinfo_db_xml (guestfs_h *g, const char *pathname, void *data);
+static int read_osinfo_db_flat (guestfs_h *g, const char *directory, 
read_osinfo_db_callback callback, void *opaque);
+static int read_osinfo_db_three_levels (guestfs_h *g, const char *directory, 
read_osinfo_db_callback callback, void *opaque);
+static int read_osinfo_db_directory (guestfs_h *g, const char *directory, 
read_osinfo_db_callback callback, void *opaque);
 
 static int
-read_osinfo_db (guestfs_h *g)
+read_osinfo_db (guestfs_h *g,
+read_osinfo_db_callback callback, void *opaque)
 {
   int r;
-  size_t i;
-
-  assert (osinfo_db_size == 0);
 
   /* (1) Try the shared osinfo directory, using either the
* $OSINFO_SYSTEM_DIR envvar or its default value.
@@ -181,59 +190,47 @@ read_osinfo_db (guestfs_h *g)
 if (path == NULL)
   path = "/usr/share/osinfo";
 os_path = safe_asprintf (g, "%s/os", path);
-r = read_osinfo_db_three_levels (g, os_path);
+r = read_osinfo_db_three_levels (g, os_path, callback, opaque);
   }
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* (2) Try the libosinfo directory, using the newer three-directory
* layout ($LIBOSINFO_DB_PATH / "os" / $group-ID / [file.xml]).
*/
-  r = read_osinfo_db_three_levels (g, LIBOSINFO_DB_PATH "/os");
+  r = read_osinfo_db_three_levels (g, LIBOSINFO_DB_PATH "/os", callback, 
opaque);
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* (3) Try the libosinfo directory, using the old flat directory
* layout ($LIBOSINFO_DB_PATH / "oses" / [file.xml]).
*/
-  r = read_osinfo_db_flat (g, LIBOSINFO_DB_PATH "/oses");
+  r = read_osinfo_db_flat (g, LIBOSINFO_DB_PATH "/oses", callback, opaque);
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* Nothing found. */
   return 0;
-
- error:
-  /* Fatal error: free any database entries which have been read, and
-   * mark the database as having a permanent error.
-   */
-  if (osinfo_db_size > 0) {
-for (i = 0; i < (size_t) osinfo_db_size; 

[Libguestfs] [PATCH v7 6/9] builder: add Index.write_entry function

[Cédric Bosdonnat]

Add a function to properly write virt-builder source index entries.
Note that this function is very similar to Index.print_entry that is
meant for debugging purposes.
---
 .gitignore|   1 +
 builder/Makefile.am   |  36 +++-
 builder/index.mli |   3 +
 builder/index_parser.ml   |  54 ++
 builder/index_parser.mli  |   4 ++
 builder/index_parser_tests.ml | 129 ++
 6 files changed, 225 insertions(+), 2 deletions(-)
 create mode 100644 builder/index_parser_tests.ml

diff --git a/.gitignore b/.gitignore
index 69e1ae160..b9e00ee04 100644
--- a/.gitignore
+++ b/.gitignore
@@ -106,6 +106,7 @@ Makefile.in
 /builder/virt-index-validate
 /builder/virt-index-validate.1
 /builder/*.xz
+/builder/index_parser_tests
 /builder/yajl_tests
 /cat/stamp-virt-*.pod
 /cat/virt-cat
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 218f64b4c..bf4ccb7d7 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -217,13 +217,36 @@ yajl_tests_BOBJECTS = \
yajl_tests.cmo
 yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx)
 
+index_parser_tests_SOURCES = \
+   index-scan.c \
+   index-struct.c \
+   index-parser-c.c \
+   index-parse.c
+index_parser_tests_CPPFLAGS = $(virt_builder_CPPFLAGS)
+index_parser_tests_BOBJECTS = \
+   utils.cmo \
+   cache.cmo \
+   downloader.cmo \
+   sigchecker.cmo \
+   index.cmo \
+   ini_reader.cmo \
+   index_parser.cmo \
+   index_parser_tests.cmo
+index_parser_tests_XOBJECTS = $(index_parser_tests_BOBJECTS:.cmo=.cmx)
+
 # Can't call the following as _OBJECTS because automake gets confused.
 if HAVE_OCAMLOPT
 yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS)
 yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_XOBJECTS)
+index_parser_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 else
 yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS)
 yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_BOBJECTS)
+index_parser_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 endif
 
 yajl_tests_DEPENDENCIES = \
@@ -236,6 +259,15 @@ yajl_tests_LINK = \
  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
  $(yajl_tests_THEOBJECTS) -o $@
 
+index_parser_tests_DEPENDENCIES = \
+   $(index_parser_tests_THEOBJECTS) \
+   ../mllib/mllib.$(MLARCHIVE) \
+   $(top_srcdir)/ocaml-link.sh
+index_parser_tests_LINK = \
+   $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
+ $(index_parser_tests_THEOBJECTS) -o $@
+
 TESTS = \
test-docs.sh \
test-virt-builder-list.sh \
@@ -249,8 +281,8 @@ if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
 endif ENABLE_APPLIANCE
 if HAVE_OCAML_PKG_OUNIT
-check_PROGRAMS += yajl_tests
-TESTS += yajl_tests
+check_PROGRAMS += yajl_tests index_parser_tests
+TESTS += yajl_tests index_parser_tests
 endif
 
 check-valgrind:
diff --git a/builder/index.mli b/builder/index.mli
index ff5ec4a35..6202d636e 100644
--- a/builder/index.mli
+++ b/builder/index.mli
@@ -39,3 +39,6 @@ and entry = {
 }
 
 val print_entry : out_channel -> (string * entry) -> unit
+(** Debugging helper function dumping an index entry to a stream.
+To write entries for non-debugging purpose, use the
+[Index_parser.write_entry] function. *)
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index c70909e44..c56c4b8f8 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -236,3 +236,57 @@ let get_index ~downloader ~sigchecker ~template
   in
 
   get_index ()
+
+let write_entry chan (name, { Index.printable_name = printable_name;
+  file_uri = file_uri;
+  arch = arch;
+  osinfo = osinfo;
+  signature_uri = signature_uri;
+  checksums = checksums;
+  revision = revision;
+  format = format;
+  size = size;
+  compressed_size = compressed_size;
+  expand = expand;
+  lvexpand = lvexpand;
+  notes = notes;
+  aliases = aliases;
+  hidden = hidden }) =
+  let fp fs = fprintf chan fs in
+  fp "[%s]\n" name;
+  may (fp "name=%s\n") printable_name;
+  may (fp "osinfo=%s\n") osinfo;
+  fp "file=%s\n" file_uri;
+  fp "arch=%s\n" arch;
+  may (fp "sig=%s\n") signature_uri;
+  (match checksums with
+  | None -> ()
+  | Some checksums ->
+List.iter (
+  fun c ->
+fp "checksum[%s]=%s\n"
+  

[Libguestfs] [PATCH v7 8/9] mllib: add XPath helper xpath_get_nodes()

[Cédric Bosdonnat]

This function will allow more OCaml-ish processing of XPath queries
with multiple results.
---
 mllib/xpath_helpers.ml  | 9 +
 mllib/xpath_helpers.mli | 4 
 2 files changed, 13 insertions(+)

diff --git a/mllib/xpath_helpers.ml b/mllib/xpath_helpers.ml
index d651fab23..c66a21c44 100644
--- a/mllib/xpath_helpers.ml
+++ b/mllib/xpath_helpers.ml
@@ -51,3 +51,12 @@ let xpath_eval_default parsefn xpath expr default =
 let xpath_string_default = xpath_eval_default identity
 let xpath_int_default = xpath_eval_default int_of_string
 let xpath_int64_default = xpath_eval_default Int64.of_string
+
+let xpath_get_nodes xpathctx expr =
+  let obj = Xml.xpath_eval_expression xpathctx expr in
+  let nodes = ref [] in
+  for i = 0 to Xml.xpathobj_nr_nodes obj - 1 do
+let node = Xml.xpathobj_node obj i in
+push_back nodes node
+  done;
+  !nodes
diff --git a/mllib/xpath_helpers.mli b/mllib/xpath_helpers.mli
index 7434ba645..83c770281 100644
--- a/mllib/xpath_helpers.mli
+++ b/mllib/xpath_helpers.mli
@@ -31,3 +31,7 @@ val xpath_int_default : Xml.xpathctx -> string -> int -> int
 val xpath_int64_default : Xml.xpathctx -> string -> int64 -> int64
 (** Parse an xpath expression and return a string/int; if the expression
 doesn't match, return the default. *)
+
+val xpath_get_nodes : Xml.xpathctx -> string -> Xml.node list
+(** Parse an XPath expression and return a list with the matching
+XML nodes. *)
-- 
2.12.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v7 3/9] mllib: ocaml wrapper for lib/osinfo

[Cédric Bosdonnat]

Provide osinfo database parsing API in OCaml.
---
 lib/osinfo.c  |  39 +
 mllib/Makefile.am |  11 --
 mllib/osinfo-c.c  | 103 ++
 mllib/osinfo.ml   |  26 ++
 mllib/osinfo.mli  |  31 
 5 files changed, 208 insertions(+), 2 deletions(-)
 create mode 100644 mllib/osinfo-c.c
 create mode 100644 mllib/osinfo.ml
 create mode 100644 mllib/osinfo.mli

diff --git a/lib/osinfo.c b/lib/osinfo.c
index 5ccb554be..9a411b28d 100644
--- a/lib/osinfo.c
+++ b/lib/osinfo.c
@@ -52,6 +52,45 @@
 
 #include "osinfo.h"
 
+#ifndef GUESTFS_PRIVATE
+#undef perrorf
+static void perrorf(guestfs_h *g, const char *fmt, ...)
+__attribute__((format (printf,2,3)));
+
+static void perrorf(guestfs_h *g, const char *fmt, ...)
+{
+  va_list args;
+  CLEANUP_FREE char *msg = NULL;
+  CLEANUP_FREE char *fs = NULL;
+
+  ignore_value (asprintf (, "%s\n", fmt));
+
+  va_start (args, fmt);
+  /* Ignoring the result is fine since perror
+   * can take NULL input */
+  ignore_value (vasprintf (, fs, args));
+  va_end (args);
+  perror (msg);
+}
+
+#undef debug
+static void debug(guestfs_h *g, const char *fmt, ...)
+__attribute__((format (printf,2,3)));
+
+static void
+debug(guestfs_h *g, const char *fmt, ...)
+{
+  va_list args;
+  CLEANUP_FREE char *fs = NULL;
+
+  ignore_value (asprintf (, "%s\n", fmt));
+
+  va_start (args, fmt);
+  vfprintf (stderr, fs, args);
+  va_end (args);
+}
+#endif /* GUESTFS_PRIVATE */
+
 
 /* Read the libosinfo XML database files.  The lock is held while
  * this is called.
diff --git a/mllib/Makefile.am b/mllib/Makefile.am
index ee2f1a7a8..ee16fe7ef 100644
--- a/mllib/Makefile.am
+++ b/mllib/Makefile.am
@@ -36,6 +36,7 @@ SOURCES_MLI = \
curl.mli \
getopt.mli \
JSON.mli \
+   osinfo.mli \
planner.mli \
progress.mli \
regedit.mli \
@@ -63,7 +64,8 @@ SOURCES_ML = \
curl.ml \
checksums.ml \
xml.ml \
-   xpath_helpers.ml
+   xpath_helpers.ml \
+   osinfo.ml
 
 SOURCES_C = \
../common/visit/visit.c \
@@ -71,8 +73,12 @@ SOURCES_C = \
../common/options/keys.c \
../common/options/uri.c \
../common/progress/progress.c \
+   ../lib/alloc.c \
+   ../lib/osinfo.c \
+   ../lib/osinfo.h \
common_utils-c.c \
getopt-c.c \
+   osinfo-c.c \
progress-c.c \
unix_utils-c.c \
uri-c.c \
@@ -106,7 +112,8 @@ libmllib_a_CPPFLAGS = \
-I$(top_srcdir)/lib \
-I$(top_srcdir)/common/visit \
-I$(top_srcdir)/common/options \
-   -I$(top_srcdir)/common/progress
+   -I$(top_srcdir)/common/progress \
+   -DLIBOSINFO_DB_PATH='"$(datadir)/libosinfo/db"'
 libmllib_a_CFLAGS = \
$(WARN_CFLAGS) $(WERROR_CFLAGS) \
$(LIBVIRT_CFLAGS) $(LIBXML2_CFLAGS) \
diff --git a/mllib/osinfo-c.c b/mllib/osinfo-c.c
new file mode 100644
index 0..84760a85f
--- /dev/null
+++ b/mllib/osinfo-c.c
@@ -0,0 +1,103 @@
+/* Bindings for osinfo db reading function.
+ * Copyright (C) 2016 Red Hat Inc.
+ * Copyright (C) 2017 SUSE Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 
USA.
+ */
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "guestfs.h"
+#include "guestfs-internal.h"
+#include "osinfo.h"
+
+#pragma GCC diagnostic ignored "-Wmissing-prototypes"
+
+struct callback_wrapper_args {
+  /* In both case we are pointing to local roots, hence why these are
+   * value* not value.
+   */
+  value *exnp;  /* Safe place to store any exception
+   raised by callback */
+  value *fvp;   /* callback. */
+};
+
+static int read_osinfo_db_callback_wrapper (guestfs_h *g, const char *path, 
void *opaque);
+
+value
+guestfs_int_mllib_read_osinfo_db (value gv, value fv)
+{
+  CAMLparam2 (gv, fv);
+  guestfs_h *g = (guestfs_h *) Int64_val (gv);
+  struct callback_wrapper_args args;
+
+  /* This stack address is used to point to the exception, if one is
+   * raised in the visitor_function.  Note that the macro initializes
+   * this to Val_unit, which is how we know if an exception was set.
+   */
+  CAMLlocal1 (exn);
+
+  exn = Val_unit;
+
+  

[Libguestfs] [PATCH v7 7/9] mllib: add do_mv helper function to Common_utils

[Cédric Bosdonnat]

---
 mllib/common_utils.ml  | 6 ++
 mllib/common_utils.mli | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/mllib/common_utils.ml b/mllib/common_utils.ml
index 6a9b08973..8ed7c7554 100644
--- a/mllib/common_utils.ml
+++ b/mllib/common_utils.ml
@@ -1188,3 +1188,9 @@ let inspect_decrypt g =
* function.
*)
   c_inspect_decrypt g#ocaml_handle (Guestfs.c_pointer g#ocaml_handle)
+
+let do_mv src dest =
+  let cmd = [ "mv"; src; dest ] in
+  let r = run_command cmd in
+  if r <> 0 then
+error (f_"moving file '%s' to '%s' failed") src dest
diff --git a/mllib/common_utils.mli b/mllib/common_utils.mli
index c088f8497..2f01cdeae 100644
--- a/mllib/common_utils.mli
+++ b/mllib/common_utils.mli
@@ -498,3 +498,6 @@ val inspect_decrypt : Guestfs.guestfs -> unit
 (** Simple implementation of decryption: look for any [crypto_LUKS]
 partitions and decrypt them, then rescan for VGs.  This only works
 for Fedora whole-disk encryption. *)
+
+val do_mv : string -> string -> unit
+(** Run the mv command, and exit with an error if it failed *)
-- 
2.12.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v7 4/9] builder: rename docs test script

[Cédric Bosdonnat]

Rename test-virt-builder-docs.sh into test-docs.sh to include test
for another tool's documentation.
---
 builder/Makefile.am | 4 ++--
 builder/{test-virt-builder-docs.sh => test-docs.sh} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (100%)

diff --git a/builder/Makefile.am b/builder/Makefile.am
index d56b394b7..218f64b4c 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -28,7 +28,7 @@ EXTRA_DIST = \
test-simplestreams/streams/v1/index.json \
test-simplestreams/streams/v1/net.cirros-cloud_released_download.json \
test-virt-builder.sh \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-builder-list-simplestreams.sh \
test-virt-builder-planner.sh \
@@ -237,7 +237,7 @@ yajl_tests_LINK = \
  $(yajl_tests_THEOBJECTS) -o $@
 
 TESTS = \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-index-validate.sh \
$(SLOW_TESTS)
diff --git a/builder/test-virt-builder-docs.sh b/builder/test-docs.sh
similarity index 100%
rename from builder/test-virt-builder-docs.sh
rename to builder/test-docs.sh
-- 
2.12.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v7 0/9] Introducing virt-builder-repository

[Cédric Bosdonnat]

Hi all,

Here is an update of the series fixing Pino's latest comment.
It just doesn't implement the change based on never-accepted
run commands patch.

Cédric Bosdonnat (9):
  lib/osinfo.c: Extract xml processing into a callback
  lib: extract osinfo DB traversing API
  mllib: ocaml wrapper for lib/osinfo
  builder: rename docs test script
  builder: add a template parameter to get_index
  builder: add Index.write_entry function
  mllib: add do_mv helper function to Common_utils
  mllib: add XPath helper xpath_get_nodes()
  Add a virt-builder-repository tool

 .gitignore |   4 +
 builder/Makefile.am| 124 -
 builder/builder.ml |   2 +-
 builder/index.mli  |   3 +
 builder/index_parser.ml|  80 ++-
 builder/index_parser.mli   |   8 +-
 builder/index_parser_tests.ml  | 129 +
 builder/repository_main.ml | 584 +
 .../{test-virt-builder-docs.sh => test-docs.sh}|   2 +
 builder/virt-builder-repository.pod| 213 
 lib/Makefile.am|   2 +
 lib/osinfo-iso.c   | 462 
 lib/osinfo.c   | 489 ++---
 lib/osinfo.h   |  27 +
 mllib/Makefile.am  |  11 +-
 mllib/common_utils.ml  |   6 +
 mllib/common_utils.mli |   3 +
 mllib/osinfo-c.c   | 103 
 mllib/osinfo.ml|  26 +
 mllib/osinfo.mli   |  31 ++
 mllib/xpath_helpers.ml |   9 +
 mllib/xpath_helpers.mli|   4 +
 22 files changed, 1869 insertions(+), 453 deletions(-)
 create mode 100644 builder/index_parser_tests.ml
 create mode 100644 builder/repository_main.ml
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (93%)
 create mode 100644 builder/virt-builder-repository.pod
 create mode 100644 lib/osinfo-iso.c
 create mode 100644 lib/osinfo.h
 create mode 100644 mllib/osinfo-c.c
 create mode 100644 mllib/osinfo.ml
 create mode 100644 mllib/osinfo.mli

-- 
2.12.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v7 2/9] lib: extract osinfo DB traversing API

[Cédric Bosdonnat]

Split lib/osinfo.c to provide an API for other pieces of code (namely
mllib) to reuse it. The ISO-related processing is thus moved into a
lib/osinfo-iso.c file.
---
 lib/Makefile.am  |   2 +
 lib/osinfo-iso.c | 462 +++
 lib/osinfo.c | 420 +-
 lib/osinfo.h |  27 
 4 files changed, 493 insertions(+), 418 deletions(-)
 create mode 100644 lib/osinfo-iso.c
 create mode 100644 lib/osinfo.h

diff --git a/lib/Makefile.am b/lib/Makefile.am
index 360ce9c92..2cb83f2bb 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -113,7 +113,9 @@ libguestfs_la_SOURCES = \
lpj.c \
match.c \
mountable.c \
+   osinfo.h \
osinfo.c \
+   osinfo-iso.c \
private-data.c \
proto.c \
qemu.c \
diff --git a/lib/osinfo-iso.c b/lib/osinfo-iso.c
new file mode 100644
index 0..059d72def
--- /dev/null
+++ b/lib/osinfo-iso.c
@@ -0,0 +1,462 @@
+/* libguestfs
+ * Copyright (C) 2012 Red Hat Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Read libosinfo XML files to parse out just the
+ * os/media/iso/system-id and os/media/iso/volume-id fields, which we
+ * can then use to map install media to operating systems.
+ *
+ * Note some assumptions here:
+ *
+ * (1) We have to do some translation of the distro names and versions
+ * stored in the libosinfo files and the standard names returned by
+ * libguestfs.
+ *
+ * (2) Media detection is only part of the story.  We may still need
+ * to inspect inside the image.
+ *
+ * (3) We only read the XML database files (at most) once per process,
+ * and keep them cached.  They are only read at all if someone tries
+ * to inspect a CD/DVD/ISO.
+ *
+ * XXX Currently the database is not freed when the program exits /
+ * library is unloaded, although we should probably do that.
+ */
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+#include "ignore-value.h"
+#include "glthread/lock.h"
+#include "c-ctype.h"
+
+#include "guestfs.h"
+#include "guestfs-internal.h"
+
+#include "osinfo.h"
+
+gl_lock_define_initialized (static, osinfo_db_lock);
+static ssize_t osinfo_db_size = 0; /* 0 = unread, -1 = error, >= 1 = #records 
*/
+static struct osinfo *osinfo_db = NULL;
+
+static void free_osinfo_db_entry (struct osinfo *);
+
+#define XMLSTREQ(a,b) (xmlStrEqual((a),(b)) == 1)
+
+static int read_osinfo_db_xml (guestfs_h *g, const char *pathname, void *data);
+
+/* Given one or more fields from the header of a CD/DVD/ISO, look up
+ * the media in the libosinfo database and return our best guess for
+ * the operating system.
+ *
+ * This returns:
+ *   -1 => a fatal error ('error' has been called, caller must not ignore it)
+ *   0  => could not locate the OS
+ *   1  => matching OS found, the osinfo_ret pointer has been filled in
+ */
+int
+guestfs_int_osinfo_map (guestfs_h *g, const struct guestfs_isoinfo *isoinfo,
+const struct osinfo **osinfo_ret)
+{
+  size_t i;
+
+  /* We only need to lock the database when reading it for the first time. */
+  gl_lock_lock (osinfo_db_lock);
+  if (osinfo_db_size == 0) {
+if (read_osinfo_db (g, read_osinfo_db_xml, NULL) == -1) {
+  /* Fatal error: free any database entries which have been read, and
+   * mark the database as having a permanent error.
+   */
+  if (osinfo_db_size > 0) {
+for (i = 0; i < (size_t) osinfo_db_size; ++i)
+  free_osinfo_db_entry (_db[i]);
+  }
+  free (osinfo_db);
+  osinfo_db = NULL;
+  osinfo_db_size = -1;
+  gl_lock_unlock (osinfo_db_lock);
+  return -1;
+}
+  }
+  gl_lock_unlock (osinfo_db_lock);
+
+  if (osinfo_db_size <= 0)
+return 0;
+
+  /* Look in the database to see if we can find a match. */
+  for (i = 0; i < (size_t) osinfo_db_size; ++i) {
+if (osinfo_db[i].re_system_id) {
+  if (!isoinfo->iso_system_id ||
+  !match (g, isoinfo->iso_system_id, osinfo_db[i].re_system_id))
+continue;
+}
+
+if (osinfo_db[i].re_volume_id) {
+  if (!isoinfo->iso_volume_id ||
+  !match (g, isoinfo->iso_volume_id, osinfo_db[i].re_volume_id))
+

[Libguestfs] [PATCH] v2v: tell v2v the real root device to mkinitrd

[Cédric Bosdonnat]

From: Pino Toscano <ptosc...@redhat.com>

Complementary fix of commit 2d25872df3619a3077006ad0f91c029602db6780.
On SLES 11 SP4 with kdump enabled mkinitrd calls mkdumprd which calls
mkinitrd, but mkdumprd doesn't have any clue of the root device.

Call mkinitrd with rootdev environment variable to tell them all
what device to use as root.

Tested-By: Cédric Bosdonnat <cbosdon...@suse.com>
---
 v2v/convert_linux.ml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/v2v/convert_linux.ml b/v2v/convert_linux.ml
index f8dfa00d7..42a19947b 100644
--- a/v2v/convert_linux.ml
+++ b/v2v/convert_linux.ml
@@ -514,7 +514,9 @@ let rec convert (g : G.guestfs) inspect source output rcaps 
=
   else if family = `SUSE_family
&& g#is_file ~followsymlinks:true "/sbin/mkinitrd" then (
 ignore (
-  g#command [| "/sbin/mkinitrd";
+  g#command [| "/usr/bin/env";
+   "rootdev=" ^ inspect.i_root;
+   "/sbin/mkinitrd";
"-m"; String.concat " " modules;
"-i"; initrd;
"-k"; kernel.ki_vmlinuz;
-- 
2.12.2

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v6 08/10] mllib: add do_mv helper function to Common_utils

[Cédric Bosdonnat]

---
 mllib/common_utils.ml  | 6 ++
 mllib/common_utils.mli | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/mllib/common_utils.ml b/mllib/common_utils.ml
index ceac57711..5cc865659 100644
--- a/mllib/common_utils.ml
+++ b/mllib/common_utils.ml
@@ -1191,3 +1191,9 @@ let do_cp src destdir =
   let cmd = [ "cp"; "-t"; destdir; "-a"; src ] in
   if run_command cmd <> 0 then
 error (f_"copy of %s to %s failed") src destdir
+
+let do_mv src dest =
+  let cmd = [ "mv"; src; dest ] in
+  let r = run_command cmd in
+  if r <> 0 then
+error (f_"moving file '%s' to '%s' failed") src dest
diff --git a/mllib/common_utils.mli b/mllib/common_utils.mli
index 937ef818b..64a0e8b9a 100644
--- a/mllib/common_utils.mli
+++ b/mllib/common_utils.mli
@@ -499,3 +499,6 @@ val inspect_decrypt : Guestfs.guestfs -> unit
 
 val do_cp : string -> string -> unit
 (** Run the cp command, and exit with an error if it failed *)
+
+val do_mv : string -> string -> unit
+(** Run the mv command, and exit with an error if it failed *)
-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v6 07/10] dib: move do_cp to mllib.Commun_utils

[Cédric Bosdonnat]

---
 dib/utils.ml   | 4 
 mllib/common_utils.ml  | 5 +
 mllib/common_utils.mli | 3 +++
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/dib/utils.ml b/dib/utils.ml
index afa2ec944..2fe70e7fc 100644
--- a/dib/utils.ml
+++ b/dib/utils.ml
@@ -98,10 +98,6 @@ let get_required_tool tool =
 let require_tool tool =
   ignore (get_required_tool tool)
 
-let do_cp src destdir =
-  let cmd = [ "cp"; "-t"; destdir; "-a"; src ] in
-  if run_command cmd <> 0 then exit 1
-
 let ensure_trailing_newline str =
   if String.length str > 0 && str.[String.length str - 1] <> '\n' then str ^ 
"\n"
   else str
diff --git a/mllib/common_utils.ml b/mllib/common_utils.ml
index 63d8dd92e..ceac57711 100644
--- a/mllib/common_utils.ml
+++ b/mllib/common_utils.ml
@@ -1186,3 +1186,8 @@ let inspect_decrypt g =
* function.
*)
   c_inspect_decrypt g#ocaml_handle (Guestfs.c_pointer g#ocaml_handle)
+
+let do_cp src destdir =
+  let cmd = [ "cp"; "-t"; destdir; "-a"; src ] in
+  if run_command cmd <> 0 then
+error (f_"copy of %s to %s failed") src destdir
diff --git a/mllib/common_utils.mli b/mllib/common_utils.mli
index ec41a8ff8..937ef818b 100644
--- a/mllib/common_utils.mli
+++ b/mllib/common_utils.mli
@@ -496,3 +496,6 @@ val inspect_decrypt : Guestfs.guestfs -> unit
 (** Simple implementation of decryption: look for any [crypto_LUKS]
 partitions and decrypt them, then rescan for VGs.  This only works
 for Fedora whole-disk encryption. *)
+
+val do_cp : string -> string -> unit
+(** Run the cp command, and exit with an error if it failed *)
-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v6 06/10] builder: add Index.write_entry function

[Cédric Bosdonnat]

Add a function to properly write virt-builder source index entries.
Note that this function is very similar to Index.print_entry that is
meant for debugging purposes.
---
 .gitignore|   1 +
 builder/Makefile.am   |  36 +++-
 builder/index.mli |   3 +
 builder/index_parser.ml   |  54 ++
 builder/index_parser.mli  |   4 ++
 builder/index_parser_tests.ml | 129 ++
 6 files changed, 225 insertions(+), 2 deletions(-)
 create mode 100644 builder/index_parser_tests.ml

diff --git a/.gitignore b/.gitignore
index 3b50afdcd..4f04f8451 100644
--- a/.gitignore
+++ b/.gitignore
@@ -106,6 +106,7 @@ Makefile.in
 /builder/virt-index-validate
 /builder/virt-index-validate.1
 /builder/*.xz
+/builder/index_parser_tests
 /builder/yajl_tests
 /cat/stamp-virt-*.pod
 /cat/virt-cat
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 218f64b4c..bf4ccb7d7 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -217,13 +217,36 @@ yajl_tests_BOBJECTS = \
yajl_tests.cmo
 yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx)
 
+index_parser_tests_SOURCES = \
+   index-scan.c \
+   index-struct.c \
+   index-parser-c.c \
+   index-parse.c
+index_parser_tests_CPPFLAGS = $(virt_builder_CPPFLAGS)
+index_parser_tests_BOBJECTS = \
+   utils.cmo \
+   cache.cmo \
+   downloader.cmo \
+   sigchecker.cmo \
+   index.cmo \
+   ini_reader.cmo \
+   index_parser.cmo \
+   index_parser_tests.cmo
+index_parser_tests_XOBJECTS = $(index_parser_tests_BOBJECTS:.cmo=.cmx)
+
 # Can't call the following as _OBJECTS because automake gets confused.
 if HAVE_OCAMLOPT
 yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS)
 yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_XOBJECTS)
+index_parser_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 else
 yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS)
 yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_BOBJECTS)
+index_parser_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 endif
 
 yajl_tests_DEPENDENCIES = \
@@ -236,6 +259,15 @@ yajl_tests_LINK = \
  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
  $(yajl_tests_THEOBJECTS) -o $@
 
+index_parser_tests_DEPENDENCIES = \
+   $(index_parser_tests_THEOBJECTS) \
+   ../mllib/mllib.$(MLARCHIVE) \
+   $(top_srcdir)/ocaml-link.sh
+index_parser_tests_LINK = \
+   $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
+ $(index_parser_tests_THEOBJECTS) -o $@
+
 TESTS = \
test-docs.sh \
test-virt-builder-list.sh \
@@ -249,8 +281,8 @@ if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
 endif ENABLE_APPLIANCE
 if HAVE_OCAML_PKG_OUNIT
-check_PROGRAMS += yajl_tests
-TESTS += yajl_tests
+check_PROGRAMS += yajl_tests index_parser_tests
+TESTS += yajl_tests index_parser_tests
 endif
 
 check-valgrind:
diff --git a/builder/index.mli b/builder/index.mli
index ff5ec4a35..6202d636e 100644
--- a/builder/index.mli
+++ b/builder/index.mli
@@ -39,3 +39,6 @@ and entry = {
 }
 
 val print_entry : out_channel -> (string * entry) -> unit
+(** Debugging helper function dumping an index entry to a stream.
+To write entries for non-debugging purpose, use the
+[Index_parser.write_entry] function. *)
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index c70909e44..c56c4b8f8 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -236,3 +236,57 @@ let get_index ~downloader ~sigchecker ~template
   in
 
   get_index ()
+
+let write_entry chan (name, { Index.printable_name = printable_name;
+  file_uri = file_uri;
+  arch = arch;
+  osinfo = osinfo;
+  signature_uri = signature_uri;
+  checksums = checksums;
+  revision = revision;
+  format = format;
+  size = size;
+  compressed_size = compressed_size;
+  expand = expand;
+  lvexpand = lvexpand;
+  notes = notes;
+  aliases = aliases;
+  hidden = hidden }) =
+  let fp fs = fprintf chan fs in
+  fp "[%s]\n" name;
+  may (fp "name=%s\n") printable_name;
+  may (fp "osinfo=%s\n") osinfo;
+  fp "file=%s\n" file_uri;
+  fp "arch=%s\n" arch;
+  may (fp "sig=%s\n") signature_uri;
+  (match checksums with
+  | None -> ()
+  | Some checksums ->
+List.iter (
+  fun c ->
+fp "checksum[%s]=%s\n"
+  

[Libguestfs] [PATCH v6 00/10] Add a virt-builder-repository tool

[Cédric Bosdonnat]

Hi all,

Here is an updated version of that patch series.

Diff to v5:
  * Apply Pino's comments
  * Fix indentation issues
  * Add a default value for arch in builder/index_parser.ml if template
is set
  * Improved new images filtering: don't process image that didn't
change. This has been uncovered by introduction of --no-compression

Cédric Bosdonnat (10):
  lib/osinfo.c: Extract xml processing into a callback
  lib: extract osinfo DB traversing API
  mllib: ocaml wrapper for lib/osinfo
  builder: rename docs test script
  builder: add a template parameter to get_index
  builder: add Index.write_entry function
  dib: move do_cp to mllib.Commun_utils
  mllib: add do_mv helper function to Common_utils
  mllib: add XPath helper xpath_get_nodes()
  Add a virt-builder-repository tool

 .gitignore |   4 +
 builder/Makefile.am| 124 -
 builder/builder.ml |   2 +-
 builder/index.mli  |   3 +
 builder/index_parser.ml|  80 ++-
 builder/index_parser.mli   |   8 +-
 builder/index_parser_tests.ml  | 129 +
 builder/repository_main.ml | 570 +
 .../{test-virt-builder-docs.sh => test-docs.sh}|   2 +
 builder/virt-builder-repository.pod| 209 
 dib/utils.ml   |   4 -
 lib/Makefile.am|   2 +
 lib/osinfo-iso.c   | 462 +
 lib/osinfo.c   | 489 ++
 lib/osinfo.h   |  27 +
 mllib/Makefile.am  |  11 +-
 mllib/common_utils.ml  |  11 +
 mllib/common_utils.mli |   6 +
 mllib/osinfo-c.c   | 103 
 mllib/osinfo.ml|  26 +
 mllib/osinfo.mli   |  31 ++
 mllib/xpath_helpers.ml |   9 +
 mllib/xpath_helpers.mli|   4 +
 23 files changed, 1859 insertions(+), 457 deletions(-)
 create mode 100644 builder/index_parser_tests.ml
 create mode 100644 builder/repository_main.ml
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (93%)
 create mode 100644 builder/virt-builder-repository.pod
 create mode 100644 lib/osinfo-iso.c
 create mode 100644 lib/osinfo.h
 create mode 100644 mllib/osinfo-c.c
 create mode 100644 mllib/osinfo.ml
 create mode 100644 mllib/osinfo.mli

-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v6 09/10] mllib: add XPath helper xpath_get_nodes()

[Cédric Bosdonnat]

This function will allow more OCaml-ish processing of XPath queries
with multiple results.
---
 mllib/xpath_helpers.ml  | 9 +
 mllib/xpath_helpers.mli | 4 
 2 files changed, 13 insertions(+)

diff --git a/mllib/xpath_helpers.ml b/mllib/xpath_helpers.ml
index 8648596a4..c0f4a7315 100644
--- a/mllib/xpath_helpers.ml
+++ b/mllib/xpath_helpers.ml
@@ -53,3 +53,12 @@ let xpath_eval_default parsefn xpath expr default =
 let xpath_string_default = xpath_eval_default identity
 let xpath_int_default = xpath_eval_default int_of_string
 let xpath_int64_default = xpath_eval_default Int64.of_string
+
+let xpath_get_nodes xpathctx expr =
+  let obj = Xml.xpath_eval_expression xpathctx expr in
+  let nodes = ref [] in
+  for i = 0 to Xml.xpathobj_nr_nodes obj - 1 do
+let node = Xml.xpathobj_node obj i in
+push_back nodes node
+  done;
+  !nodes
diff --git a/mllib/xpath_helpers.mli b/mllib/xpath_helpers.mli
index 7434ba645..83c770281 100644
--- a/mllib/xpath_helpers.mli
+++ b/mllib/xpath_helpers.mli
@@ -31,3 +31,7 @@ val xpath_int_default : Xml.xpathctx -> string -> int -> int
 val xpath_int64_default : Xml.xpathctx -> string -> int64 -> int64
 (** Parse an xpath expression and return a string/int; if the expression
 doesn't match, return the default. *)
+
+val xpath_get_nodes : Xml.xpathctx -> string -> Xml.node list
+(** Parse an XPath expression and return a list with the matching
+XML nodes. *)
-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v6 04/10] builder: rename docs test script

[Cédric Bosdonnat]

Rename test-virt-builder-docs.sh into test-docs.sh to include test
for another tool's documentation.
---
 builder/Makefile.am | 4 ++--
 builder/{test-virt-builder-docs.sh => test-docs.sh} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (100%)

diff --git a/builder/Makefile.am b/builder/Makefile.am
index d56b394b7..218f64b4c 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -28,7 +28,7 @@ EXTRA_DIST = \
test-simplestreams/streams/v1/index.json \
test-simplestreams/streams/v1/net.cirros-cloud_released_download.json \
test-virt-builder.sh \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-builder-list-simplestreams.sh \
test-virt-builder-planner.sh \
@@ -237,7 +237,7 @@ yajl_tests_LINK = \
  $(yajl_tests_THEOBJECTS) -o $@
 
 TESTS = \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-index-validate.sh \
$(SLOW_TESTS)
diff --git a/builder/test-virt-builder-docs.sh b/builder/test-docs.sh
similarity index 100%
rename from builder/test-virt-builder-docs.sh
rename to builder/test-docs.sh
-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v6 02/10] lib: extract osinfo DB traversing API

[Cédric Bosdonnat]

Split lib/osinfo.c to provide an API for other pieces of code (namely
mllib) to reuse it. The ISO-related processing is thus moved into a
lib/osinfo-iso.c file.
---
 lib/Makefile.am  |   2 +
 lib/osinfo-iso.c | 462 +++
 lib/osinfo.c | 420 +-
 lib/osinfo.h |  27 
 4 files changed, 493 insertions(+), 418 deletions(-)
 create mode 100644 lib/osinfo-iso.c
 create mode 100644 lib/osinfo.h

diff --git a/lib/Makefile.am b/lib/Makefile.am
index 063706f8f..dd5f9fb92 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -112,7 +112,9 @@ libguestfs_la_SOURCES = \
lpj.c \
match.c \
mountable.c \
+   osinfo.h \
osinfo.c \
+   osinfo-iso.c \
private-data.c \
proto.c \
qemu.c \
diff --git a/lib/osinfo-iso.c b/lib/osinfo-iso.c
new file mode 100644
index 0..059d72def
--- /dev/null
+++ b/lib/osinfo-iso.c
@@ -0,0 +1,462 @@
+/* libguestfs
+ * Copyright (C) 2012 Red Hat Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Read libosinfo XML files to parse out just the
+ * os/media/iso/system-id and os/media/iso/volume-id fields, which we
+ * can then use to map install media to operating systems.
+ *
+ * Note some assumptions here:
+ *
+ * (1) We have to do some translation of the distro names and versions
+ * stored in the libosinfo files and the standard names returned by
+ * libguestfs.
+ *
+ * (2) Media detection is only part of the story.  We may still need
+ * to inspect inside the image.
+ *
+ * (3) We only read the XML database files (at most) once per process,
+ * and keep them cached.  They are only read at all if someone tries
+ * to inspect a CD/DVD/ISO.
+ *
+ * XXX Currently the database is not freed when the program exits /
+ * library is unloaded, although we should probably do that.
+ */
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+#include "ignore-value.h"
+#include "glthread/lock.h"
+#include "c-ctype.h"
+
+#include "guestfs.h"
+#include "guestfs-internal.h"
+
+#include "osinfo.h"
+
+gl_lock_define_initialized (static, osinfo_db_lock);
+static ssize_t osinfo_db_size = 0; /* 0 = unread, -1 = error, >= 1 = #records 
*/
+static struct osinfo *osinfo_db = NULL;
+
+static void free_osinfo_db_entry (struct osinfo *);
+
+#define XMLSTREQ(a,b) (xmlStrEqual((a),(b)) == 1)
+
+static int read_osinfo_db_xml (guestfs_h *g, const char *pathname, void *data);
+
+/* Given one or more fields from the header of a CD/DVD/ISO, look up
+ * the media in the libosinfo database and return our best guess for
+ * the operating system.
+ *
+ * This returns:
+ *   -1 => a fatal error ('error' has been called, caller must not ignore it)
+ *   0  => could not locate the OS
+ *   1  => matching OS found, the osinfo_ret pointer has been filled in
+ */
+int
+guestfs_int_osinfo_map (guestfs_h *g, const struct guestfs_isoinfo *isoinfo,
+const struct osinfo **osinfo_ret)
+{
+  size_t i;
+
+  /* We only need to lock the database when reading it for the first time. */
+  gl_lock_lock (osinfo_db_lock);
+  if (osinfo_db_size == 0) {
+if (read_osinfo_db (g, read_osinfo_db_xml, NULL) == -1) {
+  /* Fatal error: free any database entries which have been read, and
+   * mark the database as having a permanent error.
+   */
+  if (osinfo_db_size > 0) {
+for (i = 0; i < (size_t) osinfo_db_size; ++i)
+  free_osinfo_db_entry (_db[i]);
+  }
+  free (osinfo_db);
+  osinfo_db = NULL;
+  osinfo_db_size = -1;
+  gl_lock_unlock (osinfo_db_lock);
+  return -1;
+}
+  }
+  gl_lock_unlock (osinfo_db_lock);
+
+  if (osinfo_db_size <= 0)
+return 0;
+
+  /* Look in the database to see if we can find a match. */
+  for (i = 0; i < (size_t) osinfo_db_size; ++i) {
+if (osinfo_db[i].re_system_id) {
+  if (!isoinfo->iso_system_id ||
+  !match (g, isoinfo->iso_system_id, osinfo_db[i].re_system_id))
+continue;
+}
+
+if (osinfo_db[i].re_volume_id) {
+  if (!isoinfo->iso_volume_id ||
+  !match (g, isoinfo->iso_volume_id, osinfo_db[i].re_volume_id))
+

[Libguestfs] [PATCH v6 03/10] mllib: ocaml wrapper for lib/osinfo

[Cédric Bosdonnat]

Provide osinfo database parsing API in OCaml.
---
 lib/osinfo.c  |  39 +
 mllib/Makefile.am |  11 --
 mllib/osinfo-c.c  | 103 ++
 mllib/osinfo.ml   |  26 ++
 mllib/osinfo.mli  |  31 
 5 files changed, 208 insertions(+), 2 deletions(-)
 create mode 100644 mllib/osinfo-c.c
 create mode 100644 mllib/osinfo.ml
 create mode 100644 mllib/osinfo.mli

diff --git a/lib/osinfo.c b/lib/osinfo.c
index 5ccb554be..9a411b28d 100644
--- a/lib/osinfo.c
+++ b/lib/osinfo.c
@@ -52,6 +52,45 @@
 
 #include "osinfo.h"
 
+#ifndef GUESTFS_PRIVATE
+#undef perrorf
+static void perrorf(guestfs_h *g, const char *fmt, ...)
+__attribute__((format (printf,2,3)));
+
+static void perrorf(guestfs_h *g, const char *fmt, ...)
+{
+  va_list args;
+  CLEANUP_FREE char *msg = NULL;
+  CLEANUP_FREE char *fs = NULL;
+
+  ignore_value (asprintf (, "%s\n", fmt));
+
+  va_start (args, fmt);
+  /* Ignoring the result is fine since perror
+   * can take NULL input */
+  ignore_value (vasprintf (, fs, args));
+  va_end (args);
+  perror (msg);
+}
+
+#undef debug
+static void debug(guestfs_h *g, const char *fmt, ...)
+__attribute__((format (printf,2,3)));
+
+static void
+debug(guestfs_h *g, const char *fmt, ...)
+{
+  va_list args;
+  CLEANUP_FREE char *fs = NULL;
+
+  ignore_value (asprintf (, "%s\n", fmt));
+
+  va_start (args, fmt);
+  vfprintf (stderr, fs, args);
+  va_end (args);
+}
+#endif /* GUESTFS_PRIVATE */
+
 
 /* Read the libosinfo XML database files.  The lock is held while
  * this is called.
diff --git a/mllib/Makefile.am b/mllib/Makefile.am
index ee2f1a7a8..ee16fe7ef 100644
--- a/mllib/Makefile.am
+++ b/mllib/Makefile.am
@@ -36,6 +36,7 @@ SOURCES_MLI = \
curl.mli \
getopt.mli \
JSON.mli \
+   osinfo.mli \
planner.mli \
progress.mli \
regedit.mli \
@@ -63,7 +64,8 @@ SOURCES_ML = \
curl.ml \
checksums.ml \
xml.ml \
-   xpath_helpers.ml
+   xpath_helpers.ml \
+   osinfo.ml
 
 SOURCES_C = \
../common/visit/visit.c \
@@ -71,8 +73,12 @@ SOURCES_C = \
../common/options/keys.c \
../common/options/uri.c \
../common/progress/progress.c \
+   ../lib/alloc.c \
+   ../lib/osinfo.c \
+   ../lib/osinfo.h \
common_utils-c.c \
getopt-c.c \
+   osinfo-c.c \
progress-c.c \
unix_utils-c.c \
uri-c.c \
@@ -106,7 +112,8 @@ libmllib_a_CPPFLAGS = \
-I$(top_srcdir)/lib \
-I$(top_srcdir)/common/visit \
-I$(top_srcdir)/common/options \
-   -I$(top_srcdir)/common/progress
+   -I$(top_srcdir)/common/progress \
+   -DLIBOSINFO_DB_PATH='"$(datadir)/libosinfo/db"'
 libmllib_a_CFLAGS = \
$(WARN_CFLAGS) $(WERROR_CFLAGS) \
$(LIBVIRT_CFLAGS) $(LIBXML2_CFLAGS) \
diff --git a/mllib/osinfo-c.c b/mllib/osinfo-c.c
new file mode 100644
index 0..84760a85f
--- /dev/null
+++ b/mllib/osinfo-c.c
@@ -0,0 +1,103 @@
+/* Bindings for osinfo db reading function.
+ * Copyright (C) 2016 Red Hat Inc.
+ * Copyright (C) 2017 SUSE Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 
USA.
+ */
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "guestfs.h"
+#include "guestfs-internal.h"
+#include "osinfo.h"
+
+#pragma GCC diagnostic ignored "-Wmissing-prototypes"
+
+struct callback_wrapper_args {
+  /* In both case we are pointing to local roots, hence why these are
+   * value* not value.
+   */
+  value *exnp;  /* Safe place to store any exception
+   raised by callback */
+  value *fvp;   /* callback. */
+};
+
+static int read_osinfo_db_callback_wrapper (guestfs_h *g, const char *path, 
void *opaque);
+
+value
+guestfs_int_mllib_read_osinfo_db (value gv, value fv)
+{
+  CAMLparam2 (gv, fv);
+  guestfs_h *g = (guestfs_h *) Int64_val (gv);
+  struct callback_wrapper_args args;
+
+  /* This stack address is used to point to the exception, if one is
+   * raised in the visitor_function.  Note that the macro initializes
+   * this to Val_unit, which is how we know if an exception was set.
+   */
+  CAMLlocal1 (exn);
+
+  exn = Val_unit;
+
+  

[Libguestfs] [PATCH v6 10/10] Add a virt-builder-repository tool

[Cédric Bosdonnat]

on and can can thus be an email address or a
+fingerprint.
+
+B: by default, virt-builder-repository searches for the key
+in the user's GPG key ring.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data. Default values are computed from the disk
+image.
+
+=item B<--keep-index>
+
+When using a GPG key, don't remove the unsigned index.
+
+=item B<--no-compression>
+
+Don't compress the template images.
+
+=item B<--machine-readable>
+
+This option is used to make the output more machine friendly
+when being parsed by other programs.  See
+L below.
+
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages.  This is the default
+when the output is a tty.  If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don't print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 MACHINE READABLE OUTPUT
+
+The I<--machine-readable> option can be used to make the output more
+machine friendly, which is useful when calling virt-builder-repository from
+other programs, GUIs etc.
+
+Use the option on its own to query the capabilities of the
+virt-builder-repository binary.  Typical output looks like this:
+
+ $ virt-builder-repository --machine-readable
+ virt-builder-repository
+
+A list of features is printed, one per line, and the program exits
+with status 0.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdon...@suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v6 05/10] builder: add a template parameter to get_index

[Cédric Bosdonnat]

get_index now gets a new template parameter. Setting it to true will
make the index parsing less picky about missing important data. This
can be used to parse a partial index file.
---
 builder/builder.ml   |  2 +-
 builder/index_parser.ml  | 26 ++
 builder/index_parser.mli |  4 +++-
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index b0a48ea89..99cd488b2 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -207,7 +207,7 @@ let main () =
   ~tmpdir in
   match source.Sources.format with
   | Sources.FormatNative ->
-Index_parser.get_index ~downloader ~sigchecker source
+Index_parser.get_index ~downloader ~sigchecker ~template:false 
source
   | Sources.FormatSimpleStreams ->
 Simplestreams_parser.get_index ~downloader ~sigchecker source
   ) sources
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index 468805cf8..c70909e44 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -24,7 +24,7 @@ open Utils
 open Printf
 open Unix
 
-let get_index ~downloader ~sigchecker
+let get_index ~downloader ~sigchecker ~template
   { Sources.uri = uri; proxy = proxy } =
   let corrupt_file () =
 error (f_"The index file downloaded from ‘%s’ is corrupt.\nYou need to ask 
the supplier of this file to fix it and upload a fixed version.") uri
@@ -99,8 +99,10 @@ let get_index ~downloader ~sigchecker
   let arch =
 try List.assoc ("arch", None) fields
 with Not_found ->
-  eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
-corrupt_file () in
+  if template then "" else (
+eprintf (f_"%s: no ‘arch’ entry for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let signature_uri =
 try Some (make_absolute_uri (List.assoc ("sig", None) fields))
 with Not_found -> None in
@@ -112,7 +114,7 @@ let get_index ~downloader ~sigchecker
   let revision =
 try Rev_int (int_of_string (List.assoc ("revision", None) fields))
 with
-| Not_found -> Rev_int 1
+| Not_found -> if template then Rev_int 0 else Rev_int 1
 | Failure _ ->
   eprintf (f_"%s: cannot parse ‘revision’ field for ‘%s’\n") prog 
n;
   corrupt_file () in
@@ -122,11 +124,19 @@ let get_index ~downloader ~sigchecker
 try Int64.of_string (List.assoc ("size", None) fields)
 with
 | Not_found ->
-  eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file ()
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: no ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  )
 | Failure _ ->
-  eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
-  corrupt_file () in
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: cannot parse ‘size’ field for ‘%s’\n") prog n;
+corrupt_file ()
+  ) in
   let compressed_size =
 try Some (Int64.of_string (List.assoc ("compressed_size", None) 
fields))
 with
diff --git a/builder/index_parser.mli b/builder/index_parser.mli
index b8d8ddf3d..aa5f84730 100644
--- a/builder/index_parser.mli
+++ b/builder/index_parser.mli
@@ -16,4 +16,6 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
Sources.source -> Index.index
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
template:bool -> Sources.source -> Index.index
+(** [get_index download sigchecker source] will parse the source index file
+ into an index entry list. *)
-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v6 01/10] lib/osinfo.c: Extract xml processing into a callback

[Cédric Bosdonnat]

In order to further reuse the osinfo database parsing in OCAML, this
commit extracts the XML processing for the distro ISOs and places it
into a newly created callback.

This will later help other code to traverse the osinfo DB files and
let them extract what they need from them.
---
 lib/osinfo.c | 80 +---
 1 file changed, 39 insertions(+), 41 deletions(-)

diff --git a/lib/osinfo.c b/lib/osinfo.c
index ea2a7659a..3514585c7 100644
--- a/lib/osinfo.c
+++ b/lib/osinfo.c
@@ -52,6 +52,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -71,10 +72,11 @@ gl_lock_define_initialized (static, osinfo_db_lock);
 static ssize_t osinfo_db_size = 0; /* 0 = unread, -1 = error, >= 1 = #records 
*/
 static struct osinfo *osinfo_db = NULL;
 
-static int read_osinfo_db (guestfs_h *g);
-static void free_osinfo_db_entry (struct osinfo *);
-
 #define XMLSTREQ(a,b) (xmlStrEqual((a),(b)) == 1)
+typedef int (*read_osinfo_db_callback) (guestfs_h *g, const char *path, void 
*opaque);
+
+static int read_osinfo_db (guestfs_h *g, read_osinfo_db_callback callback, 
void *opaque);
+static void free_osinfo_db_entry (struct osinfo *);
 
 /* Given one or more fields from the header of a CD/DVD/ISO, look up
  * the media in the libosinfo database and return our best guess for
@@ -87,14 +89,24 @@ static void free_osinfo_db_entry (struct osinfo *);
  */
 int
 guestfs_int_osinfo_map (guestfs_h *g, const struct guestfs_isoinfo *isoinfo,
-   const struct osinfo **osinfo_ret)
+const struct osinfo **osinfo_ret)
 {
   size_t i;
 
   /* We only need to lock the database when reading it for the first time. */
   gl_lock_lock (osinfo_db_lock);
   if (osinfo_db_size == 0) {
-if (read_osinfo_db (g) == -1) {
+if (read_osinfo_db (g, read_osinfo_db_xml, NULL) == -1) {
+  /* Fatal error: free any database entries which have been read, and
+   * mark the database as having a permanent error.
+   */
+  if (osinfo_db_size > 0) {
+for (i = 0; i < (size_t) osinfo_db_size; ++i)
+  free_osinfo_db_entry (_db[i]);
+  }
+  free (osinfo_db);
+  osinfo_db = NULL;
+  osinfo_db_size = -1;
   gl_lock_unlock (osinfo_db_lock);
   return -1;
 }
@@ -156,19 +168,16 @@ guestfs_int_osinfo_map (guestfs_h *g, const struct 
guestfs_isoinfo *isoinfo,
  * Try to use the shared osinfo database layout (and location) first:
  * https://gitlab.com/libosinfo/libosinfo/blob/master/docs/database-layout.txt
  */
-static int read_osinfo_db_xml (guestfs_h *g, const char *filename);
-
-static int read_osinfo_db_flat (guestfs_h *g, const char *directory);
-static int read_osinfo_db_three_levels (guestfs_h *g, const char *directory);
-static int read_osinfo_db_directory (guestfs_h *g, const char *directory);
+static int read_osinfo_db_xml (guestfs_h *g, const char *pathname, void *data);
+static int read_osinfo_db_flat (guestfs_h *g, const char *directory, 
read_osinfo_db_callback callback, void *opaque);
+static int read_osinfo_db_three_levels (guestfs_h *g, const char *directory, 
read_osinfo_db_callback callback, void *opaque);
+static int read_osinfo_db_directory (guestfs_h *g, const char *directory, 
read_osinfo_db_callback callback, void *opaque);
 
 static int
-read_osinfo_db (guestfs_h *g)
+read_osinfo_db (guestfs_h *g,
+read_osinfo_db_callback callback, void *opaque)
 {
   int r;
-  size_t i;
-
-  assert (osinfo_db_size == 0);
 
   /* (1) Try the shared osinfo directory, using either the
* $OSINFO_SYSTEM_DIR envvar or its default value.
@@ -181,59 +190,47 @@ read_osinfo_db (guestfs_h *g)
 if (path == NULL)
   path = "/usr/share/osinfo";
 os_path = safe_asprintf (g, "%s/os", path);
-r = read_osinfo_db_three_levels (g, os_path);
+r = read_osinfo_db_three_levels (g, os_path, callback, opaque);
   }
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* (2) Try the libosinfo directory, using the newer three-directory
* layout ($LIBOSINFO_DB_PATH / "os" / $group-ID / [file.xml]).
*/
-  r = read_osinfo_db_three_levels (g, LIBOSINFO_DB_PATH "/os");
+  r = read_osinfo_db_three_levels (g, LIBOSINFO_DB_PATH "/os", callback, 
opaque);
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* (3) Try the libosinfo directory, using the old flat directory
* layout ($LIBOSINFO_DB_PATH / "oses" / [file.xml]).
*/
-  r = read_osinfo_db_flat (g, LIBOSINFO_DB_PATH "/oses");
+  r = read_osinfo_db_flat (g, LIBOSINFO_DB_PATH "/oses", callback, opaque);
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* Nothing found. */
   return 0;
-
- error:
-  /* Fatal error: free any database entries which have been read, and
-   * mark the database as having a permanent error.
-   */
-  if (osinfo_db_size > 0) {
-for (i = 0; i < (size_t) osinfo_db_size; 

[Libguestfs] [PATCH v5 04/10] builder: rename docs test script

[Cédric Bosdonnat]

Rename test-virt-builder-docs.sh into test-docs.sh to include test
for another tool's documentation.
---
 builder/Makefile.am | 4 ++--
 builder/{test-virt-builder-docs.sh => test-docs.sh} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (100%)

diff --git a/builder/Makefile.am b/builder/Makefile.am
index d56b394b7..218f64b4c 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -28,7 +28,7 @@ EXTRA_DIST = \
test-simplestreams/streams/v1/index.json \
test-simplestreams/streams/v1/net.cirros-cloud_released_download.json \
test-virt-builder.sh \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-builder-list-simplestreams.sh \
test-virt-builder-planner.sh \
@@ -237,7 +237,7 @@ yajl_tests_LINK = \
  $(yajl_tests_THEOBJECTS) -o $@
 
 TESTS = \
-   test-virt-builder-docs.sh \
+   test-docs.sh \
test-virt-builder-list.sh \
test-virt-index-validate.sh \
$(SLOW_TESTS)
diff --git a/builder/test-virt-builder-docs.sh b/builder/test-docs.sh
similarity index 100%
rename from builder/test-virt-builder-docs.sh
rename to builder/test-docs.sh
-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v5 03/10] mllib: ocaml wrapper for lib/osinfo

[Cédric Bosdonnat]

Provide osinfo database parsing API in OCAML.
---
 lib/osinfo.c  |  15 
 mllib/Makefile.am |  11 --
 mllib/osinfo-c.c  | 102 ++
 mllib/osinfo.ml   |  26 ++
 mllib/osinfo.mli  |  31 +
 5 files changed, 183 insertions(+), 2 deletions(-)
 create mode 100644 mllib/osinfo-c.c
 create mode 100644 mllib/osinfo.ml
 create mode 100644 mllib/osinfo.mli

diff --git a/lib/osinfo.c b/lib/osinfo.c
index 5ccb554be..083872669 100644
--- a/lib/osinfo.c
+++ b/lib/osinfo.c
@@ -52,6 +52,21 @@
 
 #include "osinfo.h"
 
+#ifndef GUESTFS_PRIVATE
+#undef perrorf
+#define perrorf(g,...)  \
+{   \
+  CLEANUP_FREE char *msg = NULL;\
+  ignore_value (asprintf (, __VA_ARGS__));  \
+  perror (msg); \
+  /* Ignoring the result is fine since perror   \
+   * can take NULL input */ \
+}
+
+#undef debug
+#define debug(g,...) fprintf (stderr, __VA_ARGS__)
+#endif /* GUESTFS_PRIVATE */
+
 
 /* Read the libosinfo XML database files.  The lock is held while
  * this is called.
diff --git a/mllib/Makefile.am b/mllib/Makefile.am
index ee2f1a7a8..ee16fe7ef 100644
--- a/mllib/Makefile.am
+++ b/mllib/Makefile.am
@@ -36,6 +36,7 @@ SOURCES_MLI = \
curl.mli \
getopt.mli \
JSON.mli \
+   osinfo.mli \
planner.mli \
progress.mli \
regedit.mli \
@@ -63,7 +64,8 @@ SOURCES_ML = \
curl.ml \
checksums.ml \
xml.ml \
-   xpath_helpers.ml
+   xpath_helpers.ml \
+   osinfo.ml
 
 SOURCES_C = \
../common/visit/visit.c \
@@ -71,8 +73,12 @@ SOURCES_C = \
../common/options/keys.c \
../common/options/uri.c \
../common/progress/progress.c \
+   ../lib/alloc.c \
+   ../lib/osinfo.c \
+   ../lib/osinfo.h \
common_utils-c.c \
getopt-c.c \
+   osinfo-c.c \
progress-c.c \
unix_utils-c.c \
uri-c.c \
@@ -106,7 +112,8 @@ libmllib_a_CPPFLAGS = \
-I$(top_srcdir)/lib \
-I$(top_srcdir)/common/visit \
-I$(top_srcdir)/common/options \
-   -I$(top_srcdir)/common/progress
+   -I$(top_srcdir)/common/progress \
+   -DLIBOSINFO_DB_PATH='"$(datadir)/libosinfo/db"'
 libmllib_a_CFLAGS = \
$(WARN_CFLAGS) $(WERROR_CFLAGS) \
$(LIBVIRT_CFLAGS) $(LIBXML2_CFLAGS) \
diff --git a/mllib/osinfo-c.c b/mllib/osinfo-c.c
new file mode 100644
index 0..393208244
--- /dev/null
+++ b/mllib/osinfo-c.c
@@ -0,0 +1,102 @@
+/* Bindings for osinfo db reading function.
+ * Copyright (C) 2017 SUSE Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 
USA.
+ */
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "guestfs.h"
+#include "guestfs-internal.h"
+#include "osinfo.h"
+
+#pragma GCC diagnostic ignored "-Wmissing-prototypes"
+
+struct callback_wrapper_args {
+  /* In both case we are pointing to local roots, hence why these are
+   * value* not value.
+   */
+  value *exnp;  /* Safe place to store any exception
+   raised by callback */
+  value *fvp;   /* callback. */
+};
+
+static int read_osinfo_db_callback_wrapper (guestfs_h *g, const char *path, 
void *opaque);
+
+value
+guestfs_int_mllib_read_osinfo_db (value gv, value fv)
+{
+  CAMLparam2 (gv, fv);
+  guestfs_h *g = (guestfs_h *) Int64_val (gv);
+  struct callback_wrapper_args args;
+
+  /* This stack address is used to point to the exception, if one is
+   * raised in the visitor_function.  Note that the macro initializes
+   * this to Val_unit, which is how we know if an exception was set.
+   */
+  CAMLlocal1 (exn);
+
+  exn = Val_unit;
+
+  args.exnp = 
+  args.fvp = 
+
+  if (read_osinfo_db (g, read_osinfo_db_callback_wrapper, ) == -1) {
+if (exn != Val_unit) {
+  /* The failure was caused by the callback raising an
+   * exception.  Re-raise it here.
+   */
+  caml_raise (exn);
+}
+
+caml_failwith ("read_osinfo_db");
+}
+
+  CAMLreturn (Val_unit);
+}
+
+static int
+read_osinfo_db_callback_wrapper (guestfs_h *g, const char *path, void *opaque)
+{
+  

[Libguestfs] [PATCH v5 07/10] dib: move do_cp to mllib.Commun_utils

[Cédric Bosdonnat]

---
 dib/utils.ml   | 4 
 mllib/common_utils.ml  | 5 +
 mllib/common_utils.mli | 3 +++
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/dib/utils.ml b/dib/utils.ml
index 967754d95..92296d173 100644
--- a/dib/utils.ml
+++ b/dib/utils.ml
@@ -98,10 +98,6 @@ let get_required_tool tool =
 let require_tool tool =
   ignore (get_required_tool tool)
 
-let do_cp src destdir =
-  let cmd = [ "cp"; "-t"; destdir; "-a"; src ] in
-  if run_command cmd <> 0 then exit 1
-
 let ensure_trailing_newline str =
   if String.length str > 0 && str.[String.length str - 1] <> '\n' then str ^ 
"\n"
   else str
diff --git a/mllib/common_utils.ml b/mllib/common_utils.ml
index e1d63292e..945728b5e 100644
--- a/mllib/common_utils.ml
+++ b/mllib/common_utils.ml
@@ -1167,3 +1167,8 @@ let inspect_decrypt g =
* function.
*)
   c_inspect_decrypt g#ocaml_handle (Guestfs.c_pointer g#ocaml_handle)
+
+let do_cp src destdir =
+  let cmd = [ "cp"; "-t"; destdir; "-a"; src ] in
+  if run_command cmd <> 0 then
+error (f_"copy of %s to %s failed") src destdir
diff --git a/mllib/common_utils.mli b/mllib/common_utils.mli
index 1cd38ba83..5c376fcb3 100644
--- a/mllib/common_utils.mli
+++ b/mllib/common_utils.mli
@@ -492,3 +492,6 @@ val inspect_decrypt : Guestfs.guestfs -> unit
 (** Simple implementation of decryption: look for any [crypto_LUKS]
 partitions and decrypt them, then rescan for VGs.  This only works
 for Fedora whole-disk encryption. *)
+
+val do_cp : string -> string -> unit
+(** Run the cp command, and exit with an error if it failed *)
-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v5 08/10] mllib: add do_mv helper function to Common_utils

[Cédric Bosdonnat]

---
 mllib/common_utils.ml  | 6 ++
 mllib/common_utils.mli | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/mllib/common_utils.ml b/mllib/common_utils.ml
index 945728b5e..7932707c9 100644
--- a/mllib/common_utils.ml
+++ b/mllib/common_utils.ml
@@ -1172,3 +1172,9 @@ let do_cp src destdir =
   let cmd = [ "cp"; "-t"; destdir; "-a"; src ] in
   if run_command cmd <> 0 then
 error (f_"copy of %s to %s failed") src destdir
+
+let do_mv src dest =
+  let cmd = [ "mv"; src; dest ] in
+  let r = run_command cmd in
+  if r <> 0 then
+error (f_"moving file '%s' to '%s' failed") src dest
diff --git a/mllib/common_utils.mli b/mllib/common_utils.mli
index 5c376fcb3..a98daad03 100644
--- a/mllib/common_utils.mli
+++ b/mllib/common_utils.mli
@@ -495,3 +495,6 @@ val inspect_decrypt : Guestfs.guestfs -> unit
 
 val do_cp : string -> string -> unit
 (** Run the cp command, and exit with an error if it failed *)
+
+val do_mv : string -> string -> unit
+(** Run the mv command, and exit with an error if it failed *)
-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v5 06/10] builder: add Index.write_entry function

[Cédric Bosdonnat]

Add a function to properly write virt-builder source index entries.
Note that this function is very similar to Index.print_entry that is
meant for debugging purposes.
---
 .gitignore|   1 +
 builder/Makefile.am   |  36 +++-
 builder/index.mli |   3 +
 builder/index_parser.ml   |  54 ++
 builder/index_parser.mli  |   4 ++
 builder/index_parser_tests.ml | 129 ++
 6 files changed, 225 insertions(+), 2 deletions(-)
 create mode 100644 builder/index_parser_tests.ml

diff --git a/.gitignore b/.gitignore
index c82745ee8..12322e56b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -104,6 +104,7 @@ Makefile.in
 /builder/virt-index-validate
 /builder/virt-index-validate.1
 /builder/*.xz
+/builder/index_parser_tests
 /builder/yajl_tests
 /cat/stamp-virt-*.pod
 /cat/virt-cat
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 218f64b4c..bf4ccb7d7 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -217,13 +217,36 @@ yajl_tests_BOBJECTS = \
yajl_tests.cmo
 yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx)
 
+index_parser_tests_SOURCES = \
+   index-scan.c \
+   index-struct.c \
+   index-parser-c.c \
+   index-parse.c
+index_parser_tests_CPPFLAGS = $(virt_builder_CPPFLAGS)
+index_parser_tests_BOBJECTS = \
+   utils.cmo \
+   cache.cmo \
+   downloader.cmo \
+   sigchecker.cmo \
+   index.cmo \
+   ini_reader.cmo \
+   index_parser.cmo \
+   index_parser_tests.cmo
+index_parser_tests_XOBJECTS = $(index_parser_tests_BOBJECTS:.cmo=.cmx)
+
 # Can't call the following as _OBJECTS because automake gets confused.
 if HAVE_OCAMLOPT
 yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS)
 yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_XOBJECTS)
+index_parser_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 else
 yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS)
 yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_BOBJECTS)
+index_parser_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 endif
 
 yajl_tests_DEPENDENCIES = \
@@ -236,6 +259,15 @@ yajl_tests_LINK = \
  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
  $(yajl_tests_THEOBJECTS) -o $@
 
+index_parser_tests_DEPENDENCIES = \
+   $(index_parser_tests_THEOBJECTS) \
+   ../mllib/mllib.$(MLARCHIVE) \
+   $(top_srcdir)/ocaml-link.sh
+index_parser_tests_LINK = \
+   $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
+ $(index_parser_tests_THEOBJECTS) -o $@
+
 TESTS = \
test-docs.sh \
test-virt-builder-list.sh \
@@ -249,8 +281,8 @@ if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
 endif ENABLE_APPLIANCE
 if HAVE_OCAML_PKG_OUNIT
-check_PROGRAMS += yajl_tests
-TESTS += yajl_tests
+check_PROGRAMS += yajl_tests index_parser_tests
+TESTS += yajl_tests index_parser_tests
 endif
 
 check-valgrind:
diff --git a/builder/index.mli b/builder/index.mli
index ff5ec4a35..6202d636e 100644
--- a/builder/index.mli
+++ b/builder/index.mli
@@ -39,3 +39,6 @@ and entry = {
 }
 
 val print_entry : out_channel -> (string * entry) -> unit
+(** Debugging helper function dumping an index entry to a stream.
+To write entries for non-debugging purpose, use the
+[Index_parser.write_entry] function. *)
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index 26dd1f653..5bfd50b28 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -234,3 +234,57 @@ let get_index ~downloader ~sigchecker ~template
   in
 
   get_index ()
+
+let write_entry chan (name, { Index.printable_name = printable_name;
+  file_uri = file_uri;
+  arch = arch;
+  osinfo = osinfo;
+  signature_uri = signature_uri;
+  checksums = checksums;
+  revision = revision;
+  format = format;
+  size = size;
+  compressed_size = compressed_size;
+  expand = expand;
+  lvexpand = lvexpand;
+  notes = notes;
+  aliases = aliases;
+  hidden = hidden }) =
+  let fp fs = fprintf chan fs in
+  fp "[%s]\n" name;
+  may (fp "name=%s\n") printable_name;
+  may (fp "osinfo=%s\n") osinfo;
+  fp "file=%s\n" file_uri;
+  fp "arch=%s\n" arch;
+  may (fp "sig=%s\n") signature_uri;
+  (match checksums with
+  | None -> ()
+  | Some checksums ->
+List.iter (
+  fun c ->
+fp "checksum[%s]=%s\n"
+  

[Libguestfs] [PATCH v5 01/10] lib/osinfo.c: Extract xml processing into a callback

[Cédric Bosdonnat]

In order to further reuse the osinfo database parsing in OCAML, this
commit extracts the XML processing for the distro ISOs and places it
into a newly created callback.

This will later help other code to traverse the osinfo DB files and
let them extract what they need from them.
---
 lib/osinfo.c | 80 +---
 1 file changed, 39 insertions(+), 41 deletions(-)

diff --git a/lib/osinfo.c b/lib/osinfo.c
index ea2a7659a..3514585c7 100644
--- a/lib/osinfo.c
+++ b/lib/osinfo.c
@@ -52,6 +52,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -71,10 +72,11 @@ gl_lock_define_initialized (static, osinfo_db_lock);
 static ssize_t osinfo_db_size = 0; /* 0 = unread, -1 = error, >= 1 = #records 
*/
 static struct osinfo *osinfo_db = NULL;
 
-static int read_osinfo_db (guestfs_h *g);
-static void free_osinfo_db_entry (struct osinfo *);
-
 #define XMLSTREQ(a,b) (xmlStrEqual((a),(b)) == 1)
+typedef int (*read_osinfo_db_callback) (guestfs_h *g, const char *path, void 
*opaque);
+
+static int read_osinfo_db (guestfs_h *g, read_osinfo_db_callback callback, 
void *opaque);
+static void free_osinfo_db_entry (struct osinfo *);
 
 /* Given one or more fields from the header of a CD/DVD/ISO, look up
  * the media in the libosinfo database and return our best guess for
@@ -87,14 +89,24 @@ static void free_osinfo_db_entry (struct osinfo *);
  */
 int
 guestfs_int_osinfo_map (guestfs_h *g, const struct guestfs_isoinfo *isoinfo,
-   const struct osinfo **osinfo_ret)
+const struct osinfo **osinfo_ret)
 {
   size_t i;
 
   /* We only need to lock the database when reading it for the first time. */
   gl_lock_lock (osinfo_db_lock);
   if (osinfo_db_size == 0) {
-if (read_osinfo_db (g) == -1) {
+if (read_osinfo_db (g, read_osinfo_db_xml, NULL) == -1) {
+  /* Fatal error: free any database entries which have been read, and
+   * mark the database as having a permanent error.
+   */
+  if (osinfo_db_size > 0) {
+for (i = 0; i < (size_t) osinfo_db_size; ++i)
+  free_osinfo_db_entry (_db[i]);
+  }
+  free (osinfo_db);
+  osinfo_db = NULL;
+  osinfo_db_size = -1;
   gl_lock_unlock (osinfo_db_lock);
   return -1;
 }
@@ -156,19 +168,16 @@ guestfs_int_osinfo_map (guestfs_h *g, const struct 
guestfs_isoinfo *isoinfo,
  * Try to use the shared osinfo database layout (and location) first:
  * https://gitlab.com/libosinfo/libosinfo/blob/master/docs/database-layout.txt
  */
-static int read_osinfo_db_xml (guestfs_h *g, const char *filename);
-
-static int read_osinfo_db_flat (guestfs_h *g, const char *directory);
-static int read_osinfo_db_three_levels (guestfs_h *g, const char *directory);
-static int read_osinfo_db_directory (guestfs_h *g, const char *directory);
+static int read_osinfo_db_xml (guestfs_h *g, const char *pathname, void *data);
+static int read_osinfo_db_flat (guestfs_h *g, const char *directory, 
read_osinfo_db_callback callback, void *opaque);
+static int read_osinfo_db_three_levels (guestfs_h *g, const char *directory, 
read_osinfo_db_callback callback, void *opaque);
+static int read_osinfo_db_directory (guestfs_h *g, const char *directory, 
read_osinfo_db_callback callback, void *opaque);
 
 static int
-read_osinfo_db (guestfs_h *g)
+read_osinfo_db (guestfs_h *g,
+read_osinfo_db_callback callback, void *opaque)
 {
   int r;
-  size_t i;
-
-  assert (osinfo_db_size == 0);
 
   /* (1) Try the shared osinfo directory, using either the
* $OSINFO_SYSTEM_DIR envvar or its default value.
@@ -181,59 +190,47 @@ read_osinfo_db (guestfs_h *g)
 if (path == NULL)
   path = "/usr/share/osinfo";
 os_path = safe_asprintf (g, "%s/os", path);
-r = read_osinfo_db_three_levels (g, os_path);
+r = read_osinfo_db_three_levels (g, os_path, callback, opaque);
   }
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* (2) Try the libosinfo directory, using the newer three-directory
* layout ($LIBOSINFO_DB_PATH / "os" / $group-ID / [file.xml]).
*/
-  r = read_osinfo_db_three_levels (g, LIBOSINFO_DB_PATH "/os");
+  r = read_osinfo_db_three_levels (g, LIBOSINFO_DB_PATH "/os", callback, 
opaque);
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* (3) Try the libosinfo directory, using the old flat directory
* layout ($LIBOSINFO_DB_PATH / "oses" / [file.xml]).
*/
-  r = read_osinfo_db_flat (g, LIBOSINFO_DB_PATH "/oses");
+  r = read_osinfo_db_flat (g, LIBOSINFO_DB_PATH "/oses", callback, opaque);
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* Nothing found. */
   return 0;
-
- error:
-  /* Fatal error: free any database entries which have been read, and
-   * mark the database as having a permanent error.
-   */
-  if (osinfo_db_size > 0) {
-for (i = 0; i < (size_t) osinfo_db_size; 

[Libguestfs] [PATCH v5 09/10] mllib: add XPath helper xpath_get_nodes()

[Cédric Bosdonnat]

This function will allow more OCAML-ish processing of xpath queries
with multiple results.
---
 mllib/xpath_helpers.ml  | 9 +
 mllib/xpath_helpers.mli | 4 
 2 files changed, 13 insertions(+)

diff --git a/mllib/xpath_helpers.ml b/mllib/xpath_helpers.ml
index 8648596a4..f12156f45 100644
--- a/mllib/xpath_helpers.ml
+++ b/mllib/xpath_helpers.ml
@@ -53,3 +53,12 @@ let xpath_eval_default parsefn xpath expr default =
 let xpath_string_default = xpath_eval_default identity
 let xpath_int_default = xpath_eval_default int_of_string
 let xpath_int64_default = xpath_eval_default Int64.of_string
+
+let xpath_get_nodes xpathctx expr =
+  let obj = Xml.xpath_eval_expression xpathctx expr in
+  let nodes = ref [] in
+  for i = 0 to Xml.xpathobj_nr_nodes obj - 1 do
+let node = Xml.xpathobj_node obj i in
+nodes := List.append !nodes [node]
+  done;
+  !nodes
diff --git a/mllib/xpath_helpers.mli b/mllib/xpath_helpers.mli
index 7434ba645..ab176351f 100644
--- a/mllib/xpath_helpers.mli
+++ b/mllib/xpath_helpers.mli
@@ -31,3 +31,7 @@ val xpath_int_default : Xml.xpathctx -> string -> int -> int
 val xpath_int64_default : Xml.xpathctx -> string -> int64 -> int64
 (** Parse an xpath expression and return a string/int; if the expression
 doesn't match, return the default. *)
+
+val xpath_get_nodes : Xml.xpathctx -> string -> Xml.node list
+(** Parse an xpath expression and return a list with the matching
+XML nodes. *)
-- 
2.12.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v4 1/9] lib/osinfo.c: Extract xml processing into a callback

[Cédric Bosdonnat]

In order to further reuse the osinfo database parsing in OCAML, this
commit extracts the XML processing for the distro ISOs and places it
into a newly created callback.

This will later help other code to traverse the osinfo DB files and
let them extract what they need from them.
---
 lib/osinfo.c | 85 +++-
 1 file changed, 44 insertions(+), 41 deletions(-)

diff --git a/lib/osinfo.c b/lib/osinfo.c
index ea2a7659a..121fac1ba 100644
--- a/lib/osinfo.c
+++ b/lib/osinfo.c
@@ -52,6 +52,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -71,10 +72,14 @@ gl_lock_define_initialized (static, osinfo_db_lock);
 static ssize_t osinfo_db_size = 0; /* 0 = unread, -1 = error, >= 1 = #records 
*/
 static struct osinfo *osinfo_db = NULL;
 
-static int read_osinfo_db (guestfs_h *g);
-static void free_osinfo_db_entry (struct osinfo *);
-
 #define XMLSTREQ(a,b) (xmlStrEqual((a),(b)) == 1)
+typedef int (*read_osinfo_db_callback) (guestfs_h *g, const char *path, void 
*opaque);
+
+static int
+read_osinfo_db (guestfs_h *g,
+read_osinfo_db_callback callback, void *opaque);
+static int read_osinfo_db_xml (guestfs_h *g, const char *pathname, void *data);
+static void free_osinfo_db_entry (struct osinfo *);
 
 /* Given one or more fields from the header of a CD/DVD/ISO, look up
  * the media in the libosinfo database and return our best guess for
@@ -87,14 +92,24 @@ static void free_osinfo_db_entry (struct osinfo *);
  */
 int
 guestfs_int_osinfo_map (guestfs_h *g, const struct guestfs_isoinfo *isoinfo,
-   const struct osinfo **osinfo_ret)
+const struct osinfo **osinfo_ret)
 {
   size_t i;
 
   /* We only need to lock the database when reading it for the first time. */
   gl_lock_lock (osinfo_db_lock);
   if (osinfo_db_size == 0) {
-if (read_osinfo_db (g) == -1) {
+if (read_osinfo_db (g, read_osinfo_db_xml, NULL) == -1) {
+  /* Fatal error: free any database entries which have been read, and
+   * mark the database as having a permanent error.
+   */
+  if (osinfo_db_size > 0) {
+for (i = 0; i < (size_t) osinfo_db_size; ++i)
+  free_osinfo_db_entry (_db[i]);
+  }
+  free (osinfo_db);
+  osinfo_db = NULL;
+  osinfo_db_size = -1;
   gl_lock_unlock (osinfo_db_lock);
   return -1;
 }
@@ -156,19 +171,18 @@ guestfs_int_osinfo_map (guestfs_h *g, const struct 
guestfs_isoinfo *isoinfo,
  * Try to use the shared osinfo database layout (and location) first:
  * https://gitlab.com/libosinfo/libosinfo/blob/master/docs/database-layout.txt
  */
-static int read_osinfo_db_xml (guestfs_h *g, const char *filename);
-
-static int read_osinfo_db_flat (guestfs_h *g, const char *directory);
-static int read_osinfo_db_three_levels (guestfs_h *g, const char *directory);
-static int read_osinfo_db_directory (guestfs_h *g, const char *directory);
+static int read_osinfo_db_flat (guestfs_h *g, const char *directory,
+read_osinfo_db_callback callback, void 
*opaque);
+static int read_osinfo_db_three_levels (guestfs_h *g, const char *directory,
+read_osinfo_db_callback callback, void 
*opaque);
+static int read_osinfo_db_directory (guestfs_h *g, const char *directory,
+ read_osinfo_db_callback callback, void 
*opaque);
 
 static int
-read_osinfo_db (guestfs_h *g)
+read_osinfo_db (guestfs_h *g,
+read_osinfo_db_callback callback, void *opaque)
 {
   int r;
-  size_t i;
-
-  assert (osinfo_db_size == 0);
 
   /* (1) Try the shared osinfo directory, using either the
* $OSINFO_SYSTEM_DIR envvar or its default value.
@@ -181,59 +195,47 @@ read_osinfo_db (guestfs_h *g)
 if (path == NULL)
   path = "/usr/share/osinfo";
 os_path = safe_asprintf (g, "%s/os", path);
-r = read_osinfo_db_three_levels (g, os_path);
+r = read_osinfo_db_three_levels (g, os_path, callback, opaque);
   }
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* (2) Try the libosinfo directory, using the newer three-directory
* layout ($LIBOSINFO_DB_PATH / "os" / $group-ID / [file.xml]).
*/
-  r = read_osinfo_db_three_levels (g, LIBOSINFO_DB_PATH "/os");
+  r = read_osinfo_db_three_levels (g, LIBOSINFO_DB_PATH "/os", callback, 
opaque);
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* (3) Try the libosinfo directory, using the old flat directory
* layout ($LIBOSINFO_DB_PATH / "oses" / [file.xml]).
*/
-  r = read_osinfo_db_flat (g, LIBOSINFO_DB_PATH "/oses");
+  r = read_osinfo_db_flat (g, LIBOSINFO_DB_PATH "/oses", callback, opaque);
   if (r == -1)
-goto error;
+return -1;
   else if (r == 1)
 return 0;
 
   /* Nothing found. */
   return 0;
-
- error:
-  /* Fatal error: free any database entries which have been read, and
-   * 

[Libguestfs] [PATCH v4 3/9] mllib: ocaml wrapper for lib/osinfo

[Cédric Bosdonnat]

Provide osinfo database parsing API in OCAML.
---
 lib/osinfo.c  |  13 +++
 mllib/Makefile.am |  11 --
 mllib/osinfo-c.c  | 100 ++
 mllib/osinfo.ml   |  26 ++
 mllib/osinfo.mli  |  31 +
 5 files changed, 179 insertions(+), 2 deletions(-)
 create mode 100644 mllib/osinfo-c.c
 create mode 100644 mllib/osinfo.ml
 create mode 100644 mllib/osinfo.mli

diff --git a/lib/osinfo.c b/lib/osinfo.c
index 11b50d903..126a645c0 100644
--- a/lib/osinfo.c
+++ b/lib/osinfo.c
@@ -52,6 +52,19 @@
 
 #include "osinfo.h"
 
+#ifndef GUESTFS_PRIVATE
+#undef perrorf
+#define perrorf(g,...)  \
+{   \
+  CLEANUP_FREE char *msg = NULL;\
+  ignore_value (asprintf (, __VA_ARGS__));  \
+  perror (msg); \
+}
+
+#undef debug
+#define debug(g,...) fprintf (stderr, __VA_ARGS__)
+#endif /* GUESTFS_PRIVATE */
+
 
 /* Read the libosinfo XML database files.  The lock is held while
  * this is called.
diff --git a/mllib/Makefile.am b/mllib/Makefile.am
index ee2f1a7a8..ee16fe7ef 100644
--- a/mllib/Makefile.am
+++ b/mllib/Makefile.am
@@ -36,6 +36,7 @@ SOURCES_MLI = \
curl.mli \
getopt.mli \
JSON.mli \
+   osinfo.mli \
planner.mli \
progress.mli \
regedit.mli \
@@ -63,7 +64,8 @@ SOURCES_ML = \
curl.ml \
checksums.ml \
xml.ml \
-   xpath_helpers.ml
+   xpath_helpers.ml \
+   osinfo.ml
 
 SOURCES_C = \
../common/visit/visit.c \
@@ -71,8 +73,12 @@ SOURCES_C = \
../common/options/keys.c \
../common/options/uri.c \
../common/progress/progress.c \
+   ../lib/alloc.c \
+   ../lib/osinfo.c \
+   ../lib/osinfo.h \
common_utils-c.c \
getopt-c.c \
+   osinfo-c.c \
progress-c.c \
unix_utils-c.c \
uri-c.c \
@@ -106,7 +112,8 @@ libmllib_a_CPPFLAGS = \
-I$(top_srcdir)/lib \
-I$(top_srcdir)/common/visit \
-I$(top_srcdir)/common/options \
-   -I$(top_srcdir)/common/progress
+   -I$(top_srcdir)/common/progress \
+   -DLIBOSINFO_DB_PATH='"$(datadir)/libosinfo/db"'
 libmllib_a_CFLAGS = \
$(WARN_CFLAGS) $(WERROR_CFLAGS) \
$(LIBVIRT_CFLAGS) $(LIBXML2_CFLAGS) \
diff --git a/mllib/osinfo-c.c b/mllib/osinfo-c.c
new file mode 100644
index 0..9546c5984
--- /dev/null
+++ b/mllib/osinfo-c.c
@@ -0,0 +1,100 @@
+/* Bindings for osinfo db reading function.
+ * Copyright (C) 2017 SUSE Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 
USA.
+ */
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "guestfs.h"
+#include "guestfs-internal.h"
+#include "osinfo.h"
+
+#pragma GCC diagnostic ignored "-Wmissing-prototypes"
+
+struct callback_wrapper_args {
+  /* In both case we are pointing to local roots, hence why these are
+   * value* not value.
+   */
+  value *exnp;  /* Safe place to store any exception
+   raised by callback */
+  value *fvp;   /* callback. */
+};
+
+static int read_osinfo_db_callback_wrapper (guestfs_h *g, const char *path, 
void *opaque);
+
+value
+guestfs_int_mllib_read_osinfo_db (value gv, value fv)
+{
+  CAMLparam2 (gv, fv);
+  guestfs_h *g = (guestfs_h *) Int64_val (gv);
+  struct callback_wrapper_args args;
+
+  /* This stack address is used to point to the exception, if one is
+   * raised in the visitor_function.  Note that the macro initializes
+   * this to Val_unit, which is how we know if an exception was set.
+   */
+  CAMLlocal1 (exn);
+
+  args.exnp = 
+  args.fvp = 
+
+  if (read_osinfo_db (g, read_osinfo_db_callback_wrapper, ) == -1) {
+if (exn != Val_unit) {
+  /* The failure was caused by the callback raising an
+   * exception.  Re-raise it here.
+   */
+  caml_raise (exn);
+}
+
+caml_failwith ("read_osinfo_db");
+}
+
+  CAMLreturn (Val_unit);
+}
+
+static int
+read_osinfo_db_callback_wrapper (guestfs_h *g, const char *path, void *opaque)
+{
+  CAMLparam0 ();
+  CAMLlocal2 (pathv, v);
+  struct callback_wrapper_args *args = opaque;
+
+  assert (path != NULL);
+  

[Libguestfs] [PATCH v4 7/9] dib: move do_cp to mllib.Commun_utils

[Cédric Bosdonnat]

---
 dib/utils.ml   | 4 
 mllib/common_utils.ml  | 5 +
 mllib/common_utils.mli | 3 +++
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/dib/utils.ml b/dib/utils.ml
index da5e738ad..e769ebe28 100644
--- a/dib/utils.ml
+++ b/dib/utils.ml
@@ -95,10 +95,6 @@ let require_tool tool =
   with Executable_not_found tool ->
 error (f_"%s needed but not found") tool
 
-let do_cp src destdir =
-  let cmd = [ "cp"; "-t"; destdir; "-a"; src ] in
-  if run_command cmd <> 0 then exit 1
-
 let ensure_trailing_newline str =
   if String.length str > 0 && str.[String.length str - 1] <> '\n' then str ^ 
"\n"
   else str
diff --git a/mllib/common_utils.ml b/mllib/common_utils.ml
index e1d63292e..945728b5e 100644
--- a/mllib/common_utils.ml
+++ b/mllib/common_utils.ml
@@ -1167,3 +1167,8 @@ let inspect_decrypt g =
* function.
*)
   c_inspect_decrypt g#ocaml_handle (Guestfs.c_pointer g#ocaml_handle)
+
+let do_cp src destdir =
+  let cmd = [ "cp"; "-t"; destdir; "-a"; src ] in
+  if run_command cmd <> 0 then
+error (f_"copy of %s to %s failed") src destdir
diff --git a/mllib/common_utils.mli b/mllib/common_utils.mli
index 1cd38ba83..5c376fcb3 100644
--- a/mllib/common_utils.mli
+++ b/mllib/common_utils.mli
@@ -492,3 +492,6 @@ val inspect_decrypt : Guestfs.guestfs -> unit
 (** Simple implementation of decryption: look for any [crypto_LUKS]
 partitions and decrypt them, then rescan for VGs.  This only works
 for Fedora whole-disk encryption. *)
+
+val do_cp : string -> string -> unit
+(** Run the cp command, and exit with an error if it failed *)
-- 
2.11.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v4 0/9] Introducing virt-builder-repository

[Cédric Bosdonnat]

Hi all,

Here is a v4 of my series. It includes the changes according to
Pino and Richard's comments.

However, the perrorf/debug problem is addressed differently:
instead of adding an implementation for the internal function
names when building for mllib, I redefine these macros. Obviously
this is not perfect, but at least easier to understand.

Pino's comment about the Notes regex wasn't quite right... thus
I added a unit test for the Index_parser.write_entry

As some more code has been moved around, the number of commits
in the series increased a little bit ;)  

Cédric Bosdonnat (9):
  lib/osinfo.c: Extract xml processing into a callback
  lib: extract osinfo DB traversing API
  mllib: ocaml wrapper for lib/osinfo
  builder: rename docs test script
  builder: add Index_parser.write_entry function
  builder: add a template parameter to get_index
  dib: move do_cp to mllib.Commun_utils
  mllib: add do_mv helper function to Common_utils
  Add a virt-builder-repository tool

 .gitignore |   4 +
 builder/Makefile.am| 121 +-
 builder/builder.ml |   2 +-
 builder/index.mli  |   3 +
 builder/index_parser.ml|  72 +++-
 builder/index_parser.mli   |   8 +-
 builder/index_parser_tests.ml  |  95 
 builder/repository_main.ml | 466 
 .../{test-virt-builder-docs.sh => test-docs.sh}|   2 +
 builder/virt-builder-repository.pod| 183 
 dib/utils.ml   |   4 -
 lib/Makefile.am|   2 +
 lib/osinfo-iso.c   | 462 
 lib/osinfo.c   | 476 ++---
 lib/osinfo.h   |  27 ++
 mllib/Makefile.am  |  11 +-
 mllib/common_utils.ml  |  11 +
 mllib/common_utils.mli |   6 +
 mllib/osinfo-c.c   | 100 +
 mllib/osinfo.ml|  26 ++
 mllib/osinfo.mli   |  31 ++
 21 files changed, 1654 insertions(+), 458 deletions(-)
 create mode 100644 builder/index_parser_tests.ml
 create mode 100644 builder/repository_main.ml
 rename builder/{test-virt-builder-docs.sh => test-docs.sh} (93%)
 create mode 100644 builder/virt-builder-repository.pod
 create mode 100644 lib/osinfo-iso.c
 create mode 100644 lib/osinfo.h
 create mode 100644 mllib/osinfo-c.c
 create mode 100644 mllib/osinfo.ml
 create mode 100644 mllib/osinfo.mli

-- 
2.11.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v4 5/9] builder: add Index_parser.write_entry function

[Cédric Bosdonnat]

Add a function to properly write virt-builder source index entries.
Note that this function is very similar to Index.print_entry that is
meant for debugging purposes.
---
 .gitignore|  1 +
 builder/Makefile.am   | 36 +++-
 builder/index.mli |  3 ++
 builder/index_parser.ml   | 52 +++
 builder/index_parser.mli  |  6 +++
 builder/index_parser_tests.ml | 95 +++
 6 files changed, 191 insertions(+), 2 deletions(-)
 create mode 100644 builder/index_parser_tests.ml

diff --git a/.gitignore b/.gitignore
index e3b6d7b1c..06b16a49a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -104,6 +104,7 @@ Makefile.in
 /builder/virt-index-validate
 /builder/virt-index-validate.1
 /builder/*.xz
+/builder/index_parser_tests
 /builder/yajl_tests
 /cat/stamp-virt-*.pod
 /cat/virt-cat
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 218f64b4c..bf4ccb7d7 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -217,13 +217,36 @@ yajl_tests_BOBJECTS = \
yajl_tests.cmo
 yajl_tests_XOBJECTS = $(yajl_tests_BOBJECTS:.cmo=.cmx)
 
+index_parser_tests_SOURCES = \
+   index-scan.c \
+   index-struct.c \
+   index-parser-c.c \
+   index-parse.c
+index_parser_tests_CPPFLAGS = $(virt_builder_CPPFLAGS)
+index_parser_tests_BOBJECTS = \
+   utils.cmo \
+   cache.cmo \
+   downloader.cmo \
+   sigchecker.cmo \
+   index.cmo \
+   ini_reader.cmo \
+   index_parser.cmo \
+   index_parser_tests.cmo
+index_parser_tests_XOBJECTS = $(index_parser_tests_BOBJECTS:.cmo=.cmx)
+
 # Can't call the following as _OBJECTS because automake gets confused.
 if HAVE_OCAMLOPT
 yajl_tests_THEOBJECTS = $(yajl_tests_XOBJECTS)
 yajl_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_XOBJECTS)
+index_parser_tests.cmx: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 else
 yajl_tests_THEOBJECTS = $(yajl_tests_BOBJECTS)
 yajl_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
+
+index_parser_tests_THEOBJECTS = $(index_parser_tests_BOBJECTS)
+index_parser_tests.cmo: OCAMLPACKAGES += $(OCAMLPACKAGES_TESTS)
 endif
 
 yajl_tests_DEPENDENCIES = \
@@ -236,6 +259,15 @@ yajl_tests_LINK = \
  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
  $(yajl_tests_THEOBJECTS) -o $@
 
+index_parser_tests_DEPENDENCIES = \
+   $(index_parser_tests_THEOBJECTS) \
+   ../mllib/mllib.$(MLARCHIVE) \
+   $(top_srcdir)/ocaml-link.sh
+index_parser_tests_LINK = \
+   $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) 
$(OCAMLPACKAGES_TESTS) $(OCAMLLINKFLAGS) \
+ $(index_parser_tests_THEOBJECTS) -o $@
+
 TESTS = \
test-docs.sh \
test-virt-builder-list.sh \
@@ -249,8 +281,8 @@ if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
 endif ENABLE_APPLIANCE
 if HAVE_OCAML_PKG_OUNIT
-check_PROGRAMS += yajl_tests
-TESTS += yajl_tests
+check_PROGRAMS += yajl_tests index_parser_tests
+TESTS += yajl_tests index_parser_tests
 endif
 
 check-valgrind:
diff --git a/builder/index.mli b/builder/index.mli
index ff5ec4a35..6202d636e 100644
--- a/builder/index.mli
+++ b/builder/index.mli
@@ -39,3 +39,6 @@ and entry = {
 }
 
 val print_entry : out_channel -> (string * entry) -> unit
+(** Debugging helper function dumping an index entry to a stream.
+To write entries for non-debugging purpose, use the
+[Index_parser.write_entry] function. *)
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index a3cae7d1a..9b3daed24 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -226,3 +226,55 @@ let get_index ~downloader ~sigchecker
   in
 
   get_index ()
+
+let write_entry chan (name, { Index.printable_name = printable_name;
+  file_uri = file_uri;
+  arch = arch;
+  osinfo = osinfo;
+  signature_uri = signature_uri;
+  checksums = checksums;
+  revision = revision;
+  format = format;
+  size = size;
+  compressed_size = compressed_size;
+  expand = expand;
+  lvexpand = lvexpand;
+  notes = notes;
+  aliases = aliases;
+  hidden = hidden }) =
+  let fp fs = fprintf chan fs in
+  fp "[%s]\n" name;
+  may (fp "name=%s\n") printable_name;
+  may (fp "osinfo=%s\n") osinfo;
+  fp "file=%s\n" file_uri;
+  fp "arch=%s\n" arch;
+  may (fp "sig=%s\n") signature_uri;
+  (match checksums with
+  | None -> ()
+  | Some checksums ->
+List.iter (
+  fun c ->
+fp "checksum[%s]=%s\n"
+  

[Libguestfs] [PATCH v4 9/9] Add a virt-builder-repository tool

[Cédric Bosdonnat]

+left untouched.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--help>
+
+Display help.
+
+=item B<--gpg> GPG
+
+Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary.  You can
+also use this to add gpg parameters, for example to specify an
+alternate home directory:
+
+ virt-builder-repository --gpg "gpg --homedir /tmp" [...]
+
+This can also be used to avoid gpg asking for the key passphrase:
+
+ virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...]
+
+=item B<-K> KEYID
+
+=item B<--gpg-key> KEYID
+
+Specify the GPG key to be used to sign the repository index file.
+If not provided, the index will left unsigned. C is used to
+identify the GPG key to use. This value is passed to gpg's
+C<--default-key> option and can can thus be an email address or a
+fingerprint.
+
+B: by default, virt-builder-repository searches for the key
+in the user's GPG key ring.
+
+=item B<--keep-index>
+
+When using a GPG key, don't remove the unsigned index.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data.
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages.  This is the default
+when the output is a tty.  If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don't print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdon...@suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
-- 
2.11.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs

[Libguestfs] [PATCH v4 6/9] builder: add a template parameter to get_index

[Cédric Bosdonnat]

get_index now gets a new template parameter. Setting it to true will
make the index parsing less picky about missing important data. This
can be used to parse a partial index file.
---
 builder/builder.ml   |  2 +-
 builder/index_parser.ml  | 20 ++--
 builder/index_parser.mli |  2 +-
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/builder/builder.ml b/builder/builder.ml
index e59c763b2..5802b6873 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -209,7 +209,7 @@ let main () =
   ~tmpdir in
   match source.Sources.format with
   | Sources.FormatNative ->
-Index_parser.get_index ~downloader ~sigchecker source
+Index_parser.get_index ~downloader ~sigchecker ~template:false 
source
   | Sources.FormatSimpleStreams ->
 Simplestreams_parser.get_index ~downloader ~sigchecker source
   ) sources
diff --git a/builder/index_parser.ml b/builder/index_parser.ml
index 9b3daed24..3869123ed 100644
--- a/builder/index_parser.ml
+++ b/builder/index_parser.ml
@@ -24,7 +24,7 @@ open Utils
 open Printf
 open Unix
 
-let get_index ~downloader ~sigchecker
+let get_index ~downloader ~sigchecker ~template
   { Sources.uri = uri; proxy = proxy } =
   let corrupt_file () =
 error (f_"The index file downloaded from '%s' is corrupt.\nYou need to ask 
the supplier of this file to fix it and upload a fixed version.") uri
@@ -112,7 +112,7 @@ let get_index ~downloader ~sigchecker
   let revision =
 try Rev_int (int_of_string (List.assoc ("revision", None) fields))
 with
-| Not_found -> Rev_int 1
+| Not_found -> if template then Rev_int 0 else Rev_int 1
 | Failure _ ->
   eprintf (f_"%s: cannot parse 'revision' field for '%s'\n") prog 
n;
   corrupt_file () in
@@ -122,11 +122,19 @@ let get_index ~downloader ~sigchecker
 try Int64.of_string (List.assoc ("size", None) fields)
 with
 | Not_found ->
-  eprintf (f_"%s: no 'size' field for '%s'\n") prog n;
-  corrupt_file ()
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: no 'size' field for '%s'\n") prog n;
+corrupt_file ()
+  )
 | Failure _ ->
-  eprintf (f_"%s: cannot parse 'size' field for '%s'\n") prog n;
-  corrupt_file () in
+  if template then
+Int64.zero
+  else (
+eprintf (f_"%s: cannot parse 'size' field for '%s'\n") prog n;
+corrupt_file ()
+  ) in
   let compressed_size =
 try Some (Int64.of_string (List.assoc ("compressed_size", None) 
fields))
 with
diff --git a/builder/index_parser.mli b/builder/index_parser.mli
index 7c1c423ad..ae757ad6f 100644
--- a/builder/index_parser.mli
+++ b/builder/index_parser.mli
@@ -16,7 +16,7 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
Sources.source -> Index.index
+val get_index : downloader:Downloader.t -> sigchecker:Sigchecker.t -> 
template:bool -> Sources.source -> Index.index
 (** [get_index download sigchecker source] will parse the source index file
  into an index entry list. *)
 
-- 
2.11.0

___
Libguestfs mailing list
Libguestfs@redhat.com
https://www.redhat.com/mailman/listinfo/libguestfs