Hello community, here is the log from the commit of package createrepo_c for openSUSE:Factory checked in at 2020-07-15 11:19:36 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/createrepo_c (Old) and /work/SRC/openSUSE:Factory/.createrepo_c.new.3060 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "createrepo_c" Wed Jul 15 11:19:36 2020 rev:12 rq:819088 version:0.16.0 Changes: -------- --- /work/SRC/openSUSE:Factory/createrepo_c/createrepo_c.changes 2020-07-02 23:59:36.729523692 +0200 +++ /work/SRC/openSUSE:Factory/.createrepo_c.new.3060/createrepo_c.changes 2020-07-15 11:21:58.293415538 +0200 @@ -1,0 +2,7 @@ +Tue Jul 7 01:12:50 UTC 2020 - Neal Gompa <[email protected]> + +- Update to 0.16.0 + + Never do dir walk when --recycle-pkglist specified + + Add automatic module metadata handling for repos (rh#1795936) + +------------------------------------------------------------------- Old: ---- createrepo_c-0.15.11.tar.gz New: ---- createrepo_c-0.16.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ createrepo_c.spec ++++++ --- /var/tmp/diff_new_pack.F0WwwQ/_old 2020-07-15 11:22:02.009419215 +0200 +++ /var/tmp/diff_new_pack.F0WwwQ/_new 2020-07-15 11:22:02.009419215 +0200 @@ -54,7 +54,7 @@ %define devname lib%{name}-devel Name: createrepo_c -Version: 0.15.11 +Version: 0.16.0 Release: 0 Summary: RPM repository metadata generation utility License: GPL-2.0-or-later ++++++ createrepo_c-0.15.11.tar.gz -> createrepo_c-0.16.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/createrepo_c-0.15.11/VERSION.cmake new/createrepo_c-0.16.0/VERSION.cmake --- old/createrepo_c-0.15.11/VERSION.cmake 2020-06-02 10:34:46.000000000 +0200 +++ new/createrepo_c-0.16.0/VERSION.cmake 2020-07-07 02:44:02.000000000 +0200 @@ -1,3 +1,3 @@ SET(CR_MAJOR "0") -SET(CR_MINOR "15") -SET(CR_PATCH "11") +SET(CR_MINOR "16") +SET(CR_PATCH "0") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/createrepo_c-0.15.11/createrepo_c.spec new/createrepo_c-0.16.0/createrepo_c.spec --- old/createrepo_c-0.15.11/createrepo_c.spec 2020-06-02 10:34:46.000000000 +0200 +++ new/createrepo_c-0.16.0/createrepo_c.spec 2020-07-07 02:44:02.000000000 +0200 @@ -35,7 +35,7 @@ Summary: Creates a common metadata repository Name: createrepo_c -Version: 0.15.11 +Version: 0.16.0 Release: 1%{?dist} License: GPLv2+ URL: https://github.com/rpm-software-management/createrepo_c diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/createrepo_c-0.15.11/doc/createrepo_c.8 new/createrepo_c-0.16.0/doc/createrepo_c.8 --- old/createrepo_c-0.15.11/doc/createrepo_c.8 2020-06-02 10:34:46.000000000 +0200 +++ new/createrepo_c-0.16.0/doc/createrepo_c.8 2020-07-07 02:44:02.000000000 +0200 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH CREATEREPO_C "2019-07-19" "" "" +.TH CREATEREPO_C "2020-07-02" "" "" .SH NAME createrepo_c \- Create rpm-md format (xml-rpm-metadata) repository . @@ -35,6 +35,25 @@ .SH SYNOPSIS .sp createrepo_c [options] <directory> +.SH DESCRIPTION +.sp +Uses rpm packages from <directory> to create repodata. +.sp +If compiled with libmodulemd support modular metadata inside <directory> identified by the patterns below are automatically collected, merged and added to the repodata. +.sp +The patterns are: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.IP \(bu 2 +*.modulemd.yaml (recommended file name: N:S:V:C:A.modulemd.yaml) +.IP \(bu 2 +*.modulemd\-defaults.yaml (recommended file name: N.modulemd\-defaults.yaml) +.IP \(bu 2 +modules.yaml (recommended way of importing multiple documents at once) +.UNINDENT +.UNINDENT +.UNINDENT .SH OPTIONS .SS \-V \-\-version .sp diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/createrepo_c-0.15.11/src/cmd_parser.c new/createrepo_c-0.16.0/src/cmd_parser.c --- old/createrepo_c-0.15.11/src/cmd_parser.c 2020-06-02 10:34:46.000000000 +0200 +++ new/createrepo_c-0.16.0/src/cmd_parser.c 2020-07-07 02:44:02.000000000 +0200 @@ -637,5 +637,6 @@ cr_slist_free_full(options->l_update_md_paths, g_free); cr_slist_free_full(options->distro_cpeids, g_free); cr_slist_free_full(options->distro_values, g_free); + cr_slist_free_full(options->modulemd_metadata, g_free); g_slist_free(options->oldpackagedirs_paths); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/createrepo_c-0.15.11/src/cmd_parser.h new/createrepo_c-0.16.0/src/cmd_parser.h --- old/createrepo_c-0.15.11/src/cmd_parser.h 2020-06-02 10:34:46.000000000 +0200 +++ new/createrepo_c-0.16.0/src/cmd_parser.h 2020-07-07 02:44:02.000000000 +0200 @@ -135,6 +135,7 @@ is used */ char *checksum_cachedir; /*!< Path to cachedir */ GSList *oldpackagedirs_paths; /*!< paths to look for older pkgs to delta against */ + GSList *modulemd_metadata; /*!< paths to all modulemd metadata */ gboolean recycle_pkglist; }; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/createrepo_c-0.15.11/src/createrepo_c.c new/createrepo_c-0.16.0/src/createrepo_c.c --- old/createrepo_c-0.15.11/src/createrepo_c.c 2020-06-02 10:34:46.000000000 +0200 +++ new/createrepo_c-0.16.0/src/createrepo_c.c 2020-07-07 02:44:02.000000000 +0200 @@ -40,6 +40,7 @@ #include "error.h" #include "helpers.h" #include "load_metadata.h" +#include "metadata_internal.h" #include "locate_metadata.h" #include "misc.h" #include "parsepkg.h" @@ -50,6 +51,10 @@ #include "xml_dump.h" #include "xml_file.h" +#ifdef WITH_LIBMODULEMD +#include <modulemd.h> +#endif /* WITH_LIBMODULEMD */ + #define OUTDELTADIR "drpms/" // TODO: Pass only exlude_masks list here @@ -81,6 +86,18 @@ return TRUE; } +static gboolean +allowed_modulemd_module_metadata_file(const gchar *filename) +{ + if (g_str_has_suffix (filename, ".modulemd.yaml") || + g_str_has_suffix (filename, ".modulemd-defaults.yaml") || + g_str_has_suffix (filename, "modules.yaml")) + { + return TRUE; + } + return FALSE; +} + /** Function used to sort pool tasks. * This function is responsible for order of packages in metadata. @@ -132,7 +149,7 @@ } - if (cmd_options->pkglist && !cmd_options->include_pkgs) { + if ((cmd_options->pkglist || cmd_options->recycle_pkglist) && !cmd_options->include_pkgs) { g_warning("Used pkglist doesn't contain any useful items"); } else if (!(cmd_options->include_pkgs)) { // --pkglist (or --includepkg, or --recycle-pkglist) is not supplied @@ -181,12 +198,6 @@ continue; } - // Non .rpm files are ignored - if (!g_str_has_suffix (filename, ".rpm")) { - g_free(full_path); - continue; - } - // Skip symbolic links if --skip-symlinks arg is used if (cmd_options->skip_symlinks && g_file_test(full_path, G_FILE_TEST_IS_SYMLINK)) @@ -196,6 +207,25 @@ continue; } + if (allowed_modulemd_module_metadata_file(full_path)) { +#ifdef WITH_LIBMODULEMD + cmd_options->modulemd_metadata = g_slist_prepend( + cmd_options->modulemd_metadata, + (gpointer) full_path); +#else + g_warning("createrepo_c not compiled with libmodulemd support, " + "ignoring found module metadata: %s", full_path); + g_free(full_path); +#endif /* WITH_LIBMODULEMD */ + continue; + } + + // Non .rpm files are ignored + if (!g_str_has_suffix (filename, ".rpm")) { + g_free(full_path); + continue; + } + // Check filename against exclude glob masks const gchar *repo_relative_path = filename; if (in_dir_len < strlen(full_path)) @@ -232,6 +262,19 @@ for (; element; element=g_slist_next(element)) { gchar *relative_path = (gchar *) element->data; // ^^^ path from pkglist e.g. packages/i386/foobar.rpm + + if (allowed_modulemd_module_metadata_file(relative_path)) { +#ifdef WITH_LIBMODULEMD + cmd_options->modulemd_metadata = g_slist_prepend( + cmd_options->modulemd_metadata, + (gpointer) g_strdup(relative_path)); +#else + g_warning("createrepo_c not compiled with libmodulemd support, " + "ignoring found module metadata: %s", relative_path); +#endif /* WITH_LIBMODULEMD */ + continue; + } + gchar *filename; // foobar.rpm // Get index of last '/' @@ -356,8 +399,8 @@ compressed_record_type, NULL )); - //TODO(amatej): replace nth_data with ->next->data - cr_repomd_record_compress_and_fill(g_slist_nth_data(additional_metadata_rec, 1), + + cr_repomd_record_compress_and_fill(additional_metadata_rec->next->data, additional_metadata_rec->data, repomd_checksum_type, comp_type, @@ -726,6 +769,30 @@ g_slist_free(current_pkglist); current_pkglist = NULL; GSList *additional_metadata = NULL; + + // Setup compression types + const char *xml_compression_suffix = NULL; + const char *sqlite_compression_suffix = NULL; + const char *compression_suffix = NULL; + cr_CompressionType xml_compression = CR_CW_GZ_COMPRESSION; + cr_CompressionType sqlite_compression = CR_CW_BZ2_COMPRESSION; + cr_CompressionType compression = CR_CW_GZ_COMPRESSION; + + if (cmd_options->compression_type != CR_CW_UNKNOWN_COMPRESSION) { + sqlite_compression = cmd_options->compression_type; + compression = cmd_options->compression_type; + } + + if (cmd_options->general_compression_type != CR_CW_UNKNOWN_COMPRESSION) { + xml_compression = cmd_options->general_compression_type; + sqlite_compression = cmd_options->general_compression_type; + compression = cmd_options->general_compression_type; + } + + xml_compression_suffix = cr_compression_suffix(xml_compression); + sqlite_compression_suffix = cr_compression_suffix(sqlite_compression); + compression_suffix = cr_compression_suffix(compression); + cr_Metadatum *new_groupfile_metadatum = NULL; // Groupfile specified as argument @@ -749,6 +816,108 @@ } } +#ifdef WITH_LIBMODULEMD + // module metadata found in repo + if (cmd_options->modulemd_metadata) { + ModulemdModuleIndexMerger *merger = modulemd_module_index_merger_new(); + if (!merger) { + g_critical("Could not allocate module merger"); + exit(EXIT_FAILURE); + } + ModulemdModuleIndex *moduleindex; + + //load all found module metatada and associate it with merger + GSList *element = cmd_options->modulemd_metadata; + for (; element; element=g_slist_next(element)) { + moduleindex = modulemd_module_index_new(); + if (!moduleindex) { + g_critical("Could not allocate new module index"); + g_clear_pointer(&merger, g_object_unref); + exit(EXIT_FAILURE); + } + g_autoptr (GPtrArray) failures = NULL; + gboolean result = modulemd_module_index_update_from_file(moduleindex, + ((char *) element->data), + TRUE, + &failures, + &tmp_err); + if (!result) { + g_critical("Could not update module index from file %s: %s", element->data, tmp_err->message); + g_clear_error(&tmp_err); + g_clear_pointer(&moduleindex, g_object_unref); + g_clear_pointer(&merger, g_object_unref); + exit(EXIT_FAILURE); + } + modulemd_module_index_merger_associate_index(merger, moduleindex, 0); + g_clear_pointer(&moduleindex, g_object_unref); + } + + if (cmd_options->update && cmd_options->keep_all_metadata && + old_metadata_location && old_metadata_location->additional_metadata){ + //associate old metadata into the merger + if (cr_metadata_modulemd(old_metadata)){ + modulemd_module_index_merger_associate_index(merger, cr_metadata_modulemd(old_metadata), 0); + if (tmp_err) { + g_critical("%s: Cannot merge old module index with new: %s", __func__, tmp_err->message); + g_clear_error(&tmp_err); + g_clear_pointer(&merger, g_object_unref); + exit(EXIT_FAILURE); + } + } + //remove old modules (every [compressed] variant) + GSList *node_iter = old_metadata_location->additional_metadata; + while (node_iter != NULL){ + GSList *next = g_slist_next(node_iter); + cr_Metadatum *m = node_iter->data; + if(g_str_has_prefix(m->type, "modules")){ + old_metadata_location->additional_metadata = g_slist_delete_link( + old_metadata_location->additional_metadata, node_iter); + cr_metadatum_free(m); + } + node_iter = next; + } + } + + //merge module metadata and dump it to string + moduleindex = modulemd_module_index_merger_resolve (merger, &tmp_err); + g_clear_pointer(&merger, g_object_unref); + char *moduleindex_str = modulemd_module_index_dump_to_string (moduleindex, &tmp_err); + g_clear_pointer(&moduleindex, g_object_unref); + if (tmp_err) { + g_critical("%s: Cannot cannot dump module index: %s", __func__, tmp_err->message); + free(moduleindex_str); + g_clear_error(&tmp_err); + exit(EXIT_FAILURE); + } + + //compress new module metadata string to a file in temporary .repodata + gchar *modules_metadata_path = g_strconcat(tmp_out_repo, "modules.yaml", compression_suffix, NULL); + CR_FILE *modules_file = NULL; + modules_file = cr_open(modules_metadata_path, CR_CW_MODE_WRITE, compression, &tmp_err); + if (modules_file == NULL) { + g_critical("%s: Cannot open source file %s: %s", __func__, modules_metadata_path, tmp_err->message); + g_clear_error(&tmp_err); + free(moduleindex_str); + free(modules_metadata_path); + exit(EXIT_FAILURE); + } + cr_puts(modules_file, moduleindex_str, &tmp_err); + free(moduleindex_str); + cr_close(modules_file, &tmp_err); + if (tmp_err) { + g_critical("%s: Error while closing: : %s", __func__, tmp_err->message); + g_clear_error(&tmp_err); + free(modules_metadata_path); + exit(EXIT_FAILURE); + } + + //create additional metadatum for new module metadata file + cr_Metadatum *new_modules_metadatum = g_malloc0(sizeof(cr_Metadatum)); + new_modules_metadatum->name = modules_metadata_path; + new_modules_metadatum->type = g_strdup("modules"); + additional_metadata = g_slist_prepend(additional_metadata, new_modules_metadatum); + } +#endif /* WITH_LIBMODULEMD */ if (cmd_options->update && cmd_options->keep_all_metadata && old_metadata_location && old_metadata_location->additional_metadata) @@ -766,34 +935,6 @@ cr_metadatalocation_free(old_metadata_location); old_metadata_location = NULL; - - // Setup compression types - const char *xml_compression_suffix = NULL; - const char *sqlite_compression_suffix = NULL; - const char *prestodelta_compression_suffix = NULL; - cr_CompressionType xml_compression = CR_CW_GZ_COMPRESSION; - cr_CompressionType sqlite_compression = CR_CW_BZ2_COMPRESSION; - cr_CompressionType groupfile_compression = CR_CW_GZ_COMPRESSION; - cr_CompressionType prestodelta_compression = CR_CW_GZ_COMPRESSION; - - if (cmd_options->compression_type != CR_CW_UNKNOWN_COMPRESSION) { - sqlite_compression = cmd_options->compression_type; - groupfile_compression = cmd_options->compression_type; - prestodelta_compression = cmd_options->compression_type; - } - - if (cmd_options->general_compression_type != CR_CW_UNKNOWN_COMPRESSION) { - xml_compression = cmd_options->general_compression_type; - sqlite_compression = cmd_options->general_compression_type; - groupfile_compression = cmd_options->general_compression_type; - prestodelta_compression = cmd_options->general_compression_type; - } - - xml_compression_suffix = cr_compression_suffix(xml_compression); - sqlite_compression_suffix = cr_compression_suffix(sqlite_compression); - prestodelta_compression_suffix = cr_compression_suffix(prestodelta_compression); - - // Create and open new compressed files cr_XmlFile *pri_cr_file; cr_XmlFile *fil_cr_file; @@ -1362,7 +1503,7 @@ if (new_groupfile_metadatum) { additional_metadata_rec = cr_create_repomd_records_for_groupfile_metadata(new_groupfile_metadatum, additional_metadata_rec, - groupfile_compression, + compression, cmd_options->repomd_checksum_type); //NOTE(amatej): Now we can add groupfile metadata to the additional_metadata list, for unified handlig while zck compressing @@ -1620,7 +1761,7 @@ cr_ContentStat *prestodelta_zck_stat = NULL; filename = g_strconcat("prestodelta.xml", - prestodelta_compression_suffix, + compression_suffix, NULL); outdeltadir = g_build_filename(out_dir, OUTDELTADIR, NULL); prestodelta_xml_filename = g_build_filename(tmp_out_repo, @@ -1668,7 +1809,7 @@ // 3) Generate prestodelta.xml file prestodelta_stat = cr_contentstat_new(cmd_options->repomd_checksum_type, NULL); prestodelta_cr_file = cr_xmlfile_sopen_prestodelta(prestodelta_xml_filename, - prestodelta_compression, + compression, prestodelta_stat, &tmp_err); if (!prestodelta_cr_file) { @@ -1676,7 +1817,7 @@ g_clear_error(&tmp_err); goto deltaerror; } - if (cmd_options->zck_compression && prestodelta_compression != CR_CW_ZCK_COMPRESSION) { + if (cmd_options->zck_compression && compression != CR_CW_ZCK_COMPRESSION) { filename = g_strconcat("prestodelta.xml", cr_compression_suffix(CR_CW_ZCK_COMPRESSION), NULL); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/createrepo_c-0.15.11/utils/gen_rst.py new/createrepo_c-0.16.0/utils/gen_rst.py --- old/createrepo_c-0.15.11/utils/gen_rst.py 2020-06-02 10:34:46.000000000 +0200 +++ new/createrepo_c-0.16.0/utils/gen_rst.py 2020-07-07 02:44:02.000000000 +0200 @@ -43,6 +43,14 @@ rst += "%s\n\n" % line rst += "\n" + # Add description + if self.description: + rst += "DESCRIPTION\n" + rst += "===========\n\n" + for line in self.description: + rst += "%s\n\n" % line + rst += "\n" + # Add options rst += "OPTIONS\n" rst += "=======\n" @@ -177,8 +185,15 @@ info = Info(NAME, summary="Create rpm-md format (xml-rpm-metadata) repository", synopsis=["%s [options] <directory>" % (NAME,)], + description=["Uses rpm packages from <directory> to create repodata.", + "If compiled with libmodulemd support modular metadata inside <directory> identified by the patterns below are automatically collected, merged and added to the repodata.", + "The patterns are:", + " - \*.modulemd.yaml (recommended file name: N:S:V:C:A.modulemd.yaml)", + " - \*.modulemd-defaults.yaml (recommended file name: N.modulemd-defaults.yaml)", + " - modules.yaml (recommended way of importing multiple documents at once)"], options=args) + ret = info.gen_rst() if not ret: print("Error: Rst has not been generated", file=sys.stderr)
