Hi, after talking with Harald about the automation stuff and thinking a bit about the overall design of the scriptery I'm attaching a new, finished version of the patch to get the bumping of the build depends right; new in this version:
1. git-clone-all without "-r" clones ALL the repositories of frameworks,
plasma and apps.
2. The "bump-build-dep-versions" script was rewritten to:
- use the map generated by "dev-package-name-list"
- use python-debian instead of hackish regex substitutions
3. "dev-package-name-list" now works without "-r" or "-v" arguments. The "-r"
is not needed anymore and the versions are taken from the configuration file
"conf/versions.json"
4. Improved algorithm to find the build depends in "dev-package-name-list", now
it includes any binary package present in the build depends of the others;
this way we don't have to hardcode an if to handle extra-cmake-modules or
kgendesignerplugin
5. "dev-package-name-list" is now configurable via "conf/dev-package-name-
list.json", which looks like this:
{
"uversion_suffix":"~",
"include_pkg_revision":false
}
uversion_suffix is a string appended to the upstream version,
include_pkg_revision decides if the package revision will be included in the
version bumping or not.
The patch is temporarily available here:
http://gpul.grupos.udc.es/kubuntu_patches/kubuntu_automation_improvement_v3.diff
and also attached to this mail.
Regards.
=== modified file 'README' --- README 2015-09-17 16:26:15 +0000 +++ README 2015-10-17 20:31:55 +0000 @@ -54,4 +54,4 @@ make kubuntu-initial-upload move stuff to upload instead of breaking and moving to manual check over build-dep versions for differently versioned applications bits (kdepim, kdelibs) make -r releasetype arguments consistent - \ No newline at end of file + === modified file 'bump-build-dep-versions' --- bump-build-dep-versions 2015-07-14 21:01:54 +0000 +++ bump-build-dep-versions 2015-10-18 19:55:51 +0000 @@ -1,4 +1,16 @@ #!/usr/bin/python3 +# kate: space-indent on; indent-width 4; replace-tabs on; indent-mode python; remove-trailing-space modified; +# vim: expandtab ts=4 + +############################################################################ +# Copyright © 2015 Jonathan Riddell # +# Copyright © 2015 José Manuel Santamaría Lema <[email protected]> # +# # +# This program is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 2 of the License, or # +# (at your option) any later version. # +############################################################################ # script to bump build-dep versions in KDE packaging # run in directory with debian/control file @@ -7,37 +19,37 @@ import os import re import subprocess +import json -from lib.utils import * +from lib.control_edit import * parser = argparse.ArgumentParser(description="Things!") -parser.add_argument("-d", "--dist", required=True, help="Distribution name") -parser.add_argument("-v", "--version", required=True, help="Upstream version") +parser.add_argument("-d", "--dist", required�lse, default="wily", help="Distribution name") + +#FIXME: Remove these, they are there for compatibility +parser.add_argument("-v", "--version", required�lse, help="Upstream version") parser.add_argument("-r", "--releasetype", help="Type [frameworks,plasma,applications]", default="frameworks") + args = parser.parse_args() +dist = args.dist releaseType = args.releasetype -pythonDir = os.path.dirname(os.path.realpath(__file__)) - -control = readAllFromFile('debian/control') - -with open(pythonDir + '/packaging-exceptions.json') as data_file: - packagingExceptions = json.load(data_file) - noEpoch = packagingExceptions['no-epoch-dev-packages'] - -for builddep in readPackages(pythonDir + "/dev-package-name-lists/%s-%s" % (releaseType, args.dist)): - epoch = "" - if (releaseType == "plasma" or releaseType == "applications") and builddep not in noEpoch: - epoch = "4:" - version = args.version - if builddep in packagingExceptions: - version = packagingExceptions[builddep] - control = re.sub(r'%s\s*\(>=.*?\)' % builddep, '%s (>= %s%s~)' % (builddep, epoch, version), control) - -f = open("debian/control", "w") -f.write(control) -f.close() - +cwd = os.path.dirname(os.path.realpath(__file__)) +build_depends_map_file = "%s/dev-package-name-lists/%s.json" % (cwd,dist) +build_depends_map = json.load(open(build_depends_map_file, 'r')) + +#Open the control file +src_pkg, bin_pkg_list, bin_pkg_map = parse_control('./debian/control'); + +#Bump the build depends using the map from the json file +bump_version_with_map(src_pkg, 'Build-Depends', build_depends_map) +if 'Build-Depends-Indep' in src_pkg: + bump_version_with_map(src_pkg, 'Build-Depends-Indep', build_depends_map) + +#Dump the contents of the control file +dump_control('./debian/control',src_pkg, bin_pkg_list, bin_pkg_map) + +#Print the git diff print("=== bump-build-dep-versions diff start") -subprocess.check_call(["git", "diff"]) +subprocess.check_call(["git", "--no-pager", "diff"]) print("=== bump-build-dep-versions diff end") === added directory 'conf' === added file 'conf/dev-package-name-list.json' --- conf/dev-package-name-list.json 1970-01-01 00:00:00 +0000 +++ conf/dev-package-name-list.json 2015-10-18 19:55:02 +0000 @@ -0,0 +1,4 @@ +{ + "uversion_suffix":"~", + "include_pkg_revision":false +} === added file 'conf/git-clone-all.json' --- conf/git-clone-all.json 1970-01-01 00:00:00 +0000 +++ conf/git-clone-all.json 2015-10-16 09:24:47 +0000 @@ -0,0 +1,13 @@ +{ + "clone-this-one":"git-ssh-kubuntu", + "extra-remotes":["git-anon-siduction","git-anon-neon"], + + "git-ssh-kubuntu":"git+ssh://git.debian.org/git/pkg-kde/%s/%s.git", + "git-anon-kubuntu":"git://anonscm.debian.org/pkg-kde/%s/%s.git", + + "git-ssh-siduction":"[email protected]:siduction-kde-%s/%s.git", + "git-anon-siduction":"https://gitlab.com/siduction-kde-%s/%s.git";, + + "git-ssh-neon":"[email protected]:sample_url_please_replace_it_whenever_you_can/%s/%s.git", + "git-anon-neon":"git:/sample_url_please_replace_it_whenever_you_can/%s/%s.git" +} === added file 'conf/versions.json' --- conf/versions.json 1970-01-01 00:00:00 +0000 +++ conf/versions.json 2015-10-18 19:47:17 +0000 @@ -0,0 +1,6 @@ +{ + "frameworks":"5.15", + "plasma":"5.4.2", + "applications":"15.08.2", + "qt":"5.5.0" +} === added file 'dev-package-name-list' --- dev-package-name-list 1970-01-01 00:00:00 +0000 +++ dev-package-name-list 2015-10-18 19:58:10 +0000 @@ -0,0 +1,105 @@ +#!/usr/bin/python3 +# kate: space-indent on; indent-width 4; replace-tabs on; indent-mode python; remove-trailing-space modified; +# vim: expandtab ts=4 + +############################################################################ +# Copyright © 2015 Jonathan Riddell # +# Copyright © 2015 José Manuel Santamaría Lema <[email protected]> # +# # +# This program is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 2 of the License, or # +# (at your option) any later version. # +############################################################################ + +from lib.utils import * + +import argparse +import sys +import os +import json +import re +import warnings + +from debian import deb822 +from debian.changelog import Changelog, Version + + +parser = argparse.ArgumentParser(description="Update -dev package name list used by bump-build-dep-versions") +parser.add_argument("-d", "--dist", help="Distribution name", default="wily", required�lse) +args = parser.parse_args() +dist = args.dist + +#Load configuration +cwd = os.path.dirname(os.path.realpath(__file__)) +config = json.load( open(cwd + "/conf/dev-package-name-list.json", 'r') ) + +bin_package_version_map = {} +build_depends_set = set() +#Populate the vars above inspecting the control files and changelogs +_, dirs, _ = next(os.walk('.')) +for d in dirs: + #Find out source package version + changelog = Changelog() + changelog_file_name = d + '/debian/changelog' + try: + changelog.parse_changelog(open(changelog_file_name, 'r')) + except FileNotFoundError: + print("WARNING: File " + changelog_file_name + " not found!") + continue + src_package_version = changelog.get_version().upstream_version + #Find out if the package has epoch or not, if so preprend it to the version + epoch = changelog.get_version().epoch + if epoch != None: + src_package_version = epoch + ":" + src_package_version + #Add configured upstream version suffix + src_package_version += config["uversion_suffix"] + #Include package revison if required + if config["include_pkg_revision"]: + src_package_version += '-' + changelog.get_version().debian_revision + # With the info parsed from $PWD/*/debian/control populate: + # - build_depends_set <- with all the binary packages appearing in Build-Depends[-Indep] + # - dev_package_version_map <- with all the binary packages with their versions + control_file_name = d + '/debian/control' + try: + control_file = deb822.Packages.iter_paragraphs(open(control_file_name, 'r')); + except FileNotFoundError: + print("WARNING: File " + control_file_name + " not found!") + continue + warnings.simplefilter('ignore', UserWarning) #Ignore the warnings from python-debian + for pkg in control_file: + if 'Source' in pkg: + #Read the build depends of the current package and add them to build_depends_set + relation_structured = deb822.PkgRelation.parse_relations(pkg['Build-Depends']) + for i in relation_structured: + for j in i: + build_depends_set.add(j['name']) + if 'Build-Depends-Indep' in pkg: + relation_structured = deb822.PkgRelation.parse_relations(pkg['Build-Depends-Indep']) + for i in relation_structured: + for j in i: + build_depends_set.add(j['name']) + if 'Package' in pkg: + #Find out binary package names and add them to dev_package_version_map with their version + bin_package_name = pkg['Package'] + bin_package_version_map[bin_package_name] = src_package_version + control_file.close() + warnings.simplefilter('default', UserWarning) #Reset the user warnings + +#Now its easy, we have bin_package_version_map mapping all (binary_package -> version) +#so we just have to discard the binary packages which are not build depends and we got it +dev_package_version_map = dict(bin_package_version_map) +for bin_package_name in bin_package_version_map: + if bin_package_name not in build_depends_set: + if re.match('.*-dev$', bin_package_name) == None: + dev_package_version_map.pop(bin_package_name) +dev_package_version_map["_comment"] = "This file was generated automatically by dev-package-name-list." + +#Dump the thing to a json file +json_str = json.dumps(dev_package_version_map, indent=4, sort_keys=True) + +cwd = os.path.dirname(os.path.realpath(__file__)) +outFile = cwd + "/dev-package-name-lists/" + dist + ".json" +f = open(outFile, 'w') +f.write(json_str) +print("write " + outFile) === added file 'git-clone-all' --- git-clone-all 1970-01-01 00:00:00 +0000 +++ git-clone-all 2015-10-18 19:56:49 +0000 @@ -0,0 +1,97 @@ +#!/usr/bin/python3 + +############################################################################ +# Copyright © 2015 José Manuel Santamaría Lema <[email protected]> # +# # +# This program is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 2 of the License, or # +# (at your option) any later version. # +############################################################################ + +import os +import sys +import argparse +import subprocess + +from lib.utils import * + +#Argument parser +parser = argparse.ArgumentParser( + description="Clones all the frameworks or plasma or apps git repositories" + " to the current directory") +parser.add_argument("-r", "--releasetype", + help="KDE Release Type [frameworks,plasma,applications]" + "if not specified downloads all the git repositories", + required�lse) +parser.add_argument("-d", "--dist", + help="KDE Release Type [frameworks,plasma,applications]", + default="wily", required�lse) + +#Check arguments +args = parser.parse_args() + +releaseTypes = [] + +if args.releasetype != None: + if args.releasetype not in ["frameworks", "plasma", "applications"]: + print("Invalid releasetype %s" % args.releasetype) + print("Accepted release types are frameworks, plasma, applications") + sys.exit(1) + else: + releaseTypes.add(args.releasetype) +else: + releaseTypes = ["frameworks", "plasma", "applications"] + + +#Find out the base url to clone and the extra remotes +cwd = os.path.dirname(os.path.realpath(__file__)) +config_file = open(cwd + "/conf/git-clone-all.json") +config_map = json.load(config_file) +template_url_clone = config_map[config_map["clone-this-one"]] +print("Template url to clone: %s" % template_url_clone) +remotes_list = config_map["extra-remotes"] + +packages_not_cloned = [] + +#Clone the repositories and add the extra remotes +for releaseType in releaseTypes: + #Get package list + packages = getFtpVersionMap(releaseType) + for package in packages: + repo_name = package + command = "git clone " + (template_url_clone % (releaseType,repo_name)) + print("Executing: " + command) + try: + subprocess.check_output(command.split()) + except KeyboardInterrupt: + print("abort by user request") + sys.exit(130) + except subprocess.CalledProcessError: + #In any other exception we continue, this way we can use + #git-clone-all against a directory with some repositories already cloned. + if not os.path.exists(os.getcwd() + '/' + package): + packages_not_cloned.append(package) + continue + for i in remotes_list: + template_url_remote = str(config_map[i]) + remote_name = i.split('-')[-1] + try: + command = "git remote add " + remote_name + " " + (template_url_remote % (releaseType,repo_name)) + print("Executing: " + command) + old_cwd = os.getcwd() + os.chdir(repo_name) + subprocess.check_call(command.split()) + os.chdir(old_cwd) + except TypeError: + print("The remote %s couldn't be added using the template url:\n%s" % (remote_name,template_url_remote)) + +print("Summary:") +if not packages_not_cloned: + print("All packages were cloned succesfully") +else: + print("The following packages couldn't be cloned:") + for p in packages_not_cloned: + print(p) + +# vim: expandtab ts=4 === added file 'lib/control_edit.py' --- lib/control_edit.py 1970-01-01 00:00:00 +0000 +++ lib/control_edit.py 2015-10-18 19:59:41 +0000 @@ -0,0 +1,121 @@ +# -*- coding: utf8 -*- + +############################################################################ +# Copyright © 2015 José Manuel Santamaría Lema <[email protected]> # +# # +# This program is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 2 of the License, or # +# (at your option) any later version. # +############################################################################ + +from debian import deb822 + +import re + +from lib.wrap_control import * + +#Parses a control file and returns a tuple(src_pkg,bin_pkg_list,bin_pkg_map) where +# * src_pkg is the deb822 paragraph object of the source package +# * bin_pkg_list is a list of the binary packages file listed in the same order +# they appear in the control file +# * bin_pkg_map is a hash indexed by binary package name which returns the +# corresponding deb822 paragraph object for each binary package +def parse_control(filename): + control_file = deb822.Packages.iter_paragraphs(open(filename, 'r')); + bin_pkg_list = [] + bin_pkg_map = {} + for pkg in control_file: + if 'Source' in pkg: + src_pkg = pkg + else: + pkg_name = pkg['Package'] + bin_pkg_list.append(pkg_name) + bin_pkg_map[pkg_name] = pkg + control_file.close() + return (src_pkg, bin_pkg_list, bin_pkg_map) + +# Dumps the content of a control file. +def dump_control(filename,src_pkg,bin_pkg_list,bin_pkg_map): + new_control_file = open(filename, "w") + new_control_file.write(src_pkg.dump()) + for i in bin_pkg_list: + new_control_file.write('\n') + new_control_file.write(bin_pkg_map[i].dump()) + new_control_file.close() + +# Examples of deb822.PkgRelation.parse_relations() output +# +# "emacs | emacsen, make, debianutils (>= 1.7)" becomes +# [ [ {'name': 'emacs'}, {'name': 'emacsen'} ], +# [ {'name': 'make'} ], +# [ {'name': 'debianutils', 'version': ('>=', '1.7')} ] ] +# +# "tcl8.4-dev, procps [!hurd-i386]" becomes +# [ [ {'name': 'tcl8.4-dev'} ], +# [ {'name': 'procps', 'arch': (false, 'hurd-i386')} ] ] + + +def remove_from_relations( package, relation, target, wrap_relation�lse ): + if relation in package: + relation_structured = deb822.PkgRelation.parse_relations(package[relation]) + list_to_delete = [] + for i in range(len(relation_structured)): + alternatives_list = relation_structured[i] + for j in range(len(alternatives_list)): + if alternatives_list[j]['name'] == target: + list_to_delete.append( (i,j) ) + for i, j in list_to_delete: + del relation_structured[i][j] + #Delete empty lists. + relation_structured = filter(None, relation_structured) + #Convert to string + package[relation] = deb822.PkgRelation.str(relation_structured) + #Delete the relation completely if it's empty. + if package[relation].strip() == "": + del package[relation] + else: + if wrap_relation: + wrap_field(package,relation,False,False,False) + +def add_to_relations( package, relation, target, wrap_relation�lse): + if relation in package: + #Search the target in the relation. + target_found = False + relation_structured = deb822.PkgRelation.parse_relations(package[relation]) + for i in range(len(relation_structured)): + alternatives_list = relation_structured[i] + for j in range(len(alternatives_list)): + if alternatives_list[j]['name'] == target: + target_found = True + break + #If the target wasn't found, add it. + if not target_found: + package[relation] += ", " + target + else: + package[relation] = target + if wrap_relation: + wrap_field(package,relation,False,False,False) + +def bump_version( package, relation, target, version, operator='>='): + relation_structured = deb822.PkgRelation.parse_relations(package[relation]) + for i in range(len(relation_structured)): + alternatives_list = relation_structured[i] + for j in range(len(alternatives_list)): + if alternatives_list[j]['name'] == target: + alternatives_list[j]['version'] = ( operator, version ) + package[relation] = deb822.PkgRelation.str(relation_structured) + wrap_field(package,relation,True,False,False) + +def bump_version_with_map( package, relation, pkg_version_map ): + relation_structured = deb822.PkgRelation.parse_relations(package[relation]) + for i in range(len(relation_structured)): + alternatives_list = relation_structured[i] + for j in range(len(alternatives_list)): + pkg_name = alternatives_list[j]['name'] + if pkg_name in pkg_version_map: + alternatives_list[j]['version'] = ( '>=', pkg_version_map[pkg_name] ) + package[relation] = deb822.PkgRelation.str(relation_structured) + wrap_field(package,relation,False,False,False) + +# vim: expandtab ts=4 === modified file 'lib/utils.py' --- lib/utils.py 2015-06-29 14:07:29 +0000 +++ lib/utils.py 2015-10-18 19:48:25 +0000 @@ -1,5 +1,20 @@ +############################################################################ +# Copyright © 2014 Harald Sitter # +# Copyright © 2015 Philip Muskovac # +# Copyright © 2015 Jonathan Riddell # +# Copyright © 2015 José Manuel Santamaría Lema <[email protected]> # +# # +# This program is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 2 of the License, or # +# (at your option) any later version. # +############################################################################ + + import json import os +import subprocess +import re def readAllFromFile(filename): f = open(filename, "r") @@ -25,3 +40,55 @@ return pkgmap[package] else: return package + +def repoName(package): + cwd = os.path.dirname(os.path.realpath(__file__)) + with open(cwd + "/../repo-names.json") as repofile: + pkgmap = json.load(repofile) + if package in pkgmap: + return pkgmap[package] + else: + return package + +# ReleaseType example: +# ["frameworks","plasma","applications"] +# it would get a map of everything of frameworks/plasma/applications +def getFtpVersionMap(releaseType): + #Populate and return the map + packageVersionMap = {} + #Find out which subdirectories we have to inspeact in the ftp + ftp_subdirs = [] + if releaseType == "frameworks": + ftp_subdirs = ["","portingAids"] + elif releaseType == "plasma": + ftp_subdirs = [""] + elif releaseType == "applications": + ftp_subdirs = ["src"] + #Find out the version + cwd = os.path.dirname(os.path.realpath(__file__)) + config_file = open(cwd + "/../conf/versions.json") + config_map = json.load(config_file) + version = config_map[releaseType] + #Find out the stability + versionParts = version.split(".") + lastDigit = int(versionParts[-1]) + if lastDigit >= 80: + stability = "unstable" + else: + stability = "stable" + #Inspect the ftp + for subdir in ftp_subdirs: + p = subprocess.Popen(["sftp", "-b", "-", "depot.kde.org:%s/%s/%s/%s" % (stability, releaseType, version, subdir)], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + output, _ = p.communicate(bytes("ls *xz", 'utf-8')) + + for line in output.splitlines(): + line = line.decode('utf-8') + match = re.search(r'([a-zA-Z0-9\-]+)-' + '([\d.]*)' + r'\.tar\.', line) + if match: + package = match.group(1) + package_version = match.group(2) + packageVersionMap[package] = package_version + return packageVersionMap + +# vim: expandtab ts=4 === added file 'lib/wrap_control.py' --- lib/wrap_control.py 1970-01-01 00:00:00 +0000 +++ lib/wrap_control.py 2015-10-18 20:00:17 +0000 @@ -0,0 +1,61 @@ +# Copyright (C) 2010-2014, Benjamin Drung <[email protected]> +# 2010, Stefano Rivera <[email protected]> +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +import re + +# This code was copied from wrap-and-sort, it's helpful when converting a +# package relation from the structured deb822.PkgRelation to string, because +# deb822.PkgRelation.str(relation_structured) returns a string in a single line +# so it's nice to get it in the wrap-and-sort-like format again + +def sort_list(unsorted_list): + packages = [x for x in unsorted_list if re.match("[a-z0-9]", x)] + special = [x for x in unsorted_list if not re.match("[a-z0-9]", x)] + return sorted(packages) + sorted(special) + +def wrap_field(control, entry, wrap_always, short_indent, + trailing_comma, sort=True): + max_line_length = 79 #wrap-and-sort current default + packages = [x.strip() for x in control[entry].split(",")] + + # Sanitize alternative packages. E.g. "a|b |c" -> "a | b | c" + packages = [" | ".join([x.strip() for x in p.split("|")]) for p in packages] + + if sort: + # Remove duplicate entries + packages = set(packages) + # Not explicitly disallowed by Policy but known to break QA tools: + if "" in packages: + packages.remove("") + packages = sort_list(packages) + + length = len(entry) + sum([2 + len(package) for package in packages]) + if wrap_always or length > max_line_length: + indentation = " " + if not short_indent: + indentation *= len(entry) + 2 + packages_with_indention = [indentation + x for x in packages] + packages_with_indention = ",\n".join(packages_with_indention) + if trailing_comma: + packages_with_indention += ',' + if short_indent: + control[entry] = "\n" + packages_with_indention + else: + control[entry] = packages_with_indention.strip() + else: + control[entry] = ", ".join(packages).strip() + +# vim: expandtab ts=4 === added file 'repo-names.json' --- repo-names.json 1970-01-01 00:00:00 +0000 +++ repo-names.json 2015-10-16 09:27:15 +0000 @@ -0,0 +1,9 @@ +{ + "__coment": "The attribute name is the source package and the value is the alioth repository name", + "attica-kf5": "attica", + "kactivities-kf5": "kactivities", + "kwallet-kf5": "kwallet", + "kdnssd-kf5": "kdnssd", + "baloo-kf5": "baloo", + "kfilemetadata-kf5": "kfilemetadata" +}
signature.asc
Description: This is a digitally signed message part.
-- kubuntu-devel mailing list [email protected] Modify settings or unsubscribe at: https://lists.ubuntu.com/mailman/listinfo/kubuntu-devel
