commit: 033f242da9b9f031d8260aca0bf7b70ce8cd93fa
Author: Mart Raudsepp <leio <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 3 23:46:21 2017 +0000
Commit: Mart Raudsepp <leio <AT> gentoo <DOT> org>
CommitDate: Fri Mar 3 23:46:21 2017 +0000
URL:
https://gitweb.gentoo.org/proj/gentoo-bumpchecker.git/commit/?id=033f242d
gnome: Implement latest version retrieval based on GNOME infra cache.json files
Compared to the no longer working ~vuntz devspace versions file, this is
now doing HTTPS retrievals for each package again; not quite full scraping,
but almost (considering that if we'd hit up the right subdir directly, it's
pretty much the same due to one GET per package again). So it takes a bit
to retrieve the files.
This can probably be made trivially faster by anyone who knows async io
for python-requests or something, but at least it works again.
gnome-bumpchecker.py | 2 +-
modules/gnome_module.py | 45 +++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 44 insertions(+), 3 deletions(-)
diff --git a/gnome-bumpchecker.py b/gnome-bumpchecker.py
index e7ceb16..c9656e9 100755
--- a/gnome-bumpchecker.py
+++ b/gnome-bumpchecker.py
@@ -25,7 +25,7 @@ if __name__ == '__main__':
# Quick versions file parsing
release_packages = gnome.generate_data_release()
- latest_packages = gnome.generate_data_individual()
+ latest_packages = gnome.generate_data_individual(release_packages)
# Old FTP scraping way:
# release_packages, latest_packages = gnome.generate_data_ftp()
diff --git a/modules/gnome_module.py b/modules/gnome_module.py
index 7c8f613..06fe6d8 100644
--- a/modules/gnome_module.py
+++ b/modules/gnome_module.py
@@ -1,4 +1,5 @@
# Copyright John N. Laliberte <[email protected]>
+# Copyright Mart Raudsepp <[email protected]>
# LICENSE - GPL2
# vim: set sts=4 sw=4 et tw=0 :
@@ -8,6 +9,15 @@ import package_module, clioptions_module
DEBUG = False
+# TODO: Figure out some better handling of mappings together with
package_module
+# TODO: package_module has made the reverse mapping of what we need, because
we consume the names from release_packages
+# TODO: So this reverses it back again for the modules we care for this until
a proper fix
+name_mapping = {
+ "cantarell": "cantarell-fonts",
+ "nm-applet": "network-manager-applet",
+ "networkmanager": "NetworkManager",
+}
+
class GNOME:
def __init__(self, nextrev=False):
options = clioptions_module.Options()
@@ -49,8 +59,39 @@ class GNOME:
print "Warning: Ignoring package %s because parsing of its
name or version string '%s' failed" % (components[1], components[2])
return ret
- def generate_data_individual(self):
- return self.generate_data_release()
+ def generate_data_individual(self, release_packages):
+ ret = []
+ for pkg in release_packages:
+ name = pkg.name.split('/')[-1]
+ if name in name_mapping:
+ name = name_mapping[name]
+ data = self.http.get(self.url_base + '/sources/' + name +
'/cache.json')
+ if not data:
+ print("Warning: Unable to read cache.json for %s" % pkg.name)
+ continue
+ data = data.json()
+ if data[0] != 4:
+ print("Warning: unknown cache.json version for package %s" %
name)
+ continue
+ if pkg.major_minor not in data[3]:
+ print("Warning: can't find latest version for %s-%s" % (name,
pkg.major-minor))
+ continue
+ latest = False
+ # Some modules contain more than LATEST-IS-* for some reason, so
we need to iterate and find the correct item instead of [0] (even though it is
firsy always, but lets be future-proof)
+ for tarball in data[3][pkg.major_minor]:
+ if tarball.startswith('LATEST-IS-'):
+ latest = tarball[10:] # len('LATEST-IS-') == 10
+ break
+ if not latest:
+ print("Warning: couldn't find latest version from within
cache.json[3]")
+ continue
+ latest_pkg = package_module.Package(name + "-" + latest)
+ if pkg.name and pkg.version:
+ ret.append(latest_pkg)
+ else:
+ print("Warning: Ignoring package %s because parsing of its
name or version string '%s' failed" % (name, latest))
+ continue
+ return ret
def generate_data_release(self):
return
self.generate_data_from_versions_markup(self.release_versions_file_path +
self.full_release + '/versions')
