Colin Watson has proposed merging lp:~cjwatson/launchpad/remove-archive-override-check into lp:launchpad.
Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/launchpad/remove-archive-override-check/+merge/99514 archive-override-check was written at our request: https://launchpad.net/launchpad/+spec/overrides-consistency-check However, we have in fact been using a different tool for some time, which doesn't talk to the Launchpad database - it's sufficient to do Sources/Packages file analysis for this - and so doesn't need to run on ftpmaster, and which is orders of magnitude faster to run (output at http://people.canonical.com/~ubuntu-archive/architecture-mismatches.txt, currently empty). Given that we want to stop using tools that only work on ftpmaster, we aren't going to use this again, and so we should remove it. -- https://code.launchpad.net/~cjwatson/launchpad/remove-archive-override-check/+merge/99514 Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/remove-archive-override-check into lp:launchpad.
=== removed file 'lib/lp/soyuz/doc/archive-override-check.txt' --- lib/lp/soyuz/doc/archive-override-check.txt 2011-12-29 05:29:36 +0000 +++ lib/lp/soyuz/doc/archive-override-check.txt 1970-01-01 00:00:00 +0000 @@ -1,44 +0,0 @@ -Check for discrepancies in overrides between architectures -========================================================== - -This script looks for out-of-sync overrides, checking for -discrepancies in overrides between architectures. - -It should run daily via cronjob, sending the STDOUT result report via -email to distribution maintainers to ensure that only necessary -discrepancies are present during the development cycle. - -See also: - https://launchpad.net/products/soyuz/+spec/overrides-consistency-check - https://wiki.ubuntu.com/MilestoneRhythm - -XXX cprov 20060714: we need better populate archive/"publishing -history" in the sampledata to test those tools properly. - - >>> import os - >>> import subprocess - >>> import sys - >>> from lp.services.config import config - - >>> script = os.path.join( - ... config.root, "scripts", "ftpmaster-tools", - ... "archive-override-check.py") - - >>> process = subprocess.Popen( - ... [sys.executable, script, "-v", "-d", "ubuntu", "-s", "warty"], - ... stdout=subprocess.PIPE, stderr=subprocess.PIPE) - >>> stdout, stderr = process.communicate() - >>> process.returncode - 0 - >>> print stderr - INFO Creating lockfile: ... - DEBUG Considering: ubuntu/warty/RELEASE/CURRENT. - DEBUG ... published sources - DEBUG Rolling back any remaining transactions. - DEBUG Removing lock file: ... - <BLANKLINE> - - -Since its data is sane, empty STDOUT is okay. - - >>> print stdout === removed file 'lib/lp/soyuz/scripts/pubsourcechecker.py' --- lib/lp/soyuz/scripts/pubsourcechecker.py 2012-01-19 03:06:04 +0000 +++ lib/lp/soyuz/scripts/pubsourcechecker.py 1970-01-01 00:00:00 +0000 @@ -1,214 +0,0 @@ -# Copyright 2009-2012 Canonical Ltd. This software is licensed under the -# GNU Affero General Public License version 3 (see the file LICENSE). - -"""FTPMaster utilities.""" - -__metaclass__ = type - -__all__ = ['PubSourceChecker'] - - -class PubBinaryContent: - """Binary publication container. - - Currently used for auxiliary storage in PubSourceChecker. - """ - - def __init__(self, name, version, arch, component, section, priority): - self.name = name - self.version = version - self.arch = arch - self.component = component - self.section = section - self.priority = priority - self.messages = [] - - def warn(self, message): - """Append a warning in the message list.""" - self.messages.append('W: %s' % message) - - def error(self, message): - """Append a error in the message list.""" - self.messages.append('E: %s' % message) - - def renderReport(self): - """Render a report with the appended messages (self.messages). - - Return None if no message was found, otherwise return - a properly formatted string, including - - <TAB>BinaryName_Version Arch Component/Section/Priority - <TAB><TAB>MESSAGE - """ - if not len(self.messages): - return - - report = [('\t%s_%s %s %s/%s/%s' - % (self.name, self.version, self.arch, - self.component, self.section, self.priority))] - - for message in self.messages: - report.append('\t\t%s' % message) - - return "\n".join(report) - - -class PubBinaryDetails: - """Store the component, section and priority of binary packages and, for - each binary package the most frequent component, section and priority. - - These are stored in the following attributes: - - - components: A dictionary mapping binary package names to other - dictionaries mapping component names to binary packages published - in this component. - - sections: The same as components, but for sections. - - priorities: The same as components, but for priorities. - - correct_components: a dictionary mapping binary package name - to the most frequent (considered the correct) component name. - - correct_sections: same as correct_components, but for sections - - correct_priorities: same as correct_components, but for priorities - """ - - def __init__(self): - self.components = {} - self.sections = {} - self.priorities = {} - self.correct_components = {} - self.correct_sections = {} - self.correct_priorities = {} - - def addBinaryDetails(self, bin): - """Include a binary publication and update internal registers.""" - name_components = self.components.setdefault(bin.name, {}) - bin_component = name_components.setdefault(bin.component, []) - bin_component.append(bin) - - name_sections = self.sections.setdefault(bin.name, {}) - bin_section = name_sections.setdefault(bin.section, []) - bin_section.append(bin) - - name_priorities = self.priorities.setdefault(bin.name, {}) - bin_priority = name_priorities.setdefault(bin.priority, []) - bin_priority.append(bin) - - def _getMostFrequentValue(self, data): - """Return a dict of name and the most frequent value. - - Used for self.{components, sections, priorities} - """ - results = {} - - for name, items in data.iteritems(): - highest = 0 - for item, occurrences in items.iteritems(): - if len(occurrences) > highest: - highest = len(occurrences) - results[name] = item - - return results - - def setCorrectValues(self): - """Find out the correct values for the same binary name - - Consider correct the most frequent. - """ - self.correct_components = self._getMostFrequentValue(self.components) - self.correct_sections = self._getMostFrequentValue(self.sections) - self.correct_priorities = self._getMostFrequentValue(self.priorities) - - -class PubSourceChecker: - """Map and probe a Source/Binaries publication couple. - - Receive the source publication data and its binaries and perform - a group of heuristic consistency checks. - """ - - def __init__(self, name, version, component, section, urgency): - self.name = name - self.version = version - self.component = component - self.section = section - self.urgency = urgency - self.binaries = [] - self.binaries_details = PubBinaryDetails() - - def addBinary(self, name, version, architecture, component, section, - priority): - """Append the binary data to the current publication list.""" - bin = PubBinaryContent( - name, version, architecture, component, section, priority) - - self.binaries.append(bin) - - self.binaries_details.addBinaryDetails(bin) - - def check(self): - """Setup check environment and perform the required checks.""" - self.binaries_details.setCorrectValues() - - for bin in self.binaries: - self._checkComponent(bin) - self._checkSection(bin) - self._checkPriority(bin) - - def _checkComponent(self, bin): - """Check if the binary component matches the correct component. - - 'correct' is the most frequent component in this binary package - group - """ - correct_component = self.binaries_details.correct_components[bin.name] - if bin.component != correct_component: - bin.warn('Component mismatch: %s != %s' - % (bin.component, correct_component)) - - def _checkSection(self, bin): - """Check if the binary section matches the correct section. - - 'correct' is the most frequent section in this binary package - group - """ - correct_section = self.binaries_details.correct_sections[bin.name] - if bin.section != correct_section: - bin.warn('Section mismatch: %s != %s' - % (bin.section, correct_section)) - - def _checkPriority(self, bin): - """Check if the binary priority matches the correct priority. - - 'correct' is the most frequent priority in this binary package - group - """ - correct_priority = self.binaries_details.correct_priorities[bin.name] - if bin.priority != correct_priority: - bin.warn('Priority mismatch: %s != %s' - % (bin.priority, correct_priority)) - - def renderReport(self): - """Render a formatted report for the publication group. - - Return None if no issue was annotated or an formatted string - including: - - SourceName_Version Component/Section/Urgency | # bin - <BINREPORTS> - """ - report = [] - - for bin in self.binaries: - bin_report = bin.renderReport() - if bin_report: - report.append(bin_report) - - if not len(report): - return - - result = [('%s_%s %s/%s/%s | %s bin' - % (self.name, self.version, self.component, - self.section, self.urgency, len(self.binaries)))] - - result.extend(report) - - return "\n".join(result) === removed file 'lib/lp/soyuz/scripts/tests/test_overrides_checker.py' --- lib/lp/soyuz/scripts/tests/test_overrides_checker.py 2012-01-19 03:09:38 +0000 +++ lib/lp/soyuz/scripts/tests/test_overrides_checker.py 1970-01-01 00:00:00 +0000 @@ -1,174 +0,0 @@ -# Copyright 2009-2012 Canonical Ltd. This software is licensed under the -# GNU Affero General Public License version 3 (see the file LICENSE). - -"""archive-override-check tool base class tests.""" - -__metaclass__ = type - -from unittest import TestCase - -from lp.soyuz.scripts.pubsourcechecker import ( - PubBinaryContent, - PubBinaryDetails, - PubSourceChecker, - ) - - -class TestPubBinaryDetails(TestCase): - - def setUp(self): - self.binary_details = PubBinaryDetails() - - def test_single_binary(self): - """Single binary inclusion.""" - bin = PubBinaryContent('foo-dev', '1.0', 'i386', 'main', - 'base', 'REQUIRED') - - self.binary_details.addBinaryDetails(bin) - - # components/sections/priorities have symetric behaviour - - # priorities[name] -> list of added priorities - self.assertEqual( - 1, len(self.binary_details.priorities['foo-dev'])) - # not correct value was set yet - self.assertEqual( - False, 'foo-dev' in self.binary_details.correct_priorities) - # set correct values - self.binary_details.setCorrectValues() - # now we have the correct value in place - self.assertEqual( - 'REQUIRED', self.binary_details.correct_priorities['foo-dev']) - - def test_multi_binaries(self): - """Multiple binaries inclusion.""" - values_map = [ - ('i386', 'REQUIRED'), - ('amd64', 'REQUIRED'), - ('powerpc', 'REQUIRED'), - ('sparc', 'REQUIRED'), - ('hppa', 'IMPORTANT'), - ('ia64', 'IMPORTANT'), - ] - # add multiple binaries systematically according values_map - for arch, priority in values_map: - bin = PubBinaryContent('foo-dev', '1.0', arch, 'main', - 'base', priority) - - self.binary_details.addBinaryDetails(bin) - - # expects 2 distinct priorities - self.assertEqual( - 2, len(self.binary_details.priorities['foo-dev'])) - # set correct values - self.binary_details.setCorrectValues() - # 'REQUIRED' is the most frequent priority in this group of binary - self.assertEqual( - 'REQUIRED', self.binary_details.correct_priorities['foo-dev']) - - -class TestPubSourceChecker(TestCase): - - def setUp(self): - """Initialize useful constant values.""" - self.name = 'foo' - self.version = '1.0' - self.component = 'main' - self.section = 'python' - self.urgency = 'URGENT' - self.default_checker = PubSourceChecker( - self.name, self.version, self.component, - self.section, self.urgency) - - def test_initialization(self): - """Check PubSourceChecker class initialization.""" - checker = self.default_checker - self.assertEqual(self.name, checker.name) - self.assertEqual(self.version, checker.version) - self.assertEqual(self.component, checker.component) - self.assertEqual(self.section, checker.section) - self.assertEqual(self.urgency, checker.urgency) - self.assertEqual(0, len(checker.binaries)) - - def test_single_binary_ok(self): - """Probe single correct binary addition.""" - checker = self.default_checker - - checker.addBinary('foo-dev', self.version, 'i386', self.component, - self.section, 'REQUIRED') - - checker.check() - - self.assertEqual(None, checker.renderReport()) - - # inspect PubBinaryDetails attributesm check if they are populated - # correctly see TestPubBinaryDetails above. - self.assertEqual( - 1, len(checker.binaries_details.components['foo-dev'])) - self.assertEqual( - 1, len(checker.binaries_details.sections['foo-dev'])) - self.assertEqual( - 1, len(checker.binaries_details.priorities['foo-dev'])) - self.assertEqual( - self.component, - checker.binaries_details.correct_components['foo-dev']) - self.assertEqual( - self.section, - checker.binaries_details.correct_sections['foo-dev']) - self.assertEqual( - 'REQUIRED', - checker.binaries_details.correct_priorities['foo-dev']) - - def test_multi_binary_component_failure(self): - """Probe multi binary with wrong component.""" - checker = self.default_checker - - checker.addBinary('foo-dev', self.version, 'i386', 'universe', - self.section, 'REQUIRED') - checker.addBinary('foo-dev', self.version, 'amd64', 'multiverse', - self.section, 'REQUIRED') - - checker.check() - - self.assertEqual( - "foo_1.0 main/python/URGENT | 2 bin\n\t" - "foo-dev_1.0 amd64 multiverse/python/REQUIRED\n\t\t" - "W: Component mismatch: multiverse != universe", - checker.renderReport()) - - def test_multi_binary_priority_failure(self): - """Probe multiple binaries with priority conflict.""" - checker = self.default_checker - - checker.addBinary('foo-dev', self.version, 'i386', self.component, - self.section, 'REQUIRED') - checker.addBinary('foo-dbg', self.version, 'i386', self.component, - self.section, 'EXTRA') - checker.addBinary('foo-dev', self.version, 'amd64', self.component, - self.section, 'EXTRA') - - checker.check() - - self.assertEqual( - "foo_1.0 main/python/URGENT | 3 bin\n" - "\tfoo-dev_1.0 amd64 main/python/EXTRA\n" - "\t\tW: Priority mismatch: EXTRA != REQUIRED", - checker.renderReport()) - - def test_multi_binary_priority_success(self): - """Probe multiple binaries with correct priorities. - - Following UNIX approach, no output is produce for correct input. - """ - checker = self.default_checker - - checker.addBinary('foo-dev', self.version, 'i386', self.component, - self.section, 'EXTRA') - checker.addBinary('foo-dbg', self.version, 'i386', self.component, - self.section, 'EXTRA') - checker.addBinary('foo-dev', self.version, 'amd64', self.component, - self.section, 'EXTRA') - - checker.check() - - self.assertEqual(None, checker.renderReport()) === removed file 'scripts/ftpmaster-tools/archive-override-check.py' --- scripts/ftpmaster-tools/archive-override-check.py 2012-01-19 03:09:38 +0000 +++ scripts/ftpmaster-tools/archive-override-check.py 1970-01-01 00:00:00 +0000 @@ -1,99 +0,0 @@ -#!/usr/bin/python -S -# -# Copyright 2009-2012 Canonical Ltd. This software is licensed under the -# GNU Affero General Public License version 3 (see the file LICENSE). - -"""Archive Override Check - -Given a distribution to run on, report any override inconsistence found. -It basically check if all published source and binaries are coherent. -""" - -import _pythonpath - -import transaction -from zope.component import getUtility - -from lp.app.errors import NotFoundError -from lp.registry.interfaces.distribution import IDistributionSet -from lp.registry.interfaces.pocket import PackagePublishingPocket -from lp.services.config import config -from lp.services.scripts.base import LaunchpadScript -from lp.soyuz.enums import PackagePublishingStatus -from lp.soyuz.scripts.pubsourcechecker import PubSourceChecker - - -class ArchiveOverrideCheckScript(LaunchpadScript): - - def add_my_options(self): - self.parser.add_option( - "-d", "--distribution", action="store", - dest="distribution", metavar="DISTRO", default="ubuntu", - help="Distribution to consider") - self.parser.add_option( - "-s", "--suite", action="store", - dest="suite", metavar="SUITE", default=None, - help=("Suite to consider, if not passed consider the " - "currentseries and the RELEASE pocket")) - - def main(self): - try: - try: - distribution = getUtility(IDistributionSet)[ - self.options.distribution] - if self.options.suite is None: - distroseries = distribution.currentseries - pocket = PackagePublishingPocket.RELEASE - else: - distroseries, pocket = ( - distribution.getDistroSeriesAndPocket( - self.options.suite)) - - self.logger.debug( - "Considering: %s/%s/%s/%s." - % (distribution.name, distroseries.name, pocket.name, - distroseries.status.name)) - - checkOverrides(distroseries, pocket, self.logger) - except NotFoundError, info: - self.logger.error('Not found: %s' % info) - finally: - self.logger.debug("Rolling back any remaining transactions.") - transaction.abort() - - -def checkOverrides(distroseries, pocket, log): - """Initialize and handle PubSourceChecker. - - Iterate over PUBLISHED sources and perform PubSourceChecker.check() - on each published Source/Binaries couple. - """ - spps = distroseries.getSourcePackagePublishing( - status=PackagePublishingStatus.PUBLISHED, - pocket=pocket) - - log.debug('%s published sources' % spps.count()) - - for spp in spps: - spr = spp.sourcepackagerelease - checker = PubSourceChecker( - spr.name, spr.version, spp.component.name, spp.section.name, - spr.urgency.name) - - for bpp in spp.getPublishedBinaries(): - bpr = bpp.binarypackagerelease - checker.addBinary( - bpr.name, bpr.version, bpp.distroarchseries.architecturetag, - bpp.component.name, bpp.section.name, bpr.priority.name) - - checker.check() - - report = checker.renderReport() - - if report: - print report - -if __name__ == '__main__': - script = ArchiveOverrideCheckScript( - 'archive-override-check', config.archivepublisher.dbuser) - script.lock_and_run()
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : [email protected] Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
