Robert Collins has proposed merging lp:~lifeless/python-oops-datedir-repo/oops-prune into lp:python-oops-datedir-repo.
Requested reviews: Launchpad code reviewers (launchpad-reviewers) Related bugs: Bug #890875 in Python OOPS Date-dir repository: "no way to remove cruft from the repository." https://bugs.launchpad.net/python-oops-datedir-repo/+bug/890875 For more details, see: https://code.launchpad.net/~lifeless/python-oops-datedir-repo/oops-prune/+merge/82324 Implement incremental OOPS pruning, finishing the decoupling of Launchpad from OOPS storage. I'm pretty happy with this branch, the only untested code is the CLI glue, which is traditionally a PITA to test - and exacerbated here by the need to talk to Launchpad itself. For now, I've deliberately left it untested - all the complex logic is in the fully TDD and unit tested repository object. -- https://code.launchpad.net/~lifeless/python-oops-datedir-repo/oops-prune/+merge/82324 Your team Launchpad code reviewers is requested to review the proposed merge of lp:~lifeless/python-oops-datedir-repo/oops-prune into lp:python-oops-datedir-repo.
=== modified file 'NEWS' --- NEWS 2011-11-13 21:03:43 +0000 +++ NEWS 2011-11-15 21:50:28 +0000 @@ -6,6 +6,24 @@ NEXT ---- +* Repository has a simple generic config API. See the set_config and get_config + methods. (Robert Collins) + +* Repository can now answer 'what is the oldest date in the repository' which + is useful for incremental report pruning. See the oldest_date method. + (Robert Collins) + +* Repository can perform garbage collection of a date range if a list of + references to keep is supplied. See the prune_unreferenced method. + (Robert Collins) + +* There is a new script bin/prune which will prune reports from a repository + keeping only those referenced in a given Launchpad project or project group. + This adds a dependency on launchpadlib, which should be pypi installable + and is included in the Ubuntu default install - so should be a low barrier + for use. If this is an issue a future release can split this out into a + new addon package. (Robert Collins, #890875) + 0.0.11 ------ === added file 'oops_datedir_repo/prune.py' --- oops_datedir_repo/prune.py 1970-01-01 00:00:00 +0000 +++ oops_datedir_repo/prune.py 2011-11-15 21:50:28 +0000 @@ -0,0 +1,155 @@ +# +# Copyright (c) 2011, Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, version 3 only. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# GNU Lesser General Public License version 3 (see the file LICENSE). + +"""Delete OOPSes that are not referenced in the bugtracker. + +Currently only has support for the Launchpad bug tracker. +""" + +__metaclass__ = type + +import datetime +import logging +import optparse +from textwrap import dedent +import sys + +from launchpadlib.launchpad import Launchpad +from launchpadlib.uris import lookup_service_root +from pytz import utc + +import oops_datedir_repo + +__all__ = [ + 'main', + ] + + +class LaunchpadTracker: + """Abstracted bug tracker/forums etc - permits testing of main().""" + + def __init__(self, options): + self.lp = Launchpad.login_anonymously( + 'oops-prune', options.lpinstance, version='devel') + + def find_oops_references(self, start_time, end_time, project=None, + projectgroup=None): + projects = set([]) + if project is not None: + projects.add(project) + if projectgroup is not None: + [projects.add(lp_proj.name) + for lp_proj in self.lp.project_groups[projectgroup].projects] + result = set() + lp_projects = self.lp.projects + one_week = datetime.timedelta(weeks=1) + for project in projects: + lp_project = lp_projects[project] + current_start = start_time + while current_start < end_time: + current_end = current_start + one_week + if current_end > end_time: + current_end = end_time + logging.info( + "Querying OOPS references on %s from %s to %s", + project, current_start, current_end) + result.update(lp_project.findReferencedOOPS( + start_date=current_start, end_date=current_end)) + current_start = current_end + return result + + +def main(argv=None, tracker=LaunchpadTracker, logging=logging): + """Console script entry point.""" + if argv is None: + argv = sys.argv + usage = dedent("""\ + %prog [options] + + The following options must be supplied: + --repo + + And either + --project + or + --projectgroup + + e.g. + %prog --repo . --projectgroup launchpad-project + + Will process every member project of launchpad-project. + + When run this program will ask Launchpad for OOPS references made since + the last date it pruned up to, with an upper limit of one week from + today. It then looks in the repository for all oopses created during + that date range, and if they are not in the set returned by Launchpad, + deletes them. If the repository has never been pruned before, it will + pick the earliest datedir present in the repository as the start date. + """) + description = \ + "Delete OOPS reports that are not referenced in a bug tracker." + parser = optparse.OptionParser( + description=description, usage=usage) + parser.add_option('--project', + help="Launchpad project to find references in.") + parser.add_option('--projectgroup', + help="Launchpad project group to find references in.") + parser.add_option('--repo', help="Path to the repository to read from.") + parser.add_option( + '--lpinstance', help="Launchpad instance to use", default="production") + options, args = parser.parse_args(argv[1:]) + def needed(*optnames): + present = set() + for optname in optnames: + if getattr(options, optname, None) is not None: + present.add(optname) + if not present: + if len(optnames) == 1: + raise ValueError('Option "%s" must be supplied' % optname) + else: + raise ValueError( + 'One of options %s must be supplied' % (optnames,)) + elif len(present) != 1: + raise ValueError( + 'Only one of options %s can be supplied' % (optnames,)) + needed('repo') + needed('project', 'projectgroup') + logging.basicConfig( + filename='prune.log', filemode='w', level=logging.DEBUG) + repo = oops_datedir_repo.DateDirRepo(options.repo) + one_week = datetime.timedelta(weeks=1) + one_day = datetime.timedelta(days=1) + # max date to scan for + prune_until = datetime.datetime.now(utc) - one_week + # Get min date to scan for + try: + prune_from = repo.get_config('pruned-until') + except KeyError: + try: + oldest_oops = repo.oldest_date() + except ValueError: + logging.info("No OOPSes in repo, nothing to do.") + return 0 + prune_from = datetime.datetime.fromordinal(oldest_oops.toordinal()) + # get references from date range + finder = tracker(options) + references = finder.find_oops_references( + prune_from, prune_until, options.project, options.projectgroup) + # delete oops files on disk + repo.prune_unreferenced(prune_from, prune_until, references) + # stash most recent date + repo.set_config('pruned-until', prune_until) + return 0 === modified file 'oops_datedir_repo/repository.py' --- oops_datedir_repo/repository.py 2011-11-11 04:21:05 +0000 +++ oops_datedir_repo/repository.py 2011-11-15 21:50:28 +0000 @@ -23,11 +23,13 @@ ] import datetime +import errno from functools import partial from hashlib import md5 import os.path import stat +import bson from pytz import utc import serializer @@ -37,7 +39,19 @@ class DateDirRepo: - """Publish oopses to a date-dir repository.""" + """Publish oopses to a date-dir repository. + + A date-dir repository is a directory containing: + + * Zero or one directories called 'metadata'. If it exists this directory + contains any housekeeping material needed (such as a metadata.conf ini + file). + + * Zero or more directories named like YYYY-MM-DD, which contain zero or + more OOPS reports. OOPS file names can take various forms, but must not + end in .tmp - those are considered to be OOPS reports that are currently + being written. + """ def __init__(self, error_dir, instance_id=None, serializer=None, inherit_id=False, stash_path=False): @@ -70,12 +84,14 @@ ) else: self.log_namer = None - self.root = error_dir + self.root = error_dir if serializer is None: serializer = serializer_bson self.serializer = serializer self.inherit_id = inherit_id self.stash_path = stash_path + self.metadatadir = os.path.join(self.root, 'metadata') + self.config_path = os.path.join(self.metadatadir, 'config.bson') def publish(self, report, now=None): """Write the report to disk. @@ -148,13 +164,8 @@ two_days = datetime.timedelta(2) now = datetime.date.today() old = now - two_days - for dirname in os.listdir(self.root): - try: - y, m, d = dirname.split('-') - except ValueError: - # Not a datedir - continue - date = datetime.date(int(y),int(m),int(d)) + for dirname, (y,m,d) in self._datedirs(): + date = datetime.date(y, m, d) prune = date < old dirpath = os.path.join(self.root, dirname) files = os.listdir(dirpath) @@ -171,3 +182,123 @@ oopsid = publisher(report) if oopsid: os.unlink(candidate) + + def _datedirs(self): + """Yield each subdir which looks like a datedir.""" + for dirname in os.listdir(self.root): + try: + y, m, d = dirname.split('-') + y = int(y) + m = int(m) + d = int(d) + except ValueError: + # Not a datedir + continue + yield dirname, (y, m, d) + + def _read_config(self): + """Return the current config document from disk.""" + try: + with open(self.config_path, 'rb') as config_file: + return bson.loads(config_file.read()) + except IOError, e: + if e.errno != errno.ENOENT: + raise + return {} + + def get_config(self, key): + """Return a key from the repository config. + + :param key: A key to read from the config. + """ + return self._read_config()[key] + + def set_config(self, key, value): + """Set config option key to value. + + This is written to the bson document root/metadata/config.bson + + :param key: The key to set - anything that can be a key in a bson + document. + :param value: The value to set - anything that can be a value in a + bson document. + """ + config = self._read_config() + config[key] = value + try: + with open(self.config_path + '.tmp', 'wb') as config_file: + config_file.write(bson.dumps(config)) + except IOError, e: + if e.errno != errno.ENOENT: + raise + os.mkdir(self.metadatadir) + with open(self.config_path + '.tmp', 'wb') as config_file: + config_file.write(bson.dumps(config)) + os.rename(self.config_path + '.tmp', self.config_path) + + def oldest_date(self): + """Return the date of the oldest datedir in the repository. + + If pruning / resubmission is working this should also be the date of + the oldest oops in the repository. + """ + dirs = list(self._datedirs()) + if not dirs: + raise ValueError("No OOPSes in repository.") + return datetime.date(*sorted(dirs)[0][1]) + + def prune_unreferenced(self, start_time, stop_time, references): + """Delete OOPS reports filed between start_time and stop_time. + + A report is deleted if all of the following are true: + + * it is in a datedir covered by [start_time, stop_time] inclusive of + the end points. + + * It is not in the set references. + + * Its timestamp falls between start_time and stop_time inclusively or + it's timestamp is outside the datedir it is in or there is no + timestamp on the report. + + :param start_time: The lower bound to prune within. + :param stop_time: The upper bound to prune within. + :param references: An iterable of OOPS ids to keep. + """ + start_date = start_time.date() + stop_date = stop_time.date() + midnight = datetime.time(tzinfo=utc) + for dirname, (y,m,d) in self._datedirs(): + dirdate = datetime.date(y, m, d) + if dirdate < start_date or dirdate > stop_date: + continue + dirpath = os.path.join(self.root, dirname) + files = os.listdir(dirpath) + deleted = 0 + for candidate in map(partial(os.path.join, dirpath), files): + if candidate.endswith('.tmp'): + # Old half-written oops: just remove. + os.unlink(candidate) + deleted += 1 + continue + with file(candidate, 'rb') as report_file: + report = serializer.read(report_file) + report_time = report.get('time', None) + if (report_time is None or + report_time.date() < dirdate or + report_time.date() > dirdate): + # The report is oddly filed or missing a precise + # datestamp. Treat it like midnight on the day of the + # directory it was placed in - this is a lower bound on + # when it was actually created. + report_time = datetime.datetime.combine( + dirdate, midnight) + if (report_time >= start_time and + report_time <= stop_time and + report['id'] not in references): + # Unreferenced and prunable + os.unlink(candidate) + deleted += 1 + if deleted == len(files): + # Everything in the directory was deleted. + os.rmdir(dirpath) === modified file 'oops_datedir_repo/tests/test_repository.py' --- oops_datedir_repo/tests/test_repository.py 2011-11-11 04:21:05 +0000 +++ oops_datedir_repo/tests/test_repository.py 2011-11-15 21:50:28 +0000 @@ -26,6 +26,7 @@ import bson from pytz import utc import testtools +from testtools.matchers import raises from oops_datedir_repo import ( DateDirRepo, @@ -234,3 +235,103 @@ repo = DateDirRepo(self.useFixture(TempDir()).path) os.mkdir(repo.root + '/foo') repo.republish([].append) + + def test_republish_ignores_metadata_dir(self): + # The metadata directory is never pruned + repo = DateDirRepo(self.useFixture(TempDir()).path) + os.mkdir(repo.root + '/metadata') + repo.republish([].append) + self.assertTrue(os.path.exists(repo.root + '/metadata')) + + def test_get_config_value(self): + # Config values can be asked for from the repository. + repo = DateDirRepo(self.useFixture(TempDir()).path) + pruned = datetime.datetime(2006, 04, 01, 00, 30, 00, tzinfo=utc) + repo.set_config('pruned-until', pruned) + # Fresh instance, no memory tricks. + repo = DateDirRepo(repo.root) + self.assertEqual(pruned, repo.get_config('pruned-until')) + + def test_set_config_value(self): + # Config values are just keys in a bson document. + repo = DateDirRepo(self.useFixture(TempDir()).path) + pruned = datetime.datetime(2006, 04, 01, 00, 30, 00, tzinfo=utc) + repo.set_config('pruned-until', pruned) + with open(repo.root + '/metadata/config.bson', 'rb') as config_file: + from_bson = bson.loads(config_file.read()) + self.assertEqual({'pruned-until': pruned}, from_bson) + + def test_set_config_preserves_other_values(self): + # E.g. setting 'a' does not affect 'b' + repo = DateDirRepo(self.useFixture(TempDir()).path) + repo.set_config('b', 'b-value') + repo = DateDirRepo(repo.root) + repo.set_config('a', 'a-value') + with open(repo.root + '/metadata/config.bson', 'rb') as config_file: + from_bson = bson.loads(config_file.read()) + self.assertEqual({'a': 'a-value', 'b': 'b-value'}, from_bson) + + def test_oldest_date_no_contents(self): + repo = DateDirRepo(self.useFixture(TempDir()).path) + self.assertThat(lambda: repo.oldest_date(), + raises(ValueError("No OOPSes in repository."))) + + def test_oldest_date_is_oldest(self): + repo = DateDirRepo(self.useFixture(TempDir()).path) + os.mkdir(repo.root + '/2006-04-12') + os.mkdir(repo.root + '/2006-04-13') + self.assertEqual(datetime.date(2006, 4, 12), repo.oldest_date()) + + def test_prune_unreferenced_no_oopses(self): + # This shouldn't crash. + repo = DateDirRepo(self.useFixture(TempDir()).path, inherit_id=True) + now = datetime.datetime(2006, 04, 01, 00, 30, 00, tzinfo=utc) + old = now - datetime.timedelta(weeks=1) + repo.prune_unreferenced(old, now, []) + + def test_prune_unreferenced_no_references(self): + # When there are no references, everything specified is zerged. + repo = DateDirRepo(self.useFixture(TempDir()).path, inherit_id=True) + now = datetime.datetime(2006, 04, 01, 00, 30, 00, tzinfo=utc) + old = now - datetime.timedelta(weeks=1) + report = {'time': now - datetime.timedelta(hours=5)} + repo.publish(report, report['time']) + repo.prune_unreferenced(old, now, []) + self.assertThat(lambda: repo.oldest_date(), raises(ValueError)) + + def test_prune_unreferenced_outside_dates_kept(self): + # Pruning only affects stuff in the datedirs selected by the dates. + repo = DateDirRepo( + self.useFixture(TempDir()).path, inherit_id=True, stash_path=True) + now = datetime.datetime(2006, 04, 01, 00, 30, 00, tzinfo=utc) + old = now - datetime.timedelta(weeks=1) + before = {'time': old - datetime.timedelta(minutes=1)} + after = {'time': now + datetime.timedelta(minutes=1)} + repo.publish(before, before['time']) + repo.publish(after, after['time']) + repo.prune_unreferenced(old, now, []) + self.assertTrue(os.path.isfile(before['datedir_repo_filepath'])) + self.assertTrue(os.path.isfile(after['datedir_repo_filepath'])) + + def test_prune_referenced_inside_dates_kept(self): + repo = DateDirRepo( + self.useFixture(TempDir()).path, inherit_id=True, stash_path=True) + now = datetime.datetime(2006, 04, 01, 00, 30, 00, tzinfo=utc) + old = now - datetime.timedelta(weeks=1) + report = {'id': 'foo', 'time': now - datetime.timedelta(minutes=1)} + repo.publish(report, report['time']) + repo.prune_unreferenced(old, now, ['foo']) + self.assertTrue(os.path.isfile(report['datedir_repo_filepath'])) + + def test_prune_report_midnight_gets_invalid_timed_reports(self): + # If a report has a wonky or missing time, pruning treats it as being + # timed on midnight of the datedir day it is on. + repo = DateDirRepo(self.useFixture(TempDir()).path, stash_path=True) + now = datetime.datetime(2006, 04, 01, 00, 01, 00, tzinfo=utc) + old = now - datetime.timedelta(minutes=2) + badtime = {'time': now - datetime.timedelta(weeks=2)} + missingtime = {} + repo.publish(badtime, now) + repo.publish(missingtime, now) + repo.prune_unreferenced(old, now, []) + self.assertThat(lambda: repo.oldest_date(), raises(ValueError)) === modified file 'setup.py' --- setup.py 2011-11-13 21:07:43 +0000 +++ setup.py 2011-11-15 21:50:28 +0000 @@ -40,6 +40,7 @@ install_requires = [ 'bson', 'iso8601', + 'launchpadlib', # Needed for pruning - perhaps should be optional. 'oops', 'pytz', ], @@ -49,4 +50,8 @@ 'testtools', ] ), + entry_points=dict( + console_scripts=[ # `console_scripts` is a magic name to setuptools + 'prune = oops_datedir_repo.prune:main', + ]), ) === modified file 'versions.cfg' --- versions.cfg 2011-10-07 05:52:09 +0000 +++ versions.cfg 2011-11-15 21:50:28 +0000 @@ -3,13 +3,25 @@ [versions] bson = 0.3.2 +elementtree = 1.2.6-20050316 fixtures = 0.3.6 +httplib2 = 0.6.0 iso8601 = 0.1.4 +keyring = 0.6.2 +launchpadlib = 1.9.9 +lazr.authentication = 0.1.1 +lazr.restfulclient = 0.12.0 +lazr.uri = 1.0.2 +oauth = 1.0.0 oops = 0.0.8 pytz = 2010o setuptools = 0.6c11 +simplejson = 2.1.3 testtools = 0.9.11 +wadllib = 1.2.0 +wsgi-intercept = 0.4 zc.recipe.egg = 1.3.2 z3c.recipe.filetemplate = 2.1.0 z3c.recipe.scripts = 1.0.1 zc.buildout = 1.5.1 +zope.interface = 3.8.0
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : [email protected] Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
