William Grant has proposed merging lp:~wgrant/launchpad/xref-model into lp:launchpad with lp:~wgrant/launchpad/xref-db as a prerequisite.
Commit message: Add lp.services.xref for generic cross-references between artifacts. Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~wgrant/launchpad/xref-model/+merge/272588 Add lp.services.xref for generic cross-references between artifacts. Schema is at <https://code.launchpad.net/~wgrant/launchpad/xref-db/+merge/272587>. It's likely that we'll end up with wrappers that automatically resolve known objects to/from their tuple representations, but for now the by-ID API is surprisingly unonerous. -- Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/xref-model into lp:launchpad.
=== modified file 'lib/lp/services/configure.zcml' --- lib/lp/services/configure.zcml 2015-05-22 07:12:35 +0000 +++ lib/lp/services/configure.zcml 2015-09-28 12:44:55 +0000 @@ -34,4 +34,5 @@ <include package=".webhooks" /> <include package=".webservice" /> <include package=".worlddata" /> + <include package=".xref" /> </configure> === added directory 'lib/lp/services/xref' === added file 'lib/lp/services/xref/__init__.py' --- lib/lp/services/xref/__init__.py 1970-01-01 00:00:00 +0000 +++ lib/lp/services/xref/__init__.py 2015-09-28 12:44:55 +0000 @@ -0,0 +1,13 @@ +# Copyright 2015 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""Generic cross references between artifacts. + +Provides infrastructure for generic information references between +artifacts, easing weak coupling of apps. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [] === added file 'lib/lp/services/xref/configure.zcml' --- lib/lp/services/xref/configure.zcml 1970-01-01 00:00:00 +0000 +++ lib/lp/services/xref/configure.zcml 2015-09-28 12:44:55 +0000 @@ -0,0 +1,13 @@ +<!-- Copyright 2015 Canonical Ltd. This software is licensed under the + GNU Affero General Public License version 3 (see the file LICENSE). +--> + +<configure xmlns="http://namespaces.zope.org/zope";> + + <securedutility + class="lp.services.xref.model.XRefSet" + provides="lp.services.xref.interfaces.IXRefSet"> + <allow interface="lp.services.xref.interfaces.IXRefSet"/> + </securedutility> + +</configure> === added file 'lib/lp/services/xref/interfaces.py' --- lib/lp/services/xref/interfaces.py 1970-01-01 00:00:00 +0000 +++ lib/lp/services/xref/interfaces.py 2015-09-28 12:44:55 +0000 @@ -0,0 +1,62 @@ +# Copyright 2015 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'IXRefSet', + ] + +from zope.interface import Interface + + +class IXRefSet(Interface): + """Manager of cross-references between objects. + + Each participant in an xref has an "object ID": a tuple of + (str type, str id). + + All xrefs are currently between local objects, so links always exist + in both directions, but this can't be assumed to hold in future. + """ + + def create(xrefs): + """Create cross-references. + + Back-links are automatically created. + + :param xrefs: A dict of + {from_object_id: {to_object_id: + {'creator': `IPerson`, 'metadata': value}}}. + The creator and metadata keys are optional. + """ + + def findFromMany(object_ids, types=None): + """Find all cross-references from multiple objects. + + :param object_ids: A collection of object IDs. + :param types: An optional collection of the types to include. + :return: A dict of + {from_object_id: {to_object_id: + {'creator': `IPerson`, 'metadata': value}}}. + The creator and metadata keys are optional. + """ + + def delete(xrefs): + """Delete cross-references. + + Back-links are automatically deleted. + + :param xrefs: A dict of {from_object_id: [to_object_id]}. + """ + + def findFrom(object_id, types=None): + """Find all cross-references from an object. + + :param object_id: An object ID. + :param types: An optional collection of the types to include. + :return: A dict of + {to_object_id: {'creator': `IPerson`, 'metadata': value}}. + The creator and metadata keys are optional. + """ === added file 'lib/lp/services/xref/model.py' --- lib/lp/services/xref/model.py 1970-01-01 00:00:00 +0000 +++ lib/lp/services/xref/model.py 2015-09-28 12:44:55 +0000 @@ -0,0 +1,131 @@ +# Copyright 2015 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + "XRefSet", + ] + +import pytz +from storm.expr import ( + And, + Or, + ) +from storm.properties import ( + DateTime, + Int, + JSON, + Unicode, + ) +from storm.references import Reference +from zope.interface import implementer + +from lp.services.database import bulk +from lp.services.database.constants import UTC_NOW +from lp.services.database.interfaces import IStore +from lp.services.database.stormbase import StormBase +from lp.services.xref.interfaces import IXRefSet + + +class XRef(StormBase): + """Cross-reference between two objects. + + For references to local objects (there is currently no other kind), + another reference in the opposite direction exists. + + The to_id_int and from_id_int columns exist for efficient SQL joins. + They are set automatically when the ID looks like an integer. + """ + + __storm_table__ = 'XRef' + __storm_primary__ = "to_type", "to_id", "from_type", "from_id" + + to_type = Unicode(allow_none=False) + to_id = Unicode(allow_none=False) + to_id_int = Int() # For efficient joins. + from_type = Unicode(allow_none=False) + from_id = Unicode(allow_none=False) + from_id_int = Int() # For efficient joins. + creator_id = Int(name="creator") + creator = Reference(creator_id, "Person.id") + date_created = DateTime(name='date_created', tzinfo=pytz.UTC) + metadata = JSON() + + +def _int_or_none(s): + if s.isdigit(): + return int(s) + else: + return None + + +@implementer(IXRefSet) +class XRefSet: + + def create(self, xrefs): + # All references are currently to local objects, so add + # backlinks as well to keep queries in both directions quick. + # The *_id_int columns are also set if the ID looks like an int. + rows = [] + for from_, tos in xrefs.items(): + for to, props in tos.items(): + rows.append(( + from_[0], from_[1], _int_or_none(from_[1]), + to[0], to[1], _int_or_none(to[1]), + props.get('creator'), props.get('date_created', UTC_NOW), + props.get('metadata'))) + rows.append(( + to[0], to[1], _int_or_none(to[1]), + from_[0], from_[1], _int_or_none(from_[1]), + props.get('creator'), props.get('date_created', UTC_NOW), + props.get('metadata'))) + bulk.create( + (XRef.from_type, XRef.from_id, XRef.from_id_int, + XRef.to_type, XRef.to_id, XRef.to_id_int, + XRef.creator, XRef.date_created, XRef.metadata), rows) + + def delete(self, xrefs): + # Delete both directions. + pairs = [] + for from_, tos in xrefs.items(): + for to in tos: + pairs.extend([(from_, to), (to, from_)]) + + IStore(XRef).find( + XRef, + Or(*[ + And(XRef.from_type == pair[0][0], + XRef.from_id == pair[0][1], + XRef.to_type == pair[1][0], + XRef.to_id == pair[1][1]) + for pair in pairs]) + ).remove() + + def findFromMany(self, object_ids, types=None): + from lp.registry.model.person import Person + + object_ids = list(object_ids) + if not object_ids: + return {} + + store = IStore(XRef) + rows = list(store.using(XRef).find( + (XRef.from_type, XRef.from_id, XRef.to_type, XRef.to_id, + XRef.creator_id, XRef.date_created, XRef.metadata), + Or(*[ + And(XRef.from_type == id[0], XRef.from_id == id[1]) + for id in object_ids]), + XRef.to_type.is_in(types) if types is not None else True)) + bulk.load(Person, [row[4] for row in rows]) + result = {} + for row in rows: + result.setdefault((row[0], row[1]), {})[(row[2], row[3])] = { + "creator": store.get(Person, row[4]) if row[4] else None, + "date_created": row[5], + "metadata": row[6]} + return result + + def findFrom(self, object_id, types=None): + return self.findFromMany([object_id], types=types).get(object_id, {}) === added directory 'lib/lp/services/xref/tests' === added file 'lib/lp/services/xref/tests/__init__.py' --- lib/lp/services/xref/tests/__init__.py 1970-01-01 00:00:00 +0000 +++ lib/lp/services/xref/tests/__init__.py 2015-09-28 12:44:55 +0000 @@ -0,0 +1,7 @@ +# Copyright 2015 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [] === added file 'lib/lp/services/xref/tests/test_model.py' --- lib/lp/services/xref/tests/test_model.py 1970-01-01 00:00:00 +0000 +++ lib/lp/services/xref/tests/test_model.py 2015-09-28 12:44:55 +0000 @@ -0,0 +1,159 @@ +# Copyright 2015 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type + +import datetime + +import pytz +from testtools.matchers import Equals +from zope.component import getUtility + +from lp.services.database.interfaces import IStore +from lp.services.xref.interfaces import IXRefSet +from lp.services.xref.model import XRef +from lp.testing import ( + StormStatementRecorder, + TestCaseWithFactory, + ) +from lp.testing.layers import DatabaseFunctionalLayer +from lp.testing.matchers import HasQueryCount + + +class TestXRefSet(TestCaseWithFactory): + + layer = DatabaseFunctionalLayer + + def test_create_sets_date_created(self): + # date_created defaults to now, but can be overridden. + old = datetime.datetime.strptime('2005-01-01', '%Y-%m-%d').replace( + tzinfo=pytz.UTC) + now = IStore(XRef).execute( + "SELECT CURRENT_TIMESTAMP AT TIME ZONE 'UTC'" + ).get_one()[0].replace(tzinfo=pytz.UTC) + getUtility(IXRefSet).create({ + ('a', '1'): {('b', 'foo'): {}}, + ('a', '2'): {('b', 'bar'): {'date_created': old}}}) + rows = IStore(XRef).find( + (XRef.from_id, XRef.to_id, XRef.date_created), + XRef.from_type == 'a') + self.assertContentEqual( + [('1', 'foo', now), ('2', 'bar', old)], rows) + + def test_create_sets_int_columns(self): + # The string ID columns have integers equivalents for quick and + # easy joins to integer PKs. They're set automatically when the + # string ID looks like an integer. + getUtility(IXRefSet).create({ + ('a', '1234'): {('b', 'foo'): {}, ('b', '2468'): {}}, + ('a', '12ab'): {('b', '1234'): {}, ('b', 'foo'): {}}}) + rows = IStore(XRef).find( + (XRef.from_type, XRef.from_id, XRef.from_id_int, XRef.to_type, + XRef.to_id, XRef.to_id_int), + XRef.from_type == 'a') + self.assertContentEqual( + [('a', '1234', 1234, 'b', 'foo', None), + ('a', '1234', 1234, 'b', '2468', 2468), + ('a', '12ab', None, 'b', '1234', 1234), + ('a', '12ab', None, 'b', 'foo', None) + ], + rows) + + def test_findFrom(self): + creator = self.factory.makePerson() + now = IStore(XRef).execute( + "SELECT CURRENT_TIMESTAMP AT TIME ZONE 'UTC'" + ).get_one()[0].replace(tzinfo=pytz.UTC) + getUtility(IXRefSet).create({ + ('a', 'bar'): { + ('b', 'foo'): {'creator': creator, 'metadata': {'test': 1}}}, + ('b', 'foo'): { + ('a', 'baz'): {'creator': creator, 'metadata': {'test': 2}}}, + }) + + with StormStatementRecorder() as recorder: + bar_refs = getUtility(IXRefSet).findFrom(('a', 'bar')) + self.assertThat(recorder, HasQueryCount(Equals(2))) + self.assertEqual( + {('b', 'foo'): { + 'creator': creator, 'date_created': now, + 'metadata': {'test': 1}}}, + bar_refs) + + with StormStatementRecorder() as recorder: + foo_refs = getUtility(IXRefSet).findFrom(('b', 'foo')) + self.assertThat(recorder, HasQueryCount(Equals(2))) + self.assertEqual( + {('a', 'bar'): { + 'creator': creator, 'date_created': now, + 'metadata': {'test': 1}}, + ('a', 'baz'): { + 'creator': creator, 'date_created': now, + 'metadata': {'test': 2}}}, + foo_refs) + + with StormStatementRecorder() as recorder: + bar_refs = getUtility(IXRefSet).findFrom(('a', 'baz')) + self.assertThat(recorder, HasQueryCount(Equals(2))) + self.assertEqual( + {('b', 'foo'): { + 'creator': creator, 'date_created': now, + 'metadata': {'test': 2}}}, + bar_refs) + + with StormStatementRecorder() as recorder: + bar_baz_refs = getUtility(IXRefSet).findFromMany( + [('a', 'bar'), ('a', 'baz')]) + self.assertThat(recorder, HasQueryCount(Equals(2))) + self.assertEqual( + {('a', 'bar'): { + ('b', 'foo'): { + 'creator': creator, 'date_created': now, + 'metadata': {'test': 1}}}, + ('a', 'baz'): { + ('b', 'foo'): { + 'creator': creator, 'date_created': now, + 'metadata': {'test': 2}}}}, + bar_baz_refs) + + def test_findFrom_types(self): + # findFrom can look for only particular types of related + # objects. + getUtility(IXRefSet).create({ + ('a', '1'): {('a', '2'): {}, ('b', '3'): {}}, + ('b', '4'): {('a', '5'): {}, ('c', '6'): {}}, + }) + self.assertContentEqual( + [('a', '2')], + getUtility(IXRefSet).findFrom(('a', '1'), types=['a', 'c']).keys()) + self.assertContentEqual( + [('a', '5'), ('c', '6')], + getUtility(IXRefSet).findFrom(('b', '4'), types=['a', 'c']).keys()) + + # Asking for no types or types that don't exist finds nothing. + self.assertContentEqual( + [], + getUtility(IXRefSet).findFrom(('b', '4'), types=[]).keys()) + self.assertContentEqual( + [], + getUtility(IXRefSet).findFrom(('b', '4'), types=['d']).keys()) + + def test_findFromMany_none(self): + self.assertEqual({}, getUtility(IXRefSet).findFromMany([])) + + def test_delete(self): + getUtility(IXRefSet).create({ + ('a', 'bar'): {('b', 'foo'): {}}, + ('b', 'foo'): {('a', 'baz'): {}}, + }) + self.assertContentEqual( + [('a', 'bar'), ('a', 'baz')], + getUtility(IXRefSet).findFrom(('b', 'foo')).keys()) + with StormStatementRecorder() as recorder: + getUtility(IXRefSet).delete({('b', 'foo'): [('a', 'bar')]}) + self.assertThat(recorder, HasQueryCount(Equals(1))) + self.assertEqual( + [('a', 'baz')], + getUtility(IXRefSet).findFrom(('b', 'foo')).keys()) === modified file 'lib/lp/testing/tests/test_standard_test_template.py' --- lib/lp/testing/tests/test_standard_test_template.py 2015-01-30 10:13:51 +0000 +++ lib/lp/testing/tests/test_standard_test_template.py 2015-09-28 12:44:55 +0000 @@ -3,6 +3,8 @@ """XXX: Module docstring goes here.""" +from __future__ import absolute_import, print_function, unicode_literals + __metaclass__ = type # or TestCaseWithFactory === modified file 'lib/lp/testing/tests/test_standard_yuixhr_test_template.py' --- lib/lp/testing/tests/test_standard_yuixhr_test_template.py 2015-01-30 10:13:51 +0000 +++ lib/lp/testing/tests/test_standard_yuixhr_test_template.py 2015-09-28 12:44:55 +0000 @@ -4,6 +4,8 @@ """{Describe your test suite here}. """ +from __future__ import absolute_import, print_function, unicode_literals + __metaclass__ = type __all__ = [] === modified file 'standard_template.py' --- standard_template.py 2015-01-30 10:13:51 +0000 +++ standard_template.py 2015-09-28 12:44:55 +0000 @@ -3,5 +3,7 @@ """XXX: Module docstring goes here.""" +from __future__ import absolute_import, print_function, unicode_literals + __metaclass__ = type __all__ = []
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : [email protected] Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
