This is an automated email from the ASF dual-hosted git repository. kentontaylor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/allura.git
commit c65ce7a46791be9af71e6de9f7f204ea5a7be2c9 Author: Dave Brondsema <[email protected]> AuthorDate: Wed Sep 29 18:05:56 2021 -0400 Convert document/collection mapping to be like other MappedClass types --- Allura/allura/model/__init__.py | 4 +- Allura/allura/model/artifact.py | 4 +- Allura/allura/model/auth.py | 39 +++++------ Allura/allura/model/discuss.py | 2 +- Allura/allura/model/index.py | 87 ++++++++++++------------ Allura/allura/model/notification.py | 2 +- Allura/allura/model/repo.py | 4 +- Allura/allura/model/repo_refresh.py | 37 ++++++---- Allura/allura/model/repository.py | 127 +++++++++++++++++++++++------------ Allura/allura/scripts/refreshrepo.py | 4 -- ForgeGit/forgegit/model/git_repo.py | 14 ++-- 11 files changed, 190 insertions(+), 134 deletions(-) diff --git a/Allura/allura/model/__init__.py b/Allura/allura/model/__init__.py index 9572e68..75d7fa6 100644 --- a/Allura/allura/model/__init__.py +++ b/Allura/allura/model/__init__.py @@ -29,7 +29,7 @@ from .artifact import VotableArtifact from .discuss import Discussion, Thread, PostHistory, Post, DiscussionAttachment from .attachments import BaseAttachment from .auth import AuthGlobals, User, ProjectRole, EmailAddress -from .auth import AuditLog, audit_log, AlluraUserProperty, UserLoginDetails +from .auth import AuditLog, AlluraUserProperty, UserLoginDetails from .filesystem import File from .notification import Notification, Mailbox, SiteNotification from .repository import Repository, RepositoryImplementation @@ -58,7 +58,7 @@ __all__ = [ 'ArtifactReference', 'Shortlink', 'Artifact', 'MovedArtifact', 'Message', 'VersionedArtifact', 'Snapshot', 'Feed', 'AwardFile', 'Award', 'AwardGrant', 'VotableArtifact', 'Discussion', 'Thread', 'PostHistory', 'Post', 'DiscussionAttachment', 'BaseAttachment', 'AuthGlobals', 'User', 'ProjectRole', 'EmailAddress', - 'AuditLog', 'audit_log', 'AlluraUserProperty', 'File', 'Notification', 'Mailbox', 'Repository', + 'AuditLog', 'AlluraUserProperty', 'File', 'Notification', 'Mailbox', 'Repository', 'RepositoryImplementation', 'MergeRequest', 'GitLikeTree', 'Stats', 'OAuthToken', 'OAuthConsumerToken', 'OAuthRequestToken', 'OAuthAccessToken', 'MonQTask', 'Webhook', 'ACE', 'ACL', 'EVERYONE', 'ALL_PERMISSIONS', 'DENY_ALL', 'MarkdownCache', 'main_doc_session', 'main_orm_session', 'project_doc_session', 'project_orm_session', diff --git a/Allura/allura/model/artifact.py b/Allura/allura/model/artifact.py index fe5992f..25b6c4b 100644 --- a/Allura/allura/model/artifact.py +++ b/Allura/allura/model/artifact.py @@ -900,7 +900,7 @@ class Feed(MappedClass): query: 'Query[Feed]' _id = FieldProperty(S.ObjectId) - ref_id = ForeignIdProperty('ArtifactReference') + ref_id: str = ForeignIdProperty('ArtifactReference') neighborhood_id = ForeignIdProperty('Neighborhood') project_id = ForeignIdProperty('Project') app_config_id = ForeignIdProperty('AppConfig') @@ -1175,7 +1175,7 @@ class SpamCheckResult(MappedClass): query: 'Query[SpamCheckResult]' _id = FieldProperty(S.ObjectId) - ref_id = ForeignIdProperty('ArtifactReference') + ref_id: str = ForeignIdProperty('ArtifactReference') ref = RelationProperty('ArtifactReference', via='ref_id') project_id = ForeignIdProperty('Project') project = RelationProperty('Project', via='project_id') diff --git a/Allura/allura/model/auth.py b/Allura/allura/model/auth.py index add60ac..fb17054 100644 --- a/Allura/allura/model/auth.py +++ b/Allura/allura/model/auth.py @@ -38,7 +38,7 @@ from tg import config from tg import tmpl_context as c, app_globals as g from tg import request from ming import schema as S -from ming import Field, collection +from ming import Field from ming.orm import session, state from ming.orm import FieldProperty, RelationProperty, ForeignIdProperty from ming.orm.declarative import MappedClass @@ -51,7 +51,7 @@ from allura.lib import plugin from allura.lib import utils from allura.lib.decorators import memoize from allura.lib.search import SearchIndexable -from .session import main_orm_session, main_doc_session, main_explicitflush_orm_session +from .session import main_orm_session, main_explicitflush_orm_session from .session import project_orm_session from .timeline import ActivityNode, ActivityObject @@ -984,18 +984,26 @@ class ProjectRole(MappedClass): user_id={'$ne': None}, roles=self._id)).all() -audit_log = collection( - str('audit_log'), main_doc_session, - Field('_id', S.ObjectId()), - Field('project_id', S.ObjectId, if_missing=None, - index=True), # main view of audit log queries by project_id - Field('user_id', S.ObjectId, if_missing=None, index=True), - Field('timestamp', datetime, if_missing=datetime.utcnow), - Field('url', str), - Field('message', str)) +class AuditLog(MappedClass): + class __mongometa__: + session = main_orm_session + name = str('audit_log') + indexes = [ + 'project_id', + 'user_id', + ] + query: 'Query[AuditLog]' + + _id = FieldProperty(S.ObjectId) + project_id = ForeignIdProperty('Project', if_missing=None) + project = RelationProperty('Project') + user_id: ObjectId = AlluraUserProperty() + user = RelationProperty('User') + timestamp = FieldProperty(datetime, if_missing=datetime.utcnow) + url = FieldProperty(str) + message = FieldProperty(str) -class AuditLog(object): @property def timestamp_str(self): return self.timestamp.strftime('%Y-%m-%d %H:%M:%S') @@ -1056,13 +1064,6 @@ class AuditLog(object): return cls.log_user(message, *args, **kwargs) -main_orm_session.mapper(AuditLog, audit_log, properties=dict( - project_id=ForeignIdProperty('Project'), - project=RelationProperty('Project'), - user_id=AlluraUserProperty(), - user=RelationProperty('User'))) - - class UserLoginDetails(MappedClass): """ Store unique entries for users' previous login details. diff --git a/Allura/allura/model/discuss.py b/Allura/allura/model/discuss.py index cfcc1e1..fd1b423 100644 --- a/Allura/allura/model/discuss.py +++ b/Allura/allura/model/discuss.py @@ -161,7 +161,7 @@ class Thread(Artifact, ActivityObject): _id = FieldProperty(str, if_missing=lambda: h.nonce(10)) discussion_id = ForeignIdProperty(Discussion) - ref_id = ForeignIdProperty('ArtifactReference') + ref_id: str = ForeignIdProperty('ArtifactReference') subject = FieldProperty(str, if_missing='') num_replies = FieldProperty(int, if_missing=0) num_views = FieldProperty(int, if_missing=0) diff --git a/Allura/allura/model/index.py b/Allura/allura/model/index.py index 790667a..27fc9dd 100644 --- a/Allura/allura/model/index.py +++ b/Allura/allura/model/index.py @@ -20,6 +20,8 @@ from __future__ import absolute_import import re import logging from itertools import groupby + +from ming.odm.property import FieldProperty from six.moves.cPickle import dumps, loads from collections import defaultdict from six.moves.urllib.parse import unquote @@ -28,51 +30,39 @@ import bson import pymongo from tg import tmpl_context as c -from ming import collection, Field, Index from ming import schema as S from ming.utils import LazyProperty -from ming.orm import session, mapper -from ming.orm import ForeignIdProperty, RelationProperty +from ming.odm import session, MappedClass +from ming.odm import ForeignIdProperty, RelationProperty from allura.lib import helpers as h -from .session import main_doc_session, main_orm_session +from .session import main_orm_session from .project import Project import six log = logging.getLogger(__name__) -# Collection definitions -ArtifactReferenceDoc = collection( - str('artifact_reference'), main_doc_session, - Field('_id', str), - Field('artifact_reference', dict( - cls=S.Binary(), - project_id=S.ObjectId(), - app_config_id=S.ObjectId(), - artifact_id=S.Anything(if_missing=None))), - Field('references', [str], index=True), - Index('artifact_reference.project_id'), # used in ReindexCommand -) -ShortlinkDoc = collection( - str('shortlink'), main_doc_session, - Field('_id', S.ObjectId()), - # index needed for from_artifact() and index_tasks.py:del_artifacts - Field('ref_id', str, index=True), - Field('project_id', S.ObjectId()), - Field('app_config_id', S.ObjectId()), - Field('link', str), - Field('url', str), - # used by from_links() More helpful to have project_id first, for other - # queries - Index('project_id', 'link'), -) +class ArtifactReference(MappedClass): + class __mongometa__: + session = main_orm_session + name = str('artifact_reference') + indexes = [ + 'references', + 'artifact_reference.project_id', # used in ReindexCommand + ] -# Class definitions + query: 'Query[ArtifactReference]' - -class ArtifactReference(object): + _id = FieldProperty(str) + artifact_reference = FieldProperty(dict( + cls=S.Binary(), + project_id=S.ObjectId(), + app_config_id=S.ObjectId(), + artifact_id=S.Anything(if_missing=None), + )) + references = FieldProperty([str]) @classmethod def from_artifact(cls, artifact): @@ -107,10 +97,29 @@ class ArtifactReference(object): self._id, aref) -class Shortlink(object): - +class Shortlink(MappedClass): '''Collection mapping shorthand_ids for artifacts to ArtifactReferences''' + class __mongometa__: + session = main_orm_session + name = str('shortlink') + indexes = [ + 'ref_id', # for from_artifact() and index_tasks.py:del_artifacts + 'project_id', 'link', # used by from_links() More helpful to have project_id first, for other queries + ] + + query: 'Query[Shortlink]' + + _id = FieldProperty(S.ObjectId()) + ref_id: str = ForeignIdProperty(ArtifactReference) + ref = RelationProperty(ArtifactReference) + project_id = ForeignIdProperty('Project') + project = RelationProperty('Project') + app_config_id = ForeignIdProperty('AppConfig') + app_config = RelationProperty('AppConfig') + link = FieldProperty(str) + url = FieldProperty(str) + # Regexes used to find shortlinks _core_re = r'''(\[ (?:(?P<project_id>.*?):)? # optional project ID @@ -262,13 +271,3 @@ class Shortlink(object): artifact=parts[0]) else: return None - -# Mapper definitions -mapper(ArtifactReference, ArtifactReferenceDoc, main_orm_session) -mapper(Shortlink, ShortlinkDoc, main_orm_session, properties=dict( - ref_id=ForeignIdProperty(ArtifactReference), - project_id=ForeignIdProperty('Project'), - app_config_id=ForeignIdProperty('AppConfig'), - project=RelationProperty('Project'), - app_config=RelationProperty('AppConfig'), - ref=RelationProperty(ArtifactReference))) diff --git a/Allura/allura/model/notification.py b/Allura/allura/model/notification.py index 97314d4..23369bd 100644 --- a/Allura/allura/model/notification.py +++ b/Allura/allura/model/notification.py @@ -93,7 +93,7 @@ class Notification(MappedClass): app_config_id = ForeignIdProperty( 'AppConfig', if_missing=lambda: c.app.config._id) tool_name = FieldProperty(str, if_missing=lambda: c.app.config.tool_name) - ref_id = ForeignIdProperty('ArtifactReference') + ref_id: str = ForeignIdProperty('ArtifactReference') topic = FieldProperty(str) # Notification Content diff --git a/Allura/allura/model/repo.py b/Allura/allura/model/repo.py index 298fcaf..f4deba1 100644 --- a/Allura/allura/model/repo.py +++ b/Allura/allura/model/repo.py @@ -22,11 +22,11 @@ from __future__ import unicode_literals from .repository import SUser, SObjType from .repository import QSIZE, README_RE, VIEWABLE_EXTENSIONS, PYPELINE_EXTENSIONS, DIFF_SIMILARITY_THRESHOLD -from .repository import CommitDoc, TreeDoc, LastCommitDoc +from .repository import CommitDoc, LastCommitDoc from .repository import RepoObject, Commit, Tree, Blob, LastCommit from .repository import ModelCache __all__ = [ 'SUser', 'SObjType', 'QSIZE', 'README_RE', 'VIEWABLE_EXTENSIONS', 'PYPELINE_EXTENSIONS', - 'DIFF_SIMILARITY_THRESHOLD', 'CommitDoc', 'TreeDoc', 'LastCommitDoc', 'RepoObject', + 'DIFF_SIMILARITY_THRESHOLD', 'CommitDoc', 'LastCommitDoc', 'RepoObject', 'Commit', 'Tree', 'Blob', 'LastCommit', 'ModelCache'] diff --git a/Allura/allura/model/repo_refresh.py b/Allura/allura/model/repo_refresh.py index fa4a9d7..5b36379 100644 --- a/Allura/allura/model/repo_refresh.py +++ b/Allura/allura/model/repo_refresh.py @@ -31,12 +31,12 @@ from tg import tmpl_context as c, app_globals as g from ming.base import Object from ming.orm import mapper, session, ThreadLocalORMSession +from ming.odm.base import ObjectState, state from allura.lib import utils from allura.lib import helpers as h -from allura.model.repository import CommitDoc -from allura.model.repository import Commit, Tree, LastCommit, ModelCache -from allura.model.index import ArtifactReferenceDoc, ShortlinkDoc +from allura.model.repository import Commit, CommitDoc +from allura.model.index import ArtifactReference, Shortlink from allura.model.auth import User from allura.model.timeline import TransientActor import six @@ -134,33 +134,46 @@ def refresh_commit_repos(all_commit_ids, repo): oid = ci._id ci.repo_ids.append(repo._id) index_id = 'allura.model.repository.Commit#' + oid - ref = ArtifactReferenceDoc(dict( + # TODO: use ArtifactReference.from_artifact? + # print(f'ref {index_id}') + # if '5c472' in index_id: 0/0 + ref = ArtifactReference( + # ref = ArtifactReferenceDoc(dict( _id=index_id, artifact_reference=dict( cls=bson.Binary(dumps(Commit, protocol=2)), project_id=repo.app.config.project_id, app_config_id=repo.app.config._id, artifact_id=oid), - references=[])) - link0 = ShortlinkDoc(dict( + references=[]) + # ) + # TODO: use Shortlink.from_artifact? + link0 = Shortlink( _id=bson.ObjectId(), ref_id=index_id, project_id=repo.app.config.project_id, app_config_id=repo.app.config._id, link=repo.shorthand_for_commit(oid)[1:-1], - url=repo.url_for_commit(oid))) + url=repo.url_for_commit(oid)) # Always create a link for the full commit ID - link1 = ShortlinkDoc(dict( + link1 = Shortlink( _id=bson.ObjectId(), ref_id=index_id, project_id=repo.app.config.project_id, app_config_id=repo.app.config._id, link=oid, - url=repo.url_for_commit(oid))) + url=repo.url_for_commit(oid)) ci.m.save(validate=False) - ref.m.save(validate=False) - link0.m.save(validate=False) - link1.m.save(validate=False) + # set to 'dirty' to force save() to be used instead of insert() (which errors if doc exists in db already) + state(ref).status = ObjectState.dirty + session(ref).flush(ref) + session(ref).expunge(ref) + state(link0).status = ObjectState.dirty + session(link0).flush(link0) + session(link0).expunge(link0) + state(link0).status = ObjectState.dirty + session(link1).flush(link0) + session(link1).expunge(link1) def refresh_children(ci): diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py index 2712d5b..83a5122 100644 --- a/Allura/allura/model/repository.py +++ b/Allura/allura/model/repository.py @@ -48,7 +48,7 @@ import six from ming import schema as S from ming import Field, collection, Index from ming.utils import LazyProperty -from ming.orm import FieldProperty, session, Mapper, mapper +from ming.odm import FieldProperty, session, Mapper, mapper, MappedClass from ming.base import Object from allura.lib import helpers as h @@ -998,41 +998,6 @@ class MergeRequest(VersionedArtifact, ActivityObject): self.discussion_thread.add_post(text=message, is_meta=True, ignore_security=True) - -# Basic commit information -# One of these for each commit in the physical repo on disk. The _id is the -# hexsha of the commit (for Git and Hg). -CommitDoc = collection( - str('repo_ci'), main_doc_session, - Field('_id', str), - Field('tree_id', str), - Field('committed', SUser), - Field('authored', SUser), - Field('message', str), - Field('parent_ids', [str], index=True), - Field('child_ids', [str], index=True), - Field('repo_ids', [S.ObjectId()], index=True)) - -# Basic tree information -TreeDoc = collection( - str('repo_tree'), main_doc_session, - Field('_id', str), - Field('tree_ids', [dict(name=str, id=str)]), - Field('blob_ids', [dict(name=str, id=str)]), - Field('other_ids', [dict(name=str, id=str, type=SObjType)])) - -# Information about the last commit to touch a tree -LastCommitDoc = collection( - str('repo_last_commit'), main_doc_session, - Field('_id', S.ObjectId()), - Field('commit_id', str), - Field('path', str), - Index('commit_id', 'path'), - Field('entries', [dict( - name=str, - commit_id=str)])) - - class RepoObject(object): def __repr__(self): # pragma no cover @@ -1068,7 +1033,44 @@ class RepoObject(object): return r, isnew -class Commit(RepoObject, ActivityObject): +# this is duplicative with the Commit model +# would be nice to get rid of this "doc" based view, but it is used a lot +CommitDoc = collection( + str('repo_ci'), main_doc_session, + Field('_id', str), + Field('tree_id', str), + Field('committed', SUser), + Field('authored', SUser), + Field('message', str), + Field('parent_ids', [str], index=True), + Field('child_ids', [str], index=True), + Field('repo_ids', [S.ObjectId()], index=True)) + + +class Commit(MappedClass, RepoObject, ActivityObject): + # Basic commit information + # One of these for each commit in the physical repo on disk + + class __mongometa__: + session = repository_orm_session + name = str('repo_ci') + indexes = [ + 'parent_ids', + 'child_ids', + 'repo_ids', + ] + + query: 'Query[Commit]' + + _id = FieldProperty(str) # hexsha of the commit (for Git and Hg) + tree_id = FieldProperty(str) + committed = FieldProperty(SUser) + authored = FieldProperty(SUser) + message = FieldProperty(str) + parent_ids = FieldProperty([str]) + child_ids = FieldProperty([str]) + repo_ids = FieldProperty([S.ObjectId()]) + type_s = 'Commit' # Ephemeral attrs repo = None @@ -1317,7 +1319,21 @@ class Commit(RepoObject, ActivityObject): } -class Tree(RepoObject): +class Tree(MappedClass, RepoObject): + # Basic tree information + class __mongometa__: + session = repository_orm_session + name = str('repo_tree') + indexes = [ + ] + + query: 'Query[Tree]' + + _id = FieldProperty(str) + tree_ids = FieldProperty([dict(name=str, id=str)]) + blob_ids = FieldProperty([dict(name=str, id=str)]) + other_ids = FieldProperty([dict(name=str, id=str, type=SObjType)]) + # Ephemeral attrs repo = None commit = None @@ -1559,7 +1575,37 @@ class EmptyBlob(Blob): return False -class LastCommit(RepoObject): +# this is duplicative with the LastCommit model +# would be nice to get rid of this "doc" based view, but it is used a lot +LastCommitDoc = collection( + str('repo_last_commit'), main_doc_session, + Field('_id', S.ObjectId()), + Field('commit_id', str), + Field('path', str), + Index('commit_id', 'path'), + Field('entries', [dict( + name=str, + commit_id=str)])) + + +class LastCommit(MappedClass, RepoObject): + # Information about the last commit to touch a tree + class __mongometa__: + session = repository_orm_session + name = str('repo_last_commit') + indexes = [ + ('commit_id', 'path'), + ] + + query: 'Query[LastCommit]' + + _id = FieldProperty(S.ObjectId) + commit_id = FieldProperty(str) + path = FieldProperty(str) + entries = FieldProperty([dict( + name=str, + commit_id=str, + )]) def __repr__(self): return '<LastCommit /%r %s>' % (self.path, self.commit_id) @@ -1963,7 +2009,4 @@ def zipdir(source, zipfile, exclude=None): "STDERR: {3}".format(command, p.returncode, stdout, stderr)) -mapper(Commit, CommitDoc, repository_orm_session) -mapper(Tree, TreeDoc, repository_orm_session) -mapper(LastCommit, LastCommitDoc, repository_orm_session) Mapper.compile_all() diff --git a/Allura/allura/scripts/refreshrepo.py b/Allura/allura/scripts/refreshrepo.py index d117138..d57fb68 100644 --- a/Allura/allura/scripts/refreshrepo.py +++ b/Allura/allura/scripts/refreshrepo.py @@ -90,10 +90,6 @@ class RefreshRepo(ScriptTask): M.repository.CommitDoc.m.remove( {"_id": {"$in": ci_ids_chunk}}) - # we used to have a TreesDoc (plural) collection to provide a mapping of commit_id to tree_id - # so that we could clear the relevant TreeDoc records - # its ok though, since they are created in refresh_tree_info() and overwrite existing records - for ci_ids_chunk in chunked_list(ci_ids, 3000): # delete LastCommitDocs i = M.repository.LastCommitDoc.m.find( diff --git a/ForgeGit/forgegit/model/git_repo.py b/ForgeGit/forgegit/model/git_repo.py index 07662ca..ed328ee 100644 --- a/ForgeGit/forgegit/model/git_repo.py +++ b/ForgeGit/forgegit/model/git_repo.py @@ -36,7 +36,8 @@ from paste.deploy.converters import asbool import six from ming.base import Object -from ming.orm import Mapper, session +from ming.odm import state, Mapper, session +from ming.odm.base import ObjectState from ming.utils import LazyProperty from allura.lib import helpers as h @@ -314,15 +315,15 @@ class GitImplementation(M.RepositoryImplementation): return True def refresh_tree_info(self, tree, seen, lazy=True): - from allura.model.repository import TreeDoc + from allura.model.repository import Tree if lazy and tree.binsha in seen: return seen.add(tree.binsha) - doc = TreeDoc(dict( + doc = Tree( _id=tree.hexsha, tree_ids=[], blob_ids=[], - other_ids=[])) + other_ids=[]) for o in tree: if o.type == 'submodule': continue @@ -337,7 +338,10 @@ class GitImplementation(M.RepositoryImplementation): else: obj.type = o.type doc.other_ids.append(obj) - doc.m.save() + # set to 'dirty' to force save() to be used instead of insert() (which errors if doc exists in db already) + state(doc).status = ObjectState.dirty + session(doc).flush(doc) + session(doc).expunge(doc) return doc def log(self, revs=None, path=None, exclude=None, id_only=True, limit=None, **kw):
