This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/allura.git
commit a23ae3560f10b0efcdccf4c918d6d506053f7e23 Author: Dave Brondsema <[email protected]> AuthorDate: Wed May 20 16:22:54 2026 -0400 [#8607] replace last pickle usage with our own bit of compatible code --- Allura/allura/model/index.py | 33 ++++++++++++++++++-- Allura/allura/model/repo_refresh.py | 5 ++- Allura/allura/tests/unit/test_index.py | 56 ++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/Allura/allura/model/index.py b/Allura/allura/model/index.py index bd0e54241..b88998baf 100644 --- a/Allura/allura/model/index.py +++ b/Allura/allura/model/index.py @@ -16,12 +16,12 @@ # under the License. import re +import importlib import logging from itertools import groupby import typing from ming.odm.property import FieldProperty -from pickle import dumps, loads from collections import defaultdict from urllib.parse import unquote @@ -47,6 +47,33 @@ log = logging.getLogger(__name__) +def _dump_cls(cls: type) -> bytes: + # byte-for-byte equivalent of pickle.dumps(cls, protocol=2) for a top-level class: + # \x80\x02 protocol header, GLOBAL(c) <module>\n<qualname>\n, BINPUT(q) \x00, STOP(.) + return b'\x80\x02c' + cls.__module__.encode('utf-8') + b'\n' + cls.__qualname__.encode('utf-8') + b'\nq\x00.' + + +def _load_cls(data) -> type: + # parses what pickle wrote for a class reference, without using pickle. + # handles protocol 0 (c<module>\n<class>\np<n>\n.) and protocol 2 (\x80\x02 ... q\x00.) + b = bytes(data) + if b[:2] == b'\x80\x02': + b = b[2:] + if b[:1] != b'c': + raise ValueError(f'unrecognized class reference encoding: {bytes(data)!r}') + rest = b[1:] + module_name, sep, rest = rest.partition(b'\n') + if not sep: + raise ValueError(f'malformed class reference: {bytes(data)!r}') + class_name, sep, _ = rest.partition(b'\n') + if not sep: + raise ValueError(f'malformed class reference: {bytes(data)!r}') + obj = importlib.import_module(module_name.decode('utf-8')) + for part in class_name.decode('utf-8').split('.'): + obj = getattr(obj, part) + return obj + + class ArtifactReference(MappedClass): class __mongometa__: session = main_orm_session @@ -77,7 +104,7 @@ def from_artifact(cls, artifact): obj = cls( _id=artifact.index_id(), artifact_reference=dict( - cls=bson.Binary(dumps(artifact.__class__, protocol=2)), + cls=bson.Binary(_dump_cls(artifact.__class__)), project_id=artifact.app_config.project_id, app_config_id=artifact.app_config._id, artifact_id=artifact._id)) @@ -92,7 +119,7 @@ def artifact(self): '''Look up the artifact referenced''' aref = self.artifact_reference try: - cls = loads(bytes(aref.cls)) # noqa: S301 + cls = _load_cls(aref.cls) with h.push_context(aref.project_id): return cls.query.get(_id=aref.artifact_id) except Exception: diff --git a/Allura/allura/model/repo_refresh.py b/Allura/allura/model/repo_refresh.py index b2ff40f88..f1744f212 100644 --- a/Allura/allura/model/repo_refresh.py +++ b/Allura/allura/model/repo_refresh.py @@ -16,7 +16,6 @@ # under the License. import logging -from pickle import dumps import bson import tg @@ -30,7 +29,7 @@ from allura.lib import utils from allura.lib.search import find_shortlinks from allura.model.repository import Commit, CommitDoc -from allura.model.index import ArtifactReference, Shortlink +from allura.model.index import ArtifactReference, Shortlink, _dump_cls from allura.model.auth import User from allura.model.timeline import TransientActor @@ -143,7 +142,7 @@ def refresh_commit_repos(all_commit_ids, repo): ref = ArtifactReference( _id=index_id, artifact_reference=dict( - cls=bson.Binary(dumps(Commit, protocol=2)), + cls=bson.Binary(_dump_cls(Commit)), project_id=repo.app.config.project_id, app_config_id=repo.app.config._id, artifact_id=oid), diff --git a/Allura/allura/tests/unit/test_index.py b/Allura/allura/tests/unit/test_index.py new file mode 100644 index 000000000..e89b48211 --- /dev/null +++ b/Allura/allura/tests/unit/test_index.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import bson +import pytest + +from allura.model.index import _dump_cls, _load_cls +from allura.model.repository import Commit + + +def test_dump_cls(): + assert _dump_cls(Commit) == b'\x80\x02callura.model.repository\nCommit\nq\x00.' + + +def test_roundtrip(): + assert _load_cls(_dump_cls(Commit)) is Commit + + +def test_load_cls_pickle_protocol_2(): + data = b'\x80\x02callura.model.repository\nCommit\nq\x00.' + assert _load_cls(data) is Commit + + +def test_load_cls_pickle_protocol_0(): + data = bson.Binary(b'callura.model.repository\nCommit\np0\n.') + assert _load_cls(data) is Commit + + +def test_load_cls_legacy_repo_module(): + # legacy data that still may be in mongo records: + # the pre-rename module path allura.model.repo (now a backward-compat shim but still must load) + data = bson.Binary(b'callura.model.repo\nCommit\np1\n.') + assert _load_cls(data) is Commit + + +def test_load_cls_rejects_unknown_prefix(): + with pytest.raises(ValueError): + _load_cls(b'xnot a pickle') + + +def test_load_cls_rejects_truncated(): + with pytest.raises(ValueError): + _load_cls(b'callura.model.repository')
