This is an automated email from the ASF dual-hosted git repository. github-bot pushed a commit to branch jennis/introduce_artifact_delete in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 404fa91a7e0af026a7a2aeb2bd4ac2d2dc100f23 Author: James Ennis <[email protected]> AuthorDate: Tue Mar 5 16:02:19 2019 +0000 cli.py: Add artifact delete command This command provides a --no-prune option because or a large cache, pruning can be an expensive operation. If a developer wishes to quicky rebuild an artifact, they may consider using this option. --- buildstream/_frontend/cli.py | 14 ++++++ buildstream/_stream.py | 42 ++++++++++++++++- tests/frontend/artifact.py | 104 ++++++++++++++++++++++++++++++++++++++++++ tests/frontend/completions.py | 1 + 4 files changed, 160 insertions(+), 1 deletion(-) diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index 02ca52e..d8c46ce 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -1106,6 +1106,20 @@ def artifact_log(app, artifacts): click.echo_via_pager(data) +################################################################### +# Artifact Delete Command # +################################################################### [email protected](name='delete', short_help="Remove artifacts from the local cache") [email protected]('--no-prune', 'no_prune', default=False, is_flag=True, + help="Do not prune the local cache of unreachable refs") [email protected]('artifacts', type=click.Path(), nargs=-1) [email protected]_obj +def artifact_delete(app, artifacts, no_prune): + """Remove artifacts from the local cache""" + with app.initialized(): + app.stream.artifact_delete(artifacts, no_prune) + + ################################################################## # DEPRECATED Commands # ################################################################## diff --git a/buildstream/_stream.py b/buildstream/_stream.py index b0fce38..5c88042 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -30,11 +30,12 @@ from contextlib import contextmanager, suppress from fnmatch import fnmatch from ._artifactelement import verify_artifact_ref -from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, set_last_task_error +from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, CASCacheError, set_last_task_error from ._message import Message, MessageType from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue from ._pipeline import Pipeline, PipelineSelection from ._profile import Topics, profile_start, profile_end +from .types import _KeyStrength from . import utils, _yaml, _site from . import Scope, Consistency @@ -520,6 +521,45 @@ class Stream(): return logsdirs + # artifact_delete() + # + # Remove artifacts from the local cache + # + # Args: + # targets (str): Targets to remove + # no_prune (bool): Whether to prune the unreachable refs, default False + # + def artifact_delete(self, targets, no_prune): + # Return list of Element and/or ArtifactElement objects + target_objects = self.load_selection(targets, selection=PipelineSelection.NONE, load_refs=True) + + # Some of the targets may refer to the same key, so first obtain a + # set of the refs to be removed. + remove_refs = set() + for obj in target_objects: + for key_strength in [_KeyStrength.STRONG, _KeyStrength.WEAK]: + key = obj._get_cache_key(strength=key_strength) + remove_refs.add(obj.get_artifact_name(key=key)) + + ref_removed = False + for ref in remove_refs: + try: + self._artifacts.remove(ref, defer_prune=True) + except CASCacheError as e: + self._message(MessageType.WARN, "{}".format(e)) + continue + + self._message(MessageType.INFO, "Removed: {}".format(ref)) + ref_removed = True + + # Prune the artifact cache + if ref_removed and not no_prune: + with self._context.timed_activity("Pruning artifact cache"): + self._artifacts.prune() + + if not ref_removed: + self._message(MessageType.INFO, "No artifacts were removed") + # source_checkout() # # Checkout sources of the target element to the specified location diff --git a/tests/frontend/artifact.py b/tests/frontend/artifact.py index c8301c5..3c3203d 100644 --- a/tests/frontend/artifact.py +++ b/tests/frontend/artifact.py @@ -22,6 +22,7 @@ import os import pytest from buildstream.plugintestutils import cli +from tests.testutils import create_artifact_share # Project directory @@ -62,3 +63,106 @@ def test_artifact_log(cli, datafiles): assert result.exit_code == 0 # The artifact is cached under both a strong key and a weak key assert (log + log) == result.output + + +# Test that we can delete the artifact of the element which corresponds +# to the current project state [email protected](DATA_DIR) +def test_artifact_delete_element(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + element = 'target.bst' + + # Build the element and ensure it's cached + result = cli.run(project=project, args=['build', element]) + result.assert_success() + assert cli.get_element_state(project, element) == 'cached' + + result = cli.run(project=project, args=['artifact', 'delete', element]) + result.assert_success() + assert cli.get_element_state(project, element) != 'cached' + + +# Test that we can delete an artifact by specifying its ref. [email protected](DATA_DIR) +def test_artifact_delete_artifact(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + element = 'target.bst' + + # Configure a local cache + local_cache = os.path.join(str(tmpdir), 'artifacts') + cli.configure({'cachedir': local_cache}) + + # First build an element so that we can find its artifact + result = cli.run(project=project, args=['build', element]) + result.assert_success() + + # Obtain the artifact ref + cache_key = cli.get_element_key(project, element) + artifact = os.path.join('test', os.path.splitext(element)[0], cache_key) + + # Explicitly check that the ARTIFACT exists in the cache + assert os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact)) + + # Delete the artifact + result = cli.run(project=project, args=['artifact', 'delete', artifact]) + result.assert_success() + + # Check that the ARTIFACT is no longer in the cache + assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact)) + + +# Test the `bst artifact delete` command with multiple, different arguments. [email protected](DATA_DIR) +def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + element = 'target.bst' + dep = 'compose-all.bst' + + # Configure a local cache + local_cache = os.path.join(str(tmpdir), 'artifacts') + cli.configure({'cachedir': local_cache}) + + # First build an element so that we can find its artifact + result = cli.run(project=project, args=['build', element]) + result.assert_success() + assert cli.get_element_state(project, element) == 'cached' + assert cli.get_element_state(project, dep) == 'cached' + + # Obtain the artifact ref + cache_key = cli.get_element_key(project, element) + artifact = os.path.join('test', os.path.splitext(element)[0], cache_key) + + # Explicitly check that the ARTIFACT exists in the cache + assert os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact)) + + # Delete the artifact + result = cli.run(project=project, args=['artifact', 'delete', artifact, dep]) + result.assert_success() + + # Check that the ARTIFACT is no longer in the cache + assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact)) + + # Check that the dependency ELEMENT is no longer cached + assert cli.get_element_state(project, dep) != 'cached' + + +# Test that we receive the appropriate stderr when we try to delete an artifact +# that is not present in the cache. [email protected](DATA_DIR) +def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + element = 'target.bst' + + # delete it, just in case it's there + _ = cli.run(project=project, args=['artifact', 'delete', element]) + + # Ensure the element is not cached + assert cli.get_element_state(project, element) != 'cached' + + # Now try and remove it again (now we know its not there) + result = cli.run(project=project, args=['artifact', 'delete', element]) + + cache_key = cli.get_element_key(project, element) + artifact = os.path.join('test', os.path.splitext(element)[0], cache_key) + expected_err = "WARNING Could not find ref '{}'".format(artifact) + assert expected_err in result.stderr diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py index 1f29fda..7810a06 100644 --- a/tests/frontend/completions.py +++ b/tests/frontend/completions.py @@ -57,6 +57,7 @@ SOURCE_COMMANDS = [ ARTIFACT_COMMANDS = [ 'checkout ', + 'delete ', 'push ', 'pull ', 'log ',
