This is an automated email from the ASF dual-hosted git repository. github-bot pushed a commit to branch jennis/introduce_artifact_delete in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e53cfa19e1ed43a0ac9259819f4522f780361f59 Author: James Ennis <[email protected]> AuthorDate: Fri Jan 11 11:14:34 2019 +0000 cascache.py: Ensure path exists before trying to update the mtime If an artifact is pulled from the cache without its buildtree, CASCache.prune() will fail when it tries to update the mtimes of the build tree's object files. A new integration test has been added to tests/integration/artifact.py which reflects this. --- buildstream/_cas/cascache.py | 16 ++++++++++------ tests/frontend/artifact.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py index 802fc13..02030bb 100644 --- a/buildstream/_cas/cascache.py +++ b/buildstream/_cas/cascache.py @@ -791,16 +791,20 @@ class CASCache(): def _reachable_refs_dir(self, reachable, tree, update_mtime=False): if tree.hash in reachable: return + try: + if update_mtime: + os.utime(self.objpath(tree)) - if update_mtime: - os.utime(self.objpath(tree)) + reachable.add(tree.hash) - reachable.add(tree.hash) + directory = remote_execution_pb2.Directory() - directory = remote_execution_pb2.Directory() + with open(self.objpath(tree), 'rb') as f: + directory.ParseFromString(f.read()) - with open(self.objpath(tree), 'rb') as f: - directory.ParseFromString(f.read()) + except FileNotFoundError: + # Just exit early if the file doesn't exist + return for filenode in directory.files: if update_mtime: diff --git a/tests/frontend/artifact.py b/tests/frontend/artifact.py index 3c3203d..b6f8fb9 100644 --- a/tests/frontend/artifact.py +++ b/tests/frontend/artifact.py @@ -166,3 +166,41 @@ def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles): artifact = os.path.join('test', os.path.splitext(element)[0], cache_key) expected_err = "WARNING Could not find ref '{}'".format(artifact) assert expected_err in result.stderr + + +# Test that an artifact pulled from it's remote cache (without it's buildtree) will not +# throw an Exception when trying to prune the cache. [email protected](DATA_DIR) +def test_artifact_delete_pulled_artifact_without_buildtree(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + element = 'target.bst' + + # Set up remote and local shares + local_cache = os.path.join(str(tmpdir), 'artifacts') + with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as remote: + cli.configure({ + 'artifacts': {'url': remote.repo, 'push': True}, + 'cachedir': local_cache, + }) + + # Build the element + result = cli.run(project=project, args=['build', element]) + result.assert_success() + + # Make sure it's in the share + cache_key = cli.get_element_key(project, element) + assert remote.has_artifact('test', element, cache_key) + + # Delete and then pull the artifact (without its buildtree) + result = cli.run(project=project, args=['artifact', 'delete', element]) + result.assert_success() + assert cli.get_element_state(project, element) != 'cached' + result = cli.run(project=project, args=['artifact', 'pull', element]) + result.assert_success() + assert cli.get_element_state(project, element) == 'cached' + + # Now delete it again (it should have been pulled without the buildtree, but + # a digest of the buildtree is pointed to in the artifact's metadata + result = cli.run(project=project, args=['artifact', 'delete', element]) + result.assert_success() + assert cli.get_element_state(project, element) != 'cached'
