This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tpollard/752 in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 51bb1341e7db35538aca0aa0a1ca273ee373be0c Author: Tom Pollard <[email protected]> AuthorDate: Wed Nov 14 17:33:24 2018 +0000 Add ability to build without interacting with available artifact servers. By adding a new user context and bst main option, we can provide more granular control of what remotes are utilised during relevant bst pull and push actions. _artifactcache/artifactcache.py: Expand setup_remotes() to consider the user context value of 'use_remotes'. Allowing the user to omit interacting with all, or all non user defined remotes. tests/frontend: pull.py & push.py addition of tests for checking expected behaviour. Ensures interaction with user, project or no remotes. --- NEWS | 9 ++++++ buildstream/_artifactcache.py | 17 ++++++---- tests/frontend/pull.py | 72 +++++++++++++++++++++++++++++++++++++++++++ tests/frontend/push.py | 65 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index 449a23f..cf5a22d 100644 --- a/NEWS +++ b/NEWS @@ -122,6 +122,15 @@ buildstream 1.3.1 'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset' commands are affected. + o bst interaction with defined artifact servers can be controlled more granularly. + This can be done via the user configuration option `useremotes` or via the bst cli + main option '--use-remotes'. This can be set as 'none', 'user' or the default value + 'all'. Unless specifically overriden, when considering wether to pull or push to + available artifact servers (be it user or project defined) this optional config option + will be used. Setting this value to 'user' for example and performing a build would + lead to any project or junction defined artifact server to be ignored, whilst still + attempting to any user defined remotes. + ================= buildstream 1.1.5 diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py index 48ab278..c162c39 100644 --- a/buildstream/_artifactcache.py +++ b/buildstream/_artifactcache.py @@ -148,7 +148,7 @@ class ArtifactCache(): # Sets up which remotes to use # # Args: - # use_config (bool): Whether to use project configuration + # use_config (bool): Whether to use configuration # remote_url (str): Remote artifact cache URL # # This requires that all of the projects which are to be processed in the session @@ -167,11 +167,16 @@ class ArtifactCache(): self._set_remotes([ArtifactCacheSpec(remote_url, push=True)]) has_remote_caches = True if use_config: - for project in self.context.get_projects(): - artifact_caches = _configured_remote_artifact_cache_specs(self.context, project) - if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances - self._set_remotes(artifact_caches, project=project) - has_remote_caches = True + if self.context.use_remotes == 'all': + for project in self.context.get_projects(): + artifact_caches = _configured_remote_artifact_cache_specs(self.context, project) + if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances + self._set_remotes(artifact_caches, project=project) + has_remote_caches = True + # If configured to only use user configured remotes, pass existing user cache spec + elif self.context.use_remotes == 'user' and self.context.artifact_cache_specs: + self._set_remotes(self.context.artifact_cache_specs) + has_remote_caches = True if has_remote_caches: self._initialize_remotes() diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index 20b7409..57e1bfd 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -408,3 +408,75 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles): assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr assert "SKIPPED Pull" in result.stderr + + +# Tests that: +# +# * The bst main option --use-remotes limits remote action +# as expected for pull jobs +# [email protected](DATA_DIR) +def test_useremotes_cli_options(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\ + create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject: + + # Add shareproject repo url to project.conf + with open(os.path.join(project, "project.conf"), "a") as projconf: + projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo)) + + # First build the target element and push to the remotes. + # We need the artifact available in the remotes to test against. + cli.configure({ + 'artifacts': {'url': shareuser.repo, 'push': True} + }) + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() + assert cli.get_element_state(project, 'target.bst') == 'cached' + + # Assert that everything is now cached in the remotes. + all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst'] + for element_name in all_elements: + assert_shared(cli, shareuser, project, element_name) + assert_shared(cli, shareproject, project, element_name) + + # Now we've pushed, delete the user's local artifact cache + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) + + # Assert that nothing is cached locally anymore + for element_name in all_elements: + assert cli.get_element_state(project, element_name) != 'cached' + + # Attempt bst build with --use-remotes set as none, this should lead to + # a complete rebuild without pulling from either artifact remote cache + result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst']) + result.assert_success() + for element_name in all_elements: + assert element_name not in result.get_pulled_elements() + + # Delete local cache again + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) + + # Attempt bst build with --use-remotes set as user, as the shareuser is + # passed in as user config and not via a project, assert project remote + # was not attempted by it not being in the output + result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst']) + result.assert_success() + for element_name in all_elements: + assert element_name in result.get_pulled_elements() + assert shareproject.repo not in result.stderr + + # Delete local cache again + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) + + # Attempt bst build with --use-remotes set as all, this time + # assert that project remote is attempted and in the output + result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst']) + result.assert_success() + for element_name in all_elements: + assert element_name in result.get_pulled_elements() + assert shareproject.repo in result.stderr diff --git a/tests/frontend/push.py b/tests/frontend/push.py index f3f9b52..b728a2f 100644 --- a/tests/frontend/push.py +++ b/tests/frontend/push.py @@ -416,3 +416,68 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles): assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated" assert "INFO Remote ({}) already has ".format(share.repo) in result.stderr assert "SKIPPED Push" in result.stderr + + +# Tests that: +# +# * The bst main option --use-remotes limits remote action +# as expected for push jobs +# [email protected](DATA_DIR) +def test_useremotes_cli_options(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\ + create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject: + + # Add shareproject repo url to project.conf + with open(os.path.join(project, "project.conf"), "a") as projconf: + projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo)) + + # Configure shareuser remote in user conf + cli.configure({ + 'artifacts': {'url': shareuser.repo, 'push': True} + }) + + # First build the target element with --use-remotes set as none. + # This should lead to a complete build without pushing to either artifact + # remote cache + result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst']) + result.assert_success() + assert not result.get_pushed_elements() + assert cli.get_element_state(project, 'target.bst') == 'cached' + + # Delete the artifacts from the local artifact cache + all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst'] + for element_name in all_elements: + cli.remove_artifact_from_cache(project, element_name) + + # Assert that nothing is cached locally anymore + for element_name in all_elements: + assert cli.get_element_state(project, element_name) != 'cached' + + # Attempt bst build with --use-remotes set as user, this should lead to + # a complete rebuild, with artifacts pushed to the shareuser remote artifact cache + # only. Assert project remote was not attempted by it not being in the output + result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst']) + result.assert_success() + for element_name in all_elements: + assert element_name in result.get_pushed_elements() + for element_name in all_elements: + assert_shared(cli, shareuser, project, element_name) + assert shareproject.repo not in result.stderr + + # Delete the artifacts from the local artifact cache + all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst'] + for element_name in all_elements: + cli.remove_artifact_from_cache(project, element_name) + + # Attempt bst build with --use-remotes set as all, this should lead to + # a complete rebuild, with artifacts pushed to both the shareuser and + # shareproject remote artifacts caches + result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst']) + result.assert_success() + for element_name in all_elements: + assert element_name in result.get_pushed_elements() + for element_name in all_elements: + assert_shared(cli, shareproject, project, element_name)
