This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tristan/change-remote-config in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5a0f30dddaa5f53c4f9f9a34eb3c82b77be0efe9 Author: Tristan van Berkom <[email protected]> AuthorDate: Fri Jan 22 17:41:32 2021 +0900 plugins/elements/junction.py: Remove ambiguous cache related junction parameters This removes the "cache-junction-elements" and "ignore-junction-remotes" parameters from the junction YAML configuration. As referenced by the mailing list thread[0], these parameters do not belong in element configuration, fine grained configuration of these parameters are not needed in the context of a pipeline with many subprojects in play, and the documentation and expected behavior with deep and complex subproject hierarches is unclear, especially when considering the ability to override junction configurations. Further, these currently cannot even be overridden by user configuration. This will be replaced by a simpler configuration in the regular way, with recommendations (not hard decisions) in project.conf and the final decision made by user configuration files. Summary of changes: * plugins/elements/junction.py: Removed the configuration and docs * _project.py: Stop observing these configurations * tests/artifactcache/junctions.py: Stop testing these configurations [0]: https://lists.apache.org/thread.html/rf2da9830e2fa918357f99a6021e55fc43df876f0b19d43f68802f083%40%3Cdev.buildstream.apache.org%3E --- src/buildstream/_project.py | 10 -- src/buildstream/plugins/elements/junction.py | 13 +-- tests/artifactcache/junctions.py | 157 --------------------------- 3 files changed, 1 insertion(+), 179 deletions(-) diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index f1d1fa6..6dbc120 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -867,16 +867,6 @@ class Project: # Load artifacts pull/push configuration for this project self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory) - # If there is a junction Element which specifies that we want to remotely cache - # its elements, append the junction's remotes to the artifact cache specs list - if self.junction: - parent = self.junction._get_project() - if self.junction.ignore_junction_remotes: - self.artifact_cache_specs = [] - - if self.junction.cache_junction_elements: - self.artifact_cache_specs = parent.artifact_cache_specs + self.artifact_cache_specs - # Load source caches with pull/push config self.source_cache_specs = SourceCache.specs_from_config_node(config, self.directory) diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py index cc980be..a8e3f22 100644 --- a/src/buildstream/plugins/elements/junction.py +++ b/src/buildstream/plugins/elements/junction.py @@ -53,15 +53,6 @@ Overview overrides: subproject-junction.bst: local-junction.bst - # Optionally declare whether elements within the junction project - # should interact with project remotes (default: False). - cache-junction-elements: False - - # Optionally ignore junction remotes, this means that BuildStream - # will not attempt to pull artifacts from the junction project's - # remote(s) (default: False). - ignore-junction-remotes: False - With a junction element in place, local elements can depend on elements in the other BuildStream project using :ref:`element paths <format_element_names>`. For example, if you have a ``toolchain.bst`` junction element referring to @@ -350,12 +341,10 @@ class JunctionElement(Element): def configure(self, node): - node.validate_keys(["path", "options", "cache-junction-elements", "ignore-junction-remotes", "overrides"]) + node.validate_keys(["path", "options", "overrides"]) self.path = node.get_str("path", default="") self.options = node.get_mapping("options", default={}) - self.cache_junction_elements = node.get_bool("cache-junction-elements", default=False) - self.ignore_junction_remotes = node.get_bool("ignore-junction-remotes", default=False) # The overrides dictionary has the target junction # to override as a key, and the ScalarNode of the diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py index 18c48b4..e1b7dbf 100644 --- a/tests/artifactcache/junctions.py +++ b/tests/artifactcache/junctions.py @@ -82,160 +82,3 @@ def test_push_pull(cli, tmpdir, datafiles): assert state == "cached" state = cli.get_element_state(base_project, "base-element.bst") assert state == "cached" - - [email protected](DATA_DIR) -def test_caching_junction_elements(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), "parent") - base_project = os.path.join(str(project), "base") - - # Load the junction element - junction_element = os.path.join(project, "base.bst") - junction_data = _yaml.roundtrip_load(junction_element) - - # Add the "cache-junction-elements" boolean to the junction Element - junction_data["config"] = {"cache-junction-elements": True} - _yaml.roundtrip_dump(junction_data, junction_element) - - with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share( - os.path.join(str(tmpdir), "artifactshare-base") - ) as base_share: - - # First build it without the artifact cache configured - result = cli.run(project=project, args=["build", "target.bst"]) - assert result.exit_code == 0 - - # Assert that we are now cached locally - state = cli.get_element_state(project, "target.bst") - assert state == "cached" - state = cli.get_element_state(base_project, "base-element.bst") - assert state == "cached" - - project_set_artifacts(project, share.repo) - project_set_artifacts(base_project, base_share.repo) - - # Now try bst artifact push - result = cli.run(project=project, args=["artifact", "push", "--deps", "all", "target.bst"]) - assert result.exit_code == 0 - - # And finally assert that the artifacts are in the right shares - # - # The parent project's cache should *also* contain elements from the junction - assert_shared(cli, share, project, "target.bst", project_name="parent") - assert_shared(cli, share, project, "app.bst", project_name="parent") - assert_shared(cli, share, base_project, "base-element.bst", project_name="base") - - # The junction project's cache should only contain elements in the junction project - assert_not_shared(cli, base_share, project, "target.bst", project_name="parent") - assert_not_shared(cli, base_share, project, "app.bst", project_name="parent") - assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base") - - [email protected](DATA_DIR) -def test_ignore_junction_remotes(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), "parent") - base_project = os.path.join(str(project), "base") - - # Load the junction element - junction_element = os.path.join(project, "base.bst") - junction_data = _yaml.roundtrip_load(junction_element) - - with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share( - os.path.join(str(tmpdir), "artifactshare-base") - ) as base_share: - - # Immediately declare the artifact caches in the appropriate project configs - project_set_artifacts(project, share.repo) - project_set_artifacts(base_project, base_share.repo) - - # Build and populate the project remotes with their respective elements - result = cli.run(project=project, args=["build", "target.bst"]) - assert result.exit_code == 0 - - # And finally assert that the artifacts are in the right shares - # - # The parent project's cache should only contain project elements - assert_shared(cli, share, project, "target.bst", project_name="parent") - assert_shared(cli, share, project, "app.bst", project_name="parent") - assert_not_shared(cli, share, base_project, "base-element.bst", project_name="base") - - # The junction project's cache should only contain elements in the junction project - assert_not_shared(cli, base_share, project, "target.bst", project_name="parent") - assert_not_shared(cli, base_share, project, "app.bst", project_name="parent") - assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base") - - # Ensure that, from now on, we ignore junction element remotes - junction_data["config"] = {"ignore-junction-remotes": True} - _yaml.roundtrip_dump(junction_data, junction_element) - - # Now delete everything from the local cache and try to - # redownload from the shares. - # - cas = os.path.join(cli.directory, "cas") - shutil.rmtree(cas) - artifact_dir = os.path.join(cli.directory, "artifacts") - shutil.rmtree(artifact_dir) - - # Assert that nothing is cached locally anymore - state = cli.get_element_state(project, "target.bst") - assert state != "cached" - state = cli.get_element_state(base_project, "base-element.bst") - assert state != "cached" - - # Now try bst artifact pull - result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"]) - assert result.exit_code == 0 - - # And assert that they are again in the local cache, without having built - state = cli.get_element_state(project, "target.bst") - assert state == "cached" - # We shouldn't be able to download base-element! - state = cli.get_element_state(base_project, "base-element.bst") - assert state != "cached" - - [email protected](DATA_DIR) -def test_caching_elements_ignoring_remotes(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), "parent") - base_project = os.path.join(str(project), "base") - - # Load the junction element - junction_element = os.path.join(project, "base.bst") - junction_data = _yaml.roundtrip_load(junction_element) - - # Configure to push everything to the project's remote and nothing to the junction's - junction_data["config"] = {"cache-junction-elements": True, "ignore-junction-remotes": True} - _yaml.roundtrip_dump(junction_data, junction_element) - - with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share( - os.path.join(str(tmpdir), "artifactshare-base") - ) as base_share: - - # First build it without the artifact cache configured - result = cli.run(project=project, args=["build", "target.bst"]) - assert result.exit_code == 0 - - # Assert that we are now cached locally - state = cli.get_element_state(project, "target.bst") - assert state == "cached" - state = cli.get_element_state(base_project, "base-element.bst") - assert state == "cached" - - project_set_artifacts(project, share.repo) - project_set_artifacts(base_project, base_share.repo) - - # Push to the remote(s)) - result = cli.run(project=project, args=["artifact", "push", "--deps", "all", "target.bst"]) - assert result.exit_code == 0 - - # And finally assert that the artifacts are in the right shares - # - # The parent project's cache should *also* contain elements from the junction - assert_shared(cli, share, project, "target.bst", project_name="parent") - assert_shared(cli, share, project, "app.bst", project_name="parent") - assert_shared(cli, share, base_project, "base-element.bst", project_name="base") - - # The junction project's cache should be empty - assert_not_shared(cli, base_share, project, "target.bst", project_name="parent") - assert_not_shared(cli, base_share, project, "app.bst", project_name="parent") - assert_not_shared(cli, base_share, base_project, "base-element.bst", project_name="base")
