This is an automated email from the ASF dual-hosted git repository. github-bot pushed a commit to branch jonathan/mirror-client in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit d41bbc5c68c225c4c821d8ce8be3dff38476f049 Author: Jonathan Maw <[email protected]> AuthorDate: Thu May 3 17:30:50 2018 +0100 tests: Add a test for fetching when a source uses multiple aliases --- tests/frontend/mirror.py | 63 ++++++++++++++++++++++ tests/frontend/project/sources/fetch_source.py | 73 ++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py index b45aa29..9fdbd38 100644 --- a/tests/frontend/mirror.py +++ b/tests/frontend/mirror.py @@ -68,3 +68,66 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind): # But at least we can be sure it succeeds result = cli.run(project=project_dir, args=['fetch', element_name]) result.assert_success() + + [email protected](DATA_DIR) +def test_mirror_fetch_multi(cli, tmpdir, datafiles): + output_file = os.path.join(str(tmpdir), "output.txt") + project_dir = str(tmpdir) + element_dir = os.path.join(project_dir, 'elements') + os.makedirs(element_dir, exist_ok=True) + element_name = "test.bst" + element_path = os.path.join(element_dir, element_name) + element = { + 'kind': 'import', + 'sources': [ + { + 'kind': 'fetch_source', + "output-text": output_file, + "urls": ["foo:repo1", "bar:repo2"], + "fetch-succeeds": { + "FOO/repo1": True, + "BAR/repo2": False, + "OOF/repo1": False, + "RAB/repo2": True + } + } + ] + } + _yaml.dump(element, element_path) + + project_file = os.path.join(project_dir, 'project.conf') + project = { + 'name': 'test', + 'element-path': 'elements', + 'aliases': { + "foo": "FOO/", + "bar": "BAR/" + }, + 'mirrors': [ + { + 'location-name': 'middle-earth', + 'aliases': { + "foo": ["OOF/"], + "bar": ["RAB/"] + }, + }, + ], + 'plugins': [ + { + 'origin': 'local', + 'path': 'sources', + 'sources': { + 'fetch_source': 0 + } + } + ] + } + _yaml.dump(project, project_file) + + result = cli.run(project=project_dir, args=['fetch', element_name]) + result.assert_success() + with open(output_file) as f: + contents = f.read() + assert "Fetch foo:repo1 succeeded from FOO/repo1" in contents + assert "Fetch bar:repo2 succeeded from RAB/repo2" in contents diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py new file mode 100644 index 0000000..81b9777 --- /dev/null +++ b/tests/frontend/project/sources/fetch_source.py @@ -0,0 +1,73 @@ +import os +import sys + +from buildstream import Source, Consistency, SourceError + +# Expected config +# sources: +# - output-text: $FILE +# urls: +# - foo:bar +# - baz:quux +# fetch-succeeds: +# Foo/bar: true +# ooF/bar: false + + +class FetchSource(Source): + # Read config to know which URLs to fetch + def configure(self, node): + self.original_urls = self.node_get_member(node, list, 'urls') + self.urls = [self.translate_url(url) for url in self.original_urls] + self.output_file = self.node_get_member(node, str, 'output-text') + self.fetch_succeeds = {} + if 'fetch-succeeds' in node: + self.fetch_succeeds = {x[0]: x[1] for x in self.node_items(node['fetch-succeeds'])} + self.urls_cached = False + + def preflight(self): + output_dir = os.path.dirname(self.output_file) + if not os.path.exists(output_dir): + raise SourceError("Directory '{}' does not exist".format(output_dir)) + + def get_unique_key(self): + return {"urls": self.original_urls, "output_file": self.output_file} + + def get_consistency(self): + if not os.path.exists(self.output_file): + return Consistency.RESOLVED + + all_fetched = True + with open(self.output_file, "r") as f: + contents = f.read() + for url in self.original_urls: + if url not in contents: + return Consistency.RESOLVED + + return Consistency.CACHED + + # We dont have a ref, we're a local file... + def load_ref(self, node): + pass + + def get_ref(self): + return None # pragma: nocover + + def set_ref(self, ref, node): + pass # pragma: nocover + + def fetch(self): + with open(self.output_file, "a") as f: + for i, url in enumerate(self.urls): + origin_url = self.original_urls[i] + success = url in self.fetch_succeeds and self.fetch_succeeds[url] + message = "Fetch {} {} from {}\n".format(origin_url, + "succeeded" if success else "failed", + url) + f.write(message) + if not success: + raise SourceError("Failed to fetch {}".format(url)) + + +def setup(): + return FetchSource
