This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch becky/locally_downloaded_files in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3795f16a84407d3c5792d6bcff77bd980d134e4f Author: Rebecca Grayson <[email protected]> AuthorDate: Mon Jun 24 16:09:51 2019 +0100 Allowing the tar plugin to use a local path _downloadablefilesource.py: Added ability to download from either url or path Added changes to the code so that now a downloadable file can be sourced from either a given url or a local path. If both are given, error occurs. tar.py: Made changes to accept the path sourcing When a path is given, the tar can be found from the local directory rather than the mirror directory. Made changes to make the code less specific to urls so that now it will accept both url and path. --- .../plugins/sources/_downloadablefilesource.py | 136 +++++++++++++++------ src/buildstream/plugins/sources/tar.py | 43 +++++-- 2 files changed, 133 insertions(+), 46 deletions(-) diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py index b9b15e2..127047c 100644 --- a/src/buildstream/plugins/sources/_downloadablefilesource.py +++ b/src/buildstream/plugins/sources/_downloadablefilesource.py @@ -72,58 +72,100 @@ class _NetrcPasswordManager: class DownloadableFileSource(Source): # pylint: disable=attribute-defined-outside-init - COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag'] + COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag', 'path'] __urlopener = None def configure(self, node): - self.original_url = self.node_get_member(node, str, 'url') + self.original_url = self.node_get_member(node, str, 'url', None) + self.original_path = self.node_get_member(node, str, 'path', None) self.ref = self.node_get_member(node, str, 'ref', None) - self.url = self.translate_url(self.original_url) self._warn_deprecated_etag(node) + # XXX: Looks like this should be in preflight but child classes are not calling super() + if self.original_url is None and self.original_path is None: + raise SourceError("Please specify either a 'path' or a 'url'.") + if self.original_url is not None and self.original_path is not None: + self.url_provenance = self.node_provenance(node, member_name='url') + self.path_provenance = self.node_provenance(node, member_name='path') + + raise SourceError("You cannot specify both 'path' ({}) and 'url' ({})" + .format(self.path_provenance, self.url_provenance)) + + # If using a url, use appropriate checks + if self.original_url is not None: + self.url = self.translate_url(self.original_url) + + # If using path, do checks + if self.original_path is not None: + self.path = self.node_get_project_path(node, 'path') + self.fullpath = os.path.join(self.get_project_directory(), self.path) + self.sha = unique_key(self.fullpath) + else: + self.sha = self.ref + def preflight(self): return def get_unique_key(self): - return [self.original_url, self.ref] + self.__unique_key = None + if self.original_url is not None: + return [self.original_url, self.ref] + else: + return [(os.path.basename(self.original_path)), self.sha] def get_consistency(self): - if self.ref is None: - return Consistency.INCONSISTENT + if self.original_url is not None: + if self.ref is None: + return Consistency.INCONSISTENT - if os.path.isfile(self._get_mirror_file()): - return Consistency.CACHED + if os.path.isfile(self._get_mirror_file()): + return Consistency.CACHED + + else: + return Consistency.RESOLVED else: - return Consistency.RESOLVED + return Consistency.CACHED def load_ref(self, node): - self.ref = self.node_get_member(node, str, 'ref', None) - self._warn_deprecated_etag(node) + if self.original_url is not None: + self.ref = self.node_get_member(node, str, 'ref', None) + self._warn_deprecated_etag(node) + else: + pass def get_ref(self): - return self.ref + if self.original_url is not None: + return self.ref + else: + return None def set_ref(self, ref, node): - node['ref'] = self.ref = ref + if self.original_url is not None: + node['ref'] = self.ref = self.sha = ref + else: + pass def track(self): # there is no 'track' field in the source to determine what/whether # or not to update refs, because tracking a ref is always a conscious # decision by the user. - with self.timed_activity("Tracking {}".format(self.url), - silent_nested=True): - new_ref = self._ensure_mirror() - - if self.ref and self.ref != new_ref: - detail = "When tracking, new ref differs from current ref:\n" \ - + " Tracked URL: {}\n".format(self.url) \ - + " Current ref: {}\n".format(self.ref) \ - + " New ref: {}\n".format(new_ref) - self.warn("Potential man-in-the-middle attack!", detail=detail) - - return new_ref + if self.original_url is not None: + with self.timed_activity("Tracking {}".format(self.url), + silent_nested=True): + new_ref = self._ensure_mirror() + + if self.ref and self.ref != new_ref: + detail = "When tracking, new ref differs from current ref:\n" \ + + " Tracked URL: {}\n".format(self.url) \ + + " Current ref: {}\n".format(self.ref) \ + + " New ref: {}\n".format(new_ref) + self.warn("Potential man-in-the-middle attack!", detail=detail) + + return new_ref + else: + pass def fetch(self): @@ -131,16 +173,22 @@ class DownloadableFileSource(Source): # file to be already cached because Source.fetch() will # not be called if the source is already Consistency.CACHED. # - if os.path.isfile(self._get_mirror_file()): - return # pragma: nocover + if self.original_url is not None: + if os.path.isfile(self._get_mirror_file()): + return # pragma: nocover + + # Download the file, raise hell if the sha256sums don't match, + # and mirror the file otherwise. + with self.timed_activity("Fetching {}".format(self.url), silent_nested=True): + sha256 = self._ensure_mirror() + if sha256 != self.ref: + raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!" + .format(self.url, sha256, self.ref)) + else: + pass - # Download the file, raise hell if the sha256sums don't match, - # and mirror the file otherwise. - with self.timed_activity("Fetching {}".format(self.url), silent_nested=True): - sha256 = self._ensure_mirror() - if sha256 != self.ref: - raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!" - .format(self.url, sha256, self.ref)) + def _get_local_path(self): + return self.path def _warn_deprecated_etag(self, node): etag = self.node_get_member(node, str, 'etag', None) @@ -221,8 +269,12 @@ class DownloadableFileSource(Source): .format(self, self.url, e), temporary=True) from e def _get_mirror_dir(self): + if self.original_url is not None: + directory_name = utils.url_directory_name(self.original_url) + else: + directory_name = self.original_path return os.path.join(self.get_mirror_directory(), - utils.url_directory_name(self.original_url)) + directory_name) def _get_mirror_file(self, sha=None): return os.path.join(self._get_mirror_dir(), sha or self.ref) @@ -248,3 +300,17 @@ class DownloadableFileSource(Source): ftp_handler = _NetrcFTPOpener(netrc_config) DownloadableFileSource.__urlopener = urllib.request.build_opener(http_auth, ftp_handler) return DownloadableFileSource.__urlopener + + +# Create a unique key for a file +def unique_key(filename): + + # Return some hard coded things for files which + # have no content to calculate a key for + if os.path.islink(filename): + # For a symbolic link, use the link target as its unique identifier + return os.readlink(filename) + elif os.path.isdir(filename): + return "0" + + return utils.sha256sum(filename) diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py index c90de74..0ba9e72 100644 --- a/src/buildstream/plugins/sources/tar.py +++ b/src/buildstream/plugins/sources/tar.py @@ -77,8 +77,12 @@ class TarSource(DownloadableFileSource): self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir']) def preflight(self): + if self.original_url is not None: + file_source = self.url + else: + file_source = self.path self.host_lzip = None - if self.url.endswith('.lz'): + if file_source.endswith('.lz'): self.host_lzip = utils.get_host_tool('lzip') def get_unique_key(self): @@ -88,23 +92,40 @@ class TarSource(DownloadableFileSource): def _run_lzip(self): assert self.host_lzip with TemporaryFile() as lzip_stdout: - with open(self._get_mirror_file(), 'r') as lzip_file: - self.call([self.host_lzip, '-d'], - stdin=lzip_file, - stdout=lzip_stdout) - - lzip_stdout.seek(0, 0) - yield lzip_stdout + if self.original_url is not None: + with open(self._get_mirror_file(), 'r') as lzip_file: + self.call([self.host_lzip, '-d'], + stdin=lzip_file, + stdout=lzip_stdout) + + lzip_stdout.seek(0, 0) + yield lzip_stdout + else: + with open(self.fullpath, 'r') as lzip_file: + self.call([self.host_lzip, '-d'], + stdin=lzip_file, + stdout=lzip_stdout) + + lzip_stdout.seek(0, 0) + yield lzip_stdout @contextmanager def _get_tar(self): - if self.url.endswith('.lz'): + if self.original_url is not None: + file_source = self.url + else: + file_source = self.path + if file_source.endswith('.lz'): with self._run_lzip() as lzip_dec: with tarfile.open(fileobj=lzip_dec, mode='r:') as tar: yield tar else: - with tarfile.open(self._get_mirror_file()) as tar: - yield tar + if self.original_url is not None: + with tarfile.open(self._get_mirror_file()) as tar: + yield tar + else: + with tarfile.open(self.fullpath) as tar: + yield tar def stage(self, directory): try:
