This is an automated email from the ASF dual-hosted git repository. github-bot pushed a commit to branch jmac/googlecas_and_virtual_directories_1 in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2fc18a9680d70535402c1c7ec0b73b88bea55694 Author: Jim MacArthur <[email protected]> AuthorDate: Tue May 8 17:34:25 2018 +0100 Directory.py: Change the mtime interface into mark_unmodified/list_modified_paths --- buildstream/plugins/elements/compose.py | 25 +++++++------------------ buildstream/sandbox/Directory.py | 21 ++++++++++++++++----- buildstream/sandbox/_filebaseddirectory.py | 29 ++++++++++++++++++++++++----- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py index b6a1a29..c6ac1bf 100644 --- a/buildstream/plugins/elements/compose.py +++ b/buildstream/plugins/elements/compose.py @@ -119,36 +119,25 @@ class ComposeElement(Element): if require_split: # Make a snapshot of all the files before integration-commands are run. - snapshot = vbasedir.list_relative_paths_with_mtimes() + snapshot = set(vbasedir.list_relative_paths()) + vbasedir.mark_unmodified() + for dep in self.dependencies(Scope.BUILD): dep.integrate(sandbox) if require_split: # Calculate added, modified and removed files - post_integration_snapshot = vbasedir.list_relative_paths_with_mtimes() - - basedir_contents = set(post_integration_snapshot.keys()) + post_integration_snapshot = vbasedir.list_relative_paths() + modified_files = set(vbasedir.list_modified_paths()) + basedir_contents = set(post_integration_snapshot) for path in manifest: - if path in basedir_contents: - if path in snapshot: - preintegration_mtime = snapshot[path] - if preintegration_mtime != post_integration_snapshot[path]: - modified_files.add(path) - else: - # If the path appears in the manifest but not the initial snapshot, - # it may be a file staged inside a directory symlink. In this case - # the path we got from the manifest won't show up in the snapshot - # because utils.list_relative_paths() doesn't recurse into symlink - # directories. - pass - elif path in snapshot: + if path in snapshot and not path in basedir_contents: removed_files.add(path) for path in basedir_contents: if path not in snapshot: added_files.add(path) - self.info("Integration modified {}, added {} and removed {} files" .format(len(modified_files), len(added_files), len(removed_files))) diff --git a/buildstream/sandbox/Directory.py b/buildstream/sandbox/Directory.py index 07f22ba..4107496 100644 --- a/buildstream/sandbox/Directory.py +++ b/buildstream/sandbox/Directory.py @@ -110,11 +110,22 @@ class Directory(): """ raise NotImplementedError() - def list_relative_paths_with_mtimes(self) -> Dict[str, float]: - """Provide a list of relative paths with modification times for - each. Used to detect changed changed files during a Compose - operation. + def mark_unmodified(self) -> None: + """ Marks all files in this directory (recursively) as unmodified. + """ + raise NotImplementedError() + + def list_modified_paths(self) -> List[str]: + """Provide a list of relative paths which have been modified since the + last call to mark_unmodified. + + Return value: List(str) - dictionary with all paths + """ + raise NotImplementedError() + + def list_relative_paths(self) -> List[str]: + """Provide a list of all relative paths in this directory. - Return value: Dict(str->float) - dictionary with all paths and mtime in seconds. + Return value: List(str) - dictionary with all paths """ raise NotImplementedError() diff --git a/buildstream/sandbox/_filebaseddirectory.py b/buildstream/sandbox/_filebaseddirectory.py index 640e254..975a4cc 100644 --- a/buildstream/sandbox/_filebaseddirectory.py +++ b/buildstream/sandbox/_filebaseddirectory.py @@ -30,6 +30,7 @@ See also: :ref:`sandboxing`. from typing import List from collections import OrderedDict +import calendar import os import time from .._exceptions import BstError, ErrorDomain @@ -210,11 +211,29 @@ class FileBasedDirectory(Directory): self._populate_index() return len(self.index) == 0 - def list_relative_paths_with_mtimes(self) -> Dict[str, float]: - return { - f: getmtime(os.path.join(self.external_directory, f)) - for f in list_relative_paths(self.external_directory) - } + def mark_unmodified(self) -> None: + """ Marks all files in this directory (recursively) as unmodified. + """ + _set_deterministic_mtime(self.external_directory) + + def list_modified_paths(self) -> List[str]: + """Provide a list of relative paths which have been modified since the + last call to mark_unmodified. + + Return value: List(str) - list of modified paths + """ + magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11]) + + return [f for f in list_relative_paths(self.external_directory) + if getmtime(os.path.join(self.external_directory, f)) != magic_timestamp] + + def list_relative_paths(self) -> List[str]: + """Provide a list of all relative paths. + + Return value: List(str) - list of all paths + """ + + return list_relative_paths(self.external_directory) def __str__(self) -> str: # This returns the whole path (since we don't know where the directory started)
