This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tristan/sourcecache-retries in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 03fdede8a69e95f4ee9c4fd7fd12d63b5fc48d7f Author: Seppo Yli-Olli <[email protected]> AuthorDate: Fri May 6 22:27:31 2022 +0300 Make source cache errors temporary --- src/buildstream/_elementsourcescache.py | 20 ++++++++++++++------ src/buildstream/_exceptions.py | 4 ++-- src/buildstream/_sourcecache.py | 12 ++++++++---- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/buildstream/_elementsourcescache.py b/src/buildstream/_elementsourcescache.py index 5f1178d30..e5587b1db 100644 --- a/src/buildstream/_elementsourcescache.py +++ b/src/buildstream/_elementsourcescache.py @@ -103,7 +103,9 @@ class ElementSourcesCache(AssetCache): if errors and not source_digest: raise SourceCacheError( - "Failed to pull source {}".format(display_key), detail="\n".join(str(e) for e in errors) + "Failed to pull source {}".format(display_key), + detail="\n".join(str(e) for e in errors), + temporary=True, ) # If we don't have a source proto, we can't pull source files @@ -230,7 +232,7 @@ class ElementSourcesCache(AssetCache): except grpc.RpcError as e: if e.code() != grpc.StatusCode.RESOURCE_EXHAUSTED: raise SourceCacheError( - "Failed to push source blobs with status {}: {}".format(e.code().name, e.details()) + "Failed to push source blobs with status {}: {}".format(e.code().name, e.details()), temporary=True ) return False @@ -261,7 +263,7 @@ class ElementSourcesCache(AssetCache): except grpc.RpcError as e: if e.code() != grpc.StatusCode.NOT_FOUND: raise SourceCacheError( - "Error checking source cache with status {}: {}".format(e.code().name, e.details()) + "Error checking source cache with status {}: {}".format(e.code().name, e.details()), temporary=True ) referenced_directories = [source_proto.files] @@ -273,7 +275,9 @@ class ElementSourcesCache(AssetCache): references_directories=referenced_directories, ) except grpc.RpcError as e: - raise SourceCacheError("Failed to push source with status {}: {}".format(e.code().name, e.details())) + raise SourceCacheError( + "Failed to push source with status {}: {}".format(e.code().name, e.details()), temporary=True + ) return True @@ -308,7 +312,9 @@ class ElementSourcesCache(AssetCache): self.cas._fetch_directory(remote, source.files) except grpc.RpcError as e: if e.code() != grpc.StatusCode.NOT_FOUND: - raise SourceCacheError("Failed to pull source with status {}: {}".format(e.code().name, e.details())) + raise SourceCacheError( + "Failed to pull source with status {}: {}".format(e.code().name, e.details()), temporary=True + ) return False return True @@ -324,5 +330,7 @@ class ElementSourcesCache(AssetCache): except grpc.RpcError as e: if e.code() != grpc.StatusCode.RESOURCE_EXHAUSTED: - raise SourceCacheError("Failed to push source with status {}: {}".format(e.code().name, e.details())) + raise SourceCacheError( + "Failed to push source with status {}: {}".format(e.code().name, e.details()), temporary=True + ) return False diff --git a/src/buildstream/_exceptions.py b/src/buildstream/_exceptions.py index 46f231783..75badf037 100644 --- a/src/buildstream/_exceptions.py +++ b/src/buildstream/_exceptions.py @@ -184,8 +184,8 @@ class AssetCacheError(BstError): # Raised when errors are encountered in the source caches # class SourceCacheError(BstError): - def __init__(self, message, detail=None, reason=None): - super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason) + def __init__(self, message, detail=None, reason=None, temporary=False): + super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason, temporary=temporary) # ArtifactError diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py index 46e1a6e85..dc706062f 100644 --- a/src/buildstream/_sourcecache.py +++ b/src/buildstream/_sourcecache.py @@ -128,7 +128,7 @@ class SourceCache(AssetCache): ) continue except CASError as e: - raise SourceCacheError("Failed to pull source {}: {}".format(display_key, e)) from e + raise SourceCacheError("Failed to pull source {}: {}".format(display_key, e), temporary=True) from e if not source_digest: return False @@ -148,7 +148,7 @@ class SourceCache(AssetCache): source.info("Remote cas ({}) does not have blob {} cached".format(remote, e.blob)) continue except CASError as e: - raise SourceCacheError("Failed to pull source {}: {}".format(display_key, e)) from e + raise SourceCacheError("Failed to pull source {}: {}".format(display_key, e), temporary=True) from e return False @@ -240,7 +240,9 @@ class SourceCache(AssetCache): except grpc.RpcError as e: if e.code() != grpc.StatusCode.NOT_FOUND: - raise SourceCacheError("Failed to pull source with status {}: {}".format(e.code().name, e.details())) + raise SourceCacheError( + "Failed to pull source with status {}: {}".format(e.code().name, e.details()), temporary=True + ) return None def _push_source(self, source_ref, remote): @@ -254,5 +256,7 @@ class SourceCache(AssetCache): except grpc.RpcError as e: if e.code() != grpc.StatusCode.RESOURCE_EXHAUSTED: - raise SourceCacheError("Failed to push source with status {}: {}".format(e.code().name, e.details())) + raise SourceCacheError( + "Failed to push source with status {}: {}".format(e.code().name, e.details()), temporary=True + ) return False
