This is an automated email from the ASF dual-hosted git repository. eamonford pushed a commit to branch granule-ingester-error-handling in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit edbafd7e995112194e9d96f7b5e6e5cf8e220e3a Author: Eamon Ford <[email protected]> AuthorDate: Mon Aug 10 21:54:14 2020 -0700 handle the error where a granule cant be opened --- .../granule_ingester/exceptions/Exceptions.py | 7 ++++++- .../granule_ingester/exceptions/__init__.py | 19 ++++++++----------- .../granule_ingester/granule_loaders/GranuleLoader.py | 7 ++++++- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/granule_ingester/granule_ingester/exceptions/Exceptions.py b/granule_ingester/granule_ingester/exceptions/Exceptions.py index c648b99..fdd03e5 100644 --- a/granule_ingester/granule_ingester/exceptions/Exceptions.py +++ b/granule_ingester/granule_ingester/exceptions/Exceptions.py @@ -6,7 +6,11 @@ class PipelineRunningError(Exception): pass -class TileProcessingError(Exception): +class TileProcessingError(PipelineRunningError): + pass + + +class GranuleLoadingError(PipelineRunningError): pass @@ -21,6 +25,7 @@ class RabbitMQLostConnectionError(LostConnectionError): class CassandraLostConnectionError(LostConnectionError): pass + class SolrLostConnectionError(LostConnectionError): pass diff --git a/granule_ingester/granule_ingester/exceptions/__init__.py b/granule_ingester/granule_ingester/exceptions/__init__.py index ea0969f..f2429b1 100644 --- a/granule_ingester/granule_ingester/exceptions/__init__.py +++ b/granule_ingester/granule_ingester/exceptions/__init__.py @@ -1,11 +1,8 @@ -from .Exceptions import CassandraFailedHealthCheckError -from .Exceptions import CassandraLostConnectionError -from .Exceptions import FailedHealthCheckError -from .Exceptions import LostConnectionError -from .Exceptions import PipelineBuildingError -from .Exceptions import PipelineRunningError -from .Exceptions import RabbitMQFailedHealthCheckError -from .Exceptions import RabbitMQLostConnectionError -from .Exceptions import SolrFailedHealthCheckError -from .Exceptions import SolrLostConnectionError -from .Exceptions import TileProcessingError +from .Exceptions import (CassandraFailedHealthCheckError, + CassandraLostConnectionError, FailedHealthCheckError, + GranuleLoadingError, LostConnectionError, + PipelineBuildingError, PipelineRunningError, + RabbitMQFailedHealthCheckError, + RabbitMQLostConnectionError, + SolrFailedHealthCheckError, SolrLostConnectionError, + TileProcessingError) diff --git a/granule_ingester/granule_ingester/granule_loaders/GranuleLoader.py b/granule_ingester/granule_ingester/granule_loaders/GranuleLoader.py index c28ffbb..0311f49 100644 --- a/granule_ingester/granule_ingester/granule_loaders/GranuleLoader.py +++ b/granule_ingester/granule_ingester/granule_loaders/GranuleLoader.py @@ -21,6 +21,8 @@ from urllib import parse import aioboto3 import xarray as xr +from granule_ingester.exceptions import GranuleLoadingError + logger = logging.getLogger(__name__) @@ -52,7 +54,10 @@ class GranuleLoader: raise RuntimeError("Granule path scheme '{}' is not supported.".format(resource_url.scheme)) granule_name = os.path.basename(self._resource) - return xr.open_dataset(file_path, lock=False), granule_name + try: + return xr.open_dataset(file_path, lock=False), granule_name + except Exception: + raise GranuleLoadingError(f"The granule {self._resource} is not a valid NetCDF file.") @staticmethod async def _download_s3_file(url: str):
