This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 619b8c53b95bb4a67a6a527fbf41a63b0522173e Author: Hussain Towaileb <[email protected]> AuthorDate: Thu Jun 11 17:49:45 2020 +0300 [ASTERIXDB-2743][EXT] Skip not found files when querying S3 external dataset - user model changes: no - storage format changes: no - interface changes: no Details: - When querying an S3 external dataset, if some files are not found due to possiblity getting deleted while querying, skip the filies and continue to the next ones instead of failing. Change-Id: I6d3e691a1714228f1844faf05095cf8aa439ea1d Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6723 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- .../external/input/record/reader/aws/AwsS3InputStream.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java index bcbf540..9e10e6a 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java @@ -33,13 +33,19 @@ import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.ExternalDataUtils; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.util.CleanupUtils; +import org.apache.hyracks.util.LogRedactionUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; public class AwsS3InputStream extends AbstractMultipleInputStream { + private static final Logger LOGGER = LogManager.getLogger(); + // Configuration private final Map<String, String> configuration; @@ -83,6 +89,11 @@ public class AwsS3InputStream extends AbstractMultipleInputStream { // the header, then the S3 stream gets closed in the close method try { in = s3Client.getObject(getObjectRequest); + } catch (NoSuchKeyException ex) { + LOGGER.debug(() -> "Key " + LogRedactionUtil.userData(getObjectRequest.key()) + " was not found in bucket " + + getObjectRequest.bucket()); + nextFileIndex++; + return advance(); } catch (SdkException ex) { throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage()); }
