[asterixdb] branch master updated: [NO ISSUE][EXT][AZDL]: Skip instead of failing when json file not found

[htowaileb]

This is an automated email from the ASF dual-hosted git repository.

htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 6b099b0  [NO ISSUE][EXT][AZDL]: Skip instead of failing when json file 
not found
6b099b0 is described below

commit 6b099b0fb0fbea84d237514b6035d84334921197
Author: Hussain Towaileb <hussain.towai...@couchbase.com>
AuthorDate: Fri Dec 17 17:25:11 2021 +0300

    [NO ISSUE][EXT][AZDL]: Skip instead of failing when json file not found
    
    Details:
    - When a JSON file is not found while reading external azure
      datalake dataset, skip the file and continue reading, do
      not fail.
    
    Change-Id: Ic9b04e418483cc245379e35c9a20f1a4c4389e87
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14583
    Tested-by: Hussain Towaileb <hussai...@gmail.com>
    Integration-Tests: Hussain Towaileb <hussai...@gmail.com>
    Reviewed-by: Hussain Towaileb <hussai...@gmail.com>
---
 .../input/record/reader/azure/datalake/AzureDataLakeInputStream.java | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
index f0c185e..b7d142f 100644
--- 
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
+++ 
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/datalake/AzureDataLakeInputStream.java
@@ -35,10 +35,10 @@ import 
org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.util.LogRedactionUtil;
 
 import com.azure.storage.blob.models.BlobErrorCode;
+import com.azure.storage.blob.models.BlobStorageException;
 import com.azure.storage.file.datalake.DataLakeFileClient;
 import com.azure.storage.file.datalake.DataLakeFileSystemClient;
 import com.azure.storage.file.datalake.DataLakeServiceClient;
-import com.azure.storage.file.datalake.models.DataLakeStorageException;
 
 public class AzureDataLakeInputStream extends AbstractExternalInputStream {
 
@@ -67,8 +67,7 @@ public class AzureDataLakeInputStream extends 
AbstractExternalInputStream {
             if (lowerCaseFileName.endsWith(".gz") || 
lowerCaseFileName.endsWith(".gzip")) {
                 in = new GZIPInputStream(in, 
ExternalDataConstants.DEFAULT_BUFFER_SIZE);
             }
-        } catch (DataLakeStorageException ex) {
-            // TODO(htowaileb): need to find the right error for Azure Data 
Lake
+        } catch (BlobStorageException ex) {
             if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
                 LOGGER.debug(() -> "Key " + 
LogRedactionUtil.userData(filePaths.get(nextFileIndex)) + " was not "
                         + "found in container " + container);