>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21316?usp=email )
( 3 is the latest approved patch-set. No files were changed between the latest approved patch-set and the submitted one. )Change subject: [NO ISSUE][EXT]: fixes for azure ...................................................................... [NO ISSUE][EXT]: fixes for azure Details: - use isDirectory to filter out directories for Data Lake. - build the client property to get the endpoint from the client itsel. Data Lake fails if no credentials are passed, i.e., does not allow using anonymous authentication. Ext-ref: MB-72274 Change-Id: I4391e6f749896d2f1c511f9a8022dbe26076bf54 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21316 Reviewed-by: Michael Blow <[email protected]> Tested-by: Hussain Towaileb <[email protected]> Integration-Tests: Hussain Towaileb <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> --- M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java 3 files changed, 13 insertions(+), 27 deletions(-) Approvals: Jenkins: Verified Hussain Towaileb: Looks good to me, but someone else must approve; Verified; Verified Michael Blow: Looks good to me, approved diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index dba9bd7..7eb1b9c 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -1144,7 +1144,7 @@ public static boolean evaluate(String key, BiPredicate<List<Matcher>, String> predicate, List<Matcher> matchers, ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator, IWarningCollector warningCollector) throws HyracksDataException { - return !key.endsWith("/") && predicate.test(matchers, key) + return !key.isEmpty() && !key.endsWith("/") && predicate.test(matchers, key) && externalDataPrefix.evaluate(key, evaluator, warningCollector); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java index a8d17a2..1fe137d 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java @@ -361,18 +361,8 @@ } public static String getEndpointFromClient(Map<String, String> configuration) throws CompilationException { - String endpoint = configuration.get(ENDPOINT_FIELD_NAME); - if (endpoint == null) { - throw new CompilationException(PARAMETERS_REQUIRED, ENDPOINT_FIELD_NAME); - } - - BlobServiceClientBuilder builder = new BlobServiceClientBuilder(); - try { - builder.endpoint(endpoint); - } catch (Exception ex) { - throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex)); - } - return AzureUtils.extractEndPoint(builder.buildClient().getAccountUrl()); + BlobServiceClient client = buildClient(null, configuration); + return AzureUtils.extractEndPoint(client.getAccountUrl()); } public static boolean isBlobAdapter(String type) { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java index b469678..d16213b 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java @@ -54,7 +54,6 @@ import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.ExternalDataPrefix; -import org.apache.asterix.external.util.ExternalDataUtils; import org.apache.asterix.external.util.azure.AzureConstants; import org.apache.asterix.external.util.azure.AzureUtils; import org.apache.asterix.external.util.iceberg.IcebergUtils; @@ -278,12 +277,19 @@ IWarningCollector warningCollector) throws HyracksDataException { for (PathItem item : items) { String uri = item.getName(); - if (ExternalDataUtils.evaluate(uri, predicate, matchers, externalDataPrefix, evaluator, warningCollector)) { + if (evaluate(item, uri, predicate, matchers, externalDataPrefix, evaluator, warningCollector)) { filesOnly.add(item); } } } + private static boolean evaluate(PathItem item, String key, BiPredicate<List<Matcher>, String> predicate, + List<Matcher> matchers, ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator, + IWarningCollector warningCollector) throws HyracksDataException { + return !item.isDirectory() && predicate.test(matchers, key) + && externalDataPrefix.evaluate(key, evaluator, warningCollector); + } + /** * Validate external dataset properties * @@ -330,18 +336,8 @@ } public static String getEndpointFromClient(Map<String, String> configuration) throws CompilationException { - String endpoint = configuration.get(ENDPOINT_FIELD_NAME); - if (endpoint == null) { - throw new CompilationException(PARAMETERS_REQUIRED, ENDPOINT_FIELD_NAME); - } - - DataLakeServiceClientBuilder builder = new DataLakeServiceClientBuilder(); - try { - builder.endpoint(endpoint); - } catch (Exception ex) { - throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex)); - } - return AzureUtils.extractEndPoint(builder.buildClient().getAccountUrl()); + DataLakeServiceClient client = buildClient(null, configuration); + return AzureUtils.extractEndPoint(client.getAccountUrl()); } public static boolean isDatalakeAdapter(String adapter) { -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21316?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: lumina Gerrit-Change-Id: I4391e6f749896d2f1c511f9a8022dbe26076bf54 Gerrit-Change-Number: 21316 Gerrit-PatchSet: 5 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]>
