>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21316?usp=email )
Change subject: [ASTERIXDB-XXXX][EXT]: fixes for azure
......................................................................
[ASTERIXDB-XXXX][EXT]: fixes for azure
Details:
- use isDirectory to filter out directories
for datalake.
- build the client property to get the endpoing
from the client itself. datalake fails if no credentials
are passed, i.e., does not allow using anonymous authentication.
Ext-ref: MB-XXXXX
Change-Id: I4391e6f749896d2f1c511f9a8022dbe26076bf54
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java
2 files changed, 12 insertions(+), 26 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/16/21316/1
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java
index a8d17a2..1fe137d 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java
@@ -361,18 +361,8 @@
}
public static String getEndpointFromClient(Map<String, String>
configuration) throws CompilationException {
- String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
- if (endpoint == null) {
- throw new CompilationException(PARAMETERS_REQUIRED,
ENDPOINT_FIELD_NAME);
- }
-
- BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
- try {
- builder.endpoint(endpoint);
- } catch (Exception ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR,
ex, getMessageOrToString(ex));
- }
- return
AzureUtils.extractEndPoint(builder.buildClient().getAccountUrl());
+ BlobServiceClient client = buildClient(null, configuration);
+ return AzureUtils.extractEndPoint(client.getAccountUrl());
}
public static boolean isBlobAdapter(String type) {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java
index b469678..d16213b 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java
@@ -54,7 +54,6 @@
import
org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataPrefix;
-import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.azure.AzureConstants;
import org.apache.asterix.external.util.azure.AzureUtils;
import org.apache.asterix.external.util.iceberg.IcebergUtils;
@@ -278,12 +277,19 @@
IWarningCollector warningCollector) throws HyracksDataException {
for (PathItem item : items) {
String uri = item.getName();
- if (ExternalDataUtils.evaluate(uri, predicate, matchers,
externalDataPrefix, evaluator, warningCollector)) {
+ if (evaluate(item, uri, predicate, matchers, externalDataPrefix,
evaluator, warningCollector)) {
filesOnly.add(item);
}
}
}
+ private static boolean evaluate(PathItem item, String key,
BiPredicate<List<Matcher>, String> predicate,
+ List<Matcher> matchers, ExternalDataPrefix externalDataPrefix,
IExternalFilterEvaluator evaluator,
+ IWarningCollector warningCollector) throws HyracksDataException {
+ return !item.isDirectory() && predicate.test(matchers, key)
+ && externalDataPrefix.evaluate(key, evaluator,
warningCollector);
+ }
+
/**
* Validate external dataset properties
*
@@ -330,18 +336,8 @@
}
public static String getEndpointFromClient(Map<String, String>
configuration) throws CompilationException {
- String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
- if (endpoint == null) {
- throw new CompilationException(PARAMETERS_REQUIRED,
ENDPOINT_FIELD_NAME);
- }
-
- DataLakeServiceClientBuilder builder = new
DataLakeServiceClientBuilder();
- try {
- builder.endpoint(endpoint);
- } catch (Exception ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR,
ex, getMessageOrToString(ex));
- }
- return
AzureUtils.extractEndPoint(builder.buildClient().getAccountUrl());
+ DataLakeServiceClient client = buildClient(null, configuration);
+ return AzureUtils.extractEndPoint(client.getAccountUrl());
}
public static boolean isDatalakeAdapter(String adapter) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21316?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: lumina
Gerrit-Change-Id: I4391e6f749896d2f1c511f9a8022dbe26076bf54
Gerrit-Change-Number: 21316
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
