This is an automated email from the ASF dual-hosted git repository. alsuliman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 9263bca316bbf183568e32979aec0f089952ccd0 Author: Hussain Towaileb <[email protected]> AuthorDate: Fri Feb 3 12:09:01 2023 +0300 [ASTERIXDB-3112][EXT]: Add support to GCS Default Authentictation Provider Details: - Selecting default authentication provider enables the SDK to search for credentials in the following locatinos (in order): - environment variable - gcloud cli credentials file - attach service account to google compute engine - attach service account to google app engine Change-Id: I93eb3a19d799f70c00c0a40efe3d484c1967ad3e Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17379 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> --- .../asterix/common/exceptions/ErrorCode.java | 3 +- .../src/main/resources/asx_errormsg/en.properties | 3 +- .../external/util/ExternalDataConstants.java | 1 + .../asterix/external/util/ExternalDataUtils.java | 35 ++++++++++++++++++++-- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index db47df6a18..44471a299d 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -265,7 +265,8 @@ public enum ErrorCode implements IError { S3_REGION_NOT_SUPPORTED(1170), COMPILATION_SET_OPERATION_ERROR(1171), INVALID_TIMEZONE(1172), - SAMPLE_HAS_ZERO_ROWS(1173), + INVALID_PARAM_VALUE_ALLOWED_VALUE(1173), + SAMPLE_HAS_ZERO_ROWS(1174), // Feed errors DATAFLOW_ILLEGAL_STATE(3001), diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 4087bf9ee2..c9ab080646 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -267,7 +267,8 @@ 1170 = Provided S3 region is not supported: '%1$s' 1171 = Unable to process %1$s clause. %2$s 1172 = Provided timezone is invalid: '%1$s' -1173 = Sample has zero rows +1173 = Invalid value for parameter '%1$s', allowed value(s): %2$s +1174 = Sample has zero rows # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java index f0b9c90428..2097b4bcff 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java @@ -434,6 +434,7 @@ public class ExternalDataConstants { throw new AssertionError("do not instantiate"); } + public static final String APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME = "applicationDefaultCredentials"; public static final String JSON_CREDENTIALS_FIELD_NAME = "jsonCredentials"; } } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 8e38eed270..ba07629b05 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -21,6 +21,7 @@ package org.apache.asterix.external.util; import static com.google.cloud.storage.Storage.BlobListOption; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR; +import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE; import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_REQ_PARAM_VAL; import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_NOT_ALLOWED_AT_SAME_TIME; import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_REQUIRED; @@ -54,6 +55,7 @@ import static org.apache.asterix.external.util.ExternalDataConstants.Azure.MANAG import static org.apache.asterix.external.util.ExternalDataConstants.Azure.RECURSIVE_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.Azure.SHARED_ACCESS_SIGNATURE_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.Azure.TENANT_ID_FIELD_NAME; +import static org.apache.asterix.external.util.ExternalDataConstants.GCS.APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.GCS.JSON_CREDENTIALS_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ADAPTER_NAME_GCS; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER; @@ -147,7 +149,7 @@ import com.azure.storage.file.datalake.DataLakeServiceClientBuilder; import com.azure.storage.file.datalake.models.ListPathsOptions; import com.azure.storage.file.datalake.models.PathItem; import com.google.api.gax.paging.Page; -import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.storage.Blob; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; @@ -1838,19 +1840,46 @@ public class ExternalDataUtils { * @throws CompilationException CompilationException */ public static Storage buildClient(Map<String, String> configuration) throws CompilationException { + String applicationDefaultCredentials = configuration.get(APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME); String jsonCredentials = configuration.get(JSON_CREDENTIALS_FIELD_NAME); + String endpoint = configuration.get(ENDPOINT_FIELD_NAME); StorageOptions.Builder builder = StorageOptions.newBuilder(); - // Use credentials if available + // default credentials provider + if (applicationDefaultCredentials != null) { + // only "true" value is allowed + if (!applicationDefaultCredentials.equalsIgnoreCase("true")) { + throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, + APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME, "true"); + } + + // no other authentication parameters are allowed + if (jsonCredentials != null) { + throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, JSON_CREDENTIALS_FIELD_NAME, + APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME); + } + + try { + builder.setCredentials(GoogleCredentials.getApplicationDefault()); + } catch (IOException ex) { + throw CompilationException.create(EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex)); + } + } + + // json credentials if (jsonCredentials != null) { try (InputStream credentialsStream = new ByteArrayInputStream(jsonCredentials.getBytes())) { - builder.setCredentials(ServiceAccountCredentials.fromStream(credentialsStream)); + builder.setCredentials(GoogleCredentials.fromStream(credentialsStream)); } catch (IOException ex) { throw new CompilationException(EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex)); } } + if (endpoint != null) { + builder.setHost(endpoint); + } + return builder.build().getService(); }
