>From Michael Blow <[email protected]>: Michael Blow has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21206?usp=email )
Change subject: [NO ISSUE][MISC] Force compatible service version when Azurite ...................................................................... [NO ISSUE][MISC] Force compatible service version when Azurite Details: - Set BlobServiceVersion.V2025_11_05 when using Azurite, to enable upgrades to SDKs which default to a newer version. Ext-ref: MB-71737 Change-Id: I149f7443f4be9845dbafb7bddfad9186a282d395 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21206 Reviewed-by: Hussain Towaileb <[email protected]> Tested-by: Michael Blow <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageAzTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java M asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/AzureUtils.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java 6 files changed, 39 insertions(+), 4 deletions(-) Approvals: Hussain Towaileb: Looks good to me, approved Michael Blow: Verified diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageAzTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageAzTest.java index 659e8cf..5ce1f433 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageAzTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageAzTest.java @@ -55,6 +55,7 @@ import com.azure.core.http.netty.NettyAsyncHttpClientBuilder; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.BlobServiceVersion; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; @@ -94,7 +95,7 @@ SslContext insecureSslContext = SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build(); BlobServiceClient blobServiceClient = new BlobServiceClientBuilder() - .connectionString(azBlob.getConnectionString()) + .serviceVersion(BlobServiceVersion.V2025_07_05).connectionString(azBlob.getConnectionString()) .httpClient(new NettyAsyncHttpClientBuilder( HttpClient.create().secure(sslSpec -> sslSpec.sslContext(insecureSslContext).build())).build()) .buildClient(); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java index 4ecf973..1b069ad6 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java @@ -75,6 +75,7 @@ import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.BlobServiceVersion; import com.azure.storage.blob.models.PublicAccessType; import com.azure.storage.common.sas.AccountSasPermission; import com.azure.storage.common.sas.AccountSasResourceType; @@ -213,6 +214,7 @@ builder.connectionString(azuriteContainer.getConnectionString()); builder.httpClient(new NettyAsyncHttpClientBuilder( HttpClient.create().secure(sslSpec -> sslSpec.sslContext(insecureSslContext).build())).build()); + builder.serviceVersion(BlobServiceVersion.V2025_07_05); blobServiceClient = builder.buildClient(); LOGGER.info("Azurite Blob Service client created successfully"); diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java index e9cc098..e9e4131 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java @@ -19,6 +19,7 @@ package org.apache.asterix.cloud.clients.azure.blobstorage; +import static org.apache.asterix.external.util.azure.AzureUtils.IS_AZURITE_EMULATOR; import static org.apache.asterix.external.util.azure.blob.BlobUtils.disableSslVerify; import java.io.ByteArrayInputStream; @@ -52,6 +53,7 @@ import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.exceptions.RuntimeDataException; import org.apache.asterix.external.util.azure.AzureConstants; +import org.apache.asterix.external.util.azure.AzureUtils; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.cloud.io.ICloudProperties; @@ -433,6 +435,9 @@ BlobServiceClientBuilder blobServiceClientBuilder = new BlobServiceClientBuilder(); blobServiceClientBuilder.endpoint(getEndpoint(config)); blobServiceClientBuilder.httpLogOptions(AzureConstants.HTTP_LOG_OPTIONS); + if (IS_AZURITE_EMULATOR) { + AzureUtils.fixupForAzuriteEmulator(blobServiceClientBuilder); + } configCredentialsToAzClient(blobServiceClientBuilder, config); ConnectionProvider.Builder builder = ConnectionProvider.builder("azblob-client-" + (async ? "async" : "sync")) @@ -460,8 +465,8 @@ private static void configCredentialsToAzClient(BlobServiceClientBuilder builder, AzBlobStorageClientConfig config) { - String storageAccount = System.getenv("AZURE_STORAGE_ACCOUNT"); - String storageKey = System.getenv("AZURE_STORAGE_KEY"); + String storageAccount = System.getenv(AzureUtils.AZURE_STORAGE_ACCOUNT); + String storageKey = System.getenv(AzureUtils.AZURE_STORAGE_KEY); if (storageAccount != null && storageKey != null) { builder.credential(new StorageSharedKeyCredential(storageAccount, storageKey)); diff --git a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java index dc0e4f4..94ed6dc 100644 --- a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java +++ b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java @@ -35,6 +35,7 @@ import org.apache.asterix.cloud.clients.ICloudGuardian; import org.apache.asterix.cloud.clients.azure.blobstorage.AzBlobStorageClientConfig; import org.apache.asterix.cloud.clients.azure.blobstorage.AzBlobStorageCloudClient; +import org.apache.asterix.external.util.azure.AzureUtils; import org.apache.hyracks.util.StorageUtil; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo; @@ -105,6 +106,7 @@ SslContext insecureSslContext = SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build(); blobServiceClient = new BlobServiceClientBuilder().connectionString(azBlob.getConnectionString()) + .serviceVersion(AzureUtils.AZURITE_BLOB_SERVICE_VERSION) .httpClient(new NettyAsyncHttpClientBuilder( HttpClient.create().secure(sslSpec -> sslSpec.sslContext(insecureSslContext).build())).build()) .buildClient(); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/AzureUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/AzureUtils.java index d8c09bb..1fa6852 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/AzureUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/AzureUtils.java @@ -47,7 +47,26 @@ import org.apache.asterix.external.util.HDFSUtils; import org.apache.hadoop.mapred.JobConf; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.BlobServiceVersion; +import com.azure.storage.common.implementation.Constants; + public class AzureUtils { + public static final String AZURE_STORAGE_ACCOUNT = "AZURE_STORAGE_ACCOUNT"; + public static final String AZURE_STORAGE_KEY = "AZURE_STORAGE_KEY"; + + // MB-71737: Azurite does not support the current service version + public static final BlobServiceVersion AZURITE_BLOB_SERVICE_VERSION = BlobServiceVersion.V2025_07_05; + + public static final boolean IS_AZURITE_EMULATOR; + + static { + String storageAccount = System.getenv(AZURE_STORAGE_ACCOUNT); + String storageKey = System.getenv(AZURE_STORAGE_KEY); + IS_AZURITE_EMULATOR = Constants.ConnectionStringConstants.EMULATOR_ACCOUNT_NAME.equals(storageAccount) + && Constants.ConnectionStringConstants.EMULATOR_ACCOUNT_KEY.equals(storageKey); + } + private AzureUtils() { throw new AssertionError("do not instantiate"); } @@ -119,4 +138,8 @@ //We need to Remove the protocol (i.e., http(s)://) from the URI return uri.substring(uri.indexOf("//") + "//".length()); } + + public static void fixupForAzuriteEmulator(BlobServiceClientBuilder blobServiceClientBuilder) { + blobServiceClientBuilder.serviceVersion(AZURITE_BLOB_SERVICE_VERSION); + } } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java index 72a5def..a8d17a2 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java @@ -214,7 +214,9 @@ provided.get()); } } - + if (AzureUtils.IS_AZURITE_EMULATOR) { + AzureUtils.fixupForAzuriteEmulator(builder); + } try { if (type == BlobServiceClient.class) { return type.cast(builder.buildClient()); -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21206?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: lumina Gerrit-Change-Id: I149f7443f4be9845dbafb7bddfad9186a282d395 Gerrit-Change-Number: 21206 Gerrit-PatchSet: 5 Gerrit-Owner: Janhavi Tripurwar <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]>
