This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new b94390b Adding Shared Access resource support for azure (#12266)
b94390b is described below
commit b94390ba33fe4e1cb5588f166edeb94d76b2edcf
Author: Karan Kumar <[email protected]>
AuthorDate: Tue Feb 22 18:27:43 2022 +0530
Adding Shared Access resource support for azure (#12266)
Azure Blob storage has multiple modes of authentication. One of them is
Shared access resource
. This is very useful in cases when we do not want to add the account key
in the druid properties .
---
docs/development/extensions-core/azure.md | 3 +-
.../druid/storage/azure/AzureAccountConfig.java | 15 +++-
.../storage/azure/AzureStorageDruidModule.java | 46 ++++++++---
.../storage/azure/AzureStorageDruidModuleTest.java | 94 +++++++++++++++++++---
website/.spelling | 1 +
5 files changed, 134 insertions(+), 25 deletions(-)
diff --git a/docs/development/extensions-core/azure.md
b/docs/development/extensions-core/azure.md
index 116b3d5..d63e74d 100644
--- a/docs/development/extensions-core/azure.md
+++ b/docs/development/extensions-core/azure.md
@@ -33,7 +33,8 @@ To use this Apache Druid extension,
[include](../../development/extensions.md#lo
|--------|---------------|-----------|-------|
|`druid.storage.type`|azure||Must be set.|
|`druid.azure.account`||Azure Storage account name.|Must be set.|
-|`druid.azure.key`||Azure Storage account key.|Must be set.|
+|`druid.azure.key`||Azure Storage account key.|Optional. Either set key or
sharedAccessStorageToken but not both.|
+|`druid.azure.sharedAccessStorageToken`||Azure Shared Storage access
token|Optional. Either set key or sharedAccessStorageToken but not both.|
|`druid.azure.container`||Azure Storage container name.|Must be set.|
|`druid.azure.prefix`|A prefix string that will be prepended to the blob names
for the segments published to Azure deep storage| |""|
|`druid.azure.protocol`|the protocol to use|http or https|https|
diff --git
a/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java
b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java
index a6b0888..235ae6f 100644
---
a/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java
+++
b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java
@@ -41,9 +41,11 @@ public class AzureAccountConfig
private String account;
@JsonProperty
- @NotNull
private String key;
+ @JsonProperty
+ private String sharedAccessStorageToken;
+
@SuppressWarnings("unused") // Used by Jackson deserialization?
public void setProtocol(String protocol)
{
@@ -86,4 +88,15 @@ public class AzureAccountConfig
{
return key;
}
+
+ public String getSharedAccessStorageToken()
+ {
+ return sharedAccessStorageToken;
+ }
+
+ @SuppressWarnings("unused") // Used by Jackson deserialization?
+ public void setSharedAccessStorageToken(String sharedAccessStorageToken)
+ {
+ this.sharedAccessStorageToken = sharedAccessStorageToken;
+ }
}
diff --git
a/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorageDruidModule.java
b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorageDruidModule.java
index 2a1b373..e870aa0 100644
---
a/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorageDruidModule.java
+++
b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorageDruidModule.java
@@ -38,6 +38,7 @@ import org.apache.druid.guice.Binders;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.guice.LazySingleton;
import org.apache.druid.initialization.DruidModule;
+import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.storage.azure.blob.ListBlobItemHolderFactory;
@@ -52,7 +53,10 @@ public class AzureStorageDruidModule implements DruidModule
{
static final String SCHEME = "azure";
- public static final String STORAGE_CONNECTION_STRING =
"DefaultEndpointsProtocol=%s;AccountName=%s;AccountKey=%s";
+ public static final String
+ STORAGE_CONNECTION_STRING_WITH_KEY =
"DefaultEndpointsProtocol=%s;AccountName=%s;AccountKey=%s";
+ public static final String
+ STORAGE_CONNECTION_STRING_WITH_TOKEN =
"DefaultEndpointsProtocol=%s;AccountName=%s;SharedAccessSignature=%s";
public static final String INDEX_ZIP_FILE_NAME = "index.zip";
@Override
@@ -125,17 +129,39 @@ public class AzureStorageDruidModule implements
DruidModule
@LazySingleton
public Supplier<CloudBlobClient> getCloudBlobClient(final AzureAccountConfig
config)
{
+ if ((config.getKey() != null && config.getSharedAccessStorageToken() !=
null)
+ ||
+ (config.getKey() == null && config.getSharedAccessStorageToken() ==
null)) {
+ throw new ISE("Either set 'key' or 'sharedAccessStorageToken' in the
azure config but not both."
+ + " Please refer to azure documentation.");
+ }
return Suppliers.memoize(() -> {
try {
- CloudStorageAccount account = CloudStorageAccount.parse(
- StringUtils.format(
- STORAGE_CONNECTION_STRING,
- config.getProtocol(),
- config.getAccount(),
- config.getKey()
- )
- );
- return account.createCloudBlobClient();
+ final CloudStorageAccount account;
+ if (config.getKey() != null) {
+ account = CloudStorageAccount.parse(
+ StringUtils.format(
+ STORAGE_CONNECTION_STRING_WITH_KEY,
+ config.getProtocol(),
+ config.getAccount(),
+ config.getKey()
+ )
+
+ );
+ return account.createCloudBlobClient();
+ } else if (config.getSharedAccessStorageToken() != null) {
+ account = CloudStorageAccount.parse(StringUtils.format(
+ STORAGE_CONNECTION_STRING_WITH_TOKEN,
+ config.getProtocol(),
+ config.getAccount(),
+ config.getSharedAccessStorageToken()
+ ));
+ return account.createCloudBlobClient();
+ } else {
+ throw new ISE(
+ "None of 'key' or 'sharedAccessStorageToken' is set in the azure
config."
+ + " Please refer to azure extension documentation.");
+ }
}
catch (URISyntaxException | InvalidKeyException e) {
throw new RuntimeException(e);
diff --git
a/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureStorageDruidModuleTest.java
b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureStorageDruidModuleTest.java
index c8ce639..d811000 100644
---
a/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureStorageDruidModuleTest.java
+++
b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureStorageDruidModuleTest.java
@@ -26,6 +26,7 @@ import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Module;
+import com.google.inject.ProvisionException;
import com.google.inject.TypeLiteral;
import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.azure.storage.blob.CloudBlobClient;
@@ -43,7 +44,9 @@ import org.easymock.EasyMock;
import org.easymock.EasyMockSupport;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import javax.validation.Validation;
import javax.validation.Validator;
@@ -54,8 +57,12 @@ import java.util.Properties;
public class AzureStorageDruidModuleTest extends EasyMockSupport
{
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
private static final String AZURE_ACCOUNT_NAME;
private static final String AZURE_ACCOUNT_KEY;
+ private static final String AZURE_SHARED_ACCESS_TOKEN;
private static final String AZURE_CONTAINER;
private static final String AZURE_PREFIX;
private static final int AZURE_MAX_LISTING_LENGTH;
@@ -67,8 +74,6 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
private CloudObjectLocation cloudObjectLocation2;
private ListBlobItem blobItem1;
private ListBlobItem blobItem2;
-
-
private Injector injector;
static {
@@ -76,6 +81,7 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
AZURE_ACCOUNT_NAME = "azureAccount1";
AZURE_ACCOUNT_KEY = Base64.getUrlEncoder()
.encodeToString("azureKey1".getBytes(StandardCharsets.UTF_8.toString()));
+ AZURE_SHARED_ACCESS_TOKEN = "dummyToken";
AZURE_CONTAINER = "azureContainer1";
AZURE_PREFIX = "azurePrefix1";
AZURE_MAX_LISTING_LENGTH = 10;
@@ -96,7 +102,7 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void test_getAzureAccountConfig_expectedConfig()
+ public void testGetAzureAccountConfigExpectedConfig()
{
injector = makeInjectorWithProperties(PROPERTIES);
AzureAccountConfig azureAccountConfig =
injector.getInstance(AzureAccountConfig.class);
@@ -106,7 +112,21 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void test_getAzureDataSegmentConfig_expectedConfig()
+ public void testGetAzureAccountConfigExpectedConfigWithSAS()
+ {
+ Properties properties = initializePropertes();
+ properties.setProperty("druid.azure.sharedAccessStorageToken",
AZURE_SHARED_ACCESS_TOKEN);
+ properties.remove("druid.azure.key");
+
+ injector = makeInjectorWithProperties(properties);
+ AzureAccountConfig azureAccountConfig =
injector.getInstance(AzureAccountConfig.class);
+
+ Assert.assertEquals(AZURE_ACCOUNT_NAME, azureAccountConfig.getAccount());
+ Assert.assertEquals(AZURE_SHARED_ACCESS_TOKEN,
azureAccountConfig.getSharedAccessStorageToken());
+ }
+
+ @Test
+ public void testGetAzureDataSegmentConfigExpectedConfig()
{
injector = makeInjectorWithProperties(PROPERTIES);
AzureDataSegmentConfig segmentConfig =
injector.getInstance(AzureDataSegmentConfig.class);
@@ -116,7 +136,7 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void test_getAzureInputDataConfig_expectedConfig()
+ public void testGetAzureInputDataConfigExpectedConfig()
{
injector = makeInjectorWithProperties(PROPERTIES);
AzureInputDataConfig inputDataConfig =
injector.getInstance(AzureInputDataConfig.class);
@@ -125,7 +145,7 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void test_getBlobClient_expectedClient()
+ public void testGetBlobClientExpectedClient()
{
injector = makeInjectorWithProperties(PROPERTIES);
@@ -138,7 +158,7 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void test_getAzureStorageContainer_expectedClient()
+ public void testGetAzureStorageContainerExpectedClient()
{
injector = makeInjectorWithProperties(PROPERTIES);
@@ -154,7 +174,27 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void test_getAzureCloudBlobToLocationConverter_expectedConverted()
+ public void testGetAzureStorageContainerWithSASExpectedClient()
+ {
+ Properties properties = initializePropertes();
+ properties.setProperty("druid.azure.sharedAccessStorageToken",
AZURE_SHARED_ACCESS_TOKEN);
+ properties.remove("druid.azure.key");
+
+ injector = makeInjectorWithProperties(properties);
+
+ Supplier<CloudBlobClient> cloudBlobClient = injector.getInstance(
+ Key.get(new TypeLiteral<Supplier<CloudBlobClient>>(){})
+ );
+
+ AzureAccountConfig azureAccountConfig =
injector.getInstance(AzureAccountConfig.class);
+ Assert.assertEquals(AZURE_SHARED_ACCESS_TOKEN,
azureAccountConfig.getSharedAccessStorageToken());
+
+ AzureStorage azureStorage = injector.getInstance(AzureStorage.class);
+ Assert.assertSame(cloudBlobClient.get(),
azureStorage.getCloudBlobClient());
+ }
+
+ @Test
+ public void testGetAzureCloudBlobToLocationConverterExpectedConverted()
{
injector = makeInjectorWithProperties(PROPERTIES);
AzureCloudBlobHolderToCloudObjectLocationConverter
azureCloudBlobLocationConverter1 = injector.getInstance(
@@ -165,7 +205,7 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void test_getAzureByteSourceFactory_canCreateAzureByteSource()
+ public void testGetAzureByteSourceFactoryCanCreateAzureByteSource()
{
injector = makeInjectorWithProperties(PROPERTIES);
AzureByteSourceFactory factory =
injector.getInstance(AzureByteSourceFactory.class);
@@ -177,7 +217,7 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void test_getAzureEntityFactory_canCreateAzureEntity()
+ public void testGetAzureEntityFactoryCanCreateAzureEntity()
{
EasyMock.expect(cloudObjectLocation1.getBucket()).andReturn(AZURE_CONTAINER);
EasyMock.expect(cloudObjectLocation2.getBucket()).andReturn(AZURE_CONTAINER);
@@ -195,7 +235,7 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void
test_getAzureCloudBlobIteratorFactory_canCreateAzureCloudBlobIterator()
+ public void
testGetAzureCloudBlobIteratorFactoryCanCreateAzureCloudBlobIterator()
{
injector = makeInjectorWithProperties(PROPERTIES);
AzureCloudBlobIteratorFactory factory =
injector.getInstance(AzureCloudBlobIteratorFactory.class);
@@ -207,7 +247,7 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void
test_getAzureCloudBlobIterableFactory_canCreateAzureCloudBlobIterable()
+ public void
testGetAzureCloudBlobIterableFactoryCanCreateAzureCloudBlobIterable()
{
injector = makeInjectorWithProperties(PROPERTIES);
AzureCloudBlobIterableFactory factory =
injector.getInstance(AzureCloudBlobIterableFactory.class);
@@ -219,7 +259,7 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
}
@Test
- public void test_getListBlobItemDruidFactory_canCreateListBlobItemDruid()
+ public void testGetListBlobItemDruidFactoryCanCreateListBlobItemDruid()
{
injector = makeInjectorWithProperties(PROPERTIES);
ListBlobItemHolderFactory factory =
injector.getInstance(ListBlobItemHolderFactory.class);
@@ -246,6 +286,34 @@ public class AzureStorageDruidModuleTest extends
EasyMockSupport
);
}
+ @Test
+ public void testBothAccountKeyAndSAStokenSet()
+ {
+ Properties properties = initializePropertes();
+ properties.setProperty("druid.azure.sharedAccessStorageToken",
AZURE_SHARED_ACCESS_TOKEN);
+ expectedException.expect(ProvisionException.class);
+ expectedException.expectMessage("Either set 'key' or
'sharedAccessStorageToken' in the azure config but not both");
+ makeInjectorWithProperties(properties).getInstance(
+ Key.get(new TypeLiteral<Supplier<CloudBlobClient>>()
+ {
+ })
+ );
+ }
+
+ @Test
+ public void testBothAccountKeyAndSAStokenUnset()
+ {
+ Properties properties = initializePropertes();
+ properties.remove("druid.azure.key");
+ expectedException.expect(ProvisionException.class);
+ expectedException.expectMessage("Either set 'key' or
'sharedAccessStorageToken' in the azure config but not both");
+ makeInjectorWithProperties(properties).getInstance(
+ Key.get(new TypeLiteral<Supplier<CloudBlobClient>>()
+ {
+ })
+ );
+ }
+
private Injector makeInjectorWithProperties(final Properties props)
{
return Guice.createInjector(
diff --git a/website/.spelling b/website/.spelling
index f67272f..e3bc69e 100644
--- a/website/.spelling
+++ b/website/.spelling
@@ -629,6 +629,7 @@ maxFetchCapacityBytes
maxFetchRetry
prefetchTriggerBytes
shardSpecs
+sharedAccessStorageToken
- ../docs/development/extensions-contrib/cloudfiles.md
StaticCloudFilesFirehose
cloudfiles
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
