HADOOP-15694. ABFS: Allow OAuth credentials to not be tied to accounts. Contributed by Sean Mackrory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5593cbd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5593cbd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5593cbd Branch: refs/heads/HADOOP-15407 Commit: e5593cbd8323399359b3e8da46bd58e8364cbf22 Parents: 13c70e9 Author: Thomas Marquardt <tm...@microsoft.com> Authored: Wed Sep 12 22:51:41 2018 +0000 Committer: Thomas Marquardt <tm...@microsoft.com> Committed: Mon Sep 17 19:54:01 2018 +0000 ---------------------------------------------------------------------- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 208 ++++++++++---- .../fs/azurebfs/AzureBlobFileSystemStore.java | 69 ++--- .../azurebfs/constants/ConfigurationKeys.java | 44 +-- .../oauth2/UserPasswordTokenProvider.java | 10 - .../services/ShellDecryptionKeyProvider.java | 14 +- .../fs/azurebfs/services/SimpleKeyProvider.java | 18 +- .../hadoop-azure/src/site/markdown/abfs.md | 10 + .../azurebfs/AbstractAbfsIntegrationTest.java | 63 +++-- .../fs/azurebfs/AbstractAbfsScaleTest.java | 8 +- .../hadoop/fs/azurebfs/ITestAbfsClient.java | 6 +- .../fs/azurebfs/ITestAbfsReadWriteAndSeek.java | 4 +- .../ITestAzureBlobFileSystemAppend.java | 3 +- .../ITestAzureBlobFileSystemBackCompat.java | 3 +- .../azurebfs/ITestAzureBlobFileSystemCopy.java | 3 +- .../ITestAzureBlobFileSystemCreate.java | 3 +- .../ITestAzureBlobFileSystemDelete.java | 3 +- .../azurebfs/ITestAzureBlobFileSystemE2E.java | 5 +- .../ITestAzureBlobFileSystemE2EScale.java | 2 +- .../ITestAzureBlobFileSystemFileStatus.java | 2 +- .../ITestAzureBlobFileSystemFinalize.java | 8 +- .../azurebfs/ITestAzureBlobFileSystemFlush.java | 2 +- .../ITestAzureBlobFileSystemInitAndCreate.java | 2 +- .../ITestAzureBlobFileSystemListStatus.java | 3 +- .../azurebfs/ITestAzureBlobFileSystemMkDir.java | 3 +- .../azurebfs/ITestAzureBlobFileSystemOauth.java | 20 +- .../ITestAzureBlobFileSystemRename.java | 4 +- .../ITestAzureBlobFileSystemRenameUnicode.java | 2 +- .../azurebfs/ITestFileSystemInitialization.java | 8 +- .../fs/azurebfs/ITestFileSystemProperties.java | 2 +- .../azurebfs/ITestFileSystemRegistration.java | 15 +- .../TestAbfsConfigurationFieldsValidation.java | 24 +- .../fs/azurebfs/TestAccountConfiguration.java | 273 +++++++++++++++++++ .../constants/TestConfigurationKeys.java | 2 +- .../contract/ABFSContractTestBinding.java | 8 +- .../ITestAbfsFileSystemContractAppend.java | 2 +- .../ITestAbfsFileSystemContractConcat.java | 2 +- .../ITestAbfsFileSystemContractCreate.java | 2 +- .../ITestAbfsFileSystemContractDelete.java | 2 +- ...TestAbfsFileSystemContractGetFileStatus.java | 2 +- .../ITestAbfsFileSystemContractMkdir.java | 2 +- .../ITestAbfsFileSystemContractOpen.java | 2 +- .../ITestAbfsFileSystemContractRename.java | 2 +- ...TestAbfsFileSystemContractRootDirectory.java | 2 +- .../ITestAbfsFileSystemContractSeek.java | 2 +- .../ITestAbfsFileSystemContractSetTimes.java | 2 +- .../fs/azurebfs/services/TestAbfsClient.java | 10 +- .../TestShellDecryptionKeyProvider.java | 1 + .../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 5 + 48 files changed, 665 insertions(+), 227 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 518fef9..927a315 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -54,6 +54,7 @@ import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.azurebfs.services.KeyProvider; import org.apache.hadoop.fs.azurebfs.services.SimpleKeyProvider; import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx; +import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.util.ReflectionUtils; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*; @@ -65,7 +66,8 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.* @InterfaceAudience.Private @InterfaceStability.Evolving public class AbfsConfiguration{ - private final Configuration configuration; + private final Configuration rawConfig; + private final String accountName; private final boolean isSecure; @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_BUFFER_SIZE, @@ -155,9 +157,12 @@ public class AbfsConfiguration{ private Map<String, String> storageAccountKeys; - public AbfsConfiguration(final Configuration configuration) throws IllegalAccessException, InvalidConfigurationValueException { - this.configuration = configuration; - this.isSecure = this.configuration.getBoolean(FS_AZURE_SECURE_MODE, false); + public AbfsConfiguration(final Configuration rawConfig, String accountName) + throws IllegalAccessException, InvalidConfigurationValueException, IOException { + this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders( + rawConfig, AzureBlobFileSystem.class); + this.accountName = accountName; + this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false); validateStorageAccountKeys(); Field[] fields = this.getClass().getDeclaredFields(); @@ -177,14 +182,130 @@ public class AbfsConfiguration{ } } + /** + * Appends an account name to a configuration key yielding the + * account-specific form. + * @param key Account-agnostic configuration key + * @return Account-specific configuration key + */ + public String accountConf(String key) { + return key + "." + accountName; + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value. + * @param key Account-agnostic configuration key + * @return value if one exists, else null + */ + public String get(String key) { + return rawConfig.get(accountConf(key), rawConfig.get(key)); + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value, and finally tries the default value. + * @param key Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @return value if one exists, else the default value + */ + public boolean getBoolean(String key, boolean defaultValue) { + return rawConfig.getBoolean(accountConf(key), rawConfig.getBoolean(key, defaultValue)); + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value, and finally tries the default value. + * @param key Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @return value if one exists, else the default value + */ + public long getLong(String key, long defaultValue) { + return rawConfig.getLong(accountConf(key), rawConfig.getLong(key, defaultValue)); + } + + /** + * Returns the account-specific password in string form if it exists, then + * looks for an account-agnostic value. + * @param key Account-agnostic configuration key + * @return value in String form if one exists, else null + * @throws IOException + */ + public String getPasswordString(String key) throws IOException { + char[] passchars = rawConfig.getPassword(accountConf(key)); + if (passchars == null) { + passchars = rawConfig.getPassword(key); + } + if (passchars != null) { + return new String(passchars); + } + return null; + } + + /** + * Returns the account-specific Class if it exists, then looks for an + * account-agnostic value, and finally tries the default value. + * @param name Account-agnostic configuration key + * @param defaultValue Class returned if none is configured + * @param xface Interface shared by all possible values + * @return Highest-precedence Class object that was found + * @throws IOException + */ + public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface) { + return rawConfig.getClass(accountConf(name), + rawConfig.getClass(name, defaultValue, xface), + xface); + } + + /** + * Returns the account-specific password in string form if it exists, then + * looks for an account-agnostic value. + * @param name Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @return value in String form if one exists, else null + * @throws IOException + */ + public <T extends Enum<T>> T getEnum(String name, T defaultValue) { + return rawConfig.getEnum(accountConf(name), + rawConfig.getEnum(name, defaultValue)); + } + + /** + * Unsets parameter in the underlying Configuration object. + * Provided only as a convenience; does not add any account logic. + * @param key Configuration key + */ + public void unset(String key) { + rawConfig.unset(key); + } + + /** + * Sets String in the underlying Configuration object. + * Provided only as a convenience; does not add any account logic. + * @param key Configuration key + * @param value Configuration value + */ + public void set(String key, String value) { + rawConfig.set(key, value); + } + + /** + * Sets boolean in the underlying Configuration object. + * Provided only as a convenience; does not add any account logic. + * @param key Configuration key + * @param value Configuration value + */ + public void setBoolean(String key, boolean value) { + rawConfig.setBoolean(key, value); + } + public boolean isSecureMode() { return isSecure; } - public String getStorageAccountKey(final String accountName) throws AzureBlobFileSystemException { + public String getStorageAccountKey() throws AzureBlobFileSystemException { String key; - String keyProviderClass = - configuration.get(AZURE_KEY_ACCOUNT_KEYPROVIDER_PREFIX + accountName); + String keyProviderClass = get(AZURE_KEY_ACCOUNT_KEYPROVIDER); KeyProvider keyProvider; if (keyProviderClass == null) { @@ -195,7 +316,7 @@ public class AbfsConfiguration{ // implements KeyProvider Object keyProviderObject; try { - Class<?> clazz = configuration.getClassByName(keyProviderClass); + Class<?> clazz = rawConfig.getClassByName(keyProviderClass); keyProviderObject = clazz.newInstance(); } catch (Exception e) { throw new KeyProviderException("Unable to load key provider class.", e); @@ -206,7 +327,7 @@ public class AbfsConfiguration{ } keyProvider = (KeyProvider) keyProviderObject; } - key = keyProvider.getStorageAccountKey(accountName, configuration); + key = keyProvider.getStorageAccountKey(accountName, rawConfig); if (key == null) { throw new ConfigurationPropertyNotFoundException(accountName); @@ -215,8 +336,8 @@ public class AbfsConfiguration{ return key; } - public Configuration getConfiguration() { - return this.configuration; + public Configuration getRawConfiguration() { + return this.rawConfig; } public int getWriteBufferSize() { @@ -292,11 +413,11 @@ public class AbfsConfiguration{ } public SSLSocketFactoryEx.SSLChannelMode getPreferredSSLFactoryOption() { - return configuration.getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE); + return getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE); } - public AuthType getAuthType(final String accountName) { - return configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + accountName, AuthType.SharedKey); + public AuthType getAuthType(String accountName) { + return getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); } public boolean isDelegationTokenManagerEnabled() { @@ -304,34 +425,34 @@ public class AbfsConfiguration{ } public AbfsDelegationTokenManager getDelegationTokenManager() throws IOException { - return new AbfsDelegationTokenManager(configuration); + return new AbfsDelegationTokenManager(getRawConfiguration()); } - public AccessTokenProvider getTokenProvider(final String accountName) throws TokenAccessProviderException { - AuthType authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + accountName, AuthType.SharedKey); + public AccessTokenProvider getTokenProvider() throws TokenAccessProviderException { + AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); if (authType == AuthType.OAuth) { try { Class<? extends AccessTokenProvider> tokenProviderClass = - configuration.getClass(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName, null, + getClass(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, null, AccessTokenProvider.class); AccessTokenProvider tokenProvider = null; if (tokenProviderClass == ClientCredsTokenProvider.class) { - String authEndpoint = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT + accountName); - String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + accountName); - String clientSecret = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + accountName); + String authEndpoint = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT); + String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); + String clientSecret = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET); tokenProvider = new ClientCredsTokenProvider(authEndpoint, clientId, clientSecret); } else if (tokenProviderClass == UserPasswordTokenProvider.class) { - String authEndpoint = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT + accountName); - String username = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_NAME + accountName); - String password = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD + accountName); + String authEndpoint = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT); + String username = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_NAME); + String password = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD); tokenProvider = new UserPasswordTokenProvider(authEndpoint, username, password); } else if (tokenProviderClass == MsiTokenProvider.class) { - String tenantGuid = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT + accountName); - String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + accountName); + String tenantGuid = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT); + String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); tokenProvider = new MsiTokenProvider(tenantGuid, clientId); } else if (tokenProviderClass == RefreshTokenBasedTokenProvider.class) { - String refreshToken = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN + accountName); - String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + accountName); + String refreshToken = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN); + String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); tokenProvider = new RefreshTokenBasedTokenProvider(clientId, refreshToken); } else { throw new IllegalArgumentException("Failed to initialize " + tokenProviderClass); @@ -345,20 +466,19 @@ public class AbfsConfiguration{ } else if (authType == AuthType.Custom) { try { - String configKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName; + String configKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; Class<? extends CustomTokenProviderAdaptee> customTokenProviderClass = - configuration.getClass(configKey, null, - CustomTokenProviderAdaptee.class); + getClass(configKey, null, CustomTokenProviderAdaptee.class); if (customTokenProviderClass == null) { throw new IllegalArgumentException( String.format("The configuration value for \"%s\" is invalid.", configKey)); } CustomTokenProviderAdaptee azureTokenProvider = ReflectionUtils - .newInstance(customTokenProviderClass, configuration); + .newInstance(customTokenProviderClass, rawConfig); if (azureTokenProvider == null) { throw new IllegalArgumentException("Failed to initialize " + customTokenProviderClass); } - azureTokenProvider.initialize(configuration, accountName); + azureTokenProvider.initialize(rawConfig, accountName); return new CustomTokenProviderAdapter(azureTokenProvider); } catch(IllegalArgumentException e) { throw e; @@ -375,7 +495,7 @@ public class AbfsConfiguration{ void validateStorageAccountKeys() throws InvalidConfigurationValueException { Base64StringConfigurationBasicValidator validator = new Base64StringConfigurationBasicValidator( FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, "", true); - this.storageAccountKeys = configuration.getValByRegex(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX); + this.storageAccountKeys = rawConfig.getValByRegex(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX); for (Map.Entry<String, String> account : storageAccountKeys.entrySet()) { validator.validate(account.getValue()); @@ -384,7 +504,7 @@ public class AbfsConfiguration{ int validateInt(Field field) throws IllegalAccessException, InvalidConfigurationValueException { IntegerConfigurationValidatorAnnotation validator = field.getAnnotation(IntegerConfigurationValidatorAnnotation.class); - String value = configuration.get(validator.ConfigurationKey()); + String value = get(validator.ConfigurationKey()); // validate return new IntegerConfigurationBasicValidator( @@ -397,7 +517,7 @@ public class AbfsConfiguration{ long validateLong(Field field) throws IllegalAccessException, InvalidConfigurationValueException { LongConfigurationValidatorAnnotation validator = field.getAnnotation(LongConfigurationValidatorAnnotation.class); - String value = configuration.get(validator.ConfigurationKey()); + String value = rawConfig.get(validator.ConfigurationKey()); // validate return new LongConfigurationBasicValidator( @@ -410,7 +530,7 @@ public class AbfsConfiguration{ String validateString(Field field) throws IllegalAccessException, InvalidConfigurationValueException { StringConfigurationValidatorAnnotation validator = field.getAnnotation(StringConfigurationValidatorAnnotation.class); - String value = configuration.get(validator.ConfigurationKey()); + String value = rawConfig.get(validator.ConfigurationKey()); // validate return new StringConfigurationBasicValidator( @@ -421,7 +541,7 @@ public class AbfsConfiguration{ String validateBase64String(Field field) throws IllegalAccessException, InvalidConfigurationValueException { Base64StringConfigurationValidatorAnnotation validator = field.getAnnotation((Base64StringConfigurationValidatorAnnotation.class)); - String value = configuration.get(validator.ConfigurationKey()); + String value = rawConfig.get(validator.ConfigurationKey()); // validate return new Base64StringConfigurationBasicValidator( @@ -432,7 +552,7 @@ public class AbfsConfiguration{ boolean validateBoolean(Field field) throws IllegalAccessException, InvalidConfigurationValueException { BooleanConfigurationValidatorAnnotation validator = field.getAnnotation(BooleanConfigurationValidatorAnnotation.class); - String value = configuration.get(validator.ConfigurationKey()); + String value = rawConfig.get(validator.ConfigurationKey()); // validate return new BooleanConfigurationBasicValidator( @@ -441,14 +561,6 @@ public class AbfsConfiguration{ validator.ThrowIfInvalid()).validate(value); } - String getPasswordString(String key) throws IOException { - char[] passchars = configuration.getPassword(key); - if (passchars != null) { - return new String(passchars); - } - return null; - } - @VisibleForTesting void setReadBufferSize(int bufferSize) { this.readBufferSize = bufferSize; @@ -463,4 +575,4 @@ public class AbfsConfiguration{ void setEnableFlush(boolean enableFlush) { this.enableFlush = enableFlush; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 6542a64..e3bea32 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -113,10 +113,15 @@ public class AzureBlobFileSystemStore { private boolean isNamespaceEnabled; public AzureBlobFileSystemStore(URI uri, boolean isSecure, Configuration configuration, UserGroupInformation userGroupInformation) - throws AzureBlobFileSystemException { + throws AzureBlobFileSystemException, IOException { this.uri = uri; + + String[] authorityParts = authorityParts(uri); + final String fileSystemName = authorityParts[0]; + final String accountName = authorityParts[1]; + try { - this.abfsConfiguration = new AbfsConfiguration(configuration); + this.abfsConfiguration = new AbfsConfiguration(configuration, accountName); } catch (IllegalAccessException exception) { throw new FileSystemOperationUnhandledException(exception); } @@ -125,7 +130,31 @@ public class AzureBlobFileSystemStore { this.azureAtomicRenameDirSet = new HashSet<>(Arrays.asList( abfsConfiguration.getAzureAtomicRenameDirs().split(AbfsHttpConstants.COMMA))); - initializeClient(uri, isSecure); + initializeClient(uri, fileSystemName, accountName, isSecure); + } + + private String[] authorityParts(URI uri) throws InvalidUriAuthorityException, InvalidUriException { + final String authority = uri.getRawAuthority(); + if (null == authority) { + throw new InvalidUriAuthorityException(uri.toString()); + } + + if (!authority.contains(AbfsHttpConstants.AZURE_DISTRIBUTED_FILE_SYSTEM_AUTHORITY_DELIMITER)) { + throw new InvalidUriAuthorityException(uri.toString()); + } + + final String[] authorityParts = authority.split(AbfsHttpConstants.AZURE_DISTRIBUTED_FILE_SYSTEM_AUTHORITY_DELIMITER, 2); + + if (authorityParts.length < 2 || authorityParts[0] != null + && authorityParts[0].isEmpty()) { + final String errMsg = String + .format("'%s' has a malformed authority, expected container name. " + + "Authority takes the form " + + FileSystemUriSchemes.ABFS_SCHEME + "://[<container name>@]<account name>", + uri.toString()); + throw new InvalidUriException(errMsg); + } + return authorityParts; } public boolean getIsNamespaceEnabled() throws AzureBlobFileSystemException { @@ -154,7 +183,7 @@ public class AzureBlobFileSystemStore { // the Azure Storage Service URI changes from // http[s]://[account][domain-suffix]/[filesystem] to // http[s]://[ip]:[port]/[account]/[filesystem]. - String endPoint = abfsConfiguration.getConfiguration().get(AZURE_ABFS_ENDPOINT); + String endPoint = abfsConfiguration.get(AZURE_ABFS_ENDPOINT); if (endPoint == null || !endPoint.contains(AbfsHttpConstants.COLON)) { uriBuilder.setHost(hostName); return uriBuilder; @@ -738,36 +767,12 @@ public class AzureBlobFileSystemStore { return isKeyForDirectorySet(key, azureAtomicRenameDirSet); } - private void initializeClient(URI uri, boolean isSeure) throws AzureBlobFileSystemException { + private void initializeClient(URI uri, String fileSystemName, String accountName, boolean isSecure) throws AzureBlobFileSystemException { if (this.client != null) { return; } - final String authority = uri.getRawAuthority(); - if (null == authority) { - throw new InvalidUriAuthorityException(uri.toString()); - } - - if (!authority.contains(AbfsHttpConstants.AZURE_DISTRIBUTED_FILE_SYSTEM_AUTHORITY_DELIMITER)) { - throw new InvalidUriAuthorityException(uri.toString()); - } - - final String[] authorityParts = authority.split(AbfsHttpConstants.AZURE_DISTRIBUTED_FILE_SYSTEM_AUTHORITY_DELIMITER, 2); - - if (authorityParts.length < 2 || authorityParts[0] != null - && authorityParts[0].isEmpty()) { - final String errMsg = String - .format("'%s' has a malformed authority, expected container name. " - + "Authority takes the form " - + FileSystemUriSchemes.ABFS_SCHEME + "://[<container name>@]<account name>", - uri.toString()); - throw new InvalidUriException(errMsg); - } - - final String fileSystemName = authorityParts[0]; - final String accountName = authorityParts[1]; - - final URIBuilder uriBuilder = getURIBuilder(accountName, isSeure); + final URIBuilder uriBuilder = getURIBuilder(accountName, isSecure); final String url = uriBuilder.toString() + AbfsHttpConstants.FORWARD_SLASH + fileSystemName; @@ -788,9 +793,9 @@ public class AzureBlobFileSystemStore { uri.toString() + " - account name is not fully qualified."); } creds = new SharedKeyCredentials(accountName.substring(0, dotIndex), - abfsConfiguration.getStorageAccountKey(accountName)); + abfsConfiguration.getStorageAccountKey()); } else { - tokenProvider = abfsConfiguration.getTokenProvider(accountName); + tokenProvider = abfsConfiguration.getTokenProvider(); } this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, new ExponentialRetryPolicy(), tokenProvider); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 5236719..13cdaeb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -27,7 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceAudience.Public @InterfaceStability.Evolving public final class ConfigurationKeys { - public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key."; + public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key"; public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)"; public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode"; @@ -54,29 +54,33 @@ public final class ConfigurationKeys { public static final String FS_AZURE_USER_AGENT_PREFIX_KEY = "fs.azure.user.agent.prefix"; public static final String FS_AZURE_SSL_CHANNEL_MODE_KEY = "fs.azure.ssl.channel.mode"; - public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider."; + public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER = "fs.azure.account.keyprovider"; public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script"; /** End point of ABFS account: {@value}. */ public static final String AZURE_ABFS_ENDPOINT = "fs.azure.abfs.endpoint"; - /** Prefix for auth type properties: {@value}. */ - public static final String FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME = "fs.azure.account.auth.type."; - /** Prefix for oauth token provider type: {@value}. */ - public static final String FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME = "fs.azure.account.oauth.provider.type."; - /** Prefix for oauth AAD client id: {@value}. */ - public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID = "fs.azure.account.oauth2.client.id."; - /** Prefix for oauth AAD client secret: {@value}. */ - public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET = "fs.azure.account.oauth2.client.secret."; - /** Prefix for oauth AAD client endpoint: {@value}. */ - public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT = "fs.azure.account.oauth2.client.endpoint."; - /** Prefix for oauth msi tenant id: {@value}. */ - public static final String FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT = "fs.azure.account.oauth2.msi.tenant."; - /** Prefix for oauth user name: {@value}. */ - public static final String FS_AZURE_ACCOUNT_OAUTH_USER_NAME = "fs.azure.account.oauth2.user.name."; - /** Prefix for oauth user password: {@value}. */ - public static final String FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD = "fs.azure.account.oauth2.user.password."; - /** Prefix for oauth refresh token: {@value}. */ - public static final String FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN = "fs.azure.account.oauth2.refresh.token."; + /** Key for auth type properties: {@value}. */ + public static final String FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME = "fs.azure.account.auth.type"; + /** Key for oauth token provider type: {@value}. */ + public static final String FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME = "fs.azure.account.oauth.provider.type"; + /** Key for oauth AAD client id: {@value}. */ + public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID = "fs.azure.account.oauth2.client.id"; + /** Key for oauth AAD client secret: {@value}. */ + public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET = "fs.azure.account.oauth2.client.secret"; + /** Key for oauth AAD client endpoint: {@value}. */ + public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT = "fs.azure.account.oauth2.client.endpoint"; + /** Key for oauth msi tenant id: {@value}. */ + public static final String FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT = "fs.azure.account.oauth2.msi.tenant"; + /** Key for oauth user name: {@value}. */ + public static final String FS_AZURE_ACCOUNT_OAUTH_USER_NAME = "fs.azure.account.oauth2.user.name"; + /** Key for oauth user password: {@value}. */ + public static final String FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD = "fs.azure.account.oauth2.user.password"; + /** Key for oauth refresh token: {@value}. */ + public static final String FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN = "fs.azure.account.oauth2.refresh.token"; + + public static String accountProperty(String property, String account) { + return property + "." + account; + } public static final String FS_AZURE_ENABLE_DELEGATION_TOKEN = "fs.azure.enable.delegation.token"; public static final String FS_AZURE_DELEGATION_TOKEN_PROVIDER_TYPE = "fs.azure.delegation.token.provider.type"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java index 7504e9d..3dad32e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.conf.Configuration; /** * Provides tokens based on username and password. */ @@ -54,13 +53,4 @@ public class UserPasswordTokenProvider extends AccessTokenProvider { LOG.debug("AADToken: refreshing user-password based token"); return AzureADAuthenticator.getTokenUsingClientCreds(authEndpoint, username, password); } - - private static String getPasswordString(Configuration conf, String key) - throws IOException { - char[] passchars = conf.getPassword(key); - if (passchars == null) { - throw new IOException("Password " + key + " not found"); - } - return new String(passchars); - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ShellDecryptionKeyProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ShellDecryptionKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ShellDecryptionKeyProvider.java index 3fc05ff..bdac922 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ShellDecryptionKeyProvider.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ShellDecryptionKeyProvider.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Arrays; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException; import org.apache.hadoop.util.Shell; @@ -36,11 +37,18 @@ public class ShellDecryptionKeyProvider extends SimpleKeyProvider { private static final Logger LOG = LoggerFactory.getLogger(ShellDecryptionKeyProvider.class); @Override - public String getStorageAccountKey(String accountName, Configuration conf) + public String getStorageAccountKey(String accountName, Configuration rawConfig) throws KeyProviderException { - String envelope = super.getStorageAccountKey(accountName, conf); + String envelope = super.getStorageAccountKey(accountName, rawConfig); - final String command = conf.get(ConfigurationKeys.AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT); + AbfsConfiguration abfsConfig; + try { + abfsConfig = new AbfsConfiguration(rawConfig, accountName); + } catch(IllegalAccessException | IOException e) { + throw new KeyProviderException("Unable to get key from credential providers.", e); + } + + final String command = abfsConfig.get(ConfigurationKeys.AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT); if (command == null) { throw new KeyProviderException( "Script path is not specified via fs.azure.shellkeyprovider.script"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java index cedae57..727e1b3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java @@ -21,10 +21,10 @@ package org.apache.hadoop.fs.azurebfs.services; import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException; -import org.apache.hadoop.security.ProviderUtils; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,19 +36,19 @@ public class SimpleKeyProvider implements KeyProvider { private static final Logger LOG = LoggerFactory.getLogger(SimpleKeyProvider.class); @Override - public String getStorageAccountKey(String accountName, Configuration conf) + public String getStorageAccountKey(String accountName, Configuration rawConfig) throws KeyProviderException { String key = null; + try { - Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders( - conf, AzureBlobFileSystem.class); - char[] keyChars = c.getPassword(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + accountName); - if (keyChars != null) { - key = new String(keyChars); - } + AbfsConfiguration abfsConfig = new AbfsConfiguration(rawConfig, accountName); + key = abfsConfig.getPasswordString(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME); + } catch(IllegalAccessException | InvalidConfigurationValueException e) { + throw new KeyProviderException("Failure to initialize configuration", e); } catch(IOException ioe) { LOG.warn("Unable to get key from credential providers. {}", ioe); } + return key; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index a4b3483..db55e67 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -62,6 +62,16 @@ The abfs client has a fully consistent view of the store, which has complete Cre * Directory Rename: `O(files)`. * Directory Delete: `O(files)`. +## Configuring ABFS + +Any configuration can be specified generally (or as the default when accessing all accounts) or can be tied to s a specific account. +For example, an OAuth identity can be configured for use regardless of which account is accessed with the property +"fs.azure.account.oauth2.client.id" +or you can configure an identity to be used only for a specific storage account with +"fs.azure.account.oauth2.client.id.\<account\_name\>.dfs.core.windows.net". + +Note that it doesn't make sense to do this with some properties, like shared keys that are inherently account-specific. + ## Testing ABFS See the relevant section in [Testing Azure](testing_azure.html). http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index f61954f..52185cd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -67,38 +67,39 @@ public abstract class AbstractAbfsIntegrationTest extends private AzureBlobFileSystem abfs; private String abfsScheme; - private Configuration configuration; + private Configuration rawConfig; + private AbfsConfiguration abfsConfig; private String fileSystemName; private String accountName; private String testUrl; private AuthType authType; - protected AbstractAbfsIntegrationTest() { + protected AbstractAbfsIntegrationTest() throws Exception { fileSystemName = TEST_CONTAINER_PREFIX + UUID.randomUUID().toString(); - configuration = new Configuration(); - configuration.addResource(TEST_CONFIGURATION_FILE_NAME); + rawConfig = new Configuration(); + rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME); - this.accountName = this.configuration.get(FS_AZURE_ACCOUNT_NAME); + this.accountName = rawConfig.get(FS_AZURE_ACCOUNT_NAME); if (accountName == null) { // check if accountName is set using different config key - accountName = configuration.get(FS_AZURE_ABFS_ACCOUNT_NAME); + accountName = rawConfig.get(FS_AZURE_ABFS_ACCOUNT_NAME); } assumeTrue("Not set: " + FS_AZURE_ABFS_ACCOUNT_NAME, accountName != null && !accountName.isEmpty()); + abfsConfig = new AbfsConfiguration(rawConfig, accountName); - authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME - + accountName, AuthType.SharedKey); + authType = abfsConfig.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); abfsScheme = authType == AuthType.SharedKey ? FileSystemUriSchemes.ABFS_SCHEME : FileSystemUriSchemes.ABFS_SECURE_SCHEME; if (authType == AuthType.SharedKey) { - String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + accountName; - assumeTrue("Not set: " + keyProperty, configuration.get(keyProperty) != null); + assumeTrue("Not set: " + FS_AZURE_ACCOUNT_KEY, + abfsConfig.get(FS_AZURE_ACCOUNT_KEY) != null); // Update credentials } else { - String accessTokenProviderKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName; - assumeTrue("Not set: " + accessTokenProviderKey, configuration.get(accessTokenProviderKey) != null); + assumeTrue("Not set: " + FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, + abfsConfig.get(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME) != null); } final String abfsUrl = this.getFileSystemName() + "@" + this.getAccountName(); @@ -111,14 +112,14 @@ public abstract class AbstractAbfsIntegrationTest extends } this.testUrl = defaultUri.toString(); - configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); - configuration.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true); + abfsConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); + abfsConfig.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true); // For testing purposes, an IP address and port may be provided to override // the host specified in the FileSystem URI. Also note that the format of // the Azure Storage Service URI changes from // http[s]://[account][domain-suffix]/[filesystem] to // http[s]://[ip]:[port]/[account]/[filesystem]. - String endPoint = configuration.get(AZURE_ABFS_ENDPOINT); + String endPoint = abfsConfig.get(AZURE_ABFS_ENDPOINT); if (endPoint != null && endPoint.contains(":") && endPoint.split(":").length == 2) { this.isIPAddress = true; } else { @@ -140,18 +141,18 @@ public abstract class AbstractAbfsIntegrationTest extends // update configuration with wasb credentials String accountNameWithoutDomain = accountName.split("\\.")[0]; String wasbAccountName = accountNameWithoutDomain + WASB_ACCOUNT_NAME_DOMAIN_SUFFIX; - String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + wasbAccountName; - if (configuration.get(keyProperty) == null) { - configuration.set(keyProperty, getAccountKey()); + String keyProperty = FS_AZURE_ACCOUNT_KEY + "." + wasbAccountName; + if (rawConfig.get(keyProperty) == null) { + rawConfig.set(keyProperty, getAccountKey()); } azureNativeFileSystemStore.initialize( wasbUri, - configuration, - new AzureFileSystemInstrumentation(getConfiguration())); + rawConfig, + new AzureFileSystemInstrumentation(rawConfig)); wasb = new NativeAzureFileSystem(azureNativeFileSystemStore); - wasb.initialize(wasbUri, configuration); + wasb.initialize(wasbUri, rawConfig); } } @@ -197,8 +198,8 @@ public abstract class AbstractAbfsIntegrationTest extends } public AzureBlobFileSystem getFileSystem(String abfsUri) throws Exception { - configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, abfsUri); - final AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(configuration); + abfsConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, abfsUri); + final AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig); return fs; } @@ -210,7 +211,7 @@ public abstract class AbstractAbfsIntegrationTest extends public AzureBlobFileSystem createFileSystem() throws IOException { Preconditions.checkState(abfs == null, "existing ABFS instance exists: %s", abfs); - abfs = (AzureBlobFileSystem) FileSystem.newInstance(configuration); + abfs = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig); return abfs; } @@ -221,7 +222,7 @@ public abstract class AbstractAbfsIntegrationTest extends protected String getHostName() { // READ FROM ENDPOINT, THIS IS CALLED ONLY WHEN TESTING AGAINST DEV-FABRIC - String endPoint = configuration.get(AZURE_ABFS_ENDPOINT); + String endPoint = abfsConfig.get(AZURE_ABFS_ENDPOINT); return endPoint.split(":")[0]; } @@ -245,13 +246,15 @@ public abstract class AbstractAbfsIntegrationTest extends } protected String getAccountKey() { - return configuration.get( - FS_AZURE_ACCOUNT_KEY_PREFIX - + accountName); + return abfsConfig.get(FS_AZURE_ACCOUNT_KEY); + } + + public AbfsConfiguration getConfiguration() { + return abfsConfig; } - protected Configuration getConfiguration() { - return configuration; + public Configuration getRawConfiguration() { + return abfsConfig.getRawConfiguration(); } protected boolean isIPAddress() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsScaleTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsScaleTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsScaleTest.java index cfda7a7..14c9bff 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsScaleTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsScaleTest.java @@ -21,6 +21,7 @@ package org.apache.hadoop.fs.azurebfs; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.azure.integration.AzureTestConstants; import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.assumeScaleTestsEnabled; @@ -34,6 +35,10 @@ public class AbstractAbfsScaleTest extends AbstractAbfsIntegrationTest { protected static final Logger LOG = LoggerFactory.getLogger(AbstractAbfsScaleTest.class); + public AbstractAbfsScaleTest() throws Exception { + super(); + } + @Override protected int getTestTimeoutMillis() { return AzureTestConstants.SCALE_TEST_TIMEOUT_MILLIS; @@ -43,7 +48,8 @@ public class AbstractAbfsScaleTest extends AbstractAbfsIntegrationTest { public void setup() throws Exception { super.setup(); LOG.debug("Scale test operation count = {}", getOperationCount()); - assumeScaleTestsEnabled(getConfiguration()); + Configuration rawConfiguration = getRawConfiguration(); + assumeScaleTestsEnabled(rawConfiguration); } protected long getOperationCount() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java index 1c2083d..f024f25 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java @@ -30,6 +30,10 @@ import org.junit.Test; public final class ITestAbfsClient extends AbstractAbfsIntegrationTest { private static final int LIST_MAX_RESULTS = 5000; + public ITestAbfsClient() throws Exception { + super(); + } + @Test public void testContinuationTokenHavingEqualSign() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); @@ -42,4 +46,4 @@ public final class ITestAbfsClient extends AbstractAbfsIntegrationTest { Assert.assertEquals("InvalidQueryParameterValue", ex.getErrorCode().getErrorCode()); } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java index f62ea6e..a270a00 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java @@ -51,7 +51,7 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest { private final int size; - public ITestAbfsReadWriteAndSeek(final int size) { + public ITestAbfsReadWriteAndSeek(final int size) throws Exception { this.size = size; } @@ -62,7 +62,7 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest { private void testReadWriteAndSeek(int bufferSize) throws Exception { final AzureBlobFileSystem fs = getFileSystem(); - final AbfsConfiguration abfsConfiguration = new AbfsConfiguration(getConfiguration()); + final AbfsConfiguration abfsConfiguration = fs.getAbfsStore().getAbfsConfiguration(); abfsConfiguration.setWriteBufferSize(bufferSize); abfsConfiguration.setReadBufferSize(bufferSize); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java index f2e26ec..cbe1939 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java @@ -34,7 +34,8 @@ public class ITestAzureBlobFileSystemAppend extends AbstractAbfsIntegrationTest { private static final Path TEST_FILE_PATH = new Path("testfile"); private static final Path TEST_FOLDER_PATH = new Path("testFolder"); - public ITestAzureBlobFileSystemAppend() { + + public ITestAzureBlobFileSystemAppend() throws Exception { super(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java index 46ac866..d8940f7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java @@ -35,7 +35,8 @@ import org.apache.hadoop.fs.Path; */ public class ITestAzureBlobFileSystemBackCompat extends AbstractAbfsIntegrationTest { - public ITestAzureBlobFileSystemBackCompat() { + + public ITestAzureBlobFileSystemBackCompat() throws Exception { super(); Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java index 90eff97..917ee9c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java @@ -39,7 +39,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile; * Test copy operation. */ public class ITestAzureBlobFileSystemCopy extends AbstractAbfsIntegrationTest { - public ITestAzureBlobFileSystemCopy() { + + public ITestAzureBlobFileSystemCopy() throws Exception { super(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java index 1e43f9a..ab01166 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java @@ -38,7 +38,8 @@ public class ITestAzureBlobFileSystemCreate extends private static final Path TEST_FILE_PATH = new Path("testfile"); private static final Path TEST_FOLDER_PATH = new Path("testFolder"); private static final String TEST_CHILD_FILE = "childFile"; - public ITestAzureBlobFileSystemCreate() { + + public ITestAzureBlobFileSystemCreate() throws Exception { super(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index 91d1723..486daca 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -41,7 +41,8 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept; */ public class ITestAzureBlobFileSystemDelete extends AbstractAbfsIntegrationTest { - public ITestAzureBlobFileSystemDelete() { + + public ITestAzureBlobFileSystemDelete() throws Exception { super(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java index f1800c0..6021e5d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; import static org.junit.Assert.assertEquals; @@ -45,9 +44,9 @@ public class ITestAzureBlobFileSystemE2E extends AbstractAbfsIntegrationTest { private static final int TEST_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final int TEST_DEFAULT_READ_BUFFER_SIZE = 1023900; - public ITestAzureBlobFileSystemE2E() { + public ITestAzureBlobFileSystemE2E() throws Exception { super(); - Configuration configuration = this.getConfiguration(); + AbfsConfiguration configuration = this.getConfiguration(); configuration.set(ConfigurationKeys.FS_AZURE_READ_AHEAD_QUEUE_DEPTH, "0"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java index 7ed9d42..fccd063 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java @@ -45,7 +45,7 @@ public class ITestAzureBlobFileSystemE2EScale extends private static final int ONE_MB = 1024 * 1024; private static final int DEFAULT_WRITE_TIMES = 100; - public ITestAzureBlobFileSystemE2EScale() { + public ITestAzureBlobFileSystemE2EScale() throws Exception { } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java index dba10f5..b08b920 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java @@ -41,7 +41,7 @@ public class ITestAzureBlobFileSystemFileStatus extends private static final Path TEST_FILE = new Path("testFile"); private static final Path TEST_FOLDER = new Path("testDir"); - public ITestAzureBlobFileSystemFileStatus() { + public ITestAzureBlobFileSystemFileStatus() throws Exception { super(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java index c1022b0..9d17388 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java @@ -41,12 +41,12 @@ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{ @Test public void testFinalize() throws Exception { // Disable the cache for filesystem to make sure there is no reference. - Configuration configuration = this.getConfiguration(); - configuration.setBoolean( + Configuration rawConfig = this.getRawConfiguration(); + rawConfig.setBoolean( this.getAuthType() == AuthType.SharedKey ? DISABLE_ABFS_CACHE_KEY : DISABLE_ABFSSS_CACHE_KEY, true); - AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(configuration); + AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig); WeakReference<Object> ref = new WeakReference<Object>(fs); fs = null; @@ -61,4 +61,4 @@ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{ Assert.assertTrue("testFinalizer didn't get cleaned up within maxTries", ref.get() == null); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java index 337f95c..23a1ab5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java @@ -60,7 +60,7 @@ public class ITestAzureBlobFileSystemFlush extends AbstractAbfsScaleTest { private static final int TEST_FILE_LENGTH = 1024 * 1024 * 8; private static final int WAITING_TIME = 1000; - public ITestAzureBlobFileSystemFlush() { + public ITestAzureBlobFileSystemFlush() throws Exception { super(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java index 874a8a3..5f08721 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java @@ -31,8 +31,8 @@ import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; */ public class ITestAzureBlobFileSystemInitAndCreate extends AbstractAbfsIntegrationTest { - public ITestAzureBlobFileSystemInitAndCreate() { + public ITestAzureBlobFileSystemInitAndCreate() throws Exception { this.getConfiguration().unset(ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java index b87abe6..60e0fbc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java @@ -42,7 +42,8 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept; public class ITestAzureBlobFileSystemListStatus extends AbstractAbfsIntegrationTest { private static final int TEST_FILES_NUMBER = 6000; - public ITestAzureBlobFileSystemListStatus() { + + public ITestAzureBlobFileSystemListStatus() throws Exception { super(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java index 1bb2c54..382d396 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java @@ -28,7 +28,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs; * Test mkdir operation. */ public class ITestAzureBlobFileSystemMkDir extends AbstractAbfsIntegrationTest { - public ITestAzureBlobFileSystemMkDir() { + + public ITestAzureBlobFileSystemMkDir() throws Exception { super(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java index f60740f..533f471 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java @@ -53,7 +53,7 @@ public class ITestAzureBlobFileSystemOauth extends AbstractAbfsIntegrationTest{ private static final Path EXISTED_FILE_PATH = new Path("/existedFile"); private static final Path EXISTED_FOLDER_PATH = new Path("/existedFolder"); - public ITestAzureBlobFileSystemOauth() { + public ITestAzureBlobFileSystemOauth() throws Exception { Assume.assumeTrue(this.getAuthType() == AuthType.OAuth); } /* @@ -161,16 +161,18 @@ public class ITestAzureBlobFileSystemOauth extends AbstractAbfsIntegrationTest{ } private AzureBlobFileSystem getBlobConributor() throws Exception { - Configuration configuration = this.getConfiguration(); - configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID)); - configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET)); - return getFileSystem(configuration); + AbfsConfiguration abfsConfig = this.getConfiguration(); + abfsConfig.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + "." + this.getAccountName(), abfsConfig.get(FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID)); + abfsConfig.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + "." + this.getAccountName(), abfsConfig.get(FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET)); + Configuration rawConfig = abfsConfig.getRawConfiguration(); + return getFileSystem(rawConfig); } private AzureBlobFileSystem getBlobReader() throws Exception { - Configuration configuration = this.getConfiguration(); - configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_READER_CLIENT_ID)); - configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET)); - return getFileSystem(configuration); + AbfsConfiguration abfsConfig = this.getConfiguration(); + abfsConfig.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + "." + this.getAccountName(), abfsConfig.get(FS_AZURE_BLOB_DATA_READER_CLIENT_ID)); + abfsConfig.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + "." + this.getAccountName(), abfsConfig.get(FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET)); + Configuration rawConfig = abfsConfig.getRawConfiguration(); + return getFileSystem(rawConfig); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java index c97e840..e0e1d89 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java @@ -41,7 +41,9 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile; */ public class ITestAzureBlobFileSystemRename extends AbstractAbfsIntegrationTest { - public ITestAzureBlobFileSystemRename() { + + public ITestAzureBlobFileSystemRename() throws Exception { + super(); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java index 0ac7fcf..044c325 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java @@ -66,7 +66,7 @@ public class ITestAzureBlobFileSystemRenameUnicode extends }); } - public ITestAzureBlobFileSystemRenameUnicode() { + public ITestAzureBlobFileSystemRenameUnicode() throws Exception { } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java index 50b1828..8b60dd8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java @@ -32,7 +32,7 @@ import org.apache.hadoop.fs.azurebfs.services.AuthType; * Test AzureBlobFileSystem initialization. */ public class ITestFileSystemInitialization extends AbstractAbfsIntegrationTest { - public ITestFileSystemInitialization() { + public ITestFileSystemInitialization() throws Exception { super(); } @@ -62,10 +62,10 @@ public class ITestFileSystemInitialization extends AbstractAbfsIntegrationTest { null, null, null); - Configuration conf = getConfiguration(); - conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); + Configuration rawConfig = getRawConfiguration(); + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); - try(SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.newInstance(conf)) { + try(SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.newInstance(rawConfig)) { assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java index 7a7e327..e6b45c8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java @@ -33,7 +33,7 @@ import org.apache.hadoop.fs.Path; public class ITestFileSystemProperties extends AbstractAbfsIntegrationTest { private static final int TEST_DATA = 100; private static final Path TEST_PATH = new Path("/testfile"); - public ITestFileSystemProperties() { + public ITestFileSystemProperties() throws Exception { } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java index 56a91d3..4393bd8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java @@ -77,14 +77,15 @@ public class ITestFileSystemRegistration extends AbstractAbfsIntegrationTest { @Test public void ensureAzureBlobFileSystemIsDefaultFileSystem() throws Exception { - AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(getConfiguration()); + Configuration rawConfig = getRawConfiguration(); + AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig); assertNotNull("filesystem", fs); if (this.getAuthType() == AuthType.OAuth) { - Abfss afs = (Abfss) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem(); + Abfss afs = (Abfss) FileContext.getFileContext(rawConfig).getDefaultFileSystem(); assertNotNull("filecontext", afs); } else { - Abfs afs = (Abfs) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem(); + Abfs afs = (Abfs) FileContext.getFileContext(rawConfig).getDefaultFileSystem(); assertNotNull("filecontext", afs); } @@ -100,13 +101,13 @@ public class ITestFileSystemRegistration extends AbstractAbfsIntegrationTest { null, null, null); - getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + Configuration rawConfig = getRawConfiguration(); + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); - SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.get( - getConfiguration()); + SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.get(rawConfig); assertNotNull("filesystem", fs); - Abfss afs = (Abfss) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem(); + Abfss afs = (Abfss) FileContext.getFileContext(rawConfig).getDefaultFileSystem(); assertNotNull("filecontext", afs); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java index bedd4f3..50f7bd9 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.azurebfs; +import java.io.IOException; import java.lang.reflect.Field; import org.apache.commons.codec.Charsets; @@ -66,6 +67,7 @@ public class TestAbfsConfigurationFieldsValidation { private static final int TEST_INT = 1234565; private static final int TEST_LONG = 4194304; + private final String accountName; private final String encodedString; private final String encodedAccountKey; @@ -96,6 +98,7 @@ public class TestAbfsConfigurationFieldsValidation { public TestAbfsConfigurationFieldsValidation() throws Exception { super(); Base64 base64 = new Base64(); + this.accountName = "testaccount1.blob.core.windows.net"; this.encodedString = new String(base64.encode("base64Value".getBytes(Charsets.UTF_8)), Charsets.UTF_8); this.encodedAccountKey = new String(base64.encode("someAccountKey".getBytes(Charsets.UTF_8)), Charsets.UTF_8); Configuration configuration = new Configuration(); @@ -105,8 +108,8 @@ public class TestAbfsConfigurationFieldsValidation { configuration.set(STRING_KEY, "stringValue"); configuration.set(BASE64_KEY, encodedString); configuration.set(BOOLEAN_KEY, "true"); - configuration.set(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "testaccount1.blob.core.windows.net", this.encodedAccountKey); - abfsConfiguration = new AbfsConfiguration(configuration); + configuration.set(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "." + accountName, this.encodedAccountKey); + abfsConfiguration = new AbfsConfiguration(configuration, accountName); } @Test @@ -143,30 +146,35 @@ public class TestAbfsConfigurationFieldsValidation { @Test public void testGetAccountKey() throws Exception { - String accountKey = abfsConfiguration.getStorageAccountKey("testaccount1.blob.core.windows.net"); + String accountKey = abfsConfiguration.getStorageAccountKey(); assertEquals(this.encodedAccountKey, accountKey); } @Test(expected = ConfigurationPropertyNotFoundException.class) public void testGetAccountKeyWithNonExistingAccountName() throws Exception { - abfsConfiguration.getStorageAccountKey("bogusAccountName"); + Configuration configuration = new Configuration(); + configuration.addResource(TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME); + configuration.unset(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME); + AbfsConfiguration abfsConfig = new AbfsConfiguration(configuration, "bogusAccountName"); + abfsConfig.getStorageAccountKey(); } @Test - public void testSSLSocketFactoryConfiguration() throws InvalidConfigurationValueException, IllegalAccessException { + public void testSSLSocketFactoryConfiguration() + throws InvalidConfigurationValueException, IllegalAccessException, IOException { assertEquals(SSLSocketFactoryEx.SSLChannelMode.Default, abfsConfiguration.getPreferredSSLFactoryOption()); assertNotEquals(SSLSocketFactoryEx.SSLChannelMode.Default_JSSE, abfsConfiguration.getPreferredSSLFactoryOption()); assertNotEquals(SSLSocketFactoryEx.SSLChannelMode.OpenSSL, abfsConfiguration.getPreferredSSLFactoryOption()); Configuration configuration = new Configuration(); configuration.setEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, SSLSocketFactoryEx.SSLChannelMode.Default_JSSE); - AbfsConfiguration localAbfsConfiguration = new AbfsConfiguration(configuration); + AbfsConfiguration localAbfsConfiguration = new AbfsConfiguration(configuration, accountName); assertEquals(SSLSocketFactoryEx.SSLChannelMode.Default_JSSE, localAbfsConfiguration.getPreferredSSLFactoryOption()); configuration = new Configuration(); configuration.setEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, SSLSocketFactoryEx.SSLChannelMode.OpenSSL); - localAbfsConfiguration = new AbfsConfiguration(configuration); + localAbfsConfiguration = new AbfsConfiguration(configuration, accountName); assertEquals(SSLSocketFactoryEx.SSLChannelMode.OpenSSL, localAbfsConfiguration.getPreferredSSLFactoryOption()); } -} \ No newline at end of file +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
