HADOOP-15663. ABFS: Simplify configuration. Contributed by Da Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81dc4a99 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81dc4a99 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81dc4a99 Branch: refs/heads/HADOOP-15407 Commit: 81dc4a995c3837b721a0c1f897698b5ed47b8fb0 Parents: df57c6c Author: Thomas Marquardt <tm...@microsoft.com> Authored: Fri Aug 31 03:24:42 2018 +0000 Committer: Thomas Marquardt <tm...@microsoft.com> Committed: Mon Sep 17 19:54:01 2018 +0000 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 12 ++ .../hadoop/fs/azurebfs/AbfsConfiguration.java | 4 - .../fs/azurebfs/AzureBlobFileSystemStore.java | 24 ++- .../azurebfs/constants/ConfigurationKeys.java | 5 +- .../hadoop/fs/azurebfs/utils/UriUtils.java | 15 +- .../src/site/markdown/testing_azure.md | 209 ++++++++++++++----- .../fs/azure/AzureBlobStorageTestAccount.java | 22 +- ...TestFileSystemOperationExceptionMessage.java | 3 +- .../fs/azure/ITestWasbUriAndConfiguration.java | 26 --- .../azure/integration/AzureTestConstants.java | 6 +- .../fs/azure/integration/AzureTestUtils.java | 18 ++ .../azure/metrics/TestRollingWindowAverage.java | 4 +- .../azurebfs/AbstractAbfsIntegrationTest.java | 73 ++++--- .../azurebfs/AbstractAbfsTestWithTimeout.java | 70 +++++++ .../ITestAzureBlobFileSystemBackCompat.java | 2 +- .../fs/azurebfs/ITestWasbAbfsCompatibility.java | 2 +- .../TestAbfsConfigurationFieldsValidation.java | 3 +- .../constants/TestConfigurationKeys.java | 13 +- .../contract/ABFSContractTestBinding.java | 3 + .../TestConfigurationValidators.java | 1 - .../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 80 +++++++ .../utils/CleanUpAbfsTestContainer.java | 77 ------- .../hadoop/fs/azurebfs/utils/TestUriUtils.java | 12 +- .../src/test/resources/azure-bfs-test.xml | 188 ----------------- .../src/test/resources/azure-test.xml | 56 ++--- 25 files changed, 473 insertions(+), 455 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 471dacc..3fcdecb 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1619,6 +1619,18 @@ <!-- Azure file system properties --> <property> + <name>fs.AbstractFileSystem.wasb.impl</name> + <value>org.apache.hadoop.fs.azure.Wasb</value> + <description>AbstractFileSystem implementation class of wasb://</description> +</property> + +<property> + <name>fs.AbstractFileSystem.wasbs.impl</name> + <value>org.apache.hadoop.fs.azure.Wasbs</value> + <description>AbstractFileSystem implementation class of wasbs://</description> +</property> + +<property> <name>fs.wasb.impl</name> <value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value> <description>The implementation class of the Native Azure Filesystem</description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 069f17a..924bc3e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -173,10 +173,6 @@ public class AbfsConfiguration{ } } - public boolean isEmulator() { - return this.getConfiguration().getBoolean(FS_AZURE_EMULATOR_ENABLED, false); - } - public boolean isSecureMode() { return isSecure; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index fc60127..6542a64 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -77,6 +77,7 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy; import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials; +import org.apache.hadoop.fs.azurebfs.utils.UriUtils; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -86,6 +87,7 @@ import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_ABFS_ENDPOINT; import static org.apache.hadoop.util.Time.now; /** @@ -146,7 +148,27 @@ public class AzureBlobFileSystemStore { final URIBuilder uriBuilder = new URIBuilder(); uriBuilder.setScheme(scheme); - uriBuilder.setHost(hostName); + + // For testing purposes, an IP address and port may be provided to override + // the host specified in the FileSystem URI. Also note that the format of + // the Azure Storage Service URI changes from + // http[s]://[account][domain-suffix]/[filesystem] to + // http[s]://[ip]:[port]/[account]/[filesystem]. + String endPoint = abfsConfiguration.getConfiguration().get(AZURE_ABFS_ENDPOINT); + if (endPoint == null || !endPoint.contains(AbfsHttpConstants.COLON)) { + uriBuilder.setHost(hostName); + return uriBuilder; + } + + // Split ip and port + String[] data = endPoint.split(AbfsHttpConstants.COLON); + if (data.length != 2) { + throw new RuntimeException(String.format("ABFS endpoint is not set correctly : %s, " + + "Do not specify scheme when using {IP}:{PORT}", endPoint)); + } + uriBuilder.setHost(data[0].trim()); + uriBuilder.setPort(Integer.parseInt(data[1].trim())); + uriBuilder.setPath("/" + UriUtils.extractAccountNameFromHostName(hostName)); return uriBuilder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 8ad0dc6..ca4c9c3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -37,9 +37,6 @@ public final class ConfigurationKeys { public static final String AZURE_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval"; public static final String AZURE_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries"; - // Remove this and use common azure storage emulator property for public release. - public static final String FS_AZURE_EMULATOR_ENABLED = "fs.azure.abfs.emulator.enabled"; - // Read and write buffer sizes defined by the user public static final String AZURE_WRITE_BUFFER_SIZE = "fs.azure.write.request.size"; public static final String AZURE_READ_BUFFER_SIZE = "fs.azure.read.request.size"; @@ -60,6 +57,8 @@ public final class ConfigurationKeys { public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider."; public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script"; + /** End point of ABFS account: {@value}. */ + public static final String AZURE_ABFS_ENDPOINT = "fs.azure.abfs.endpoint"; /** Prefix for auth type properties: {@value}. */ public static final String FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME = "fs.azure.account.auth.type."; /** Prefix for oauth token provider type: {@value}. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java index 7652adf..1bbc1b3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java @@ -41,20 +41,21 @@ public final class UriUtils { } /** - * Extracts the raw account name from account name. - * @param accountName to extract the raw account name. - * @return extracted raw account name. + * Extracts the account name from the host name. + * @param hostName the fully-qualified domain name of the storage service + * endpoint (e.g. {account}.dfs.core.windows.net. + * @return the storage service account name. */ - public static String extractRawAccountFromAccountName(final String accountName) { - if (accountName == null || accountName.isEmpty()) { + public static String extractAccountNameFromHostName(final String hostName) { + if (hostName == null || hostName.isEmpty()) { return null; } - if (!containsAbfsUrl(accountName)) { + if (!containsAbfsUrl(hostName)) { return null; } - String[] splitByDot = accountName.split("\\."); + String[] splitByDot = hostName.split("\\."); if (splitByDot.length == 0) { return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md index c148807..a4d920a 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md @@ -90,7 +90,7 @@ For example: <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <configuration> <property> - <name>fs.azure.test.account.name</name> + <name>fs.azure.wasb.account.name</name> <value>{ACCOUNTNAME}.blob.core.windows.net</value> </property> <property> @@ -126,7 +126,7 @@ Overall, to run all the tests using `mvn test`, a sample `azure-auth-keys.xml` <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <configuration> <property> - <name>fs.azure.test.account.name</name> + <name>fs.azure.wasb.account.name</name> <value>{ACCOUNTNAME}.blob.core.windows.net</value> </property> <property> @@ -576,77 +576,172 @@ This will delete the containers; the output log of the test run will provide the details and summary of the operation. -## Testing ABFS +## Testing the Azure ABFS Client -The ABFS Connector tests share the same account as the wasb tests; this is -needed for cross-connector compatibility tests. - -This makes for a somewhat complex set of configuration options. - -Here are the settings for an account `ACCOUNTNAME` +Azure Data Lake Storage Gen 2 (ADLS Gen 2) is a set of capabilities dedicated to +big data analytics, built on top of Azure Blob Storage. The ABFS and ABFSS +schemes target the ADLS Gen 2 REST API, and the WASB and WASBS schemes target +the Azure Blob Storage REST API. ADLS Gen 2 offers better performance and +scalability. ADLS Gen 2 also offers authentication and authorization compatible +with the Hadoop Distributed File System permissions model when hierarchical +namespace is enabled for the storage account. Furthermore, the metadata and data +produced by ADLS Gen 2 REST API can be consumed by Blob REST API, and vice versa. + +In order to test ABFS, please add the following configuration to your +`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include +compatibility tests which require WASB credentials, in addition to the ABFS +credentials. ```xml -<property> - <name>abfs.account.name</name> - <value>ACCOUNTNAME</value> -</property> +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <property> + <name>fs.azure.abfs.account.name</name> + <value>{ACCOUNT_NAME}.dfs.core.windows.net</value> + </property> -<property> - <name>abfs.account.full.name</name> - <value>${abfs.account.name}.dfs.core.windows.net</value> -</property> + <property> + <name>fs.azure.account.key.{ACCOUNT_NAME}.dfs.core.windows.net</name> + <value>{ACCOUNT_ACCESS_KEY}</value> + </property> -<property> - <name>abfs.account.key</name> - <value>SECRETKEY==</value> -</property> + <property> + <name>fs.azure.wasb.account.name</name> + <value>{ACCOUNT_NAME}.blob.core.windows.net</value> + </property> + + <property> + <name>fs.azure.account.key.{ACCOUNT_NAME}.blob.core.windows.net</name> + <value>{ACCOUNT_ACCESS_KEY}</value> + </property> -<property> - <name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name> - <value>${abfs.account.key}</value> -</property> + <property> + <name>fs.contract.test.fs.abfs</name> + <value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value> + <description>A file system URI to be used by the contract tests.</description> + </property> -<property> - <name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name> - <value>${abfs.account.key}</value> -</property> + <property> + <name>fs.contract.test.fs.wasb</name> + <value>wasb://{CONTAINER_NAME}@{ACCOUNT_NAME}.blob.core.windows.net</value> + <description>A file system URI to be used by the contract tests.</description> + </property> +</configuration> +``` -<property> - <name>fs.azure.test.account.key.ACCOUNTNAME.dfs.core.windows.net</name> - <value>${abfs.account.key}</value> -</property> +To run OAuth and ACL test cases you must use a storage account with the +hierarchical namespace enabled, and set the following configuration settings: +```xml +<!--=========================== AUTHENTICATION OPTIONS ===================--> +<!--ATTENTION: + TO RUN ABFS & WASB COMPATIBILITY TESTS, YOU MUST SET AUTH TYPE AS SharedKey. + OAUTH IS INTRODUCED TO ABFS ONLY.--> <property> - <name>fs.azure.test.account.key.ACCOUNTNAME.blob.core.windows.net</name> - <value>${abfs.account.key}</value> + <name>fs.azure.account.auth.type.{YOUR_ABFS_ACCOUNT_NAME}</name> + <value>{AUTH TYPE}</value> + <description>The authorization type can be SharedKey, OAuth, or Custom. The + default is SharedKey.</description> </property> -<property> - <name>fs.azure.account.key.ACCOUNTNAME</name> - <value>${abfs.account.key}</value> -</property> +<!--============================= FOR OAUTH ===========================--> +<!--IF AUTH TYPE IS SET AS OAUTH, FOLLOW THE STEPS BELOW--> +<!--NOTICE: AAD client and tenant related properties can be obtained through Azure Portal--> -<property> - <name>fs.azure.test.account.key.ACCOUNTNAME</name> - <value>${abfs.account.key}</value> -</property> + <!--1. UNCOMMENT BELOW AND CHOOSE YOUR OAUTH PROVIDER TYPE --> -<property> - <name>fs.azure.test.account.name</name> - <value>${abfs.account.full.name}</value> -</property> + <!-- + <property> + <name>fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}</name> + <value>org.apache.hadoop.fs.azurebfs.oauth2.{Token Provider Class name}</value> + <description>The full name of token provider class name.</description> + </property> + --> -<property> - <name>fs.contract.test.fs.abfs</name> - <value>abfs://testcontai...@accountname.dfs.core.windows.net</value> - <description>Container for contract tests</description> -</property> + <!--2. UNCOMMENT BELOW AND SET CREDENTIALS ACCORDING TO THE PROVIDER TYPE--> -<property> - <name>fs.contract.test.fs.abfss</name> - <value>abfss://testcontai...@accountname.dfs.core.windows.net</value> - <description>Container for contract tests</description> -</property> + <!--2.1. If "ClientCredsTokenProvider" is set as key provider, uncomment below and + set auth endpoint, client id and secret below--> + <!-- + <property> + <name>fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}</name> + <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value> + <description>Token end point, this can be found through Azure portal</description> + </property> + <property> + <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name> + <value>{client id}</value> + <description>AAD client id.</description> + </property> + + <property> + <name>fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}</name> + <value>{client secret}</value> + </property> + --> + + <!--2.2. If "UserPasswordTokenProvider" is set as key provider, uncomment below and + set auth endpoint, use name and password--> + <!-- + <property> + <name>fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}</name> + <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value> + <description>Token end point, this can be found through Azure portal</description> + </property> + <property> + <name>fs.azure.account.oauth2.user.name.{ABFS_ACCOUNT_NAME}</name> + <value>{user name}</value> + </property> + + <property> + <name>fs.azure.account.oauth2.user.password.{ABFS_ACCOUNT_NAME}</name> + <value>{user password}</value> + </property> + --> + + <!--2.3. If "MsiTokenProvider" is set as key provider, uncomment below and + set tenantGuid and client id.--> + <!-- + <property> + <name>fs.azure.account.oauth2.msi.tenant.{ABFS_ACCOUNT_NAME}</name> + <value>{tenantGuid}</value> + <description>msi tenantGuid.</description> + </property> + + <property> + <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name> + <value>{client id}</value> + <description>AAD client id.</description> + </property> + --> + + <!--2.4. If "RefreshTokenBasedTokenProvider" is set as key provider, uncomment below and + set refresh token and client id.--> + <!-- + <property> + <name>fs.azure.account.oauth2.refresh.token.{ABFS_ACCOUNT_NAME}</name> + <value>{refresh token}</value> + <description>refresh token.</description> + </property> + + <property> + <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name> + <value>{client id}</value> + <description>AAD client id.</description> + </property> + --> ``` + +If running tests against an endpoint that uses the URL format +http[s]://[ip]:[port]/[account]/[filesystem] instead of +http[s]://[account][domain-suffix]/[filesystem], please use the following: + +```xml +<property> + <name>fs.azure.abfs.endpoint</name> + <value>{IP}:{PORT}</value> +</property> +``` \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java index 5b36c87..b65ce78 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java @@ -48,6 +48,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME; import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_LOCAL_SAS_KEY_MODE; import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE; +import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig; /** * Helper class to create WASB file systems backed by either a mock in-memory @@ -58,11 +59,14 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, private static final Logger LOG = LoggerFactory.getLogger( AzureBlobStorageTestAccount.class); - private static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key."; private static final String SAS_PROPERTY_NAME = "fs.azure.sas."; private static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml"; - private static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.test.account.name"; + public static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key."; + public static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name"; + public static final String WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN = "fs.azure.wasb.account.name"; public static final String MOCK_ACCOUNT_NAME = "mockAccount.blob.core.windows.net"; + public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX = ".blob.core.windows.net"; + public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX = "\\.blob(\\.preprod)?\\.core\\.windows\\.net"; public static final String MOCK_CONTAINER_NAME = "mockContainer"; public static final String WASB_AUTHORITY_DELIMITER = "@"; public static final String WASB_SCHEME = "wasb"; @@ -379,7 +383,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, containerName); container.create(); - String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + String accountName = verifyWasbAccountNameInConfig(conf); // Ensure that custom throttling is disabled and tolerate concurrent // out-of-band appends. @@ -525,7 +529,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, static CloudStorageAccount createTestAccount(Configuration conf) throws URISyntaxException, KeyProviderException { - String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + String testAccountName = verifyWasbAccountNameInConfig(conf); if (testAccountName == null) { LOG.warn("Skipping live Azure test because of missing test account"); return null; @@ -570,16 +574,16 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, String containerName = useContainerSuffixAsContainerName ? containerNameSuffix : String.format( - "wasbtests-%s-%tQ%s", + "wasbtests-%s-%s%s", System.getProperty("user.name"), - new Date(), + UUID.randomUUID().toString(), containerNameSuffix); container = account.createCloudBlobClient().getContainerReference( containerName); if (createOptions.contains(CreateOptions.CreateContainer)) { container.createIfNotExists(); } - String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + String accountName = verifyWasbAccountNameInConfig(conf); if (createOptions.contains(CreateOptions.UseSas)) { String sas = generateSAS(container, createOptions.contains(CreateOptions.Readonly)); @@ -741,7 +745,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, CloudBlobClient blobClient = account.createCloudBlobClient(); // Capture the account URL and the account name. - String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + String accountName = verifyWasbAccountNameInConfig(conf); configureSecureModeTestSettings(conf); @@ -814,7 +818,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, CloudBlobClient blobClient = account.createCloudBlobClient(); // Capture the account URL and the account name. - String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + String accountName = verifyWasbAccountNameInConfig(conf); configureSecureModeTestSettings(conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java index 6d5e72e..af570bd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java @@ -29,6 +29,7 @@ import com.microsoft.azure.storage.CloudStorageAccount; import org.junit.Test; import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG; +import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig; /** * Test for error messages coming from SDK. @@ -46,7 +47,7 @@ public class ITestFileSystemOperationExceptionMessage AzureBlobStorageTestAccount.createTestAccount(conf); AzureTestUtils.assume("No test account", account != null); - String testStorageAccount = conf.get("fs.azure.test.account.name"); + String testStorageAccount = verifyWasbAccountNameInConfig(conf); conf = new Configuration(); conf.set("fs.AbstractFileSystem.wasb.impl", "org.apache.hadoop.fs.azure.Wasb"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java index bee0220..7783684 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java @@ -500,32 +500,6 @@ public class ITestWasbUriAndConfiguration extends AbstractWasbTestWithTimeout { } @Test - public void testNoAbstractFileSystemImplementationSpecifiedForWasbsScheme() throws Exception { - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasbs", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - - FileSystem fs = FileSystem.get(conf); - assertTrue(fs instanceof NativeAzureFileSystem); - assertEquals("wasbs", fs.getScheme()); - - // should throw if 'fs.AbstractFileSystem.wasbs.impl'' is not specified - try{ - FileContext.getFileContext(conf).getDefaultFileSystem(); - fail("Should've thrown."); - }catch(UnsupportedFileSystemException e){ - } - - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - } - - @Test public void testCredentialProviderPathExclusions() throws Exception { String providerPath = "user:///,jceks://wasb/user/hrt_qa/sqoopdbpasswd.jceks," + http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java index 0b72f06..82907c5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java @@ -143,12 +143,10 @@ public interface AzureTestConstants { - String ACCOUNT_KEY_PROPERTY_NAME - = "fs.azure.account.key."; + String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key."; + String ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name"; String SAS_PROPERTY_NAME = "fs.azure.sas."; String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml"; - String TEST_ACCOUNT_NAME_PROPERTY_NAME - = "fs.azure.test.account.name"; String MOCK_ACCOUNT_NAME = "mockAccount.blob.core.windows.net"; String MOCK_CONTAINER_NAME = "mockContainer"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java index 8d2a104..b438c8e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java @@ -36,10 +36,14 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; + +import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX; +import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN; import static org.apache.hadoop.fs.azure.integration.AzureTestConstants.*; import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; import static org.apache.hadoop.test.MetricsAsserts.getLongGauge; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; +import static org.junit.Assume.assumeTrue; /** * Utilities for the Azure tests. Based on {@code S3ATestUtils}, so @@ -476,4 +480,18 @@ public final class AzureTestUtils extends Assert { + KEY_SCALE_TESTS_ENABLED, enabled); } + + /** + * Check the account name for WASB tests is set correctly and return. + */ + public static String verifyWasbAccountNameInConfig(Configuration conf) { + String accountName = conf.get(ACCOUNT_NAME_PROPERTY_NAME); + if (accountName == null) { + accountName = conf.get(WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN); + } + assumeTrue("Account for WASB is missing or it is not in correct format", + accountName != null && !accountName.endsWith(WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX)); + return accountName; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java index 9b1fb8e..cd8b692 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java @@ -18,8 +18,8 @@ package org.apache.hadoop.fs.azure.metrics; -import static org.junit.Assert.*; -import org.junit.*; +import static org.junit.Assert.assertEquals; +import org.junit.Test; public class TestRollingWindowAverage { /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index e0afeb4..f61954f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -35,7 +35,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.services.AuthType; -import org.apache.hadoop.fs.azure.AbstractWasbTestWithTimeout; import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; @@ -45,12 +44,12 @@ import org.apache.hadoop.fs.azurebfs.utils.UriUtils; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.io.IOUtils; +import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*; import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.FILE_SYSTEM_NOT_FOUND; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.*; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assume.assumeTrue; -import static org.hamcrest.CoreMatchers.*; /** * Base for AzureBlobFileSystem Integration tests. @@ -58,12 +57,12 @@ import static org.hamcrest.CoreMatchers.*; * <I>Important: This is for integration tests only.</I> */ public abstract class AbstractAbfsIntegrationTest extends - AbstractWasbTestWithTimeout { + AbstractAbfsTestWithTimeout { private static final Logger LOG = LoggerFactory.getLogger(AbstractAbfsIntegrationTest.class); - private boolean isEmulator; + private boolean isIPAddress; private NativeAzureFileSystem wasb; private AzureBlobFileSystem abfs; private String abfsScheme; @@ -75,27 +74,28 @@ public abstract class AbstractAbfsIntegrationTest extends private AuthType authType; protected AbstractAbfsIntegrationTest() { - fileSystemName = ABFS_TEST_CONTAINER_PREFIX + UUID.randomUUID().toString(); + fileSystemName = TEST_CONTAINER_PREFIX + UUID.randomUUID().toString(); configuration = new Configuration(); - configuration.addResource(ABFS_TEST_RESOURCE_XML); - this.accountName = this.configuration.get(FS_AZURE_TEST_ACCOUNT_NAME); + configuration.addResource(TEST_CONFIGURATION_FILE_NAME); + + this.accountName = this.configuration.get(FS_AZURE_ACCOUNT_NAME); + if (accountName == null) { + // check if accountName is set using different config key + accountName = configuration.get(FS_AZURE_ABFS_ACCOUNT_NAME); + } + assumeTrue("Not set: " + FS_AZURE_ABFS_ACCOUNT_NAME, + accountName != null && !accountName.isEmpty()); + authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + accountName, AuthType.SharedKey); abfsScheme = authType == AuthType.SharedKey ? FileSystemUriSchemes.ABFS_SCHEME : FileSystemUriSchemes.ABFS_SECURE_SCHEME; - String accountName = configuration.get(FS_AZURE_TEST_ACCOUNT_NAME, ""); - assumeTrue("Not set: " + FS_AZURE_TEST_ACCOUNT_NAME, - !accountName.isEmpty()); - assertThat("The key in " + FS_AZURE_TEST_ACCOUNT_KEY_PREFIX - + " is not bound to an ABFS account", - accountName, containsString("dfs.core.windows.net")); - String fullKey = FS_AZURE_TEST_ACCOUNT_KEY_PREFIX - + accountName; - if (authType == AuthType.SharedKey) { - assumeTrue("Not set: " + fullKey, configuration.get(fullKey) != null); + String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + accountName; + assumeTrue("Not set: " + keyProperty, configuration.get(keyProperty) != null); + // Update credentials } else { String accessTokenProviderKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName; assumeTrue("Not set: " + accessTokenProviderKey, configuration.get(accessTokenProviderKey) != null); @@ -113,7 +113,17 @@ public abstract class AbstractAbfsIntegrationTest extends this.testUrl = defaultUri.toString(); configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); configuration.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true); - this.isEmulator = this.configuration.getBoolean(FS_AZURE_EMULATOR_ENABLED, false); + // For testing purposes, an IP address and port may be provided to override + // the host specified in the FileSystem URI. Also note that the format of + // the Azure Storage Service URI changes from + // http[s]://[account][domain-suffix]/[filesystem] to + // http[s]://[ip]:[port]/[account]/[filesystem]. + String endPoint = configuration.get(AZURE_ABFS_ENDPOINT); + if (endPoint != null && endPoint.contains(":") && endPoint.split(":").length == 2) { + this.isIPAddress = true; + } else { + this.isIPAddress = false; + } } @@ -122,13 +132,22 @@ public abstract class AbstractAbfsIntegrationTest extends //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem. createFileSystem(); - if (!isEmulator && authType == AuthType.SharedKey) { + if (!isIPAddress && authType == AuthType.SharedKey) { final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl())); final AzureNativeFileSystemStore azureNativeFileSystemStore = new AzureNativeFileSystemStore(); + + // update configuration with wasb credentials + String accountNameWithoutDomain = accountName.split("\\.")[0]; + String wasbAccountName = accountNameWithoutDomain + WASB_ACCOUNT_NAME_DOMAIN_SUFFIX; + String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + wasbAccountName; + if (configuration.get(keyProperty) == null) { + configuration.set(keyProperty, getAccountKey()); + } + azureNativeFileSystemStore.initialize( wasbUri, - getConfiguration(), + configuration, new AzureFileSystemInstrumentation(getConfiguration())); wasb = new NativeAzureFileSystem(azureNativeFileSystemStore); @@ -201,7 +220,9 @@ public abstract class AbstractAbfsIntegrationTest extends } protected String getHostName() { - return configuration.get(FS_AZURE_TEST_HOST_NAME); + // READ FROM ENDPOINT, THIS IS CALLED ONLY WHEN TESTING AGAINST DEV-FABRIC + String endPoint = configuration.get(AZURE_ABFS_ENDPOINT); + return endPoint.split(":")[0]; } protected void setTestUrl(String testUrl) { @@ -220,21 +241,21 @@ public abstract class AbstractAbfsIntegrationTest extends } protected String getAccountName() { - return configuration.get(FS_AZURE_TEST_ACCOUNT_NAME); + return this.accountName; } protected String getAccountKey() { return configuration.get( - FS_AZURE_TEST_ACCOUNT_KEY_PREFIX - + getAccountName()); + FS_AZURE_ACCOUNT_KEY_PREFIX + + accountName); } protected Configuration getConfiguration() { return configuration; } - protected boolean isEmulator() { - return isEmulator; + protected boolean isIPAddress() { + return isIPAddress; } protected AuthType getAuthType() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java new file mode 100644 index 0000000..fee90ab --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.rules.Timeout; + +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_TIMEOUT; + +/** + * Base class for any ABFS test with timeouts & named threads. + * This class does not attempt to bind to Azure. + */ +public class AbstractAbfsTestWithTimeout extends Assert { + /** + * The name of the current method. + */ + @Rule + public TestName methodName = new TestName(); + /** + * Set the timeout for every test. + * This is driven by the value returned by {@link #getTestTimeoutMillis()}. + */ + @Rule + public Timeout testTimeout = new Timeout(getTestTimeoutMillis()); + + /** + * Name the junit thread for the class. This will overridden + * before the individual test methods are run. + */ + @BeforeClass + public static void nameTestThread() { + Thread.currentThread().setName("JUnit"); + } + + /** + * Name the thread to the current test method. + */ + @Before + public void nameThread() { + Thread.currentThread().setName("JUnit-" + methodName.getMethodName()); + } + + /** + * Override point: the test timeout in milliseconds. + * @return a timeout in milliseconds + */ + protected int getTestTimeoutMillis() { + return TEST_TIMEOUT; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java index 800b95a..46ac866 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java @@ -69,7 +69,7 @@ public class ITestAzureBlobFileSystemBackCompat extends private String getBlobConnectionString() { String connectionString; - if (isEmulator()) { + if (isIPAddress()) { connectionString = "DefaultEndpointsProtocol=http;BlobEndpoint=http://"; + this.getHostName() + ":8880/" + this.getAccountName().split("\\.") [0] + ";AccountName=" + this.getAccountName().split("\\.")[0] http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java index e4f3ea3..c4bfee2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java @@ -50,7 +50,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { LoggerFactory.getLogger(ITestWasbAbfsCompatibility.class); public ITestWasbAbfsCompatibility() throws Exception { - Assume.assumeFalse("Emulator is not supported", isEmulator()); + Assume.assumeFalse("Emulator is not supported", isIPAddress()); Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java index 556dcdb..bedd4f3 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java @@ -23,6 +23,7 @@ import java.lang.reflect.Field; import org.apache.commons.codec.Charsets; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation; import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.BooleanConfigurationValidatorAnnotation; import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation; @@ -98,7 +99,7 @@ public class TestAbfsConfigurationFieldsValidation { this.encodedString = new String(base64.encode("base64Value".getBytes(Charsets.UTF_8)), Charsets.UTF_8); this.encodedAccountKey = new String(base64.encode("someAccountKey".getBytes(Charsets.UTF_8)), Charsets.UTF_8); Configuration configuration = new Configuration(); - configuration.addResource("azure-bfs-test.xml"); + configuration.addResource(TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME); configuration.set(INT_KEY, "1234565"); configuration.set(LONG_KEY, "4194304"); configuration.set(STRING_KEY, "stringValue"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java index 67301c7..7852f25 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java @@ -22,10 +22,9 @@ package org.apache.hadoop.fs.azurebfs.constants; * Responsible to keep all the Azure Blob File System configurations keys in Hadoop configuration file. */ public final class TestConfigurationKeys { - public static final String FS_AZURE_TEST_ACCOUNT_NAME = "fs.azure.test.account.name"; - public static final String FS_AZURE_TEST_ACCOUNT_KEY_PREFIX = "fs.azure.test.account.key."; - public static final String FS_AZURE_TEST_HOST_NAME = "fs.azure.test.host.name"; - public static final String FS_AZURE_TEST_HOST_PORT = "fs.azure.test.host.port"; + public static final String FS_AZURE_ACCOUNT_NAME = "fs.azure.account.name"; + public static final String FS_AZURE_ABFS_ACCOUNT_NAME = "fs.azure.abfs.account.name"; + public static final String FS_AZURE_ACCOUNT_KEY_PREFIX = "fs.azure.account.key."; public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs"; public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID = "fs.azure.account.oauth2.contributor.client.id"; @@ -34,9 +33,9 @@ public final class TestConfigurationKeys { public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_ID = "fs.azure.account.oauth2.reader.client.id"; public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET = "fs.azure.account.oauth2.reader.client.secret"; - public static final String ABFS_TEST_RESOURCE_XML = "azure-bfs-test.xml"; - - public static final String ABFS_TEST_CONTAINER_PREFIX = "abfs-testcontainer-"; + public static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml"; + public static final String TEST_CONTAINER_PREFIX = "abfs-testcontainer-"; + public static final int TEST_TIMEOUT = 10 * 60 * 1000; private TestConfigurationKeys() {} } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java index 5505e6a..6d089d1 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; import org.apache.hadoop.fs.azurebfs.services.AuthType; +import org.junit.Assume; /** * Bind ABFS contract tests to the Azure test setup/teardown. @@ -42,6 +43,8 @@ public class ABFSContractTestBinding extends AbstractAbfsIntegrationTest { if (useExistingFileSystem) { Configuration configuration = getConfiguration(); String testUrl = configuration.get(TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI); + Assume.assumeTrue("Contract tests are skipped because of missing config property :" + + TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI, testUrl != null); if (getAuthType() != AuthType.SharedKey) { testUrl = testUrl.replaceFirst(FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java index e9e90c9..a9556ab 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java @@ -32,7 +32,6 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.M import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE; -import static org.junit.Assert.assertEquals; /** * Test configuration validators. http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java new file mode 100644 index 0000000..9c636a2 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.utils; + +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; + +import org.junit.Assume; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; +import org.apache.hadoop.fs.azurebfs.services.AuthType; + +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONTAINER_PREFIX; + + /* + * Some Utils for ABFS tests. + * */ +public final class AbfsTestUtils extends AbstractAbfsIntegrationTest{ + private static final Logger LOG = + LoggerFactory.getLogger(AbfsTestUtils.class); + /** + * If unit tests were interrupted and crushed accidentally, the test containers won't be deleted. + * In that case, dev can use this tool to list and delete all test containers. + * By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-" + */ + + public void checkContainers() throws Throwable { + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + int count = 0; + CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount(); + CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); + Iterable<CloudBlobContainer> containers + = blobClient.listContainers(TEST_CONTAINER_PREFIX); + for (CloudBlobContainer container : containers) { + count++; + LOG.info("Container {}, URI {}", + container.getName(), + container.getUri()); + } + LOG.info("Found {} test containers", count); + } + + + public void deleteContainers() throws Throwable { + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + int count = 0; + CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount(); + CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); + Iterable<CloudBlobContainer> containers + = blobClient.listContainers(TEST_CONTAINER_PREFIX); + for (CloudBlobContainer container : containers) { + LOG.info("Container {} URI {}", + container.getName(), + container.getUri()); + if (container.deleteIfExists()) { + count++; + } + } + LOG.info("Deleted {} test containers", count); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java deleted file mode 100644 index ef4ddb9..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.fs.azurebfs.utils; - -import com.microsoft.azure.storage.CloudStorageAccount; -import com.microsoft.azure.storage.blob.CloudBlobClient; -import com.microsoft.azure.storage.blob.CloudBlobContainer; - -import org.junit.Assume; -import org.junit.Test; - -import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; -import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; -import org.apache.hadoop.fs.azurebfs.services.AuthType; - -import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.ABFS_TEST_CONTAINER_PREFIX; - -/** - * If unit tests were interrupted and crushed accidentally, the test containers won't be deleted. - * In that case, dev can use this tool to list and delete all test containers. - * By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-" - */ -public class CleanUpAbfsTestContainer extends AbstractAbfsIntegrationTest{ - - public CleanUpAbfsTestContainer() { - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); - } - - @Test - public void testEnumContainers() throws Throwable { - int count = 0; - CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount(); - CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); - Iterable<CloudBlobContainer> containers - = blobClient.listContainers(ABFS_TEST_CONTAINER_PREFIX); - for (CloudBlobContainer container : containers) { - count++; - System.out.println(String.format("Container %s URI %s", - container.getName(), - container.getUri())); - } - System.out.println(String.format("Found %d test containers", count)); - } - - @Test - public void testDeleteContainers() throws Throwable { - int count = 0; - CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount(); - CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); - Iterable<CloudBlobContainer> containers - = blobClient.listContainers(ABFS_TEST_CONTAINER_PREFIX); - for (CloudBlobContainer container : containers) { - System.out.println(String.format("Container %s URI %s", - container.getName(), - container.getUri())); - if (container.deleteIfExists()) { - count++; - } - } - System.out.println(String.format("Deleted %s test containers", count)); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java index e71e31d..690e56c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java @@ -38,11 +38,11 @@ public final class TestUriUtils { @Test public void testExtractRawAccountName() throws Exception { - Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.core.windows.net")); - Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.preprod.core.windows.net")); - Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net")); - Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("")); - Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName(null)); - Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net")); + Assert.assertEquals("abfs", UriUtils.extractAccountNameFromHostName("abfs.dfs.core.windows.net")); + Assert.assertEquals("abfs", UriUtils.extractAccountNameFromHostName("abfs.dfs.preprod.core.windows.net")); + Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net")); + Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("")); + Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName(null)); + Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net")); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml deleted file mode 100644 index 9b908b0..0000000 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml +++ /dev/null @@ -1,188 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> - - <!-- - <property> - <name>fs.azure.test.account.name</name> - <value>{YOURACCOUNT}</value> - </property> - - <property> - <name>fs.azure.account.auth.type.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{AUTH TYPE}</value> - <description>The auth type can be : SharedKey, OAuth, Custom. By default "SharedKey" is used.</description> - </property> - - <property> - <name>fs.contract.test.fs.abfs</name> - <value>abfs://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value> - <description>The name of the azure file system for testing.</description> - </property> - - <property> - <name>fs.contract.test.fs.abfss</name> - <value>abfss://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value> - <description>The name of the azure file system for testing.</description> - </property> - --> - - <!--If auth type is "SharedKey", provide SharedKey credentials below --> - <!-- - <property> - <name>fs.azure.account.key.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{ACCOUNTKEY}</value> - </property> - - <property> - <name>fs.azure.test.account.key.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{ACCOUNTKEY}</value> - </property> - --> - - <!--If auth type is "OAuth", set below properties, AAD client and tenant related properties can be obtained through Azure Portal--> - <!-- - <property> - <name>fs.azure.account.oauth.provider.type.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>org.apache.hadoop.fs.azurebfs.oauth2.{Token Provider Class name}</value> - <description>The full name of token provider class name.</description> - </property> - --> - - <!--If "ClientCredsTokenProvider" is set as key provider, set auth endpoint, client id and secret below--> - <!-- - <property> - <name>fs.azure.account.oauth2.client.endpoint.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value> - <description>Token end point, this can be found through Azure portal</description> - </property> - - <property> - <name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{client id}</value> - <description>AAD client id.</description> - </property> - - <property> - <name>fs.azure.account.oauth2.client.secret.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{client secret}</value> - </property> - --> - - <!--If "UserPasswordTokenProvider" is set as key provider, set auth endpoint, use name and password below--> - <!-- - <property> - <name>fs.azure.account.oauth2.client.endpoint.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value> - <description>Token end point, this can be found through Azure portal</description> - </property> - - <property> - <name>fs.azure.account.oauth2.user.name.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{user name}</value> - </property> - - <property> - <name>fs.azure.account.oauth2.user.password.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{user password}</value> - </property> - --> - - <!--If "MsiTokenProvider" is set as key provider, set tenantGuid and client id below--> - <!-- - <property> - <name>fs.azure.account.oauth2.msi.tenant.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{tenantGuid}</value> - <description>msi tenantGuid.</description> - </property> - - <property> - <name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{client id}</value> - <description>AAD client id.</description> - </property> - --> - - <!--If "RefreshTokenBasedTokenProvider" is set as key provider, set refresh token and client id below--> - <!-- - <property> - <name>fs.azure.account.oauth2.refresh.token.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{refresh token}</value> - <description>refresh token.</description> - </property> - - <property> - <name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name> - <value>{client id}</value> - <description>AAD client id.</description> - </property> - --> - - <!--Below four configure properties are provided for Blob Data Contributor a - nd Blob Data Reader OAuth access test "ITestAzureBlobFileSystemOauth", - using"ClientCredsTokenProvider" as key - provider. These are for test only.--> - - <!-- - <property> - <name>fs.azure.account.oauth2.contributor.client.id</name> - <value>{ID of client which has Data Contributor role}</value> - </property> - - <property> - <name>fs.azure.account.oauth2.contributor.client.secret</name> - <value>{secret of client which has Data Contributor role}</value> - </property> - - <property> - <name>fs.azure.account.oauth2.reader.client.id</name> - <value>{ID of client which has Data Reader role}</value> - </property> - - <property> - <name>fs.azure.account.oauth2.reader.client.secret</name> - <value>{Secret of client which has Data Reader role}</value> - </property> - --> - - - - <!-- Save above configuration properties in a separate file named --> - <!-- azure-bfs-auth-keys.xml in the same directory as this file. --> - <!-- DO NOT ADD azure-bfs-auth-keys.xml TO REVISION CONTROL. The keys to your --> - <!-- Azure Storage account are a secret and must not be shared. --> - - <include xmlns="http://www.w3.org/2001/XInclude"; href="azure-bfs-auth-keys.xml"> - <fallback /> - </include> - - <!--<property>--> - <!--<name>fs.azure.test.host.name</name>--> - <!--<value>{HOSTNAME}</value>--> - <!--</property>--> - - <!--<property>--> - <!--<name>fs.azure.test.host.port</name>--> - <!--<value>{PORT}</value>--> - <!--</property>--> - - <!--<property>--> - <!--<name>fs.azure.abfs.emulator.enabled</name>--> - <!--<value>false</value>--> - <!--</property>--> - -</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml index 8cea256..a36a391 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml @@ -16,49 +16,39 @@ <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <configuration xmlns:xi="http://www.w3.org/2001/XInclude";> - <!-- For tests against live azure, provide the following account information --> + <!--==================== WASB CONFIGURATION ====================--> + <!-- For tests against azure-emulator --> + <property> + <name>fs.azure.test.emulator</name> + <value>false</value> + </property> - <!-- - <property> - <name>fs.azure.test.account.name</name> - <value>{ACCOUNTNAME}.blob.core.windows.net</value> - </property> - <property> - <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name> - <value>{ACCOUNTKEY}</value> - </property> - --> + <property> + <name>fs.azure.secure.mode</name> + <value>false</value> + </property> - <!-- uncomment to test in Azure secure mode --> - <!-- - <property> - <name>fs.azure.secure.mode</name> - <value>true</value> - </property> - --> + <!--==================== ABFS CONFIGURATION ====================--> + <!-- SEE relevant section in "site/markdown/testing_azure.md"--> + <!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========--> <property> <name>fs.azure.user.agent.prefix</name> <value>MSFT</value> </property> - <!-- Save the above configuration properties in a separate file named --> - <!-- azure-auth-keys.xml in the same directory as this file. --> - <!-- DO NOT ADD azure-auth-keys.xml TO REVISION CONTROL. The keys to your --> - <!-- Azure Storage account are a secret and must not be shared. --> + <!--====================== IMPORTANT!! ========================--> + <!--WARNING!! + PLEASE CREATE A FILE NAMED "azure-auth-keys.xml" + AND SEE "site/markdown/testing_azure.md", THEN + STORE THE CONFIGURATION PROPERTIES WITHIN IT. + TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS, + THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING + IT IN PATCHES OR COMMITS. --> + <!--=============================================================--> + <include xmlns="http://www.w3.org/2001/XInclude"; href="azure-auth-keys.xml"> <fallback /> </include> - <!-- For tests against azure-emulator --> - <!-- - <property> - <name>fs.azure.test.emulator</name> - <value>true</value> - </property> - --> - <property> - <name>fs.AbstractFileSystem.wasb.impl</name> - <value>org.apache.hadoop.fs.azure.Wasb</value> - </property> </configuration> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
