HADOOP-13929. ADLS connector should not check in contract-test-options.xml. (John Zhuge via lei)
(cherry picked from commit 71c23c9fc94cfdf58de80effbc3f51c0925d0cfe) (cherry picked from commit ed4388f903d34a2c09845d200e1e717397b217e1) (cherry picked from commit 674b9cdab1e753a070dfb104f499d829f7ba4094) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b11f159 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b11f159 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b11f159 Branch: refs/heads/branch-2.8.0 Commit: 9b11f159ba302cbb4b74cb99cafcb0b724d5cb68 Parents: 9e101ca Author: Lei Xu <l...@cloudera.com> Authored: Mon Feb 13 13:33:13 2017 -0800 Committer: Chris Douglas <cdoug...@apache.org> Committed: Thu Mar 9 15:24:45 2017 -0800 ---------------------------------------------------------------------- .gitignore | 10 ++-- .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 2 +- .../src/site/markdown/index.md | 4 +- .../fs/adl/live/AdlStorageConfiguration.java | 42 +++++++------- .../src/test/resources/adls.xml | 11 ++++ .../test/resources/contract-test-options.xml | 61 -------------------- 6 files changed, 39 insertions(+), 91 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b11f159/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index eb1fc96..2a82ba7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,10 @@ target build +# Filesystem contract test options and credentials +auth-keys.xml +azure-auth-keys.xml + # External tool builders */.externalToolBuilders */maven-eclipse.xml @@ -22,12 +26,6 @@ build hadoop-common-project/hadoop-kms/downloads/ hadoop-hdfs-project/hadoop-hdfs/downloads hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads -hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml -hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.toolbox yarnregistry.pdf -hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml -hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml -hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml -hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml patchprocess/ http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b11f159/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 3d41025..303b7bc 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -72,7 +72,7 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys.*; @InterfaceAudience.Public @InterfaceStability.Evolving public class AdlFileSystem extends FileSystem { - static final String SCHEME = "adl"; + public static final String SCHEME = "adl"; static final int DEFAULT_PORT = 443; private URI uri; private String userName; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b11f159/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md index ced5cff..5037db6 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md @@ -224,7 +224,9 @@ commands demonstrate access to a storage account named `youraccount`. ## <a name="Testing_the_hadoop-azure_Module" />Testing the azure-datalake-store Module The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage. -A selection of tests can run against the Azure Data Lake Storage. To run tests against Adl storage. Please configure contract-test-options.xml with Adl account information mentioned in the above sections. Also turn on contract test execution flag to trigger tests against Azure Data Lake Storage. +A selection of tests can run against the Azure Data Lake Storage. To run these +tests, please create `src/test/resources/auth-keys.xml` with Adl account +information mentioned in the above sections and the following properties. <property> <name>dfs.adl.test.contract.enable</name> http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b11f159/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java index f7f0b71..7d6153d 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java @@ -21,6 +21,8 @@ package org.apache.hadoop.fs.adl.live; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.adl.AdlFileSystem; +import org.apache.hadoop.util.ReflectionUtils; import java.io.IOException; import java.net.URI; @@ -30,15 +32,19 @@ import java.net.URISyntaxException; * Configure Adl storage file system. */ public final class AdlStorageConfiguration { + static final String CONTRACT_XML = "adls.xml"; + private static final String CONTRACT_ENABLE_KEY = "dfs.adl.test.contract.enable"; + private static final boolean CONTRACT_ENABLE_DEFAULT = false; + + private static final String FILE_SYSTEM_KEY = + String.format("test.fs.%s.name", AdlFileSystem.SCHEME); - private static final String TEST_CONFIGURATION_FILE_NAME = - "contract-test-options.xml"; - private static final String TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME = - "adls.xml"; - private static final String KEY_FILE_SYSTEM_IMPL = "fs.contract.test.fs"; - private static final String KEY_FILE_SYSTEM = "test.fs.adl.name"; + private static final String FILE_SYSTEM_IMPL_KEY = + String.format("fs.%s.impl", AdlFileSystem.SCHEME); + private static final Class<?> FILE_SYSTEM_IMPL_DEFAULT = + AdlFileSystem.class; private static boolean isContractTestEnabled = false; private static Configuration conf = null; @@ -48,8 +54,7 @@ public final class AdlStorageConfiguration { public synchronized static Configuration getConfiguration() { Configuration newConf = new Configuration(); - newConf.addResource(TEST_CONFIGURATION_FILE_NAME); - newConf.addResource(TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME); + newConf.addResource(CONTRACT_XML); return newConf; } @@ -58,7 +63,8 @@ public final class AdlStorageConfiguration { conf = getConfiguration(); } - isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY, false); + isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY, + CONTRACT_ENABLE_DEFAULT); return isContractTestEnabled; } @@ -72,23 +78,15 @@ public final class AdlStorageConfiguration { return null; } - String fileSystem = conf.get(KEY_FILE_SYSTEM); + String fileSystem = conf.get(FILE_SYSTEM_KEY); if (fileSystem == null || fileSystem.trim().length() == 0) { throw new IOException("Default file system not configured."); } - String fileSystemImpl = conf.get(KEY_FILE_SYSTEM_IMPL); - if (fileSystemImpl == null || fileSystemImpl.trim().length() == 0) { - throw new IOException( - "Configuration " + KEY_FILE_SYSTEM_IMPL + "does not exist."); - } - FileSystem fs = null; - try { - fs = (FileSystem) Class.forName(fileSystemImpl).newInstance(); - } catch (Exception e) { - throw new IOException("Could not instantiate the filesystem."); - } - fs.initialize(new URI(conf.get(KEY_FILE_SYSTEM)), conf); + Class<?> clazz = conf.getClass(FILE_SYSTEM_IMPL_KEY, + FILE_SYSTEM_IMPL_DEFAULT); + FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf); + fs.initialize(new URI(fileSystem), conf); return fs; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b11f159/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml index 76a78b0..5bbdd6f 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml +++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml @@ -12,6 +12,17 @@ limitations under the License. See accompanying LICENSE file. --> <configuration> + <!-- + To run live tests: + # Create a file auth-keys.xml - DO NOT ADD TO REVISION CONTROL + # Add property test.fs.adl.name to point to an ADLS filesystem URL + # Add property dfs.adl.test.contract.enable with value set to true + # Add the credentials for the service you are testing against + --> + <include xmlns="http://www.w3.org/2001/XInclude" href="auth-keys.xml"> + <fallback/> + </include> + <property> <name>fs.contract.test.root-tests-enabled</name> <value>true</value> http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b11f159/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml deleted file mode 100644 index 646922e..0000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml +++ /dev/null @@ -1,61 +0,0 @@ -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> -<configuration> - <property> - <name>dfs.adls.oauth2.refresh.url</name> - <value> - </value> - </property> - - <property> - <name>dfs.adls.oauth2.credential</name> - <value></value> - </property> - - <property> - <name>dfs.adls.oauth2.client.id</name> - <value></value> - </property> - - <property> - <name>dfs.adls.oauth2.access.token.provider.type</name> - <value>ClientCredential</value> - <description> - Supported provider type: - - "ClientCredential" : Client id and client credentials(Provided - through configuration file) flow. - - "RefreshToken" : Client id and refresh token(Provided - through configuration file)flow. - - "Custom" : Custom AAD token management. - </description> - </property> - - <property> - <name>dfs.adl.test.contract.enable</name> - <value>false</value> - </property> - - <property> - <name>test.fs.adl.name</name> - <value></value> - </property> - - <property> - <name>fs.contract.test.fs</name> - <value>org.apache.hadoop.fs.adl.AdlFileSystem</value> - </property> -</configuration> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org