Repository: hadoop Updated Branches: refs/heads/branch-2 cbb1c7619 -> ff2870310
HADOOP-14174. Set default ADLS access token provider type to ClientCredential. Contributed by John Zhuge. Signed-off-by: John Zhuge <[email protected]> (cherry picked from commit 56e81f2a204d5e65d29222df1b2ba4d892f9c0d5) Conflicts: hadoop-common-project/hadoop-common/src/main/resources/core-default.xml hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff287031 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff287031 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff287031 Branch: refs/heads/branch-2 Commit: ff2870310ea60b40c7a43e8bff36c315b41ba9f7 Parents: cbb1c76 Author: John Zhuge <[email protected]> Authored: Fri Apr 7 12:12:54 2017 -0700 Committer: John Zhuge <[email protected]> Committed: Fri Apr 7 15:01:29 2017 -0700 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 75 ++++++++++++++++++++ .../conf/TestCommonConfigurationFields.java | 2 +- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 2 + .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 3 +- .../apache/hadoop/fs/adl/AdlMockWebServer.java | 3 + .../hadoop/fs/adl/TestAzureADTokenProvider.java | 3 + .../hadoop/fs/adl/TestCustomTokenProvider.java | 4 ++ .../fs/adl/TestRelativePathFormation.java | 4 ++ 8 files changed, 94 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff287031/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 3b9193a..61c9df7 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2512,6 +2512,81 @@ </description> </property> + <!-- Azure Data Lake File System Configurations --> + + <property> + <name>fs.adl.impl</name> + <value>org.apache.hadoop.fs.adl.AdlFileSystem</value> + </property> + + <property> + <name>fs.AbstractFileSystem.adl.impl</name> + <value>org.apache.hadoop.fs.adl.Adl</value> + </property> + + <property> + <name>adl.feature.ownerandgroup.enableupn</name> + <value>false</value> + <description> + When true : User and Group in FileStatus/AclStatus response is + represented as user friendly name as per Azure AD profile. + + When false (default) : User and Group in FileStatus/AclStatus + response is represented by the unique identifier from Azure AD + profile (Object ID as GUID). + + For optimal performance, false is recommended. + </description> + </property> + + <property> + <name>fs.adl.oauth2.access.token.provider.type</name> + <value>ClientCredential</value> + <description> + Defines Azure Active Directory OAuth2 access token provider type. + Supported types are ClientCredential, RefreshToken, and Custom. + The ClientCredential type requires property fs.adl.oauth2.client.id, + fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url. + The RefreshToken type requires property fs.adl.oauth2.client.id and + fs.adl.oauth2.refresh.token. + The Custom type requires property fs.adl.oauth2.access.token.provider. + </description> + </property> + + <property> + <name>fs.adl.oauth2.client.id</name> + <value></value> + <description>The OAuth2 client id.</description> + </property> + + <property> + <name>fs.adl.oauth2.credential</name> + <value></value> + <description>The OAuth2 access key.</description> + </property> + + <property> + <name>fs.adl.oauth2.refresh.url</name> + <value></value> + <description>The OAuth2 token endpoint.</description> + </property> + + <property> + <name>fs.adl.oauth2.refresh.token</name> + <value></value> + <description>The OAuth2 refresh token.</description> + </property> + + <property> + <name>fs.adl.oauth2.access.token.provider</name> + <value></value> + <description> + The class name of the OAuth2 access token provider. + </description> + </property> + + <!-- Azure Data Lake File System Configurations Ends Here--> + <property> <name>hadoop.caller.context.enabled</name> <value>false</value> http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff287031/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index dd0ebab..30e08d5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -115,7 +115,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { // ADL properties are in a different subtree // - org.apache.hadoop.hdfs.web.ADLConfKeys xmlPrefixToSkipCompare.add("adl."); - xmlPropsToSkipCompare.add("fs.adl.impl"); + xmlPrefixToSkipCompare.add("fs.adl."); xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl"); // Deprecated properties. These should eventually be removed from the http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff287031/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index d3a5565..31df222 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -40,6 +40,8 @@ public final class AdlConfKeys { "fs.adl.oauth2.client.id"; public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY = "fs.adl.oauth2.access.token.provider.type"; + public static final TokenProviderType AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT = + TokenProviderType.ClientCredential; // OAuth Refresh Token Configuration public static final String AZURE_AD_REFRESH_TOKEN_KEY = http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff287031/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 0b860b3..e63f115 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -243,7 +243,8 @@ public class AdlFileSystem extends FileSystem { Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders( config, AdlFileSystem.class); TokenProviderType type = conf.getEnum( - AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); + AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT); switch (type) { case RefreshToken: http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff287031/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java index 55c8f81..d843d55 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java @@ -28,6 +28,8 @@ import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider; import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; import com.squareup.okhttp.mockwebserver.MockWebServer; @@ -84,6 +86,7 @@ public class AdlMockWebServer { // Responses are returned in the same order that they are enqueued. fs = new TestableAdlFileSystem(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff287031/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java index 3867e74..36498c6 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java @@ -101,6 +101,7 @@ public class TestAzureADTokenProvider { public void testCustomCredTokenProvider() throws URISyntaxException, IOException { Configuration conf = new Configuration(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); @@ -115,6 +116,7 @@ public class TestAzureADTokenProvider { public void testInvalidProviderConfigurationForType() throws URISyntaxException, IOException { Configuration conf = new Configuration(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); try { @@ -136,6 +138,7 @@ public class TestAzureADTokenProvider { Configuration conf = new Configuration(); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, "wrong.classpath.CustomMockTokenProvider"); try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff287031/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java index c594c65..737534c 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java @@ -38,6 +38,8 @@ import java.util.Collection; import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; /** * Test access token provider behaviour with custom token provider and for token @@ -89,6 +91,8 @@ public class TestCustomTokenProvider extends AdlMockWebServer { */ public void init() throws IOException, URISyntaxException { Configuration configuration = new Configuration(); + configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + TokenProviderType.Custom); configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, typeOfTokenProviderClass.getName()); fileSystems = new TestableAdlFileSystem[fsObjectCount]; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff287031/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java index 908f8b8..334c372 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java @@ -29,6 +29,8 @@ import java.net.URISyntaxException; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; /** * This class verifies path conversion to SDK. @@ -39,6 +41,8 @@ public class TestRelativePathFormation { public void testToRelativePath() throws URISyntaxException, IOException { AdlFileSystem fs = new AdlFileSystem(); Configuration configuration = new Configuration(); + configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + TokenProviderType.Custom); configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, "org.apache.hadoop.fs.adl.common.CustomMockTokenProvider"); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
