HADOOP-14844. Remove requirement to specify TenantGuid for MSI Token Provider. Contributed by Atul Sikaria.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4661850 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4661850 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4661850 Branch: refs/heads/HDFS-10467 Commit: a4661850c1e0794baf493a468191e12681d68ab4 Parents: c35510a Author: John Zhuge <[email protected]> Authored: Fri Sep 8 11:51:03 2017 -0700 Committer: John Zhuge <[email protected]> Committed: Fri Sep 8 11:51:03 2017 -0700 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 16 +++------------ hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 - .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 4 +--- .../src/site/markdown/index.md | 21 ++++++-------------- .../hadoop/fs/adl/TestAzureADTokenProvider.java | 4 ---- 6 files changed, 11 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 23739b0..269a13c 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2717,8 +2717,7 @@ fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url. The RefreshToken type requires property fs.adl.oauth2.client.id and fs.adl.oauth2.refresh.token. - The MSI type requires properties fs.adl.oauth2.msi.port and - fs.adl.oauth2.msi.tenantguid. + The MSI type reads optional property fs.adl.oauth2.msi.port, if specified. The DeviceCode type requires property fs.adl.oauth2.devicecode.clientapp.id. The Custom type requires property fs.adl.oauth2.access.token.provider. @@ -2762,17 +2761,8 @@ <value></value> <description> The localhost port for the MSI token service. This is the port specified - when creating the Azure VM. - Used by MSI token provider. - </description> - </property> - - <property> - <name>fs.adl.oauth2.msi.tenantguid</name> - <value></value> - <description> - The tenant guid for the Azure AAD tenant under which the azure data lake - store account is created. + when creating the Azure VM. The default, if this setting is not specified, + is 50342. Used by MSI token provider. </description> </property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-tools/hadoop-azure-datalake/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml index b06c774..e32f9fc 100644 --- a/hadoop-tools/hadoop-azure-datalake/pom.xml +++ b/hadoop-tools/hadoop-azure-datalake/pom.xml @@ -110,7 +110,7 @@ <dependency> <groupId>com.microsoft.azure</groupId> <artifactId>azure-data-lake-store-sdk</artifactId> - <version>2.2.2</version> + <version>2.2.3</version> </dependency> <!-- ENDS HERE--> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index f77d981..790902c 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -56,7 +56,6 @@ public final class AdlConfKeys { // MSI Auth Configuration public static final String MSI_PORT = "fs.adl.oauth2.msi.port"; - public static final String MSI_TENANT_GUID = "fs.adl.oauth2.msi.tenantguid"; // DeviceCode Auth configuration public static final String DEVICE_CODE_CLIENT_APP_ID = http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index a5e31e1..a496595 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -290,9 +290,7 @@ public class AdlFileSystem extends FileSystem { private AccessTokenProvider getMsiBasedTokenProvider( Configuration conf) throws IOException { - int port = Integer.parseInt(getNonEmptyVal(conf, MSI_PORT)); - String tenantGuid = getPasswordString(conf, MSI_TENANT_GUID); - return new MsiTokenProvider(port, tenantGuid); + return new MsiTokenProvider(conf.getInt(MSI_PORT, -1)); } private AccessTokenProvider getDeviceCodeTokenProvider( http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md index e34da36..ca79321 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md @@ -164,15 +164,11 @@ Identity extension within the VM. The advantage of doing this is that the credentials are managed by the extension, and do not have to be put into core-site.xml. -To use MSI, the following two steps are needed: -1. Modify the VM deployment template to specify the port number of the token - service exposed to localhost by the identity extension in the VM. -2. Get your Azure ActiveDirectory Tenant ID: - 1. Go to [the portal](https://portal.azure.com) - 2. Under services in left nav, look for Azure Active Directory and click on it. - 3. Click on Properties - 4. Note down the GUID shown under "Directory ID" - this is your AAD tenant ID - +To use MSI, modify the VM deployment template to use the identity extension. Note the +port number you specified in the template: this is the port number for the REST endpoint +of the token service exposed to localhost by the identity extension in the VM. The default +recommended port number is 50342 - if the recommended port number is used, then the msi.port +setting below can be omitted in the configuration. ##### Configure core-site.xml Add the following properties to your `core-site.xml` @@ -185,12 +181,7 @@ Add the following properties to your `core-site.xml` <property> <name>fs.adl.oauth2.msi.port</name> - <value>PORT NUMBER FROM STEP 1 ABOVE</value> -</property> - -<property> - <name>fs.adl.oauth2.msi.TenantGuid</name> - <value>AAD TENANT ID GUID FROM STEP 2 ABOVE</value> + <value>PORT NUMBER FROM ABOVE (if different from the default of 50342)</value> </property> ``` http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4661850/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java index 929b33a..12c2e3f 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java @@ -43,8 +43,6 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; import static org.apache.hadoop.fs.adl.AdlConfKeys.DEVICE_CODE_CLIENT_APP_ID; -import static org.apache.hadoop.fs.adl.AdlConfKeys.MSI_PORT; -import static org.apache.hadoop.fs.adl.AdlConfKeys.MSI_TENANT_GUID; import static org.apache.hadoop.fs.adl.TokenProviderType.*; import static org.junit.Assert.assertEquals; @@ -107,8 +105,6 @@ public class TestAzureADTokenProvider { throws IOException, URISyntaxException { Configuration conf = new Configuration(); conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, MSI); - conf.set(MSI_PORT, "54321"); - conf.set(MSI_TENANT_GUID, "TENANT_GUID"); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
