HADOOP-15969. ABFS: getNamespaceEnabled can fail blocking user access thru ACLs.
Contributed by Da Zhou. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2523d81 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2523d81 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2523d81 Branch: refs/heads/HDFS-12943 Commit: b2523d8100844338e073531c47666d744a101caf Parents: 788e747 Author: Da Zhou <da.z...@microsoft.com> Authored: Mon Dec 17 11:04:40 2018 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Dec 17 11:04:40 2018 +0000 ---------------------------------------------------------------------- .../fs/azurebfs/AzureBlobFileSystemStore.java | 19 ++-- .../azurebfs/AbstractAbfsIntegrationTest.java | 4 + .../fs/azurebfs/ITestGetNameSpaceEnabled.java | 96 ++++++++++++++++++++ .../constants/TestConfigurationKeys.java | 1 + 4 files changed, 114 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2523d81/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 5f1692f..5d70b77 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.azurebfs; import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; @@ -181,12 +182,18 @@ public class AzureBlobFileSystemStore { public boolean getIsNamespaceEnabled() throws AzureBlobFileSystemException { if (!isNamespaceEnabledSet) { - LOG.debug("getFilesystemProperties for filesystem: {}", - client.getFileSystem()); - - final AbfsRestOperation op = client.getFilesystemProperties(); - isNamespaceEnabled = Boolean.parseBoolean( - op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_NAMESPACE_ENABLED)); + LOG.debug("Get root ACL status"); + try { + client.getAclStatus(AbfsHttpConstants.FORWARD_SLASH + AbfsHttpConstants.ROOT_PATH); + isNamespaceEnabled = true; + } catch (AbfsRestOperationException ex) { + // Get ACL status is a HEAD request, its response doesn't contain errorCode + // So can only rely on its status code to determine its account type. + if (HttpURLConnection.HTTP_BAD_REQUEST != ex.getStatusCode()) { + throw ex; + } + isNamespaceEnabled = false; + } isNamespaceEnabledSet = true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2523d81/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 382cd7f..cb9549d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -262,6 +262,10 @@ public abstract class AbstractAbfsIntegrationTest extends return this.authType; } + public String getAbfsScheme() { + return this.abfsScheme; + } + protected boolean isIPAddress() { return isIPAddress; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2523d81/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java new file mode 100644 index 0000000..b9a2315 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs; + +import java.io.IOException; +import java.util.UUID; + +import org.junit.Assume; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; +import org.apache.hadoop.fs.azurebfs.services.AuthType; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +/** + * Test getIsNamespaceEnabled call. + */ +public class ITestGetNameSpaceEnabled extends AbstractAbfsIntegrationTest { + + private boolean isUsingXNSAccount; + public ITestGetNameSpaceEnabled() throws Exception { + isUsingXNSAccount = getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); + } + + @Test + public void testXNSAccount() throws IOException { + Assume.assumeTrue("Skip this test because the account being used for test is a non XNS account", + isUsingXNSAccount); + assertTrue("Expecting getIsNamespaceEnabled() return true", + getFileSystem().getIsNamespaceEnabled()); + } + + @Test + public void testNonXNSAccount() throws IOException { + Assume.assumeFalse("Skip this test because the account being used for test is a XNS account", + isUsingXNSAccount); + assertFalse("Expecting getIsNamespaceEnabled() return false", + getFileSystem().getIsNamespaceEnabled()); + } + + @Test + public void testFailedRequestWhenFSNotExist() throws Exception { + AbfsConfiguration config = this.getConfiguration(); + config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false); + String testUri = this.getTestUrl(); + String nonExistingFsUrl = getAbfsScheme() + "://" + UUID.randomUUID() + + testUri.substring(testUri.indexOf("@")); + AzureBlobFileSystem fs = this.getFileSystem(nonExistingFsUrl); + + intercept(AbfsRestOperationException.class, + "\"The specified filesystem does not exist.\", 404", + ()-> { + fs.getIsNamespaceEnabled(); + }); + } + + @Test + public void testFailedRequestWhenCredentialsNotCorrect() throws Exception { + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + Configuration config = this.getRawConfiguration(); + config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false); + String accountName = this.getAccountName(); + String configkKey = FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "." + accountName; + // Provide a wrong sharedKey + String secret = config.get(configkKey); + secret = (char) (secret.charAt(0) + 1) + secret.substring(1); + config.set(configkKey, secret); + + AzureBlobFileSystem fs = this.getFileSystem(config); + intercept(AbfsRestOperationException.class, + "\"Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.\", 403", + ()-> { + fs.getIsNamespaceEnabled(); + }); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2523d81/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java index 5565a49..fbd13fe 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java @@ -26,6 +26,7 @@ public final class TestConfigurationKeys { public static final String FS_AZURE_ABFS_ACCOUNT_NAME = "fs.azure.abfs.account.name"; public static final String FS_AZURE_ACCOUNT_KEY = "fs.azure.account.key"; public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs"; + public static final String FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT = "fs.azure.test.namespace.enabled"; public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID = "fs.azure.account.oauth2.contributor.client.id"; public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET = "fs.azure.account.oauth2.contributor.client.secret"; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
