This is an automated email from the ASF dual-hosted git repository.
anujmodi2021 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new b0c3d89a0e5 HADOOP-19854: [ABFS] Fixing Namespace Detection for FNS
Accounts with Soft Delete Enabled (#8410) (#8459)
b0c3d89a0e5 is described below
commit b0c3d89a0e5c24049750390b68717335be1d839d
Author: manika137 <[email protected]>
AuthorDate: Mon May 4 21:29:27 2026 +0530
HADOOP-19854: [ABFS] Fixing Namespace Detection for FNS Accounts with Soft
Delete Enabled (#8410) (#8459)
Contributed by Manika Joshi
---
.../fs/azurebfs/AzureBlobFileSystemStore.java | 29 +++++-
.../hadoop/fs/azurebfs/services/AbfsErrors.java | 4 +
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 103 ++++++++++++++++++++-
3 files changed, 130 insertions(+), 6 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 055cee06256..86416cc5206 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -162,6 +162,7 @@
import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.INFINITE_LEASE_DURATION;
import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_BLOB_DOMAIN_NAME;
import static
org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_ENCRYPTION_CONTEXT;
+import static
org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_SOFT_DELETE_NOT_SUPPORTED;
import static
org.apache.hadoop.fs.azurebfs.utils.UriUtils.isKeyForDirectorySet;
/**
@@ -422,17 +423,35 @@ private synchronized boolean
getNamespaceEnabledInformationFromServer(
} catch (AbfsRestOperationException ex) {
// Get ACL status is a HEAD request, its response doesn't contain
errorCode
// So can only rely on its status code to determine account type.
- if (HttpURLConnection.HTTP_BAD_REQUEST != ex.getStatusCode()) {
+ int status = ex.getStatusCode();
+ String message = ex.getMessage();
+
+ // Case 1: 409: Soft delete not supported
+ if (status == HttpURLConnection.HTTP_CONFLICT
+ && message != null
+ && message.contains(ERR_SOFT_DELETE_NOT_SUPPORTED)) {
+ /*
+ * HTTP_CONFLICT with soft delete not supported error indicates a FNS
account.
+ * This occurs when:
+ * 1. FNS-Blob endpoint is used (due to DFS endpoint usage for getAcl
call)
+ * 2. FNS-DFS endpoint is used (later internally converted to Blob)
+ * For both cases, namespace should be disabled. The exception is
irrelevant here
+ * since we'll be using Blob endpoint ultimately for both the cases
here.
+ * No need to throw the exception.
+ */
+ LOG.debug("Ignore soft-delete error. Setting namespace enabled to
false.");
+ setNamespaceEnabled(false);
+ } else if (status == HttpURLConnection.HTTP_BAD_REQUEST) { // Case 2: 400
+ // If getAcl fails with 400, namespace is disabled.
+ LOG.debug("Failed to get ACL status with 400. Inferring namespace
disabled and ignoring error", ex);
+ setNamespaceEnabled(false);
+ } else {
// If getAcl fails with anything other than 400, namespace is enabled.
setNamespaceEnabled(true);
// Continue to throw exception as earlier.
LOG.debug("Failed to get ACL status with non 400. Inferring namespace
enabled", ex);
throw ex;
}
- // If getAcl fails with 400, namespace is disabled.
- LOG.debug("Failed to get ACL status with 400. "
- + "Inferring namespace disabled and ignoring error", ex);
- setNamespaceEnabled(false);
} catch (AzureBlobFileSystemException ex) {
throw ex;
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java
index fe7f3b5cb1b..b5744e1ba3a 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java
@@ -59,6 +59,10 @@ public final class AbfsErrors {
+ "and cannot be appended to by the Azure Data Lake Storage Service API";
public static final String CONDITION_NOT_MET = "The condition specified
using "
+ "HTTP conditional header(s) is not met.";
+ public static final String ERR_READ_ON_DIRECTORY = "Read operation not
permitted on a directory.";
+ public static final String ERR_OPENFILE_ON_DIRECTORY = "openFileForRead must
be used with files and not directories";
+ public static final String ERR_SOFT_DELETE_NOT_SUPPORTED = "This endpoint
does not support BlobStorageEvents or SoftDelete.";
+
/**
* Exception message on filesystem init if token-provider-auth-type configs
are provided
*/
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
index 9e310a8e9cd..38cde0b4137 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.UUID;
+import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.junit.jupiter.api.Test;
import org.assertj.core.api.Assertions;
import org.mockito.Mockito;
@@ -38,6 +39,7 @@
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
+import static java.net.HttpURLConnection.HTTP_CONFLICT;
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
@@ -47,6 +49,7 @@
import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_BLOB_DOMAIN_NAME;
import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DFS_DOMAIN_NAME;
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_KEY;
+import static
org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_SOFT_DELETE_NOT_SUPPORTED;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doReturn;
@@ -266,6 +269,8 @@ public void ensureGetAclDetermineHnsStatusAccurately()
throws Exception {
true, true);
ensureGetAclDetermineHnsStatusAccuratelyInternal(HTTP_UNAVAILABLE,
true, true);
+ ensureGetAclDetermineHnsStatusAccuratelyInternal(HTTP_CONFLICT,
+ false, false);
}
private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
@@ -274,8 +279,11 @@ private void
ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
AbfsClient mockClient = mock(AbfsClient.class);
store.getAbfsConfiguration().setIsNamespaceEnabledAccountForTesting(Trilean.UNKNOWN);
doReturn(mockClient).when(store).getClient(AbfsServiceType.DFS);
+ String errorMsg = statusCode == HTTP_CONFLICT
+ ? ERR_SOFT_DELETE_NOT_SUPPORTED
+ : Integer.toString(statusCode);
AbfsRestOperationException ex = new AbfsRestOperationException(
- statusCode, null, Integer.toString(statusCode), null);
+ statusCode, null, errorMsg, null);
doThrow(ex).when(mockClient).getAclStatus(anyString(),
any(TracingContext.class));
if (isExceptionExpected) {
@@ -301,6 +309,99 @@ private void
ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
.getAclStatus(anyString(), any(TracingContext.class));
}
+ /**
+ * Verify that for FNS accounts, the error code returned by the server
+ * is either HTTP 400 (Bad Request) or HTTP 409 (if soft-delete enabled).
This validates the expected
+ * server behavior when getAcl is called on FNS accounts.
+ */
+ @Test
+ public void testFNSAccountReturnsExpectedErrorCodes() throws Exception {
+ assumeHnsDisabled();
+ Configuration config = getConfigurationWithoutHnsConfig();
+
+ // Spy on the client to capture the actual exception thrown
+ try (AzureBlobFileSystem fs = (AzureBlobFileSystem)
FileSystem.newInstance(config)) {
+ AzureBlobFileSystemStore spyStore = Mockito.spy(fs.getAbfsStore());
+ AbfsClient spyClient =
Mockito.spy(spyStore.getClient(AbfsServiceType.DFS));
+
+ doReturn(spyClient).when(spyStore).getClient(AbfsServiceType.DFS);
+
+ // Force namespace to be unknown to trigger server call
+
spyStore.getAbfsConfiguration().setIsNamespaceEnabledAccountForTesting(Trilean.UNKNOWN);
+
+ // Capture any exception that might be thrown during getAclStatus
+ AbfsRestOperationException capturedException = null;
+ try {
+ spyClient.getAclStatus(AbfsHttpConstants.ROOT_PATH,
+ getTestTracingContext(fs, false));
+ } catch (AbfsRestOperationException ex) {
+ capturedException = ex;
+ }
+
+ // For FNS accounts, we expect either 400 or 409
+
+ // NOTE: 409 (soft-delete not supported error) would come explicitly
when we have
+ // the test account's soft-delete enabled. If soft-delete is disabled
for the test account,
+ // we should get 400 instead.
+ if (capturedException != null) {
+ int statusCode = capturedException.getStatusCode();
+ String errorMessage = capturedException.getMessage();
+
+ Assertions.assertThat(statusCode)
+ .describedAs("FNS account should return either 400 or 409 status
code")
+ .isIn(HTTP_BAD_REQUEST, HTTP_CONFLICT);
+
+ // If it's 409, verify it contains unsupported soft delete error
message
+ if (statusCode == HTTP_CONFLICT) {
+ Assertions.assertThat(errorMessage)
+ .describedAs("HTTP 409 response should contain soft delete
error message")
+ .contains(ERR_SOFT_DELETE_NOT_SUPPORTED);
+ }
+ }
+
+ // Verify that namespace is set to false regardless of which error was
returned
+ boolean isHnsEnabled =
spyStore.getIsNamespaceEnabled(getTestTracingContext(fs, false));
+ Assertions.assertThat(isHnsEnabled)
+ .describedAs("FNS account should have namespace disabled")
+ .isFalse();
+ }
+ }
+
+ /**
+ * Verify behavior when getAcl call fails with unexpected error codes
+ * (neither 400 nor 409). In such cases, namespace should be set to true
and exception
+ * should be propagated.
+ */
+ @Test
+ public void testErrorCodeSetsNamespaceToTrueAndThrowsException() throws
Exception {
+ // Create mock setup to simulate unexpected error code
+ AzureBlobFileSystemStore store =
Mockito.spy(getFileSystem().getAbfsStore());
+ AbfsClient mockClient = mock(AbfsClient.class);
+
store.getAbfsConfiguration().setIsNamespaceEnabledAccountForTesting(Trilean.UNKNOWN);
+ doReturn(mockClient).when(store).getClient(AbfsServiceType.DFS);
+
+ // Simulate 500 Internal Server Error (unexpected error code)
+ AbfsRestOperationException unexpectedException = new
AbfsRestOperationException(
+ HTTP_INTERNAL_ERROR, null, "Internal Server Error", null);
+ doThrow(unexpectedException).when(mockClient)
+ .getAclStatus(anyString(), any(TracingContext.class));
+
+ // Attempt to get namespace status should throw the exception
+ try {
+ store.getIsNamespaceEnabled(getTestTracingContext(getFileSystem(),
false));
+ Assertions.fail("Expected AbfsRestOperationException to be thrown");
+ } catch (AbfsRestOperationException ex) {
+ Assertions.assertThat(ex.getStatusCode())
+ .describedAs("Exception should have 500 status code")
+ .isEqualTo(HTTP_INTERNAL_ERROR);
+ }
+
+ // Even though exception was thrown, namespace should be set to true
+
Assertions.assertThat(store.getAbfsConfiguration().getIsNamespaceEnabledAccount())
+ .describedAs("Namespace should be set to TRUE for unexpected error
codes")
+ .isEqualTo(Trilean.TRUE);
+ }
+
@Test
public void testAccountSpecificConfig() throws Exception {
Configuration rawConfig = new Configuration();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
