bhattmanish98 commented on code in PR #7364:
URL: https://github.com/apache/hadoop/pull/7364#discussion_r1959107055
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java:
##########
@@ -167,4 +185,139 @@ public void testMkdirWithExistingFilename() throws
Exception {
intercept(FileAlreadyExistsException.class, () -> fs.mkdirs(new
Path("/testFilePath")));
intercept(FileAlreadyExistsException.class, () -> fs.mkdirs(new
Path("/testFilePath/newDir")));
}
+
+ /**
+ * Tests the idempotency of creating a path with retries by simulating
+ * a conflict response (HTTP 409) from the Azure Blob File System client.
+ * The method ensures that the path creation operation retries correctly
+ * with the proper transaction ID headers, verifying idempotency during
+ * failure recovery.
+ *
+ * @throws Exception if any error occurs during the operation.
+ */
+ @Test
+ public void createPathRetryIdempotency() throws Exception {
+ Configuration configuration = new Configuration(getRawConfiguration());
+ configuration.set(FS_AZURE_ENABLE_CLIENT_TRANSACTION_ID, "true");
+ try (AzureBlobFileSystem fs = getFileSystem(configuration)) {
+ assumeRecoveryThroughClientTransactionID(fs, true);
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ AbfsClientHandler clientHandler = Mockito.spy(store.getClientHandler());
+ AbfsDfsClient abfsClient = (AbfsDfsClient) Mockito.spy(
+ clientHandler.getClient());
+ fs.getAbfsStore().setClient(abfsClient);
+ fs.getAbfsStore().setClientHandler(clientHandler);
+ Mockito.doReturn(abfsClient).when(clientHandler).getIngressClient();
+ final Path nonOverwriteFile = new Path(
+ "/NonOverwriteTest_FileName_" + UUID.randomUUID());
+ final List<AbfsHttpHeader> headers = new ArrayList<>();
+ TestAbfsClient.mockAbfsOperationCreation(abfsClient,
+ new MockIntercept<AbfsRestOperation>() {
+ private int count = 0;
+
+ @Override
+ public void answer(final AbfsRestOperation mockedObj,
+ final InvocationOnMock answer)
+ throws AbfsRestOperationException {
+ if (count == 0) {
+ count = 1;
+ AbfsHttpOperation op = Mockito.mock(AbfsHttpOperation.class);
+ Mockito.doReturn("PUT").when(op).getMethod();
+ Mockito.doReturn("").when(op).getStorageErrorMessage();
+ Mockito.doReturn(true).when(mockedObj).hasResult();
+ Mockito.doReturn(op).when(mockedObj).getResult();
+ Mockito.doReturn(HTTP_CONFLICT).when(op).getStatusCode();
+ headers.addAll(mockedObj.getRequestHeaders());
+ throw new AbfsRestOperationException(HTTP_CONFLICT,
+ AzureServiceErrorCode.PATH_CONFLICT.getErrorCode(), "",
+ null, op);
+ }
+ }
+ });
+ AbfsRestOperation getPathRestOp = Mockito.mock(AbfsRestOperation.class);
+ AbfsHttpOperation op = Mockito.mock(AbfsHttpOperation.class);
+ Mockito.doAnswer(answer -> {
+ String requiredHeader = null;
+ for (AbfsHttpHeader httpHeader : headers) {
+ if (X_MS_CLIENT_TRANSACTION_ID.equalsIgnoreCase(
+ httpHeader.getName())) {
+ requiredHeader = httpHeader.getValue();
+ break;
+ }
+ }
+ return requiredHeader;
+ }).when(op).getResponseHeader(X_MS_CLIENT_TRANSACTION_ID);
+ Mockito.doReturn(true).when(getPathRestOp).hasResult();
+ Mockito.doReturn(op).when(getPathRestOp).getResult();
+ Mockito.doReturn(getPathRestOp).when(abfsClient).getPathStatus(
+ Mockito.nullable(String.class), Mockito.nullable(Boolean.class),
+ Mockito.nullable(TracingContext.class),
+ Mockito.nullable(ContextEncryptionAdapter.class));
+ fs.create(nonOverwriteFile, false);
+ }
+ }
+
+ /**
+ * Test to verify that the client transaction ID is included in the response
header
+ * during the creation of a new file in Azure Blob Storage.
+ * <p>
+ * This test ensures that when a new file is created, the Azure Blob
FileSystem client
+ * correctly includes the client transaction ID in the response header for
the created file.
+ * The test uses a configuration where client transaction ID is enabled and
verifies
+ * its presence after the file creation operation.
+ * </p>
+ *
+ * @throws Exception if any error occurs during test execution
+ */
+ @Test
+ public void getClientTransactionIdAfterCreate() throws Exception {
+ try (AzureBlobFileSystem fs = getFileSystem()) {
+ assumeRecoveryThroughClientTransactionID(fs, true);
+ AbfsDfsClient abfsDfsClient = (AbfsDfsClient) fs.getAbfsClient();
+ final Path nonOverwriteFile = new Path(
+ "/NonOverwriteTest_FileName_" + UUID.randomUUID());
+ fs.create(nonOverwriteFile, false);
+
+ final AbfsHttpOperation getPathStatusOp =
+ abfsDfsClient.getPathStatus(nonOverwriteFile.toUri().getPath(),
false,
+ getTestTracingContext(fs, true), null).getResult();
+ Assertions.assertThat(
+ getPathStatusOp.getResponseHeader(X_MS_CLIENT_TRANSACTION_ID))
+ .describedAs("Client transaction ID should be set during create")
+ .isNotNull();
+ }
+ }
+
+ /**
+ * Test to verify that the client transaction ID is included in the response
header
+ * after two consecutive create operations on the same file in Azure Blob
Storage.
+ * <p>
+ * This test ensures that even after performing two create operations (with
overwrite)
+ * on the same file, the Azure Blob FileSystem client includes the client
transaction ID
+ * in the response header for the created file. The test checks for the
presence of
+ * the client transaction ID in the response after the second create call.
+ * </p>
+ *
+ * @throws Exception if any error occurs during test execution
+ */
+ @Test
+ public void testClientTransactionIdAfterTwoCreateCalls() throws Exception {
+ try (AzureBlobFileSystem fs = getFileSystem()) {
+ assumeRecoveryThroughClientTransactionID(fs, true);
+ AbfsDfsClient abfsDfsClient = (AbfsDfsClient) fs.getAbfsClient();
+ Path testPath = path("testfile");
+ AzureBlobFileSystemStore.Permissions permissions
+ = new AzureBlobFileSystemStore.Permissions(false,
+ FsPermission.getDefault(), FsPermission.getUMask(fs.getConf()));
+ fs.create(testPath, false);
+ fs.create(testPath, true);
+ final AbfsHttpOperation getPathStatusOp =
+ abfsDfsClient.getPathStatus(testPath.toUri().getPath(), false,
+ getTestTracingContext(fs, true), null).getResult();
+ Assertions.assertThat(
+ getPathStatusOp.getResponseHeader(X_MS_CLIENT_TRANSACTION_ID))
+ .describedAs("Client transaction ID should be set during create")
+ .isNotNull();
+ }
Review Comment:
Added assert for clienttransactionid check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
