This is an automated email from the ASF dual-hosted git repository.
anujmodi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8f78af1edcd HADOOP-19580. [ABFS][BugFix] IsNonEmptyDirectory Check
should loop on listing using updated continuation token (#7716)
8f78af1edcd is described below
commit 8f78af1edcd9ef9ce67212ff7dd76e4484a846c3
Author: Anuj Modi <anujmodi2...@gmail.com>
AuthorDate: Wed May 28 23:53:13 2025 +0530
HADOOP-19580. [ABFS][BugFix] IsNonEmptyDirectory Check should loop on
listing using updated continuation token (#7716)
Contributed by Anuj Modi
Reviewed by Sneha Vijayarajan
---
.../fs/azurebfs/services/AbfsBlobClient.java | 2 +-
.../ITestAzureBlobFileSystemListStatus.java | 44 ++++++++++++++++++----
.../fs/azurebfs/services/ITestAbfsClient.java | 42 +++++++++++++++++++--
3 files changed, 76 insertions(+), 12 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java
index 5d8c2efb94e..a0567da97b2 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java
@@ -2037,7 +2037,7 @@ public boolean isNonEmptyDirectory(String path,
List<FileStatus> fileStatusList = new ArrayList<>();
// We need to loop on continuation token until we get an entry or
continuation token becomes null.
do {
- ListResponseData listResponseData = listPath(path, false, 1, null,
tracingContext, null);
+ ListResponseData listResponseData = listPath(path, false, 1,
continuationToken, tracingContext, null);
fileStatusList.addAll(listResponseData.getFileStatusList());
continuationToken = listResponseData.getContinuationToken();
} while (StringUtils.isNotEmpty(continuationToken) &&
fileStatusList.isEmpty());
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
index a533f352a97..e491362df5c 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
@@ -391,9 +391,9 @@ public void testEmptyListingInSubsequentCall() throws
IOException {
true, 2, 0);
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true,
EMPTY_STRING,
false, 2, 1);
- testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true,
TEST_CONTINUATION_TOKEN,
+ testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1,
true, TEST_CONTINUATION_TOKEN + 2,
true, 3, 0);
- testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true,
TEST_CONTINUATION_TOKEN,
+ testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1,
true, TEST_CONTINUATION_TOKEN + 2,
false, 3, 1);
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, EMPTY_STRING,
@@ -409,9 +409,9 @@ public void testEmptyListingInSubsequentCall() throws
IOException {
true, 2, 1);
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false,
EMPTY_STRING,
false, 2, 2);
- testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false,
TEST_CONTINUATION_TOKEN,
+ testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1,
false, TEST_CONTINUATION_TOKEN + 2,
true, 3, 1);
- testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false,
TEST_CONTINUATION_TOKEN,
+ testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1,
false, TEST_CONTINUATION_TOKEN + 2,
false, 3, 2);
}
@@ -453,9 +453,23 @@ private void
testEmptyListingInSubsequentCallInternal(String firstCT,
listResponseData3.setFileStatusList(new ArrayList<>());
listResponseData3.setOp(Mockito.mock(AbfsRestOperation.class));
-
Mockito.doReturn(listResponseData1).doReturn(listResponseData2).doReturn(listResponseData3)
- .when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
- any(), any(), any());
+ final int[] itr = new int[1];
+ final String[] continuationTokenUsed = new String[3];
+
+ Mockito.doAnswer(invocationOnMock -> {
+ if (itr[0] == 0) {
+ itr[0]++;
+ continuationTokenUsed[0] = invocationOnMock.getArgument(3);
+ return listResponseData1;
+ } else if (itr[0] == 1) {
+ itr[0]++;
+ continuationTokenUsed[1] = invocationOnMock.getArgument(3);
+ return listResponseData2;
+ }
+ continuationTokenUsed[2] = invocationOnMock.getArgument(3);
+ return listResponseData3;
+ }).when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
+ any(), any(TracingContext.class), any());
FileStatus[] list = spiedFs.listStatus(new Path("/testPath"));
@@ -473,6 +487,22 @@ private void
testEmptyListingInSubsequentCallInternal(String firstCT,
Mockito.verify(spiedClient, times(0))
.getPathStatus(eq("/testPath"), any(), eq(null), eq(false));
}
+
+ Assertions.assertThat(continuationTokenUsed[0])
+ .describedAs("First continuation token used is not as expected")
+ .isNull();
+
+ if (expectedInvocations > 1) {
+ Assertions.assertThat(continuationTokenUsed[1])
+ .describedAs("Second continuation token used is not as expected")
+ .isEqualTo(firstCT);
+ }
+
+ if (expectedInvocations > 2) {
+ Assertions.assertThat(continuationTokenUsed[2])
+ .describedAs("Third continuation token used is not as expected")
+ .isEqualTo(secondCT);
+ }
}
/**
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
index 1083df3ba57..cba1643478f 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
@@ -745,18 +745,18 @@ public void testIsNonEmptyDirectory() throws IOException {
true, 2, false);
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true,
EMPTY_STRING,
false, 2, true);
- testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true,
TEST_CONTINUATION_TOKEN,
+ testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, true,
TEST_CONTINUATION_TOKEN + 2,
true, 3, false);
- testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true,
TEST_CONTINUATION_TOKEN,
+ testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, true,
TEST_CONTINUATION_TOKEN + 2,
false, 2, true);
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false,
EMPTY_STRING,
true, 1, true);
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false,
EMPTY_STRING,
false, 1, true);
- testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false,
TEST_CONTINUATION_TOKEN,
+ testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, false,
TEST_CONTINUATION_TOKEN + 2,
true, 1, true);
- testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false,
TEST_CONTINUATION_TOKEN,
+ testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, false,
TEST_CONTINUATION_TOKEN + 2,
false, 1, true);
}
@@ -801,11 +801,45 @@ private void testIsNonEmptyDirectoryInternal(String
firstCT,
.when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
any(), any(), any());
+ final int[] itr = new int[1];
+ final String[] continuationTokenUsed = new String[3];
+
+ Mockito.doAnswer(invocationOnMock -> {
+ if (itr[0] == 0) {
+ itr[0]++;
+ continuationTokenUsed[0] = invocationOnMock.getArgument(3);
+ return listResponseData1;
+ } else if (itr[0] == 1) {
+ itr[0]++;
+ continuationTokenUsed[1] = invocationOnMock.getArgument(3);
+ return listResponseData2;
+ }
+ continuationTokenUsed[2] = invocationOnMock.getArgument(3);
+ return listResponseData3;
+ }).when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
+ any(), any(TracingContext.class), any());
+
Assertions.assertThat(spiedClient.isNonEmptyDirectory("/testPath",
Mockito.mock(TracingContext.class)))
.describedAs("isNonEmptyDirectory in client giving unexpected results")
.isEqualTo(isNonEmpty);
+ Assertions.assertThat(continuationTokenUsed[0])
+ .describedAs("First continuation token used is not as expected")
+ .isNull();
+
+ if (expectedInvocations > 1) {
+ Assertions.assertThat(continuationTokenUsed[1])
+ .describedAs("Second continuation token used is not as expected")
+ .isEqualTo(firstCT);
+ }
+
+ if (expectedInvocations > 2) {
+ Assertions.assertThat(continuationTokenUsed[2])
+ .describedAs("Third continuation token used is not as expected")
+ .isEqualTo(secondCT);
+ }
+
Mockito.verify(spiedClient, times(expectedInvocations))
.listPath(eq("/testPath"), eq(false), eq(1),
any(), any(TracingContext.class), any());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
