This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 0cd6b3bf8a HDDS-10399. IndexOutOfBoundsException when shallow listing
empty directory in non-FSO bucket (#6259)
0cd6b3bf8a is described below
commit 0cd6b3bf8a230b8fab71be7ecd802ef36cd7b55f
Author: SaketaChalamchala <[email protected]>
AuthorDate: Sun Feb 25 00:18:57 2024 -0800
HDDS-10399. IndexOutOfBoundsException when shallow listing empty directory
in non-FSO bucket (#6259)
---
.../apache/hadoop/ozone/client/OzoneBucket.java | 2 +-
.../org/apache/hadoop/ozone/om/TestListKeys.java | 114 +++++++++++++++++----
2 files changed, 95 insertions(+), 21 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index dc7d6cf0a7..3b76daeba4 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -1253,7 +1253,7 @@ public class OzoneBucket extends WithMetadata {
proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false,
startKey, listCacheSize, false);
- if (addedKeyPrefix) {
+ if (addedKeyPrefix && statuses.size() > 0) {
// previous round already include the startKey, so remove it
statuses.remove(0);
} else {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java
index be972557f4..204c0ee668 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java
@@ -63,6 +63,8 @@ public class TestListKeys {
private static OzoneConfiguration conf;
private static OzoneBucket legacyOzoneBucket;
+
+ private static OzoneBucket obsOzoneBucket;
private static OzoneClient client;
/**
@@ -86,6 +88,10 @@ public class TestListKeys {
legacyOzoneBucket = TestDataUtil
.createVolumeAndBucket(client, BucketLayout.LEGACY);
+ // create a volume and a OBJECT_STORE bucket
+ obsOzoneBucket = TestDataUtil
+ .createVolumeAndBucket(client, BucketLayout.OBJECT_STORE);
+
initFSNameSpace();
}
@@ -99,6 +105,7 @@ public class TestListKeys {
private static void initFSNameSpace() throws Exception {
buildNameSpaceTree(legacyOzoneBucket);
+ buildNameSpaceTree(obsOzoneBucket);
}
/**
@@ -108,9 +115,9 @@ public class TestListKeys {
* |
* a1
* |
- * -----------------------------------
- * | | |
- * b1 b2 b3
+ * --------------------------------------------------------
+ * | | | |
+ * b1 b2 b3 b4
* ------- --------- -----------
* | | | | | | | |
* c1 c2 d1 d2 d3 e1 e2 e3
@@ -125,25 +132,27 @@ public class TestListKeys {
private static void buildNameSpaceTree(OzoneBucket ozoneBucket)
throws Exception {
LinkedList<String> keys = new LinkedList<>();
- keys.add("/a1/b1/c1111.tx");
- keys.add("/a1/b1/c1222.tx");
- keys.add("/a1/b1/c1333.tx");
- keys.add("/a1/b1/c1444.tx");
- keys.add("/a1/b1/c1555.tx");
- keys.add("/a1/b1/c1/c1.tx");
- keys.add("/a1/b1/c12/c2.tx");
- keys.add("/a1/b1/c12/c3.tx");
-
- keys.add("/a1/b2/d1/d11.tx");
- keys.add("/a1/b2/d2/d21.tx");
- keys.add("/a1/b2/d2/d22.tx");
- keys.add("/a1/b2/d3/d31.tx");
-
- keys.add("/a1/b3/e1/e11.tx");
- keys.add("/a1/b3/e2/e21.tx");
- keys.add("/a1/b3/e3/e31.tx");
+ keys.add("a1/b1/c1111.tx");
+ keys.add("a1/b1/c1222.tx");
+ keys.add("a1/b1/c1333.tx");
+ keys.add("a1/b1/c1444.tx");
+ keys.add("a1/b1/c1555.tx");
+ keys.add("a1/b1/c1/c1.tx");
+ keys.add("a1/b1/c12/c2.tx");
+ keys.add("a1/b1/c12/c3.tx");
+
+ keys.add("a1/b2/d1/d11.tx");
+ keys.add("a1/b2/d2/d21.tx");
+ keys.add("a1/b2/d2/d22.tx");
+ keys.add("a1/b2/d3/d31.tx");
+
+ keys.add("a1/b3/e1/e11.tx");
+ keys.add("a1/b3/e2/e21.tx");
+ keys.add("a1/b3/e3/e31.tx");
createKeys(ozoneBucket, keys);
+
+ ozoneBucket.createDirectory("a1/b4/");
}
private static Stream<Arguments> shallowListDataWithTrailingSlash() {
@@ -186,6 +195,58 @@ public class TestListKeys {
"a1/b1/c1333.tx",
"a1/b1/c1444.tx",
"a1/b1/c1555.tx"
+ ))),
+
+ // Case-7: StartKey is empty, return key that is same as keyPrefix.
+ of("a1/b4/", "", newLinkedList(Arrays.asList(
+ "a1/b4/"
+ )))
+ );
+ }
+
+ private static Stream<Arguments> shallowListObsDataWithTrailingSlash() {
+ return Stream.of(
+
+ // Case-1: StartKey is less than prefixKey, return emptyList.
+ of("a1/b2/", "a1", newLinkedList(Collections.emptyList())),
+
+ // Case-2: StartKey is empty, return all immediate node.
+ of("a1/b2/", "", newLinkedList(Arrays.asList(
+ "a1/b2/d1/",
+ "a1/b2/d2/",
+ "a1/b2/d3/"
+ ))),
+
+ // Case-3: StartKey is same as prefixKey, return all immediate nodes.
+ of("a1/b2/", "a1/b2", newLinkedList(Arrays.asList(
+ "a1/b2/d1/",
+ "a1/b2/d2/",
+ "a1/b2/d3/"
+ ))),
+
+ // Case-4: StartKey is greater than prefixKey
+ of("a1/b2/", "a1/b2/d2/d21.tx", newLinkedList(Arrays.asList(
+ "a1/b2/d2/",
+ "a1/b2/d3/"
+ ))),
+
+ // Case-5: StartKey reaches last element, return emptyList
+ of("a1/b2/", "a1/b2/d3/d31.tx", newLinkedList(
+ Collections.emptyList()
+ )),
+
+ // Case-6: Mix result
+ of("a1/b1/", "a1/b1/c12", newLinkedList(Arrays.asList(
+ "a1/b1/c12/",
+ "a1/b1/c1222.tx",
+ "a1/b1/c1333.tx",
+ "a1/b1/c1444.tx",
+ "a1/b1/c1555.tx"
+ ))),
+
+ // Case-7: StartKey is empty, return key that is same as keyPrefix.
+ of("a1/b4/", "", newLinkedList(Arrays.asList(
+ "a1/b4/"
)))
);
}
@@ -252,6 +313,11 @@ public class TestListKeys {
of("a1/b1/c12", "", newLinkedList(Arrays.asList(
"a1/b1/c12/",
"a1/b1/c1222.tx"
+ ))),
+
+ // Case-10:
+ of("a1/b4", "", newLinkedList(Arrays.asList(
+ "a1/b4/"
)))
);
@@ -264,11 +330,19 @@ public class TestListKeys {
checkKeyShallowList(keyPrefix, startKey, expectedKeys, legacyOzoneBucket);
}
+ @ParameterizedTest
+ @MethodSource("shallowListObsDataWithTrailingSlash")
+ public void testShallowListObsKeysWithPrefixTrailingSlash(String keyPrefix,
+ String startKey, List<String> expectedKeys) throws Exception {
+ checkKeyShallowList(keyPrefix, startKey, expectedKeys, obsOzoneBucket);
+ }
+
@ParameterizedTest
@MethodSource("shallowListDataWithoutTrailingSlash")
public void testShallowListKeysWithoutPrefixTrailingSlash(String keyPrefix,
String startKey, List<String> expectedKeys) throws Exception {
checkKeyShallowList(keyPrefix, startKey, expectedKeys, legacyOzoneBucket);
+ checkKeyShallowList(keyPrefix, startKey, expectedKeys, obsOzoneBucket);
}
private void checkKeyShallowList(String keyPrefix, String startKey,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
