This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a3db2dfc50 HDDS-6992. ListKeys : SubPaths with a mixture of files and
dirs breaks sorted order (#3589)
a3db2dfc50 is described below
commit a3db2dfc504ac6dc7a22cc48762e181810b53046
Author: Rakesh Radhakrishnan <[email protected]>
AuthorDate: Wed Jul 13 03:53:53 2022 +0530
HDDS-6992. ListKeys : SubPaths with a mixture of files and dirs breaks
sorted order (#3589)
---
.../apache/hadoop/ozone/client/OzoneBucket.java | 187 ++++++-------
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 20 ++
.../hadoop/ozone/om/TestListKeysWithFSO.java | 302 ++++++++++++---------
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 18 +-
4 files changed, 284 insertions(+), 243 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index a7c3f76de1..596afa6371 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -1076,7 +1076,6 @@ public class OzoneBucket extends WithMetadata {
private class KeyIteratorWithFSO extends KeyIterator {
private Stack<Pair<String, String>> stack;
- private List<OzoneKey> pendingItemsToBeBatched;
private boolean addedKeyPrefix;
private String removeStartKey = "";
@@ -1108,13 +1107,18 @@ public class OzoneBucket extends WithMetadata {
String parentStartKeyPath = OzoneFSUtils.getParentDir(startKey);
- if (StringUtils.isNotBlank(startKey) &&
- StringUtils.compare(parentStartKeyPath, keyPrefix) >= 0) {
- seekPaths.add(new ImmutablePair<>(parentStartKeyPath, startKey));
-
- // recursively fetch all the sub-paths between keyPrefix and prevKey
- getSeekPathsBetweenKeyPrefixAndStartKey(keyPrefix, parentStartKeyPath,
- seekPaths);
+ if (StringUtils.isNotBlank(startKey)) {
+ if (StringUtils.compare(parentStartKeyPath, keyPrefix) >= 0) {
+ seekPaths.add(new ImmutablePair<>(parentStartKeyPath, startKey));
+
+ // recursively fetch all the sub-paths between keyPrefix and prevKey
+ getSeekPathsBetweenKeyPrefixAndStartKey(keyPrefix,
parentStartKeyPath,
+ seekPaths);
+ } else if (StringUtils.compare(startKey, keyPrefix) >= 0) {
+ // Both keyPrefix and startKey reached at the same level.
+ // Adds partial keyPrefix and startKey for seek.
+ seekPaths.add(new ImmutablePair<>(keyPrefix, startKey));
+ }
}
}
@@ -1122,7 +1126,6 @@ public class OzoneBucket extends WithMetadata {
List<OzoneKey> getNextListOfKeys(String prevKey) throws IOException {
if (stack == null) {
stack = new Stack();
- pendingItemsToBeBatched = new ArrayList<>();
}
// normalize paths
@@ -1153,29 +1156,16 @@ public class OzoneBucket extends WithMetadata {
List<Pair<String, String>> seekPaths = new ArrayList<>();
if (StringUtils.isNotBlank(getKeyPrefix())) {
- String parentStartKeyPath = OzoneFSUtils.getParentDir(prevKey);
- if (StringUtils.compare(parentStartKeyPath, getKeyPrefix()) >=
- 0) {
- // Add the leaf node to the seek path. The idea is to search
for
- // sub-paths if the given start key is a directory.
- seekPaths.add(new ImmutablePair<>(prevKey, ""));
- removeStartKey = prevKey;
- getSeekPathsBetweenKeyPrefixAndStartKey(getKeyPrefix(),
prevKey,
- seekPaths);
- } else if (StringUtils.compare(prevKey, getKeyPrefix()) >= 0) {
- // Add the leaf node to the seek path. The idea is to search
for
- // sub-paths if the given start key is a directory.
- seekPaths.add(new ImmutablePair<>(prevKey, ""));
- removeStartKey = prevKey;
- }
- } else {
- // Key Prefix is Blank. The seek all the keys with startKey.
- seekPaths.add(new ImmutablePair<>(prevKey, ""));
- removeStartKey = prevKey;
- getSeekPathsBetweenKeyPrefixAndStartKey(getKeyPrefix(), prevKey,
- seekPaths);
+ // If the prev key is a dir then seek its sub-paths
+ // Say, prevKey="a1/b2/d2"
+ addPrevDirectoryToSeekPath(prevKey, seekPaths);
}
+ // Key Prefix is Blank. The seek all the keys with startKey.
+ removeStartKey = prevKey;
+ getSeekPathsBetweenKeyPrefixAndStartKey(getKeyPrefix(), prevKey,
+ seekPaths);
+
// 2. Push elements in reverse order so that the FS tree traversal
// will occur in left-to-right fashion[Depth-First Search]
for (int index = seekPaths.size() - 1; index >= 0; index--) {
@@ -1183,9 +1173,10 @@ public class OzoneBucket extends WithMetadata {
stack.push(seekDirPath);
}
} else if (StringUtils.isNotBlank(getKeyPrefix())) {
- if (!OzoneFSUtils.isSibling(prevKey, getKeyPrefix())) {
+ if (!OzoneFSUtils.isAncestorPath(getKeyPrefix(), prevKey)) {
// Case-1 - sibling: keyPrefix="a1/b2", startKey="a0/b123Invalid"
// Skip traversing, if the startKey is not a sibling.
+ // "a1/b", "a1/b1/e/"
return new ArrayList<>();
} else if (StringUtils.compare(prevKey, getKeyPrefix()) < 0) {
// Case-2 - compare: keyPrefix="a1/b2", startKey="a1/b123Invalid"
@@ -1197,21 +1188,45 @@ public class OzoneBucket extends WithMetadata {
}
}
- // 3. Pop out top pair and get its immediate children
+ // 1. Pop out top pair and get its immediate children
List<OzoneKey> keysResultList = new ArrayList<>();
if (stack.isEmpty()) {
// case: startKey is empty
- getChildrenKeys(getKeyPrefix(), prevKey, keysResultList);
- } else {
- // case: startKey is non-empty
- Pair<String, String> keyPrefixPath = stack.pop();
- getChildrenKeys(keyPrefixPath.getLeft(), keyPrefixPath.getRight(),
- keysResultList);
+ if (getChildrenKeys(getKeyPrefix(), prevKey, keysResultList)) {
+ return keysResultList;
+ }
}
+ // 2. Pop element and seek for its sub-child path(s). Basically moving
+ // seek pointer to next level(depth) in FS tree.
+ // case: startKey is non-empty
+ while (!stack.isEmpty()) {
+ Pair<String, String> keyPrefixPath = stack.pop();
+ if (getChildrenKeys(keyPrefixPath.getLeft(), keyPrefixPath.getRight(),
+ keysResultList)) {
+ // reached limit batch size.
+ break;
+ }
+ }
return keysResultList;
}
+ private void addPrevDirectoryToSeekPath(String prevKey,
+ List<Pair<String, String>> seekPaths)
+ throws IOException {
+ try {
+ OzoneFileStatus prevStatus =
+ proxy.getOzoneFileStatus(volumeName, name, prevKey);
+ if (prevStatus != null) {
+ if (prevStatus.isDirectory()) {
+ seekPaths.add(new ImmutablePair<>(prevKey, ""));
+ }
+ }
+ } catch (OMException ome) {
+ // ignore exception
+ }
+ }
+
/**
* List children under the given keyPrefix and startKey path. It does
* recursive #listStatus calls to list all the sub-keys resultList.
@@ -1258,82 +1273,60 @@ public class OzoneBucket extends WithMetadata {
// listStatus API expects a not null 'startKey' value
startKey = startKey == null ? "" : startKey;
- // 1. Add pending items to the user key resultList
- if (addAllPendingItemsToResultList(keysResultList)) {
- // reached limit batch size.
- return true;
- }
-
- // 2. Get immediate children of keyPrefix, starting with startKey
+ // 1. Get immediate children of keyPrefix, starting with startKey
List<OzoneFileStatus> statuses = proxy.listStatus(volumeName, name,
- keyPrefix, false, startKey, listCacheSize, true);
+ keyPrefix, false, startKey, listCacheSize, true);
+ boolean reachedLimitCacheSize = statuses.size() == listCacheSize;
- // 3. Special case: ListKey expects keyPrefix element should present in
+ // 2. Special case: ListKey expects keyPrefix element should present in
// the resultList, only if startKey is blank. If startKey is not blank
// then resultList shouldn't contain the startKey element.
// Since proxy#listStatus API won't return keyPrefix element in the
// resultList. So, this is to add user given keyPrefix to the return
list.
addKeyPrefixInfoToResultList(keyPrefix, startKey, keysResultList);
- // 4. Special case: ListKey expects startKey shouldn't present in the
+ // 3. Special case: ListKey expects startKey shouldn't present in the
// resultList. Since proxy#listStatus API returns startKey element to
// the returnList, this function is to remove the startKey element.
removeStartKeyIfExistsInStatusList(startKey, statuses);
- boolean reachedLimitCacheSize = false;
- // This dirList is used to store paths elements in left-to-right order.
- List<String> dirList = new ArrayList<>();
-
- // 5. Iterating over the resultStatuses list and add each key to the
- // resultList. If the listCacheSize reaches then it will add the rest
- // of the statuses to pendingItemsToBeBatched
+ // 4. Iterating over the resultStatuses list and add each key to the
+ // resultList.
for (int indx = 0; indx < statuses.size(); indx++) {
OzoneFileStatus status = statuses.get(indx);
OmKeyInfo keyInfo = status.getKeyInfo();
String keyName = keyInfo.getKeyName();
+ OzoneKey ozoneKey;
// Add dir to the dirList
if (status.isDirectory()) {
- dirList.add(keyInfo.getKeyName());
// add trailing slash to represent directory
keyName = OzoneFSUtils.addTrailingSlashIfNeeded(keyName);
}
+ ozoneKey = new OzoneKey(keyInfo.getVolumeName(),
+ keyInfo.getBucketName(), keyName,
+ keyInfo.getDataSize(), keyInfo.getCreationTime(),
+ keyInfo.getModificationTime(),
+ keyInfo.getReplicationConfig());
- OzoneKey ozoneKey = new OzoneKey(keyInfo.getVolumeName(),
- keyInfo.getBucketName(), keyName,
- keyInfo.getDataSize(), keyInfo.getCreationTime(),
- keyInfo.getModificationTime(),
- keyInfo.getReplicationConfig());
-
- // 5.1) Add to the resultList till it reaches limit batch size.
- // Once it reaches limit, then add rest of the items to
- // pendingItemsToBeBatched and this will picked in next batch iteration
- if (!reachedLimitCacheSize && listCacheSize > keysResultList.size()) {
- keysResultList.add(ozoneKey);
- reachedLimitCacheSize = listCacheSize <= keysResultList.size();
- } else {
- pendingItemsToBeBatched.add(ozoneKey);
- }
- }
-
- // 6. Push elements in reverse order so that the FS tree traversal will
- // occur in left-to-right fashion.
- for (int indx = dirList.size() - 1; indx >= 0; indx--) {
- String dirPathComponent = dirList.get(indx);
- stack.push(new ImmutablePair<>(dirPathComponent, ""));
- }
-
- if (reachedLimitCacheSize) {
- return true;
- }
+ keysResultList.add(ozoneKey);
- // 7. Pop element and seek for its sub-child path(s). Basically moving
- // seek pointer to next level(depth) in FS tree.
- while (!stack.isEmpty()) {
- Pair<String, String> keyPrefixPath = stack.pop();
- if (getChildrenKeys(keyPrefixPath.getLeft(), keyPrefixPath.getRight(),
- keysResultList)) {
- // reached limit batch size.
+ if (status.isDirectory()) {
+ // Adding in-progress keyPath back to the stack to make sure
+ // all the siblings will be fetched.
+ stack.push(new ImmutablePair<>(keyPrefix, keyInfo.getKeyName()));
+ // Adding current directory to the stack, so that this dir will be
+ // the top element. Moving seek pointer to fetch sub-paths
+ stack.push(new ImmutablePair<>(keyInfo.getKeyName(), ""));
+ // Return it so that the next iteration will be
+ // started using the stacked items.
+ return true;
+ } else if (reachedLimitCacheSize && indx == statuses.size() - 1) {
+ // The last element is a FILE and reaches the listCacheSize.
+ // Now, sets next seek key to this element
+ stack.push(new ImmutablePair<>(keyPrefix, keyInfo.getKeyName()));
+ // Return it so that the next iteration will be
+ // started using the stacked items.
return true;
}
}
@@ -1364,20 +1357,6 @@ public class OzoneBucket extends WithMetadata {
}
}
- private boolean addAllPendingItemsToResultList(List<OzoneKey> keys) {
-
- Iterator<OzoneKey> ozoneKeyItr = pendingItemsToBeBatched.iterator();
- while (ozoneKeyItr.hasNext()) {
- if (listCacheSize <= keys.size()) {
- // reached limit batch size.
- return true;
- }
- keys.add(ozoneKeyItr.next());
- ozoneKeyItr.remove();
- }
- return false;
- }
-
private void addKeyPrefixInfoToResultList(String keyPrefix,
String startKey, List<OzoneKey> keysResultList) throws IOException {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
index 2750b63357..6d3268a144 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
@@ -176,6 +176,26 @@ public final class OzoneFSUtils {
return childParent == parentParent;
}
+ public static boolean isAncestorPath(String parentKey, String childKey) {
+ // Empty childKey has no parent, so just returning false.
+ if (org.apache.commons.lang3.StringUtils.isBlank(childKey)) {
+ return false;
+ }
+ java.nio.file.Path parentPath = Paths.get(parentKey);
+ java.nio.file.Path childPath = Paths.get(childKey);
+
+ java.nio.file.Path childParent = childPath.getParent();
+ java.nio.file.Path parentParent = parentPath.getParent();
+
+ if (childParent != null && parentParent != null) {
+ return childParent.startsWith(parentParent) ||
+ childParent.equals(parentParent);
+ }
+
+ return childParent == parentParent;
+ }
+
+
/**
* Verifies whether the childKey is an immediate path under the given
* parentKey.
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java
index b8a3a54e69..e3cc201eea 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java
@@ -42,9 +42,10 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
-import java.util.TreeSet;
+import java.util.ArrayList;
import java.util.UUID;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
/**
@@ -61,6 +62,8 @@ public class TestListKeysWithFSO {
private static OzoneBucket legacyOzoneBucket;
private static OzoneBucket fsoOzoneBucket;
+ private static OzoneBucket legacyOzoneBucket2;
+ private static OzoneBucket fsoOzoneBucket2;
@Rule
public Timeout timeout = new Timeout(1200000);
@@ -79,6 +82,9 @@ public class TestListKeysWithFSO {
clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();
omId = UUID.randomUUID().toString();
+ // Set the number of keys to be processed during batch operate.
+ conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 3);
+ conf.setInt(OZONE_CLIENT_LIST_CACHE_SIZE, 3);
cluster = MiniOzoneCluster.newBuilder(conf).setClusterId(clusterId)
.setScmId(scmId).setOmId(omId).build();
cluster.waitForClusterToBeReady();
@@ -88,21 +94,31 @@ public class TestListKeysWithFSO {
.createVolumeAndBucket(cluster, BucketLayout.LEGACY);
String volumeName = legacyOzoneBucket.getVolumeName();
- // create a volume and a FSO bucket
+ OzoneClient client = cluster.getClient();
+ OzoneVolume ozoneVolume = client.getObjectStore().getVolume(volumeName);
+
+ // create buckets
BucketArgs omBucketArgs;
BucketArgs.Builder builder = BucketArgs.newBuilder();
builder.setStorageType(StorageType.DISK);
builder.setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED);
omBucketArgs = builder.build();
- OzoneClient client = cluster.getClient();
- OzoneVolume ozoneVolume = client.getObjectStore().getVolume(volumeName);
String fsoBucketName = "bucket" + RandomStringUtils.randomNumeric(5);
ozoneVolume.createBucket(fsoBucketName, omBucketArgs);
fsoOzoneBucket = ozoneVolume.getBucket(fsoBucketName);
- // Set the number of keys to be processed during batch operate.
- conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 2);
+ fsoBucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ ozoneVolume.createBucket(fsoBucketName, omBucketArgs);
+ fsoOzoneBucket2 = ozoneVolume.getBucket(fsoBucketName);
+
+ builder = BucketArgs.newBuilder();
+ builder.setStorageType(StorageType.DISK);
+ builder.setBucketLayout(BucketLayout.LEGACY);
+ omBucketArgs = builder.build();
+ String legacyBucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ ozoneVolume.createBucket(legacyBucketName, omBucketArgs);
+ legacyOzoneBucket2 = ozoneVolume.getBucket(legacyBucketName);
initFSNameSpace();
}
@@ -132,6 +148,9 @@ public class TestListKeysWithFSO {
*/
buildNameSpaceTree(legacyOzoneBucket);
buildNameSpaceTree(fsoOzoneBucket);
+
+ buildNameSpaceTree2(legacyOzoneBucket2);
+ buildNameSpaceTree2(fsoOzoneBucket2);
}
@Test
@@ -139,11 +158,12 @@ public class TestListKeysWithFSO {
// case-1: StartKey LeafNode is lexographically behind than prefixKey.
// So, will return EmptyList
// a1/b2 < a1/b2Invalid
- List<String> expectedKeys = getExpectedKeyList("a1/b2", "a1");
- checkKeyList("a1/b2", "a1", expectedKeys);
+ List<String> expectedKeys =
+ getExpectedKeyList("a1/b2", "a1", legacyOzoneBucket);
+ checkKeyList("a1/b2", "a1", expectedKeys, fsoOzoneBucket);
// case-2: Same prefixKey and startKey, but with an ending slash
- expectedKeys = getExpectedKeyList("a1/b2", "a1/b2/");
+ expectedKeys = getExpectedKeyList("a1/b2", "a1/b2/", legacyOzoneBucket);
/**
* a1/b2/d1/
* a1/b2/d1/d11.tx
@@ -153,12 +173,12 @@ public class TestListKeysWithFSO {
* a1/b2/d3/
* a1/b2/d3/d31.tx
*/
- checkKeyList("a1/b2", "a1/b2/", expectedKeys);
+ checkKeyList("a1/b2", "a1/b2/", expectedKeys, fsoOzoneBucket);
// case-3: Same prefixKey and startKey, but without an ending slash.
// StartKey(dir) to be included in the finalList, if its a
// directory and not ended with trailing slash.
- expectedKeys = getExpectedKeyList("a1/b2", "a1/b2");
+ expectedKeys = getExpectedKeyList("a1/b2", "a1/b2", legacyOzoneBucket);
/**
* a1/b2/
* a1/b2/d1/
@@ -169,39 +189,42 @@ public class TestListKeysWithFSO {
* a1/b2/d3/
* a1/b2/d3/d31.tx
*/
- checkKeyList("a1/b2", "a1/b2", expectedKeys);
+ checkKeyList("a1/b2", "a1/b2", expectedKeys, fsoOzoneBucket);
// case-4: StartKey is a file with an ending slash.
// StartKey(file) with or without an ending slash
// to be excluded in the finalList.
- expectedKeys = getExpectedKeyList("a1/b2/d2", "a1/b2/d2/d22.tx/");
- checkKeyList("a1/b2/d2", "a1/b2/d2/d22.tx/", expectedKeys);
+ expectedKeys =
+ getExpectedKeyList("a1/b2/d2", "a1/b2/d2/d22.tx/", legacyOzoneBucket);
+ checkKeyList("a1/b2/d2", "a1/b2/d2/d22.tx/", expectedKeys, fsoOzoneBucket);
// case-5:
// StartKey "a1/b2/d2/d22.tx" is a file and get all the keys
lexographically
// greater than "a1/b2/d2/d22.tx".
- expectedKeys = getExpectedKeyList("a1/b2", "a1/b2/d2/d22.tx");
+ expectedKeys =
+ getExpectedKeyList("a1/b2", "a1/b2/d2/d22.tx", legacyOzoneBucket);
/**
1 = "a1/b2/d3/"
2 = "a1/b2/d3/d31.tx"
*/
- checkKeyList("a1/b2", "a1/b2/d2/d22.tx", expectedKeys);
+ checkKeyList("a1/b2", "a1/b2/d2/d22.tx", expectedKeys, fsoOzoneBucket);
// case-6:
// StartKey "a1/b2/d2" is a dir and get all the keys lexographically
// greater than "a1/b2/d2".
- expectedKeys = getExpectedKeyList("a1/b2", "a1/b2/d2/");
+ expectedKeys = getExpectedKeyList("a1/b2", "a1/b2/d2/", legacyOzoneBucket);
/**
1 = "a1/b2/d2/d21.tx"
2 = "a1/b2/d2/d22.tx"
3 = "a1/b2/d3/"
4 = "a1/b2/d3/d31.tx"
*/
- checkKeyList("a1/b2", "a1/b2/d2/", expectedKeys);
+ checkKeyList("a1/b2", "a1/b2/d2/", expectedKeys, fsoOzoneBucket);
// case-7: In below case, the startKey is a directory which is included
// in the finalList. So, should we include startKey file in the finalList ?
- expectedKeys = getExpectedKeyList("a1", "a1/b2/d2/d21.tx");
+ expectedKeys =
+ getExpectedKeyList("a1", "a1/b2/d2/d21.tx", legacyOzoneBucket);
/**
1 = "a1/b2/d2/d22.tx"
2 = "a1/b2/d3/"
@@ -214,11 +237,11 @@ public class TestListKeysWithFSO {
9 = "a1/b3/e3/"
10 = "a1/b3/e3/e31.tx"
*/
- checkKeyList("a1", "a1/b2/d2/d21.tx", expectedKeys);
+ checkKeyList("a1", "a1/b2/d2/d21.tx", expectedKeys, fsoOzoneBucket);
// case-8: StartKey(dir) to be included in the finalList, if its a
// directory and not ended with trailing slash.
- expectedKeys = getExpectedKeyList("a1", "a1/b2/d2/");
+ expectedKeys = getExpectedKeyList("a1", "a1/b2/d2/", legacyOzoneBucket);
/**
1 = "a1/b2/d2/d21.tx"
2 = "a1/b2/d2/d22.tx"
@@ -229,11 +252,12 @@ public class TestListKeysWithFSO {
10 = "a1/b3/e3/"
11 = "a1/b3/e3/e31.tx"
*/
- checkKeyList("a1", "a1/b2/d2/", expectedKeys);
+ checkKeyList("a1", "a1/b2/d2/", expectedKeys, fsoOzoneBucket);
// case-9: Reached Last Element, return EmptyList
- expectedKeys = getExpectedKeyList("a1", "a1/b3/e3/e31.tx");
- checkKeyList("a1", "a1/b3/e3/e31.tx", expectedKeys);
+ expectedKeys =
+ getExpectedKeyList("a1", "a1/b3/e3/e31.tx", legacyOzoneBucket);
+ checkKeyList("a1", "a1/b3/e3/e31.tx", expectedKeys, fsoOzoneBucket);
}
@Test
@@ -241,148 +265,139 @@ public class TestListKeysWithFSO {
// case-1: StartKey LeafNode is lexographically ahead than prefixKey.
// So, will return EmptyList
// a1/b2 < a1/b2Invalid
- List<String> expectedKeys = getExpectedKeyList("a1", "a1/a111/b111");
- checkKeyList("a1", "a1/a111/b111", expectedKeys);
+ List<String> expectedKeys =
+ getExpectedKeyList("a1", "a1/a111/b111", legacyOzoneBucket);
+ checkKeyList("a1", "a1/a111/b111", expectedKeys, fsoOzoneBucket);
// a1/b2 < a1/b20
- expectedKeys = getExpectedKeyList("a1/b2", "a1/b20");
- checkKeyList("a1/b2", "a1/b20", expectedKeys);
+ expectedKeys = getExpectedKeyList("a1/b2", "a1/b20", legacyOzoneBucket);
+ checkKeyList("a1/b2", "a1/b20", expectedKeys, fsoOzoneBucket);
// case-2: StartKey LeafNode's parent is lexographically ahead than
// prefixKey. So, will return EmptyList
// a1/b1 < a1/b2
- expectedKeys = getExpectedKeyList("a1/b1", "a1/b2/d0");
- checkKeyList("a1/b1", "a1/b2/d0", expectedKeys);
+ expectedKeys = getExpectedKeyList("a1/b1", "a1/b2/d0", legacyOzoneBucket);
+ checkKeyList("a1/b1", "a1/b2/d0", expectedKeys, fsoOzoneBucket);
// case-3:
// StartKey LeafNode's parent is not matching with than prefixKey's parent.
// So, will return EmptyList
- expectedKeys = getExpectedKeyList("a1/b2", "a0/b123Invalid");
- checkKeyList("a1/b2", "a0/b123Invalid", expectedKeys);
+ expectedKeys =
+ getExpectedKeyList("a1/b2", "a0/b123Invalid", legacyOzoneBucket);
+ checkKeyList("a1/b2", "a0/b123Invalid", expectedKeys, fsoOzoneBucket);
// case-4: StartKey LeafNode is lexographically behind prefixKey.
// So will return all the sub-paths of prefixKey
// startKey=a1/b123Invalid is lexographically before prefixKey=a1/b2
- expectedKeys = getExpectedKeyList("a1/b2", "a1/b123Invalid");
- /**
- * a1/b2/
- * a1/b2/d1/
- * a1/b2/d1/d11.tx
- * a1/b2/d2/
- * .....
- * a1/b2/d3/
- * a1/b2/d3/d31.tx
- */
- checkKeyList("a1/b2", "a1/b123Invalid", expectedKeys);
+ expectedKeys =
+ getExpectedKeyList("a1/b2", "a1/b123Invalid", legacyOzoneBucket);
+ checkKeyList("a1/b2", "a1/b123Invalid", expectedKeys, fsoOzoneBucket);
- // case-5:
- // StartKey LeafNode is a sub-directory of prefixKey.
+ // case-5: StartKey LeafNode is a sub-directory of prefixKey.
// So will fetch and return all the sub-paths after d0.
- expectedKeys = getExpectedKeyList("a1/b2", "a1/b2/d0");
- /**
- a1/b2/d1/
- a1/b2/d1/d11.tx
- .....
- a1/b2/d3/
- a1/b2/d3/d31.tx
- */
- checkKeyList("a1/b2", "a1/b2/d0", expectedKeys);
+ expectedKeys = getExpectedKeyList("a1/b2", "a1/b2/d0", legacyOzoneBucket);
+ checkKeyList("a1/b2", "a1/b2/d0", expectedKeys, fsoOzoneBucket);
- // case-6:
- // StartKey LeafNode is a sub-file of prefixKey.
+ // case-6: StartKey LeafNode is a sub-file of prefixKey.
// So will fetch and return all the sub-paths after d111.txt.
- expectedKeys = getExpectedKeyList("a1/b2", "a1/b2/d111.txt");
- /**
- * a1/b2/d2/
- * a1/b2/d2/d21.tx
- * a1/b2/d2/d22.tx
- * a1/b2/d3/
- * a1/b2/d3/d31.tx
- */
- checkKeyList("a1/b2", "a1/b2/d111.txt", expectedKeys);
+ expectedKeys =
+ getExpectedKeyList("a1/b2", "a1/b2/d111.txt", legacyOzoneBucket);
+ checkKeyList("a1/b2", "a1/b2/d111.txt", expectedKeys, fsoOzoneBucket);
- // case-7:
- // StartKey LeafNode is a sub-file of prefixKey.
+ // case-7: StartKey LeafNode is a sub-file of prefixKey.
// So will fetch and return all the sub-paths after "d3/d4111.tx".
// Since there is no sub-paths after "d3" it will return emptyList
- expectedKeys = getExpectedKeyList("a1/b2", "a1/b2/d3/d4111.tx");
- checkKeyList("a1/b2", "a1/b2/d3/d4111.tx", expectedKeys);
+ expectedKeys =
+ getExpectedKeyList("a1/b2", "a1/b2/d3/d4111.tx", legacyOzoneBucket);
+ checkKeyList("a1/b2", "a1/b2/d3/d4111.tx", expectedKeys, fsoOzoneBucket);
- // case-8:
- // StartKey LeafNode is a sub-dir of prefixKey.
+ // case-8: StartKey LeafNode is a sub-dir of prefixKey.
// So will fetch and return all the sub-paths after "d311111".
- expectedKeys = getExpectedKeyList("a1", "a1/b2/d311111");
- /**
- * a1/b3/
- * a1/b3/e1/
- * ....
- * a1/b3/e3/
- * a1/b3/e3/e31.tx
- */
- checkKeyList("a1", "a1/b2/d311111", expectedKeys);
+ expectedKeys = getExpectedKeyList("a1", "a1/b2/d311111",
legacyOzoneBucket);
+ checkKeyList("a1", "a1/b2/d311111", expectedKeys, fsoOzoneBucket);
// case-9:
// Immediate child of prefixKey is lexographically greater than "a1/b1".
// So will fetch and return all the sub-paths after "b11111",
// which is "a1/b2"
- expectedKeys = getExpectedKeyList("a1", "a1/b11111/d311111");
- /**
- 0 = "a1/b2/"
- 1 = "a1/b2/d1/"
- 2 = "a1/b2/d1/d11.tx"
- 3 = "a1/b2/d2/"
- ........
- 14 = "a1/b3/e3/e31.tx"
- */
- checkKeyList("a1", "a1/b11111/d311111", expectedKeys);
+ expectedKeys =
+ getExpectedKeyList("a1", "a1/b11111/d311111", legacyOzoneBucket);
+ checkKeyList("a1", "a1/b11111/d311111", expectedKeys, fsoOzoneBucket);
// case-10:
// StartKey "a1/b2/d2" is valid and get all the keys lexographically
// greater than "a1/b2/d2/d11111".
- expectedKeys = getExpectedKeyList("a1", "a1/b2/d2/d21111");
- /**
- 0 = "a1/b2/d2/d22.tx"
- 1 = "a1/b2/d3/"
- ......
- 7 = "a1/b3/e3/"
- 8 = "a1/b3/e3/e31.tx"
- */
- checkKeyList("a1", "a1/b2/d2/d21111", expectedKeys);
+ expectedKeys =
+ getExpectedKeyList("a1", "a1/b2/d2/d21111", legacyOzoneBucket);
+ checkKeyList("a1", "a1/b2/d2/d21111", expectedKeys, fsoOzoneBucket);
- // case-11:
- // StartKey is a sub-path of prefixKey.
+ // case-11: StartKey is a sub-path of prefixKey.
// So will fetch and return all the sub-paths after "e311111".
// Return EmptyList as we reached the end of the tree
- expectedKeys = getExpectedKeyList("a1", "a1/b3/e3/e311111.tx");
- checkKeyList("a1", "a1/b3/e3/e311111.tx", expectedKeys);
+ expectedKeys =
+ getExpectedKeyList("a1", "a1/b3/e3/e311111.tx", legacyOzoneBucket);
+ checkKeyList("a1", "a1/b3/e3/e311111.tx", expectedKeys, fsoOzoneBucket);
- // case-12:
- // StartKey is a sub-path of prefixKey.
+ // case-12: StartKey is a sub-path of prefixKey.
// So will fetch and return all the sub-paths after "e4444".
// Return EmptyList as we reached the end of the tree
- expectedKeys = getExpectedKeyList("a1/b2", "a1/b3/e4444");
- checkKeyList("a1/b2", "a1/b3/e4444", expectedKeys);
+ expectedKeys =
+ getExpectedKeyList("a1/b2", "a1/b3/e4444", legacyOzoneBucket);
+ checkKeyList("a1/b2", "a1/b3/e4444", expectedKeys, fsoOzoneBucket);
// case-13:
// StartKey is a sub-path of prefixKey and startKey with a trailing slash.
// So will fetch and return all the sub-paths after "e".
- expectedKeys = getExpectedKeyList("a1/b3", "a1/b3/e/");
- checkKeyList("a1/b3", "a1/b3/e/", expectedKeys);
+ expectedKeys = getExpectedKeyList("a1/b", "a1/b3/e/", legacyOzoneBucket);
+ checkKeyList("a1/b3", "a1/b3/e/", expectedKeys, fsoOzoneBucket);
- // case-14:
- // PrefixKey is empty and search should consider startKey.
+ // case-14: PrefixKey is empty and search should consider startKey.
// Fetch all the keys after, a1/b2/d
- expectedKeys = getExpectedKeyList("", "a1/b2/d");
- checkKeyList("", "a1/b2/d", expectedKeys);
-
- // case-15:
- // PrefixKey is empty and search should consider startKey.
- expectedKeys = getExpectedKeyList("", "a1/b2/d2/d21111");
- checkKeyList("", "a1/b2/d2/d21111", expectedKeys);
-
- // case-16:
- // PrefixKey is empty and search should consider startKey.
- expectedKeys = getExpectedKeyList("", "a0/b2/d2/d21111");
- checkKeyList("", "a0/b2/d2/d21111", expectedKeys);
+ expectedKeys = getExpectedKeyList("", "a1/b2/d", legacyOzoneBucket);
+ checkKeyList("", "a1/b2/d", expectedKeys, fsoOzoneBucket);
+
+ // case-15: PrefixKey is empty and search should consider startKey.
+ expectedKeys = getExpectedKeyList("", "a1/b2/d2/d21111",
legacyOzoneBucket);
+ checkKeyList("", "a1/b2/d2/d21111", expectedKeys, fsoOzoneBucket);
+
+ // case-16: PrefixKey is empty and search should consider startKey.
+ expectedKeys = getExpectedKeyList("", "a0/b2/d2/d21111",
legacyOzoneBucket);
+ checkKeyList("", "a0/b2/d2/d21111", expectedKeys, fsoOzoneBucket);
+
+ // case-17: Partial prefixKey and seek till prefixKey.
+ expectedKeys = getExpectedKeyList("a1/b", "a1/b1/e/", legacyOzoneBucket);
+ checkKeyList("a1/b", "a1/b1/e/", expectedKeys, fsoOzoneBucket);
+
+ // case-18: Partial prefixKey and seek till prefixKey.
+ expectedKeys =
+ getExpectedKeyList("a1/b1/c", "a1/b1/01/f/g/h", legacyOzoneBucket);
+ checkKeyList("a1/b1/c", "a1/b1/01/f/g/h", expectedKeys, fsoOzoneBucket);
+
+ // case-19: Partial prefixKey and seek till prefixKey.
+ expectedKeys =
+ getExpectedKeyList("a1/b1", "a1/01/e/f/g/h", legacyOzoneBucket);
+ checkKeyList("a1/b1", "a1/01/e/f/g/h", expectedKeys, fsoOzoneBucket);
+
+ // case-20: Partial prefixKey and seek till prefixKey.
+ expectedKeys = getExpectedKeyList("a", "a1/b1/e/", legacyOzoneBucket);
+ checkKeyList("a", "a1/b1/e/", expectedKeys, fsoOzoneBucket);
+ }
+
+ @Test
+ public void testListKeysWithMixOfDirsAndFiles() throws Exception {
+ // case-1: StartKey LeafNode is lexographically ahead than prefixKey.
+ // So, will return EmptyList
+ // a1/b2 < a1/b2Invalid
+ List<String> expectedKeys = getExpectedKeyList("", "", legacyOzoneBucket2);
+ checkKeyList("", "", expectedKeys, fsoOzoneBucket2);
+
+ expectedKeys = getExpectedKeyList("", "a", legacyOzoneBucket2);
+ checkKeyList("", "a", expectedKeys, fsoOzoneBucket2);
+
+ expectedKeys = getExpectedKeyList("a", "a", legacyOzoneBucket2);
+ checkKeyList("a", "a", expectedKeys, fsoOzoneBucket2);
+
+ expectedKeys = getExpectedKeyList("a", "a1", legacyOzoneBucket2);
+ checkKeyList("a", "a1", expectedKeys, fsoOzoneBucket2);
}
/**
@@ -430,10 +445,37 @@ public class TestListKeysWithFSO {
createKeys(ozoneBucket, keys);
}
+ private static void buildNameSpaceTree2(OzoneBucket ozoneBucket)
+ throws Exception {
+ LinkedList<String> keys = new LinkedList<>();
+ keys.add("/a1/b1/c1/c1.tx");
+ keys.add("/a110.tx");
+ keys.add("/a111.tx");
+ keys.add("/a112.tx");
+ keys.add("/a113.tx");
+ keys.add("/a114.tx");
+ keys.add("/a115.tx");
+ keys.add("/a2/b1/c1/c1.tx");
+ keys.add("/a220.tx");
+ keys.add("/a221.tx");
+ keys.add("/a222.tx");
+ keys.add("/a223.tx");
+ keys.add("/a224.tx");
+ keys.add("/a225.tx");
+ keys.add("/a3/b1/c1/c1.tx");
+
+ keys.add("/x/y/z/z1.tx");
+
+ keys.add("/dir1/dir2/dir3/d11.tx");
+
+ createKeys(ozoneBucket, keys);
+ }
+
+
private static List<String> getExpectedKeyList(String keyPrefix,
- String startKey) throws Exception {
+ String startKey, OzoneBucket legacyBucket) throws Exception {
Iterator<? extends OzoneKey> ozoneKeyIterator =
- legacyOzoneBucket.listKeys(keyPrefix, startKey);
+ legacyBucket.listKeys(keyPrefix, startKey);
List<String> keys = new LinkedList<>();
while (ozoneKeyIterator.hasNext()) {
@@ -444,17 +486,17 @@ public class TestListKeysWithFSO {
}
private void checkKeyList(String keyPrefix, String startKey,
- List<String> keys) throws Exception {
+ List<String> keys, OzoneBucket fsoBucket) throws Exception {
Iterator<? extends OzoneKey> ozoneKeyIterator =
- fsoOzoneBucket.listKeys(keyPrefix, startKey);
+ fsoBucket.listKeys(keyPrefix, startKey);
- TreeSet<String> outputKeys = new TreeSet<>();
+ List <String> keyLists = new ArrayList<>();
while (ozoneKeyIterator.hasNext()) {
OzoneKey ozoneKey = ozoneKeyIterator.next();
- outputKeys.add(ozoneKey.getName());
+ keyLists.add(ozoneKey.getName());
}
- LinkedList outputKeysList = new LinkedList(outputKeys);
+ LinkedList outputKeysList = new LinkedList(keyLists);
System.out.println("BEGIN:::keyPrefix---> " + keyPrefix + ":::---> " +
startKey);
for (String key : keys) {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
index c0a844c520..b7ea175d4a 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
@@ -450,11 +450,11 @@ public class TestObjectStoreWithFSO {
expectedKeys = new LinkedList<>();
expectedKeys.add("a/b2/");
expectedKeys.add("a/b2/d1/");
- expectedKeys.add("a/b2/d2/");
- expectedKeys.add("a/b2/d3/");
expectedKeys.add("a/b2/d1/d11.tx");
+ expectedKeys.add("a/b2/d2/");
expectedKeys.add("a/b2/d2/d21.tx");
expectedKeys.add("a/b2/d2/d22.tx");
+ expectedKeys.add("a/b2/d3/");
expectedKeys.add("a/b2/d3/d31.tx");
checkKeyList(ozoneKeyIterator, expectedKeys);
@@ -484,24 +484,24 @@ public class TestObjectStoreWithFSO {
LinkedList<String> expectedKeys = new LinkedList<>();
expectedKeys.add("a/");
expectedKeys.add("a/b1/");
- expectedKeys.add("a/b2/");
- expectedKeys.add("a/b3/");
expectedKeys.add("a/b1/c1/");
- expectedKeys.add("a/b1/c2/");
expectedKeys.add("a/b1/c1/c1.tx");
+ expectedKeys.add("a/b1/c2/");
expectedKeys.add("a/b1/c2/c2.tx");
+ expectedKeys.add("a/b2/");
expectedKeys.add("a/b2/d1/");
- expectedKeys.add("a/b2/d2/");
- expectedKeys.add("a/b2/d3/");
expectedKeys.add("a/b2/d1/d11.tx");
+ expectedKeys.add("a/b2/d2/");
expectedKeys.add("a/b2/d2/d21.tx");
expectedKeys.add("a/b2/d2/d22.tx");
+ expectedKeys.add("a/b2/d3/");
expectedKeys.add("a/b2/d3/d31.tx");
+ expectedKeys.add("a/b3/");
expectedKeys.add("a/b3/e1/");
- expectedKeys.add("a/b3/e2/");
- expectedKeys.add("a/b3/e3/");
expectedKeys.add("a/b3/e1/e11.tx");
+ expectedKeys.add("a/b3/e2/");
expectedKeys.add("a/b3/e2/e21.tx");
+ expectedKeys.add("a/b3/e3/");
expectedKeys.add("a/b3/e3/e31.tx");
checkKeyList(keyItr, expectedKeys);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
