xichen01 commented on code in PR #5003:
URL: https://github.com/apache/ozone/pull/5003#discussion_r1256110482
##########
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java:
##########
@@ -1099,9 +1132,121 @@ public OzoneKey next() {
*/
List<OzoneKey> getNextListOfKeys(String prevKey) throws
IOException {
+ // If shallow is true, only list immediate children
+ if (shallow) {
+ return getNextShallowListOfKeys(prevKey);
+ }
return proxy.listKeys(volumeName, name, keyPrefix, prevKey,
listCacheSize);
}
+
+ /**
+ * Using listStatus instead of listKeys avoiding listing all children keys.
+ * Giving the structure of keys delimited by "/":
+ *
+ * buck-1
+ * |
+ * a
+ * |
+ * -----------------------------------
+ * | | |
+ * b1 b2 b3
+ * ----- -------- ----------
+ * | | | | | | | |
+ * c1 c2 d1 d2 d3 e1 e2 e3
+ * | |
+ * -------- |
+ * | | |
+ * d21.txt d22.txt e11.txt
+ *
+ * For the above structure, the keys listed delimited "/" in order are
+ * as follows:
+ * a/
+ * a/b1/
+ * a/b1/c1/
+ * a/b1/c2/
+ * a/b2/
+ * a/b2/d1/
+ * a/b2/d2/
+ * a/b2/d2/d21.txt
+ * a/b2/d2/d22.txt
+ * a/b2/d3/
+ * a/b3/
+ * a/b3/e1/
+ * a/b3/e1/e11.txt
+ * a/b3/e2/
+ * a/b3/e3/
+ *
+ * Example 1: keyPrefix="a/b2/", prevKey=""
+ * result: [a/b2/d1/, a/b2/d2/, a/b2/d3/]
+ * Example 2: keyPrefix="a/b2/", prevKey="a/b2/d2/d21.txt"
+ * result: [a/b2/d2/, a/b2/d3/]
+ * Example 3: keyPrefix="a/b2/", prevKey="a/b2/d2/d22.txt"
+ * result: [a/b2/d3/]
+ * Say, keyPrefix="a/b" and prevKey="", the results will be
+ * [a/b1/, a/b2/, a/b3/]
+ * In implementation, the keyPrefix "a/b" can be identified in listKeys,
+ * but cannot be identified in listStatus. Therefore, keyPrefix "a/b"
+ * needs to be split into keyPrefix "a" and call listKeys method to get
+ * the next one key as the startKey in listStatus.
+ */
+ private List<OzoneKey> getNextShallowListOfKeys(String prevKey)
+ throws IOException {
+ String startKey = prevKey;
+ // handle for first round
+ if (!addedKeyPrefix) {
+ // prepare startKey
+ List<OzoneKey> nextOneKeys =
+ proxy.listKeys(volumeName, name, getKeyPrefix(), prevKey, 1);
+ if (nextOneKeys.isEmpty()) {
+ return nextOneKeys;
+ }
+ // startKey is guaranteed to be sorted after keyPrefix
+ startKey = nextOneKeys.get(0).getName();
+
+ // prepare delimiterKeyPrefix
+ delimiterKeyPrefix = getKeyPrefix();
+ if (!getKeyPrefix().endsWith(OZONE_URI_DELIMITER)) {
+ int lastIndex = getKeyPrefix().lastIndexOf(OZONE_URI_DELIMITER);
+ if (lastIndex != -1) {
+ delimiterKeyPrefix = getKeyPrefix().substring(0, lastIndex);
+ } else {
+ delimiterKeyPrefix = "";
+ }
+ }
+ }
+
+ // Elements in statuses must be sorted after startKey,
+ // which means they come after the keyPrefix.
+ List<OzoneFileStatus> statuses = proxy.listStatus(volumeName, name,
+ delimiterKeyPrefix, false, startKey, listCacheSize);
+
+ if (addedKeyPrefix) {
+ // previous round already include the startKey, so remove it
+ statuses.remove(0);
+ } else {
+ setAddedKeyPrefix(true);
+ }
+
+ // If statuses is empty after filtering, indicating that
+ // already no result matching the keyPrefix.
+ return statuses.stream()
+ .map(status -> {
+ OmKeyInfo keyInfo = status.getKeyInfo();
+ String keyName = keyInfo.getKeyName();
+ if (status.isDirectory()) {
+ // add trailing slash to represent directory
+ keyName = OzoneFSUtils.addTrailingSlashIfNeeded(keyName);
+ }
+ return new OzoneKey(keyInfo.getVolumeName(),
Review Comment:
Constructing a `OzoneKey` from `OmKeyInf` appears many times in the code,
so maybe we can create a tool function to do those. (Such as line L1608 and
L1508)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
