This is an automated email from the ASF dual-hosted git repository.
jojochuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2976b85885a HDDS-16760. Remove the per-call InfoBucket from rooted-ofs
listStatus
2976b85885a is described below
commit 2976b85885a54ba7a3da6fc429c4cdaa299a3d54
Author: Andrey Yarovoy <[email protected]>
AuthorDate: Wed Oct 14 15:28:03 2026 -0400
HDDS-16760. Remove the per-call InfoBucket from rooted-ofs listStatus
---
.../apache/hadoop/ozone/OzoneManagerVersion.java | 5 +
.../ozone/client/protocol/ClientProtocol.java | 12 +
.../client/protocol/ListStatusLightOptions.java | 20 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 19 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 19 ++
...OzoneManagerProtocolClientSideTranslatorPB.java | 6 +
.../ozone/AbstractRootedOzoneFileSystemTest.java | 48 ++++
.../fs/ozone/TestOfsListStatusRpcBenchmark.java | 308 +++++++++++++++++++++
.../src/main/proto/OmClientProtocol.proto | 3 +
.../apache/hadoop/ozone/om/OmMetadataReader.java | 51 ++--
.../protocolPB/OzoneManagerRequestHandler.java | 4 +-
.../hadoop/ozone/om/TestOMMetadataReader.java | 71 +++++
.../protocolPB/TestOzoneManagerRequestHandler.java | 36 +++
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 65 ++++-
...estBasicRootedOzoneClientAdapterListStatus.java | 277 ++++++++++++++++++
.../hadoop/ozone/client/ClientProtocolStub.java | 5 +
16 files changed, 910 insertions(+), 39 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
index bce3ab2e1e9..be5ebc7efdd 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
@@ -73,6 +73,11 @@ public enum OzoneManagerVersion implements ComponentVersion {
+ "server-side, so file system clients no longer need the "
+ "client-side InfoBucket layout check for getFileChecksum"),
ZDU(100, "OzoneManager version that supports zero downtime upgrade"),
+ LIST_STATUS_REJECTS_OBS(101,
+ "OzoneManager version that rejects listStatus on OBJECT_STORE buckets "
+ + "server-side when the request sets requireFileSystemLayout, so
file "
+ + "system clients no longer need the client-side InfoBucket layout "
+ + "check for listStatus"),
UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any
version recognized by this" +
" component, which may indicate it came from a component in a newer
version");
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 96247bb2ad1..f6eee8909a8 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -1252,6 +1252,18 @@ List<OzoneFileStatus> listStatus(String volumeName,
String bucketName,
String keyName, boolean recursive, String startKey,
long numEntries, boolean allowPartialPrefixes) throws IOException;
+ /**
+ * List the status for a file or a directory and its contents, returning the
+ * full file status. Takes the same options as {@link #listStatusLight}.
+ *
+ * @param options Encapsulates volume, bucket, key, recursive, startKey,
+ * numEntries, allowPartialPrefixes, and whether OM must
reject
+ * an OBJECT_STORE bucket.
+ * @return list of file status
+ */
+ List<OzoneFileStatus> listStatus(ListStatusLightOptions options)
+ throws IOException;
+
/**
* Lightweight listStatus API.
*
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ListStatusLightOptions.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ListStatusLightOptions.java
index 28fff17bbf5..240a0364745 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ListStatusLightOptions.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ListStatusLightOptions.java
@@ -36,6 +36,9 @@ public final class ListStatusLightOptions {
// When keyName is empty (root listing), this is the original S3/list
// prefix for STS auth. Enables LIST check on this prefix instead of "*".
private final String listPrefix;
+ // Set by file system clients: OM rejects the listing if the resolved bucket
+ // layout is OBJECT_STORE.
+ private final boolean requireFileSystemLayout;
private ListStatusLightOptions(Builder b) {
this.volumeName = b.volumeName;
@@ -46,6 +49,7 @@ private ListStatusLightOptions(Builder b) {
this.numEntries = b.numEntries;
this.allowPartialPrefixes = b.allowPartialPrefixes;
this.listPrefix = b.listPrefix;
+ this.requireFileSystemLayout = b.requireFileSystemLayout;
}
public String getVolumeName() {
@@ -80,6 +84,10 @@ public String getListPrefix() {
return listPrefix;
}
+ public boolean isRequireFileSystemLayout() {
+ return requireFileSystemLayout;
+ }
+
public static Builder builder() {
return new Builder();
}
@@ -113,6 +121,7 @@ public static final class Builder {
private long numEntries;
private boolean allowPartialPrefixes;
private String listPrefix;
+ private boolean requireFileSystemLayout;
public Builder setVolumeName(String volumeName) {
this.volumeName = volumeName;
@@ -154,6 +163,11 @@ public Builder setListPrefix(String listPrefix) {
return this;
}
+ public Builder setRequireFileSystemLayout(boolean requireFileSystemLayout)
{
+ this.requireFileSystemLayout = requireFileSystemLayout;
+ return this;
+ }
+
public ListStatusLightOptions build() {
return new ListStatusLightOptions(this);
}
@@ -169,7 +183,8 @@ public boolean equals(Object o) {
}
final ListStatusLightOptions that = (ListStatusLightOptions) o;
return recursive == that.recursive && numEntries == that.numEntries &&
- allowPartialPrefixes == that.allowPartialPrefixes &&
Objects.equals(volumeName, that.volumeName) &&
+ allowPartialPrefixes == that.allowPartialPrefixes &&
+ requireFileSystemLayout == that.requireFileSystemLayout &&
Objects.equals(volumeName, that.volumeName) &&
Objects.equals(bucketName, that.bucketName) && Objects.equals(keyName,
that.keyName) &&
Objects.equals(startKey, that.startKey) && Objects.equals(listPrefix,
that.listPrefix);
}
@@ -177,6 +192,7 @@ public boolean equals(Object o) {
@Override
public int hashCode() {
return Objects.hash(
- volumeName, bucketName, keyName, recursive, startKey, numEntries,
allowPartialPrefixes, listPrefix);
+ volumeName, bucketName, keyName, recursive, startKey, numEntries,
allowPartialPrefixes, listPrefix,
+ requireFileSystemLayout);
}
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 438ce126978..21e430e75b3 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -2535,6 +2535,14 @@ private OmKeyArgs prepareOmKeyArgs(String volumeName,
String bucketName,
return builder.build();
}
+ private OmKeyArgs prepareListStatusKeyArgs(ListStatusLightOptions options) {
+ final OmKeyArgs keyArgs = prepareOmKeyArgs(
+ options.getVolumeName(), options.getBucketName(),
options.getKeyName(), options.getListPrefix());
+ return options.isRequireFileSystemLayout()
+ ? keyArgs.toBuilder().setRequireFileSystemLayout(true).build()
+ : keyArgs;
+ }
+
@Override
public OzoneDataStreamOutput createStreamFile(String volumeName,
String bucketName, String keyName, long size,
@@ -2576,11 +2584,18 @@ public List<OzoneFileStatus> listStatus(String
volumeName, String bucketName,
allowPartialPrefixes);
}
+ @Override
+ public List<OzoneFileStatus> listStatus(ListStatusLightOptions options)
+ throws IOException {
+ final OmKeyArgs keyArgs = prepareListStatusKeyArgs(options);
+ return ozoneManagerClient.listStatus(keyArgs, options.isRecursive(),
options.getStartKey(),
+ options.getNumEntries(), options.isAllowPartialPrefixes());
+ }
+
@Override
public List<OzoneFileStatusLight> listStatusLight(ListStatusLightOptions
options)
throws IOException {
- final OmKeyArgs keyArgs = prepareOmKeyArgs(
- options.getVolumeName(), options.getBucketName(),
options.getKeyName(), options.getListPrefix());
+ final OmKeyArgs keyArgs = prepareListStatusKeyArgs(options);
if (OzoneManagerVersion.LIGHTWEIGHT_LIST_STATUS.isSupportedBy(omVersion)) {
return ozoneManagerClient.listStatusLight(
keyArgs, options.isRecursive(), options.getStartKey(),
options.getNumEntries(),
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
index 03ff4bd8f61..2280845cd3b 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
@@ -64,6 +64,9 @@ public final class OmKeyArgs extends WithMetadata implements
Auditable {
// Original S3/list prefix when keyName is empty (root listing). Used for STS
// auth to check LIST on this prefix instead of "*".
private final String listPrefix;
+ // Set by file system clients so that OM rejects a listing on an
+ // OBJECT_STORE bucket.
+ private final boolean requireFileSystemLayout;
private final String expectedETag;
private final boolean derivedKeyPiggyBacking;
@@ -88,6 +91,7 @@ private OmKeyArgs(Builder b) {
this.tags = b.tags.build();
this.expectedDataGeneration = b.expectedDataGeneration;
this.listPrefix = b.listPrefix;
+ this.requireFileSystemLayout = b.requireFileSystemLayout;
this.expectedETag = b.expectedETag;
this.derivedKeyPiggyBacking = b.derivedKeyPiggyBacking;
}
@@ -180,6 +184,14 @@ public String getListPrefix() {
return listPrefix;
}
+ /**
+ * Whether OM must reject the request if the resolved bucket layout is
+ * OBJECT_STORE, because the caller needs file system semantics.
+ */
+ public boolean isRequireFileSystemLayout() {
+ return requireFileSystemLayout;
+ }
+
public String getExpectedETag() {
return expectedETag;
}
@@ -262,6 +274,7 @@ public static class Builder extends WithMetadata.Builder {
private final MapBuilder<String, String> tags;
private Long expectedDataGeneration = null;
private String listPrefix = null;
+ private boolean requireFileSystemLayout;
private String expectedETag;
private boolean derivedKeyPiggyBacking;
@@ -314,6 +327,7 @@ public Builder(OmKeyArgs obj) {
this.tags = MapBuilder.of(obj.tags);
this.acls = AclListBuilder.of(obj.acls);
this.listPrefix = obj.listPrefix;
+ this.requireFileSystemLayout = obj.requireFileSystemLayout;
}
public Builder setVolumeName(String volume) {
@@ -452,6 +466,11 @@ public Builder setListPrefix(String prefix) {
return this;
}
+ public Builder setRequireFileSystemLayout(boolean require) {
+ this.requireFileSystemLayout = require;
+ return this;
+ }
+
public Builder setExpectedETag(String eTag) {
this.expectedETag = eTag;
return this;
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 9489c9cf5fc..5c46210c272 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -2650,6 +2650,9 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args,
boolean recursive,
final ListStatusRequest.Builder listStatusRequestBuilder =
createListStatusRequestBuilder(
keyArgs, recursive, startKey, numEntries, allowPartialPrefixes);
+ if (args.isRequireFileSystemLayout()) {
+ listStatusRequestBuilder.setRequireFileSystemLayout(true);
+ }
OMRequest omRequest = createOMRequest(Type.ListStatus)
.setListStatusRequest(listStatusRequestBuilder.build())
@@ -2683,6 +2686,9 @@ public List<OzoneFileStatusLight>
listStatusLight(OmKeyArgs args,
if (listPrefix != null && !listPrefix.isEmpty()) {
listStatusRequestBuilder.setListPrefix(listPrefix);
}
+ if (args.isRequireFileSystemLayout()) {
+ listStatusRequestBuilder.setRequireFileSystemLayout(true);
+ }
OMRequest omRequest = createOMRequest(Type.ListStatusLight)
.setListStatusRequest(listStatusRequestBuilder.build())
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
index 83661bd3d55..99234920214 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
@@ -1942,6 +1942,54 @@ void testGetFileChecksumOnNonExistentFile() throws
Exception {
() -> fs.getFileChecksum(missing));
}
+ @Test
+ void testListStatusDoesNotTriggerInfoBucket() throws Exception {
+ Path dir = new Path(bucketPath, "list-rpc-" +
RandomStringUtils.secure().nextAlphanumeric(5));
+ ContractTestUtils.touch(fs, new Path(dir, "file"));
+
+ OMMetrics metrics = getOMMetrics();
+ long bucketInfosBefore = metrics.getNumBucketInfos();
+
+ assertEquals(1, fs.listStatus(dir).length);
+ assertTrue(fs.listStatusIterator(dir).hasNext());
+
+ assertEquals(bucketInfosBefore, metrics.getNumBucketInfos(), "listStatus
must not trigger InfoBucket");
+ }
+
+ @Test
+ void testListStatusRejectsObsBucket() throws Exception {
+ OzoneBucket obsBucket = DataTestUtil.createVolumeAndBucket(client,
BucketLayout.OBJECT_STORE);
+ DataTestUtil.createKey(obsBucket, "obs-key",
"data".getBytes(StandardCharsets.UTF_8));
+ Path obsBucketPath = new Path(new Path(OZONE_URI_DELIMITER,
obsBucket.getVolumeName()), obsBucket.getName());
+
+ OMMetrics metrics = getOMMetrics();
+ long bucketInfosBefore = metrics.getNumBucketInfos();
+
+ IllegalArgumentException exception =
assertThrows(IllegalArgumentException.class,
+ () -> fs.listStatus(obsBucketPath));
+ assertThat(exception.getMessage()).contains(obsBucket.getName());
+ assertThat(exception.getMessage()).contains("OBJECT_STORE");
+ assertEquals(bucketInfosBefore, metrics.getNumBucketInfos(), "listStatus
must not trigger InfoBucket");
+ }
+
+ @Test
+ void testListStatusOrphanLinkBucketIsEmpty() throws Exception {
+ OzoneBucket sourceBucket = DataTestUtil.createVolumeAndBucket(client,
bucketLayout);
+ OzoneVolume sourceVolume =
objectStore.getVolume(sourceBucket.getVolumeName());
+ String linkBucketName = "orphan-link-" +
RandomStringUtils.secure().nextNumeric(5);
+ createLinkBucket(sourceVolume.getName(), linkBucketName,
sourceVolume.getName(), sourceBucket.getName());
+ sourceVolume.deleteBucket(sourceBucket.getName());
+ Path linkPath = new Path(new Path(OZONE_URI_DELIMITER,
sourceVolume.getName()), linkBucketName);
+
+ assertEquals(0, fs.listStatus(linkPath).length);
+ assertFalse(fs.listStatusIterator(linkPath).hasNext());
+
+ // A bucket that does not exist still fails, unlike the orphan link.
+ Path missingBucket = new Path(new Path(OZONE_URI_DELIMITER,
sourceVolume.getName()), "missing-bucket");
+ OMException exception = assertThrows(OMException.class, () ->
fs.listStatus(missingBucket));
+ assertEquals(BUCKET_NOT_FOUND, exception.getResult());
+ }
+
@Test
void testGetFileStatus() throws Exception {
String volumeNameLocal = getRandomNonExistVolumeName();
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOfsListStatusRpcBenchmark.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOfsListStatusRpcBenchmark.java
new file mode 100644
index 00000000000..1266766248a
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOfsListStatusRpcBenchmark.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Benchmark for the OFS listStatus OM RPC count (HDDS-16760).
+ *
+ * <p>The same workload runs unchanged on the baseline ({@code master}:
InfoBucket + ListStatusLight per call)
+ * and on the fix (a single ListStatusLight that asks OM to reject an
OBJECT_STORE bucket). It follows
+ * {@link TestOfsGetFileChecksumRpcBenchmark}: {@link #NUM_BUCKETS} buckets
are listed
+ * {@link #ACCESSES_PER_BUCKET} times each from a shuffled access sequence,
single-threaded and with
+ * {@link #CONCURRENT_THREADS} client threads. Each listed directory holds
{@link #FILES_PER_DIR} files, so the
+ * entries fit in one page; {@code BasicRootedOzoneFileSystem} still lists
until a page adds nothing, so each
+ * listStatus makes {@link #LISTINGS_PER_CALL} adapter listings on both arms.
+ */
+@Tag("benchmark")
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestOfsListStatusRpcBenchmark {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(TestOfsListStatusRpcBenchmark.class);
+
+ /** Distinct buckets in the working set. */
+ private static final int NUM_BUCKETS = 200;
+ /** listStatus calls per bucket. */
+ private static final int ACCESSES_PER_BUCKET = 10;
+ private static final int TOTAL_CALLS = NUM_BUCKETS * ACCESSES_PER_BUCKET;
+ private static final long SHUFFLE_SEED = 20250831L;
+ /** Client threads issuing listStatus in the concurrent comparison. */
+ private static final int CONCURRENT_THREADS = 10;
+ /** Entries in each listed directory. */
+ private static final int FILES_PER_DIR = 10;
+ /** Adapter listings per listStatus: the page with the entries, then one
that adds nothing. */
+ private static final int LISTINGS_PER_CALL = 2;
+
+ private MiniOzoneCluster cluster;
+ private OzoneClient client;
+ private OzoneConfiguration conf;
+ private String rootPath;
+
+ private final List<Path> accessSequence = new ArrayList<>(TOTAL_CALLS);
+
+ @BeforeAll
+ void init() throws IOException, InterruptedException, TimeoutException {
+ conf = new OzoneConfiguration();
+ conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
BucketLayout.FILE_SYSTEM_OPTIMIZED.name());
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ client = cluster.newClient();
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
conf.get(OZONE_OM_ADDRESS_KEY));
+
+ ObjectStore objectStore = client.getObjectStore();
+ String volName = "benchvol";
+ objectStore.createVolume(volName);
+ OzoneVolume volume = objectStore.getVolume(volName);
+ BucketArgs fsoArgs =
BucketArgs.newBuilder().setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED).build();
+ try (FileSystem setupFs = FileSystem.newInstance(URI.create(rootPath),
conf)) {
+ for (int i = 0; i < NUM_BUCKETS; i++) {
+ String buckName = "benchbucket-" + i;
+ volume.createBucket(buckName, fsoArgs);
+ Path dir = new Path("/" + volName + "/" + buckName + "/dir");
+ for (int f = 0; f < FILES_PER_DIR; f++) {
+ ContractTestUtils.touch(setupFs, new Path(dir, "file-" + f));
+ }
+ for (int a = 0; a < ACCESSES_PER_BUCKET; a++) {
+ accessSequence.add(dir);
+ }
+ }
+
assertThat(setupFs.listStatus(accessSequence.get(0))).hasSize(FILES_PER_DIR);
+ }
+ Collections.shuffle(accessSequence, new Random(SHUFFLE_SEED));
+ }
+
+ @AfterAll
+ void shutdown() {
+ IOUtils.closeQuietly(client);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ void benchmarkListStatus() throws Exception {
+ // Prime OM-side state and the JVM so the measured run pays no cold-start.
+ runWorkload(1);
+
+ Result single = runWorkload(1);
+ Result concurrent = runWorkload(CONCURRENT_THREADS);
+
+ assertWorkloadInvariants(single);
+ assertWorkloadInvariants(concurrent);
+
+ logResult("single-threaded", single);
+ logResult(CONCURRENT_THREADS + " concurrent threads", concurrent);
+ }
+
+ /**
+ * Invariants that hold on both the baseline and the fixed code, so the
identical benchmark passes in either
+ * worktree: every listStatus issues {@link #LISTINGS_PER_CALL} listing
RPCs, and the bucket and volume RPCs stay
+ * within one per listing.
+ */
+ private void assertWorkloadInvariants(Result r) {
+ long listings = (long) TOTAL_CALLS * LISTINGS_PER_CALL;
+ assertThat(r.listStatusRpcs).isEqualTo(listings);
+ assertThat(r.bucketInfoRpcs).isBetween(0L, listings);
+ assertThat(r.volumeInfoRpcs).isBetween(0L, listings);
+ }
+
+ private void logResult(String label, Result r) {
+ long omRpcs = r.volumeInfoRpcs + r.bucketInfoRpcs + r.listStatusRpcs;
+ LOG.info(String.format("%n"
+ + "BENCHRESULT OFS listStatus OM RPC benchmark (HDDS-16760) - %s%n"
+ + " buckets=%d, accesses/bucket=%d, total listStatus=%d,
threads=%d, entries/dir=%d%n"
+ + "
------------------------------------------------------------%n"
+ + " InfoVolume RPCs : %d%n"
+ + " InfoBucket RPCs : %d%n"
+ + " ListStatus RPCs : %d%n"
+ + " total OM RPCs : %d%n"
+ + " OM RPCs per call : %.2f%n"
+ + "
------------------------------------------------------------%n"
+ + " latency mean : %.3f ms%n"
+ + " latency p50 : %.3f ms%n"
+ + " latency p90 : %.3f ms%n"
+ + " latency p99 : %.3f ms%n"
+ + " latency max : %.3f ms%n"
+ + " throughput : %.1f listStatus/s%n",
+ label, NUM_BUCKETS, ACCESSES_PER_BUCKET, TOTAL_CALLS, r.threads,
FILES_PER_DIR,
+ r.volumeInfoRpcs, r.bucketInfoRpcs, r.listStatusRpcs,
+ omRpcs, omRpcs / (double) TOTAL_CALLS,
+ millis(r.mean()), millis(r.percentile(0.50)),
+ millis(r.percentile(0.90)), millis(r.percentile(0.99)),
+ millis(r.max()), r.opsPerSecond()));
+ }
+
+ private static double millis(double nanos) {
+ return nanos / (double) TimeUnit.MILLISECONDS.toNanos(1);
+ }
+
+ private Result runWorkload(int threads) throws IOException,
InterruptedException {
+ OzoneConfiguration runConf = new OzoneConfiguration(conf);
+ runConf.set(FS_DEFAULT_NAME_KEY, rootPath);
+
+ OMMetrics metrics = cluster.getOzoneManager().getMetrics();
+ long[] latencyNanos = new long[TOTAL_CALLS];
+ // A new FileSystem instance per run, so each run starts with no
client-side state carried over.
+ try (FileSystem fs = FileSystem.newInstance(URI.create(rootPath),
runConf)) {
+ long volumeInfosBefore = metrics.getNumVolumeInfos();
+ long bucketInfosBefore = metrics.getNumBucketInfos();
+ long listStatusBefore = metrics.getNumListStatus();
+ long elapsedNanos = threads == 1
+ ? runSequential(fs, latencyNanos)
+ : runConcurrent(fs, threads, latencyNanos);
+ return new Result(threads,
+ metrics.getNumVolumeInfos() - volumeInfosBefore,
+ metrics.getNumBucketInfos() - bucketInfosBefore,
+ metrics.getNumListStatus() - listStatusBefore,
+ elapsedNanos, latencyNanos);
+ }
+ }
+
+ private long runSequential(FileSystem fs, long[] latencyNanos) throws
IOException {
+ long startNanos = System.nanoTime();
+ int i = 0;
+ for (Path path : accessSequence) {
+ long callStart = System.nanoTime();
+ fs.listStatus(path);
+ latencyNanos[i++] = System.nanoTime() - callStart;
+ }
+ return System.nanoTime() - startNanos;
+ }
+
+ private long runConcurrent(FileSystem fs, int threads, long[] latencyNanos)
+ throws IOException, InterruptedException {
+ ExecutorService pool = Executors.newFixedThreadPool(threads);
+ CountDownLatch startGate = new CountDownLatch(1);
+ CountDownLatch doneGate = new CountDownLatch(threads);
+ AtomicReference<IOException> failure = new AtomicReference<>();
+ // Disjoint, contiguous slices of the shuffled sequence; each thread
writes only its own latency indices, so no
+ // synchronization is needed per call.
+ int chunk = (TOTAL_CALLS + threads - 1) / threads;
+ for (int t = 0; t < threads; t++) {
+ final int from = t * chunk;
+ final int to = Math.min(TOTAL_CALLS, from + chunk);
+ pool.submit(() -> {
+ try {
+ startGate.await();
+ for (int i = from; i < to; i++) {
+ long callStart = System.nanoTime();
+ fs.listStatus(accessSequence.get(i));
+ latencyNanos[i] = System.nanoTime() - callStart;
+ }
+ } catch (IOException e) {
+ failure.compareAndSet(null, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneGate.countDown();
+ }
+ });
+ }
+ long startNanos = System.nanoTime();
+ startGate.countDown();
+ doneGate.await();
+ long elapsedNanos = System.nanoTime() - startNanos;
+ pool.shutdownNow();
+ if (failure.get() != null) {
+ throw failure.get();
+ }
+ return elapsedNanos;
+ }
+
+ private static final class Result {
+ private final int threads;
+ private final long volumeInfoRpcs;
+ private final long bucketInfoRpcs;
+ private final long listStatusRpcs;
+ private final long elapsedNanos;
+ private final long[] sortedLatencyNanos;
+
+ Result(int threads, long volumeInfoRpcs, long bucketInfoRpcs, long
listStatusRpcs, long elapsedNanos,
+ long[] latencyNanos) {
+ this.threads = threads;
+ this.volumeInfoRpcs = volumeInfoRpcs;
+ this.bucketInfoRpcs = bucketInfoRpcs;
+ this.listStatusRpcs = listStatusRpcs;
+ this.elapsedNanos = elapsedNanos;
+ this.sortedLatencyNanos = latencyNanos.clone();
+ Arrays.sort(this.sortedLatencyNanos);
+ }
+
+ double opsPerSecond() {
+ return TOTAL_CALLS / (elapsedNanos / (double)
TimeUnit.SECONDS.toNanos(1));
+ }
+
+ double mean() {
+ long sum = 0;
+ for (long l : sortedLatencyNanos) {
+ sum += l;
+ }
+ return sum / (double) sortedLatencyNanos.length;
+ }
+
+ double percentile(double q) {
+ int idx = (int) Math.ceil(q * sortedLatencyNanos.length) - 1;
+ idx = Math.max(0, Math.min(sortedLatencyNanos.length - 1, idx));
+ return sortedLatencyNanos[idx];
+ }
+
+ double max() {
+ return sortedLatencyNanos[sortedLatencyNanos.length - 1];
+ }
+ }
+}
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 1445e1bb2bf..529ef0ffd85 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1379,6 +1379,9 @@ message ListStatusRequest {
// When keyArgs.keyName is empty (root listing), this is the original
S3/list
// prefix for STS auth. Enables LIST check on this prefix instead of "*".
optional string listPrefix = 6;
+ // Set by file system clients: OM rejects the listing with
+ // NOT_SUPPORTED_OPERATION if the resolved bucket layout is OBJECT_STORE.
+ optional bool requireFileSystemLayout = 7;
}
message ListStatusResponse {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
index 4435cf78b82..dfefbef845f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@ -252,6 +252,9 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args,
boolean recursive,
args = bucket.update(args);
try {
+ if (args.isRequireFileSystemLayout()) {
+ validateFileSystemLayout(bucket);
+ }
if (isAclEnabled) {
checkListStatusAcls(args, bucket);
}
@@ -288,6 +291,9 @@ public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs
args,
OmKeyArgs resolvedArgs = bucket.update(args);
try {
+ if (resolvedArgs.isRequireFileSystemLayout()) {
+ validateFileSystemLayout(bucket);
+ }
if (isAclEnabled) {
checkListStatusAcls(resolvedArgs, bucket);
}
@@ -310,6 +316,25 @@ startKey, maxListingPageSize, getClientAddress(),
}
}
+ /**
+ * Rejects file system reads on a bucket whose link-resolved layout is
+ * OBJECT_STORE, so file system clients need no InfoBucket to check it.
+ * listStatus and listStatusLight call it only when the client asked for it,
+ * because they also serve object store callers (S3 shallow key listing).
+ */
+ private static void validateFileSystemLayout(ResolvedBucket bucket) throws
OMException {
+ if (bucket.bucketLayout() != null) {
+ try {
+ OzoneFSUtils.validateBucketLayout(bucket.requestedBucket(),
bucket.bucketLayout());
+ } catch (IllegalArgumentException e) {
+ // Convert to an OMException so it is returned to the client as a
+ // normal (non-retryable) RPC response instead of escaping the read
+ // handler's IOException catch and triggering a client retry storm.
+ throw new OMException(e.getMessage(),
ResultCodes.NOT_SUPPORTED_OPERATION);
+ }
+ }
+ }
+
@Override
public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException {
ResolvedBucket bucket = ozoneManager.resolveBucketLink(args);
@@ -320,18 +345,7 @@ public OzoneFileStatus getFileStatus(OmKeyArgs args)
throws IOException {
args = bucket.update(args);
try {
- if (bucket.bucketLayout() != null) {
- try {
- OzoneFSUtils.validateBucketLayout(bucket.requestedBucket(),
- bucket.bucketLayout());
- } catch (IllegalArgumentException e) {
- // Convert to an OMException so it is returned to the client as a
- // normal (non-retryable) RPC response instead of escaping the read
- // handler's IOException catch and triggering a client retry storm.
- throw new OMException(e.getMessage(),
- ResultCodes.NOT_SUPPORTED_OPERATION);
- }
- }
+ validateFileSystemLayout(bucket);
if (isAclEnabled) {
checkAcls(getResourceType(args), StoreType.OZONE, ACLType.READ,
bucket, args.getKeyName());
}
@@ -361,18 +375,7 @@ public OmKeyInfo lookupFile(OmKeyArgs args) throws
IOException {
args = normalizeKeyArgs(bucket.update(args), bucket);
try {
- if (bucket.bucketLayout() != null) {
- try {
- OzoneFSUtils.validateBucketLayout(bucket.requestedBucket(),
- bucket.bucketLayout());
- } catch (IllegalArgumentException e) {
- // Convert to an OMException so it is returned to the client as a
- // normal (non-retryable) RPC response instead of escaping the read
- // handler's IOException catch and triggering a client retry storm.
- throw new OMException(e.getMessage(),
- ResultCodes.NOT_SUPPORTED_OPERATION);
- }
- }
+ validateFileSystemLayout(bucket);
if (isAclEnabled) {
checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.READ,
bucket, args.getKeyName());
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index b862bd03662..a0a4d400dfb 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -1332,6 +1332,7 @@ private ListStatusResponse listStatus(
.setKeyName(keyArgs.getKeyName())
.setLatestVersionLocation(keyArgs.getLatestVersionLocation())
.setHeadOp(keyArgs.getHeadOp())
+ .setRequireFileSystemLayout(request.getRequireFileSystemLayout())
.build();
boolean allowPartialPrefixes =
request.hasAllowPartialPrefix() && request.getAllowPartialPrefix();
@@ -1356,7 +1357,8 @@ private ListStatusLightResponse
listStatusLight(ListStatusRequest request) throw
.setKeyName(keyArgs.getKeyName())
.setSortDatanodesInPipeline(false)
.setLatestVersionLocation(true)
- .setHeadOp(keyArgs.getHeadOp());
+ .setHeadOp(keyArgs.getHeadOp())
+ .setRequireFileSystemLayout(request.getRequireFileSystemLayout());
if (request.hasListPrefix() && !request.getListPrefix().isEmpty()) {
omKeyArgsBuilder.setListPrefix(request.getListPrefix());
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java
index 01a5bad1808..4343480376f 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java
@@ -27,6 +27,8 @@
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
@@ -900,4 +902,73 @@ public void lookupFileAllowsLegacyLayout() throws
Exception {
assertEquals(expectedKeyInfo, reader.lookupFile(keyArgs));
verify(keyManager).lookupFile(any(OmKeyArgs.class), anyString());
}
+
+ private static OmMetadataReader newListStatusReader(KeyManager keyManager,
String bucketName,
+ BucketLayout layout) throws IOException {
+ OzoneManager ozoneManager = mock(OzoneManager.class);
+ when(ozoneManager.getAclsEnabled()).thenReturn(false);
+ when(ozoneManager.getConfiguration()).thenReturn(new OzoneConfiguration());
+
when(ozoneManager.getBucketManager()).thenReturn(mock(BucketManager.class));
+
when(ozoneManager.getVolumeManager()).thenReturn(mock(VolumeManager.class));
+
when(ozoneManager.getPerfMetrics()).thenReturn(mock(OMPerformanceMetrics.class));
+ when(ozoneManager.resolveBucketLink(any(OmKeyArgs.class)))
+ .thenReturn(new ResolvedBucket("vol", bucketName, "vol", bucketName,
"owner", layout));
+ return new OmMetadataReader(keyManager, mock(PrefixManager.class),
ozoneManager, mock(org.slf4j.Logger.class),
+ mock(AuditLogger.class), mock(OmMetadataReaderMetrics.class), null);
+ }
+
+ private static OmKeyArgs listStatusArgs(String bucketName, boolean
requireFileSystemLayout) {
+ return new OmKeyArgs.Builder()
+ .setVolumeName("vol")
+ .setBucketName(bucketName)
+ .setKeyName("dir")
+ .setRequireFileSystemLayout(requireFileSystemLayout)
+ .build();
+ }
+
+ @Test
+ public void listStatusRejectsObjectStoreLayout() throws Exception {
+ KeyManager keyManager = mock(KeyManager.class);
+ OmMetadataReader reader = newListStatusReader(keyManager, "obs-bucket",
BucketLayout.OBJECT_STORE);
+
+ OMException exception = assertThrows(OMException.class,
+ () -> reader.listStatus(listStatusArgs("obs-bucket", true), false, "",
100, false));
+ assertEquals(ResultCodes.NOT_SUPPORTED_OPERATION, exception.getResult());
+ assertTrue(exception.getMessage().contains("obs-bucket"));
+ assertTrue(exception.getMessage().contains("OBJECT_STORE"));
+
+ exception = assertThrows(OMException.class,
+ () -> reader.listStatusLight(listStatusArgs("obs-bucket", true),
false, "", 100, false));
+ assertEquals(ResultCodes.NOT_SUPPORTED_OPERATION, exception.getResult());
+
+ verify(keyManager, never()).listStatus(any(), anyBoolean(), any(),
anyLong(), any(), anyBoolean());
+ verify(keyManager, never()).listStatus(any(), anyBoolean(), any(),
anyLong(), any(), anyBoolean(),
+ anyBoolean());
+ }
+
+ @Test
+ public void listStatusAllowsLegacyLayout() throws Exception {
+ KeyManager keyManager = mock(KeyManager.class);
+ OmMetadataReader reader = newListStatusReader(keyManager, "legacy-bucket",
BucketLayout.LEGACY);
+
+ assertTrue(reader.listStatus(listStatusArgs("legacy-bucket", true), false,
"", 100, false).isEmpty());
+ assertTrue(reader.listStatusLight(listStatusArgs("legacy-bucket", true),
false, "", 100, false).isEmpty());
+
+ verify(keyManager).listStatus(any(OmKeyArgs.class), eq(false), eq(""),
anyLong(), any(), eq(false));
+ verify(keyManager).listStatus(any(OmKeyArgs.class), eq(false), eq(""),
anyLong(), any(), eq(false), eq(false));
+ }
+
+ @Test
+ public void listStatusServesObjectStoreLayoutWhenNotRequested() throws
Exception {
+ // Object store callers, such as S3 shallow key listing and OM trash, list
+ // OBJECT_STORE buckets without asking for a file system layout.
+ KeyManager keyManager = mock(KeyManager.class);
+ OmMetadataReader reader = newListStatusReader(keyManager, "obs-bucket",
BucketLayout.OBJECT_STORE);
+
+ assertTrue(reader.listStatus(listStatusArgs("obs-bucket", false), false,
"", 100, false).isEmpty());
+ assertTrue(reader.listStatusLight(listStatusArgs("obs-bucket", false),
false, "", 100, false).isEmpty());
+
+ verify(keyManager).listStatus(any(OmKeyArgs.class), eq(false), eq(""),
anyLong(), any(), eq(false));
+ verify(keyManager).listStatus(any(OmKeyArgs.class), eq(false), eq(""),
anyLong(), any(), eq(false), eq(false));
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java
index 0f8d9602538..e50547f93b3 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java
@@ -307,6 +307,42 @@ public void testListStatusResponseSize(int resultSize)
throws IOException {
}
}
+ /**
+ * ListStatus and ListStatusLight must forward requireFileSystemLayout from
+ * the request into the OmKeyArgs handed to the OM.
+ */
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testListStatusForwardsRequireFileSystemLayout(boolean require)
throws IOException {
+ OzoneManagerRequestHandler requestHandler = getRequestHandler(10);
+ OzoneManager ozoneManager = requestHandler.getOzoneManager();
+ OzoneManagerProtocolProtos.ListStatusRequest listStatusRequest =
+ OzoneManagerProtocolProtos.ListStatusRequest.newBuilder()
+ .setKeyArgs(OzoneManagerProtocolProtos.KeyArgs.newBuilder()
+
.setVolumeName("volume").setBucketName("bucket").setKeyName("dir"))
+ .setRecursive(false).setStartKey("").setNumEntries(10)
+ .setRequireFileSystemLayout(require)
+ .build();
+
+ for (OzoneManagerProtocolProtos.Type type : Arrays.asList(
+ OzoneManagerProtocolProtos.Type.ListStatus,
OzoneManagerProtocolProtos.Type.ListStatusLight)) {
+ OzoneManagerProtocolProtos.OMRequest request =
Mockito.mock(OzoneManagerProtocolProtos.OMRequest.class);
+ Mockito.when(request.getTraceID()).thenReturn("traceId");
+ Mockito.when(request.getCmdType()).thenReturn(type);
+
Mockito.when(request.getListStatusRequest()).thenReturn(listStatusRequest);
+ requestHandler.handleReadRequest(request);
+ }
+
+ ArgumentCaptor<OmKeyArgs> listStatusArgs =
ArgumentCaptor.forClass(OmKeyArgs.class);
+ Mockito.verify(ozoneManager).listStatus(listStatusArgs.capture(),
Mockito.anyBoolean(), Mockito.anyString(),
+ Mockito.anyLong(), Mockito.anyBoolean());
+ Assertions.assertEquals(require,
listStatusArgs.getValue().isRequireFileSystemLayout());
+ ArgumentCaptor<OmKeyArgs> listStatusLightArgs =
ArgumentCaptor.forClass(OmKeyArgs.class);
+
Mockito.verify(ozoneManager).listStatusLight(listStatusLightArgs.capture(),
Mockito.anyBoolean(),
+ Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean());
+ Assertions.assertEquals(require,
listStatusLightArgs.getValue().isRequireFileSystemLayout());
+ }
+
/**
* Test to verify BasicOmKeyInfo encryption field works in listKeysLight.
*/
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 8540929de3b..311c7bfc3a3 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -89,6 +89,7 @@
import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.client.protocol.ListStatusLightOptions;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -976,21 +977,60 @@ public List<FileStatusAdapter> listStatus(String pathStr,
boolean recursive,
// Internally we need startKey to be passed into bucket.listStatus
String startKey = ofsStartPath.getKeyName();
try {
- OzoneBucket bucket = getBucket(ofsPath, false);
List<OzoneFileStatus> statuses = Collections.emptyList();
List<OzoneFileStatusLight> lightStatuses = Collections.emptyList();
- if (bucket.isSourcePathExist()) {
- if (lite) {
- lightStatuses = bucket.listStatusLight(keyName, recursive, startKey,
numEntries);
- } else {
- statuses = bucket.listStatus(keyName, recursive, startKey,
numEntries);
+ if
(proxy.getOmVersion().compareTo(OzoneManagerVersion.LIST_STATUS_REJECTS_OBS) >=
0) {
+ // Ask OM to reject an OBJECT_STORE bucket (HDDS-16760), so list
without
+ // fetching the bucket first. The bucket is fetched only when the
listing
+ // fails because a bucket is missing, to tell an orphan link bucket
+ // (empty listing) from a bucket that does not exist.
+ ListStatusLightOptions options = ListStatusLightOptions.builder()
+ .setVolumeName(ofsPath.getVolumeName())
+ .setBucketName(ofsPath.getBucketName())
+ .setKeyName(keyName)
+ .setRecursive(recursive)
+ .setStartKey(startKey)
+ .setNumEntries(numEntries)
+ .setRequireFileSystemLayout(true)
+ .build();
+ try {
+ if (lite) {
+ lightStatuses = proxy.listStatusLight(options);
+ } else {
+ statuses = proxy.listStatus(options);
+ }
+ } catch (OMException e) {
+ if (e.getResult() ==
OMException.ResultCodes.NOT_SUPPORTED_OPERATION) {
+ // Surface it as IllegalArgumentException, matching the client-side
+ // layout check of the fallback path below.
+ throw new IllegalArgumentException(e.getMessage());
+ }
+ if (e.getResult() != OMException.ResultCodes.BUCKET_NOT_FOUND
+ && e.getResult() != OMException.ResultCodes.VOLUME_NOT_FOUND) {
+ throw e;
+ }
+ // Rethrows as before if the bucket itself does not exist.
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ if (bucket.isSourcePathExist()) {
+ throw e;
+ }
+ logOrphanLinkBucket(bucket);
}
-
} else {
- LOG.warn("Source Bucket does not exist, link bucket {} is orphan " +
- "and returning empty list of files inside it", bucket.getName());
+ // Older OM has no server-side OBJECT_STORE check; validate the bucket
+ // layout on the client, matching pre-HDDS-16760 behavior.
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ if (bucket.isSourcePathExist()) {
+ if (lite) {
+ lightStatuses = bucket.listStatusLight(keyName, recursive,
startKey, numEntries);
+ } else {
+ statuses = bucket.listStatus(keyName, recursive, startKey,
numEntries);
+ }
+ } else {
+ logOrphanLinkBucket(bucket);
+ }
}
-
+
// Note: result in statuses above doesn't have volume/bucket path since
// they are from the server.
String ofsPathPrefix = ofsPath.getNonKeyPath();
@@ -1014,6 +1054,11 @@ public List<FileStatusAdapter> listStatus(String
pathStr, boolean recursive,
}
}
+ private static void logOrphanLinkBucket(OzoneBucket bucket) {
+ LOG.warn("Source Bucket does not exist, link bucket {} is orphan " +
+ "and returning empty list of files inside it", bucket.getName());
+ }
+
@Override
public Token<OzoneTokenIdentifier> getDelegationToken(String renewer)
throws IOException {
diff --git
a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterListStatus.java
b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterListStatus.java
new file mode 100644
index 00000000000..541a42ba9a9
--- /dev/null
+++
b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterListStatus.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.URI;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OzoneManagerVersion;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.client.protocol.ListStatusLightOptions;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.InOrder;
+
+/**
+ * Unit tests for {@link BasicRootedOzoneClientAdapterImpl#listStatus} on a key
+ * path, covering the listing without InfoBucket added by HDDS-16760, the
+ * orphan link bucket handling on the error path, and the fallback used when
+ * the OM predates {@link OzoneManagerVersion#LIST_STATUS_REJECTS_OBS}. Uses a
+ * partial mock so no OM connection is required.
+ */
+public class TestBasicRootedOzoneClientAdapterListStatus {
+
+ private static final String DIR_PATH = "/vol/bucket/dir";
+ private static final URI OFS_URI = URI.create("ofs://om/");
+
+ private BasicRootedOzoneClientAdapterImpl adapter;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private OzoneBucket bucket;
+
+ @BeforeEach
+ public void setUp() throws Exception {
+ adapter = mock(BasicRootedOzoneClientAdapterImpl.class,
CALLS_REAL_METHODS);
+ proxy = mock(ClientProtocol.class);
+ objectStore = mock(ObjectStore.class);
+ bucket = mock(OzoneBucket.class);
+
+ when(proxy.getBucketDetails(anyString(), anyString())).thenReturn(bucket);
+ when(bucket.getVolumeName()).thenReturn("vol");
+ when(bucket.getName()).thenReturn("bucket");
+
when(bucket.getBucketLayout()).thenReturn(BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ // Mirror OzoneBucket: the source path exists unless link resolution
clears it.
+ AtomicBoolean sourcePathExist = new AtomicBoolean(true);
+ doAnswer(invocation -> {
+ sourcePathExist.set(invocation.getArgument(0));
+ return null;
+ }).when(bucket).setSourcePathExist(anyBoolean());
+ when(bucket.isSourcePathExist()).thenAnswer(invocation ->
sourcePathExist.get());
+
+ setField("proxy", proxy);
+ setField("objectStore", objectStore);
+ setField("config", new OzoneConfiguration());
+
+ // Default to an OM that performs the server-side OBJECT_STORE rejection.
+
when(proxy.getOmVersion()).thenReturn(OzoneManagerVersion.LIST_STATUS_REJECTS_OBS);
+ }
+
+ private void setField(String name, Object value) throws Exception {
+ Field field =
BasicRootedOzoneClientAdapterImpl.class.getDeclaredField(name);
+ field.setAccessible(true);
+ field.set(adapter, value);
+ }
+
+ private List<FileStatusAdapter> listStatus(boolean lite) throws IOException {
+ return adapter.listStatus(DIR_PATH, false, "", 100, OFS_URI, new
Path("/"), "user", lite);
+ }
+
+ private void failListing(OMException e) throws IOException {
+ when(proxy.listStatus(any(ListStatusLightOptions.class))).thenThrow(e);
+
when(proxy.listStatusLight(any(ListStatusLightOptions.class))).thenThrow(e);
+ }
+
+ /** The options the adapter sends: the listed path, asking OM to reject
OBJECT_STORE. */
+ private static ListStatusLightOptions fsListing() {
+ return argThat(o -> o.getVolumeName().equals("vol") &&
o.getBucketName().equals("bucket")
+ && o.getKeyName().equals("dir") && !o.isRecursive() &&
o.getNumEntries() == 100
+ && o.isRequireFileSystemLayout());
+ }
+
+ /** Makes the bucket a link whose source volume has been deleted. */
+ private void makeOrphanLink() throws IOException {
+ when(bucket.isLink()).thenReturn(true);
+ when(bucket.getSourceVolume()).thenReturn("srcvol");
+ when(bucket.getSourceBucket()).thenReturn("srcbucket");
+ when(objectStore.getVolume("srcvol"))
+ .thenThrow(new OMException("no volume", ResultCodes.VOLUME_NOT_FOUND));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void listingIsTheOnlyOmCall(boolean lite) throws IOException {
+ assertThat(listStatus(lite)).isEmpty();
+
+ if (lite) {
+ verify(proxy).listStatusLight(fsListing());
+ } else {
+ verify(proxy).listStatus(fsListing());
+ }
+ verify(proxy, never()).getBucketDetails(anyString(), anyString());
+ verify(objectStore, never()).getVolume(anyString());
+ }
+
+ @ParameterizedTest
+ @EnumSource(value = ResultCodes.class, names = {"BUCKET_NOT_FOUND",
"VOLUME_NOT_FOUND"})
+ public void orphanLinkReturnsEmptyListAfterFailedListing(ResultCodes code)
throws IOException {
+ // OM cannot resolve the link to its deleted source, so the listing fails
+ // and only then does the client fetch the bucket to recognise the orphan.
+ failListing(new OMException("source missing", code));
+ makeOrphanLink();
+
+ for (boolean lite : new boolean[] {true, false}) {
+ assertThat(listStatus(lite)).isEmpty();
+ }
+
+ InOrder order = inOrder(proxy);
+ order.verify(proxy).listStatusLight(fsListing());
+ order.verify(proxy).getBucketDetails("vol", "bucket");
+ order.verify(proxy).listStatus(fsListing());
+ order.verify(proxy).getBucketDetails("vol", "bucket");
+ }
+
+ @Test
+ public void orphanLinkDeclaredObjectStoreIsRejected() throws IOException {
+ // The fallback validates the link's own layout for an orphan; the error
+ // path must keep doing so.
+ failListing(new OMException("source missing",
ResultCodes.BUCKET_NOT_FOUND));
+ makeOrphanLink();
+ when(bucket.getBucketLayout()).thenReturn(BucketLayout.OBJECT_STORE);
+
+ IllegalArgumentException ex = assertThrows(IllegalArgumentException.class,
() -> listStatus(false));
+ assertThat(ex.getMessage()).contains("OBJECT_STORE");
+ }
+
+ @Test
+ public void missingBucketPropagatesFromBucketFetch() throws IOException {
+ failListing(new OMException("no bucket in listing",
ResultCodes.BUCKET_NOT_FOUND));
+ OMException missing = new OMException("no bucket",
ResultCodes.BUCKET_NOT_FOUND);
+ when(proxy.getBucketDetails("vol", "bucket")).thenThrow(missing);
+
+ OMException ex = assertThrows(OMException.class, () -> listStatus(false));
+ assertSame(missing, ex);
+ }
+
+ @Test
+ public void existingBucketOnErrorPathRethrowsListingFailure() throws
IOException {
+ // The bucket exists by the time it is fetched, so it is not an orphan
link.
+ OMException listingFailure = new OMException("no bucket",
ResultCodes.BUCKET_NOT_FOUND);
+ failListing(listingFailure);
+
+ OMException ex = assertThrows(OMException.class, () -> listStatus(false));
+ assertSame(listingFailure, ex);
+ }
+
+ @Test
+ public void obsRejectionMappedToIllegalArgumentException() throws
IOException {
+ failListing(new OMException("Bucket: bucket has layout: OBJECT_STORE",
ResultCodes.NOT_SUPPORTED_OPERATION));
+
+ IllegalArgumentException ex = assertThrows(IllegalArgumentException.class,
() -> listStatus(false));
+ assertThat(ex.getMessage()).contains("OBJECT_STORE");
+ verify(proxy, never()).getBucketDetails(anyString(), anyString());
+ }
+
+ @Test
+ public void fileNotFoundMappedToFileNotFoundException() throws IOException {
+ failListing(new OMException("no dir", ResultCodes.FILE_NOT_FOUND));
+
+ assertThrows(FileNotFoundException.class, () -> listStatus(false));
+ verify(proxy, never()).getBucketDetails(anyString(), anyString());
+ }
+
+ @Test
+ public void otherOMExceptionPropagates() throws IOException {
+ failListing(new OMException("denied", ResultCodes.PERMISSION_DENIED));
+
+ OMException ex = assertThrows(OMException.class, () -> listStatus(false));
+ assertEquals(ResultCodes.PERMISSION_DENIED, ex.getResult());
+ verify(proxy, never()).getBucketDetails(anyString(), anyString());
+ }
+
+ @Test
+ public void olderOmFallsBackToClientSideBucketCheck() throws IOException {
+ // An OM older than LIST_STATUS_REJECTS_OBS has no server-side OBJECT_STORE
+ // rejection, so the adapter must fetch the bucket and validate its layout.
+
when(proxy.getOmVersion()).thenReturn(OzoneManagerVersion.LOOKUP_FILE_REJECTS_OBS);
+
+ assertThat(listStatus(false)).isEmpty();
+
+ verify(proxy).getBucketDetails("vol", "bucket");
+ verify(bucket).listStatus("dir", false, "", 100);
+ verify(proxy, never()).listStatus(any(ListStatusLightOptions.class));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void zduOmFallsBackToClientSideBucketCheck(boolean lite) throws
IOException {
+ // An OM finalized for ZDU reports ZDU even without this change, and such
an
+ // OM ignores requireFileSystemLayout, so the client must still fetch the
+ // bucket and validate its layout.
+ when(proxy.getOmVersion()).thenReturn(OzoneManagerVersion.ZDU);
+
+ assertThat(listStatus(lite)).isEmpty();
+
+ verify(proxy).getBucketDetails("vol", "bucket");
+ if (lite) {
+ verify(bucket).listStatusLight("dir", false, "", 100);
+ } else {
+ verify(bucket).listStatus("dir", false, "", 100);
+ }
+ verify(proxy, never()).listStatus(any(ListStatusLightOptions.class));
+ verify(proxy, never()).listStatusLight(any(ListStatusLightOptions.class));
+ }
+
+ @Test
+ public void olderOmRejectsObsBucketClientSide() throws IOException {
+
when(proxy.getOmVersion()).thenReturn(OzoneManagerVersion.LOOKUP_FILE_REJECTS_OBS);
+ when(bucket.getBucketLayout()).thenReturn(BucketLayout.OBJECT_STORE);
+
+ IllegalArgumentException ex = assertThrows(IllegalArgumentException.class,
() -> listStatus(false));
+ assertThat(ex.getMessage()).contains("OBJECT_STORE");
+ verify(bucket, never()).listStatus(anyString(), anyBoolean(), anyString(),
anyLong());
+ }
+
+ @Test
+ public void olderOmOrphanLinkReturnsEmptyList() throws IOException {
+
when(proxy.getOmVersion()).thenReturn(OzoneManagerVersion.LOOKUP_FILE_REJECTS_OBS);
+ makeOrphanLink();
+
+ assertThat(listStatus(false)).isEmpty();
+ verify(bucket, never()).listStatus(anyString(), anyBoolean(), anyString(),
anyLong());
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index 8487bdb4aa1..391785a0861 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -709,6 +709,11 @@ public List<OzoneFileStatusLight>
listStatusLight(ListStatusLightOptions options
return null;
}
+ @Override
+ public List<OzoneFileStatus> listStatus(ListStatusLightOptions options)
throws IOException {
+ return null;
+ }
+
@Override
public boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException {
return false;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
