This is an automated email from the ASF dual-hosted git repository.
ivandika3 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new afa7f0e667c HDDS-13242. getTrashRoots should not be bounded by
ozone.fs.listing.page.size.max (#11002)
afa7f0e667c is described below
commit afa7f0e667cccb8cf157f07438c290002b004ffd
Author: Ivan Andika <[email protected]>
AuthorDate: Tue Aug 18 09:19:39 2026 +0800
HDDS-13242. getTrashRoots should not be bounded by
ozone.fs.listing.page.size.max (#11002)
---
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 34 +++++-
.../hadoop/ozone/om/TestTrashOzoneFileSystem.java | 133 +++++++++++++++++++++
2 files changed, 161 insertions(+), 6 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
index e147239a5ad..05485f3b8a3 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
@@ -17,6 +17,9 @@
package org.apache.hadoop.ozone.om;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE_DEFAULT;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE_MAX;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_MAX_LISTING_PAGE_SIZE;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_O3TRASH_URI_SCHEME;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static
org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.addTrailingSlashIfNeeded;
@@ -194,13 +197,32 @@ private boolean deleteFSO(OFSPath srcPath) {
public FileStatus[] listStatus(Path path) throws IOException {
ozoneManager.getMetrics().incNumTrashListStatus();
List<FileStatus> fileStatuses = new ArrayList<>();
+ int pageSize = ozoneConfiguration.getInt(
+ OZONE_FS_LISTING_PAGE_SIZE_MAX,
+ OZONE_FS_LISTING_PAGE_SIZE_DEFAULT);
+ pageSize = (int) OzoneConfigUtil.limitValue(pageSize,
+ OZONE_FS_LISTING_PAGE_SIZE_MAX,
+ OZONE_FS_LISTING_PAGE_SIZE_MAX,
+ OZONE_FS_MAX_LISTING_PAGE_SIZE);
OmKeyArgs keyArgs = constructOmKeyArgs(path);
- List<OzoneFileStatus> list = ozoneManager.
- listStatus(keyArgs, false, null, Integer.MAX_VALUE);
- for (OzoneFileStatus status : list) {
- FileStatus fileStatus = convertToFileStatus(status);
- fileStatuses.add(fileStatus);
- }
+ String startKey = null;
+ String lastKeyPath = null;
+ int entriesAdded;
+ do {
+ List<OzoneFileStatus> list = ozoneManager.
+ listStatus(keyArgs, false, startKey, pageSize);
+ entriesAdded = 0;
+ for (OzoneFileStatus status : list) {
+ // The server includes the start key itself in the response.
+ if (status.getPath().equals(lastKeyPath)) {
+ continue;
+ }
+ fileStatuses.add(convertToFileStatus(status));
+ lastKeyPath = status.getPath();
+ startKey = status.getKeyInfo().getKeyName();
+ entriesAdded++;
+ }
+ } while (entriesAdded > 0);
return fileStatuses.toArray(new FileStatus[0]);
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashOzoneFileSystem.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashOzoneFileSystem.java
new file mode 100644
index 00000000000..5ee5067bfe5
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashOzoneFileSystem.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE_MAX;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.hadoop.hdds.utils.db.DBConfigFromFile;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+/**
+ * Unit tests for {@link TrashOzoneFileSystem}.
+ */
+class TestTrashOzoneFileSystem {
+
+ private static final int PAGE_SIZE = 5;
+ private static final int TRASH_ROOT_COUNT = 12;
+
+ private final AtomicInteger objectId = new AtomicInteger();
+
+ private OmTestManagers newOmTestManagers(File testDir) throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ System.setProperty(DBConfigFromFile.CONFIG_DIR, "/");
+ ServerUtils.setOzoneMetaDirPath(conf, testDir.toString());
+ conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
TimeUnit.MILLISECONDS);
+ conf.setQuietMode(false);
+ conf.setInt(OZONE_FS_LISTING_PAGE_SIZE_MAX, PAGE_SIZE);
+ return new OmTestManagers(conf);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"})
+ void testGetTrashRootsBeyondPageSize(BucketLayout bucketLayout,
+ @TempDir File testDir) throws Exception {
+ OmTestManagers omTestManagers = newOmTestManagers(testDir);
+ try {
+ OzoneManager om = omTestManagers.getOzoneManager();
+ OzoneManagerProtocol writeClient = omTestManagers.getWriteClient();
+ final String volumeName = "vol-" + objectId.incrementAndGet();
+ final String bucketName = "bucket-" + objectId.incrementAndGet();
+ createVolumeAndBucket(omTestManagers, volumeName, bucketName,
bucketLayout,
+ writeClient);
+ // Create more trash roots than the configured listing page size so that
a
+ // single listStatus call cannot return all of them.
+ for (int i = 0; i < TRASH_ROOT_COUNT; i++) {
+ createDirectory(writeClient, volumeName, bucketName,
+ TRASH_PREFIX + "/user" + i);
+ }
+
+ try (FileSystem fs = SecurityUtil.doAsLoginUser(
+ (PrivilegedExceptionAction<FileSystem>)
+ () -> new TrashOzoneFileSystem(om))) {
+ Collection<FileStatus> trashRoots = fs.getTrashRoots(true);
+ assertEquals(TRASH_ROOT_COUNT, trashRoots.size());
+ }
+ } finally {
+ omTestManagers.stop();
+ }
+ }
+
+ private void createVolumeAndBucket(OmTestManagers omTestManagers,
+ String volumeName, String bucketName, BucketLayout bucketLayout,
+ OzoneManagerProtocol writeClient) throws IOException {
+ OMRequestTestUtils.addVolumeToOM(omTestManagers.getMetadataManager(),
+ OmVolumeArgs.newBuilder()
+ .setOwnerName("o")
+ .setAdminName("a")
+ .setVolume(volumeName)
+ .setObjectID(objectId.incrementAndGet())
+ .build());
+ OMRequestTestUtils.addBucketToOM(omTestManagers.getMetadataManager(),
+ OmBucketInfo.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setBucketLayout(bucketLayout)
+ .setOwner("o")
+ .setObjectID(objectId.incrementAndGet())
+ .build());
+ writeClient.createDirectory(new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(TRASH_PREFIX)
+ .setOwnerName("test")
+ .build());
+ }
+
+ private void createDirectory(OzoneManagerProtocol writeClient,
+ String volumeName, String bucketName, String dirName) throws IOException
{
+ OmKeyArgs keyArg = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(dirName)
+ .setOwnerName("test")
+ .build();
+ writeClient.createDirectory(keyArg);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
