This is an automated email from the ASF dual-hosted git repository.
smengcl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 07f069b72fc HDDS-16015. Make listStatusLight skip SCM pipeline
refresh. (#10894)
07f069b72fc is described below
commit 07f069b72fc8f2f2534d2ed75ae15ea0f2ee31d9
Author: Aryan Gupta <[email protected]>
AuthorDate: Wed Aug 12 01:33:30 2026 +0530
HDDS-16015. Make listStatusLight skip SCM pipeline refresh. (#10894)
---
.../hadoop/ozone/om/snapshot/OmSnapshotTests.java | 26 ++++++++++
.../org/apache/hadoop/ozone/om/KeyManager.java | 28 +++++++++++
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 33 +++++++++----
.../apache/hadoop/ozone/om/OmMetadataReader.java | 37 +++++++++++++--
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 32 +++++++++++--
.../org/apache/hadoop/ozone/om/OzoneManager.java | 11 ++---
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 55 +++++++++++++++++++++-
7 files changed, 197 insertions(+), 25 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
index 40dd14d4469..fe8f4b24e7e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
@@ -147,6 +147,7 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.service.SnapshotDiffCleanupService;
@@ -531,6 +532,31 @@ public void checkKey() throws Exception {
snapshotKeyPrefix + key1);
}
+ @Test
+ public void testSnapshotListStatusLightReturnsDenormalizedKeyName()
+ throws Exception {
+ String key = "light-list/" + "key-" + counter.incrementAndGet();
+ createFileKey(ozoneBucket, key);
+
+ String snapshotName = "snap-light-" + counter.incrementAndGet();
+ String snapshotKeyPrefix = createSnapshot(volumeName, bucketName,
+ snapshotName);
+ String snapshotKey = snapshotKeyPrefix + key;
+
+ OmKeyArgs keyArgs = genKeyArgs(snapshotKey);
+
+ List<OzoneFileStatus> fullStatuses =
+ writeClient.listStatus(keyArgs, false, "", 1, false);
+ List<OzoneFileStatusLight> lightStatuses =
+ writeClient.listStatusLight(keyArgs, false, "", 1, false);
+
+ assertEquals(1, fullStatuses.size());
+ assertEquals(1, lightStatuses.size());
+
+ assertEquals(snapshotKey, fullStatuses.get(0).getKeyInfo().getKeyName());
+ assertEquals(snapshotKey, lightStatuses.get(0).getKeyInfo().getKeyName());
+ }
+
@Test
public void testListDeleteKey() throws Exception {
String volume = "vol-" + counter.incrementAndGet();
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
index 4077ee088ef..2c959cb4c01 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
@@ -38,6 +38,7 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.service.CompactionService;
import org.apache.hadoop.ozone.om.service.DirectoryDeletingService;
@@ -90,6 +91,33 @@ OmKeyInfo lookupKey(OmKeyArgs args, ResolvedBucket
bucketLayout,
OmKeyInfo getKeyInfo(OmKeyArgs args, ResolvedBucket buctket,
String clientAddress) throws IOException;
+ /**
+ * List the status for a file or a directory and its contents.
+ * Optionally skips pipeline refresh for callers that only need metadata.
+ *
+ * @param args Key args
+ * @param recursive For a directory if true all the descendants of a
+ * particular directory are listed
+ * @param startKey Key from which listing needs to start. If startKey
exists
+ * its status is included in the final list.
+ * @param numEntries Number of entries to list from the start key
+ * @param clientAddress a hint to key manager, order the datanode in returned
+ * pipeline by distance between client and datanode.
+ * @param allowPartialPrefixes if partial prefixes should be allowed,
+ * this is needed in context of ListKeys
+ * @param refreshPipelineInfo if true, update pipeline location information
+ * using SCM-backed cache; if false, skip it
+ * @return list of file status
+ * @throws IOException if file or bucket or volume does not exist
+ */
+ default List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
+ String startKey, long numEntries, String clientAddress,
+ boolean allowPartialPrefixes, boolean refreshPipelineInfo)
+ throws IOException {
+ return listStatus(args, recursive, startKey, numEntries, clientAddress,
+ allowPartialPrefixes);
+ }
+
/**
* Returns a list of keys represented by {@link OmKeyInfo}
* in the given bucket.
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index f24a5c470fb..40ca7c3002a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -1979,6 +1979,16 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args,
boolean recursive,
public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
String startKey, long numEntries, String clientAddress,
boolean allowPartialPrefixes) throws IOException {
+ return listStatus(args, recursive, startKey, numEntries, clientAddress,
+ allowPartialPrefixes, true);
+ }
+
+ @Override
+ @SuppressWarnings("methodlength")
+ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
+ String startKey, long numEntries, String clientAddress,
+ boolean allowPartialPrefixes, boolean refreshPipelineInfo)
+ throws IOException {
Objects.requireNonNull(args, "Key args can not be null");
String volumeName = args.getVolumeName();
String bucketName = args.getBucketName();
@@ -1997,7 +2007,8 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args,
boolean recursive,
Collection<OzoneFileStatus> statuses =
statusHelper.listStatusFSO(args, startKey, numEntries,
clientAddress, allowPartialPrefixes);
- return buildFinalStatusList(statuses, args, clientAddress);
+ return buildFinalStatusList(statuses, args, clientAddress,
+ refreshPipelineInfo);
}
// A map sorted by OmKey to combine results from TableCache and DB.
@@ -2065,7 +2076,9 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args,
boolean recursive,
slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0]));
}
- refreshPipelineFromCache(keyInfoList);
+ if (refreshPipelineInfo) {
+ refreshPipelineFromCache(keyInfoList);
+ }
if (args.getSortDatanodes()) {
sortDatanodes(clientAddress, keyInfoList);
@@ -2169,7 +2182,7 @@ private void findKeyInDbWithIterator(boolean recursive,
String startKey,
private List<OzoneFileStatus> buildFinalStatusList(
Collection<OzoneFileStatus> statusesCollection, OmKeyArgs omKeyArgs,
- String clientAddress)
+ String clientAddress, boolean refreshPipelineInfo)
throws IOException {
List<OzoneFileStatus> fileStatusFinalList = new ArrayList<>();
List<OmKeyInfo> keyInfoList = new ArrayList<>();
@@ -2181,19 +2194,21 @@ private List<OzoneFileStatus> buildFinalStatusList(
fileStatusFinalList.add(fileStatus);
}
return sortPipelineInfo(fileStatusFinalList, keyInfoList,
- omKeyArgs, clientAddress);
+ omKeyArgs, clientAddress, refreshPipelineInfo);
}
private List<OzoneFileStatus> sortPipelineInfo(
List<OzoneFileStatus> fileStatusFinalList, List<OmKeyInfo> keyInfoList,
- OmKeyArgs omKeyArgs, String clientAddress) throws IOException {
+ OmKeyArgs omKeyArgs, String clientAddress, boolean refreshPipelineInfo)
+ throws IOException {
if (omKeyArgs.getLatestVersionLocation()) {
slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0]));
}
- // refreshPipeline flag check has been removed as part of
- // https://issues.apache.org/jira/browse/HDDS-3658.
- // Please refer this jira for more details.
- refreshPipelineFromCache(keyInfoList);
+ if (refreshPipelineInfo) {
+ // listStatusLight callers set this flag to false so that lightweight
+ // listings remain metadata-only and avoid SCM-backed pipeline refresh.
+ refreshPipelineFromCache(keyInfoList);
+ }
if (omKeyArgs.getSortDatanodes()) {
sortDatanodes(clientAddress, keyInfoList);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
index 64f46089c06..7df37759d1e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@ -259,12 +259,39 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args,
boolean recursive,
public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
boolean recursive, String startKey, long numEntries,
boolean allowPartialPrefixes) throws IOException {
- List<OzoneFileStatus> ozoneFileStatuses =
- listStatus(args, recursive, startKey, numEntries,
allowPartialPrefixes);
+ long maxListingPageSize = ozoneManager.getConfiguration().getInt(
+ OZONE_FS_LISTING_PAGE_SIZE_MAX,
+ OZONE_FS_LISTING_PAGE_SIZE_DEFAULT);
+ maxListingPageSize = OzoneConfigUtil.limitValue(numEntries,
+ OZONE_FS_LISTING_PAGE_SIZE, OZONE_FS_LISTING_PAGE_SIZE_MAX,
+ maxListingPageSize);
+
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(args);
+ Map<String, String> auditMap = bucket.audit(args.toAuditMap());
+ OmKeyArgs resolvedArgs = bucket.update(args);
- return ozoneFileStatuses.stream()
- .map(OzoneFileStatusLight::fromOzoneFileStatus)
- .collect(Collectors.toList());
+ try {
+ if (isAclEnabled) {
+ checkAcls(getResourceType(resolvedArgs), StoreType.OZONE, ACLType.READ,
+ bucket, resolvedArgs.getKeyName());
+ }
+ metrics.incNumListStatus();
+ List<OzoneFileStatus> ozoneFileStatuses = keyManager.listStatus(
+ resolvedArgs, recursive,
+ startKey, maxListingPageSize, getClientAddress(),
+ allowPartialPrefixes, false);
+ List<OzoneFileStatusLight> lightStatuses = ozoneFileStatuses.stream()
+ .map(OzoneFileStatusLight::fromOzoneFileStatus)
+ .collect(Collectors.toList());
+ audit.logReadSuccess(buildAuditMessageForSuccess(
+ OMAction.LIST_STATUS, auditMap));
+ return lightStatuses;
+ } catch (Exception ex) {
+ metrics.incNumListStatusFails();
+ audit.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_STATUS,
+ auditMap, ex));
+ throw ex;
+ }
}
@Override
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
index 5147eafe628..6d3a56f40ed 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
@@ -133,12 +133,13 @@ recursive, normalizeKeyName(startKey), numEntries,
public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
boolean recursive, String startKey, long numEntries,
boolean allowPartialPrefixes) throws IOException {
-
- List<OzoneFileStatus> ozoneFileStatuses =
- listStatus(args, recursive, startKey, numEntries,
allowPartialPrefixes);
+ List<OzoneFileStatusLight> ozoneFileStatuses = omMetadataReader
+ .listStatusLight(normalizeOmKeyArgs(args),
+ recursive, normalizeKeyName(startKey), numEntries,
+ allowPartialPrefixes);
return ozoneFileStatuses.stream()
- .map(OzoneFileStatusLight::fromOzoneFileStatus)
+ .map(this::denormalizeOzoneFileStatusLight)
.collect(Collectors.toList());
}
@@ -285,6 +286,29 @@ private OzoneFileStatus denormalizeOzoneFileStatus(
omKeyInfo, fileStatus.getBlockSize(), fileStatus.isDirectory());
}
+ private OzoneFileStatusLight denormalizeOzoneFileStatusLight(
+ OzoneFileStatusLight fileStatus) {
+ if (fileStatus == null || fileStatus.getKeyInfo() == null) {
+ return fileStatus;
+ }
+ BasicOmKeyInfo keyInfo = fileStatus.getKeyInfo();
+ BasicOmKeyInfo denormalized = new BasicOmKeyInfo.Builder()
+ .setVolumeName(keyInfo.getVolumeName())
+ .setBucketName(keyInfo.getBucketName())
+ .setKeyName(denormalizeKeyName(keyInfo.getKeyName()))
+ .setDataSize(keyInfo.getDataSize())
+ .setCreationTime(keyInfo.getCreationTime())
+ .setModificationTime(keyInfo.getModificationTime())
+ .setReplicationConfig(keyInfo.getReplicationConfig())
+ .setIsFile(keyInfo.isFile())
+ .setETag(keyInfo.getETag())
+ .setOwnerName(keyInfo.getOwnerName())
+ .setIsEncrypted(keyInfo.isEncrypted())
+ .build();
+ return new OzoneFileStatusLight(
+ denormalized, fileStatus.getBlockSize(), fileStatus.isDirectory());
+ }
+
private KeyInfoWithVolumeContext denormalizeKeyInfoWithVolumeContext(
KeyInfoWithVolumeContext k) {
return new KeyInfoWithVolumeContext.Builder()
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 225ddeb6c90..e1682e1a048 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -4177,12 +4177,11 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args,
boolean recursive,
public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
boolean recursive, String startKey, long numEntries,
boolean allowPartialPrefixes) throws IOException {
- List<OzoneFileStatus> ozoneFileStatuses =
- listStatus(args, recursive, startKey, numEntries,
allowPartialPrefixes);
-
- return ozoneFileStatuses.stream()
- .map(OzoneFileStatusLight::fromOzoneFileStatus)
- .collect(Collectors.toList());
+ try (UncheckedAutoCloseableSupplier<IOmMetadataReader> rcReader =
+ getReader(args)) {
+ return rcReader.get().listStatusLight(
+ args, recursive, startKey, numEntries, allowPartialPrefixes);
+ }
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
index be307c06f95..e7df8077cc3 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
@@ -79,12 +79,14 @@
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.apache.ozone.test.OzoneTestBase;
import org.apache.ratis.util.ExitUtils;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -510,7 +512,7 @@ public void testGetKeyInfo() throws IOException {
.build();
ContainerInfo ci = mock(ContainerInfo.class);
- when(ci.getContainerID()).thenReturn(1L);
+ when(ci.getContainerID()).thenReturn(containerID);
// Setup SCM containerClient so that 1st call returns pipeline1 and
// 2nd call returns pipeline2.
@@ -766,4 +768,55 @@ public void listStatus() throws Exception {
null, Long.MAX_VALUE, client);
verify(containerClient, times(1)).getContainerWithPipelineBatch(anySet());
}
+
+ @Test
+ public void listStatusLightDoesNotRefreshPipeline() throws Exception {
+ String volume = volumeName();
+ String bucket = "bucket-light";
+ String keyPrefix = "key-light-";
+
+ OMRequestTestUtils.addVolumeToDB(volume, OzoneConsts.OZONE,
metadataManager);
+ OMRequestTestUtils.addBucketToDB(volume, bucket, metadataManager);
+
+ final Pipeline pipeline = MockPipeline.createPipeline(3);
+ for (long i = 1; i <= 5; i++) {
+ final long containerID = CONTAINER_ID.incrementAndGet();
+ final OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder()
+ .setBlockID(new BlockID(containerID, 1L))
+ .setPipeline(pipeline)
+ .setOffset(0)
+ .setLength(1024)
+ .build();
+
+ OmKeyInfo keyInfo = new OmKeyInfo.Builder()
+ .setVolumeName(volume)
+ .setBucketName(bucket)
+ .setCreationTime(Time.now())
+ .setOmKeyLocationInfos(singletonList(
+ new OmKeyLocationInfoGroup(0, new ArrayList<>())))
+ .setReplicationConfig(RatisReplicationConfig
+ .getInstance(ReplicationFactor.THREE))
+ .setKeyName(keyPrefix + i)
+ .setObjectID(i)
+ .setUpdateID(i)
+ .build();
+ keyInfo.appendNewBlocks(singletonList(keyLocationInfo), false);
+ OMRequestTestUtils.addKeyToOM(metadataManager, keyInfo);
+ }
+
+ OmKeyArgs.Builder builder = new OmKeyArgs.Builder()
+ .setVolumeName(volume)
+ .setBucketName(bucket)
+ .setKeyName("");
+
+ List<OzoneFileStatusLight> fileStatusList;
+ try (UncheckedAutoCloseableSupplier<IOmMetadataReader> rcReader =
+ om.getOmMetadataReader()) {
+ fileStatusList = rcReader.get().listStatusLight(builder.build(), false,
+ null, Long.MAX_VALUE, false);
+ }
+
+ assertEquals(5, fileStatusList.size());
+ verify(containerClient, times(0)).getContainerWithPipelineBatch(anySet());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
