This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 435f87964f HDDS-9297. Avoid reading bucket/volume info multiple times
in getKeyInfo (#5427)
435f87964f is described below
commit 435f87964f9c6cc605050038b05e0b02c99f9a1b
Author: Duong Nguyen <[email protected]>
AuthorDate: Mon Nov 6 23:45:17 2023 -0800
HDDS-9297. Avoid reading bucket/volume info multiple times in getKeyInfo
(#5427)
---
.../client/rpc/TestOzoneRpcClientAbstract.java | 7 ++-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 32 ++++++----
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 2 +-
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 9 ++-
.../org/apache/hadoop/ozone/om/KeyManager.java | 6 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 29 ++++-----
.../apache/hadoop/ozone/om/OmMetadataReader.java | 50 +++++++++++++---
.../org/apache/hadoop/ozone/om/OzoneManager.java | 14 +++--
.../org/apache/hadoop/ozone/om/ResolvedBucket.java | 70 ++++++++++++++++------
.../ozone/om/request/file/OMFileRequest.java | 30 +++++++++-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 16 ++++-
.../ozone/om/request/bucket/TestBucketRequest.java | 3 +-
.../request/file/TestOMDirectoryCreateRequest.java | 5 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 5 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 9 +--
.../s3/multipart/TestS3MultipartRequest.java | 6 +-
.../ozone/om/response/TestCleanupTableInfo.java | 11 ++--
17 files changed, 218 insertions(+), 86 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index c528cb5d4c..1e5dc26d84 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -96,6 +96,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmFailoverProxyUtil;
import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.ResolvedBucket;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import org.apache.hadoop.ozone.om.ha.HadoopRpcOMFailoverProxyProvider;
@@ -3782,8 +3783,12 @@ public abstract class TestOzoneRpcClientAbstract {
.setBucketName(bucket.getName())
.setKeyName(keyName)
.build();
+ ResolvedBucket resolvedBucket = new ResolvedBucket(
+ bucket.getVolumeName(), bucket.getName(),
+ bucket.getVolumeName(), bucket.getName(),
+ "", bucket.getBucketLayout());
OmKeyInfo omKeyInfo = ozoneManager.getKeyManager().getKeyInfo(keyArgs,
- UUID.randomUUID().toString());
+ resolvedBucket, UUID.randomUUID().toString());
OmKeyLocationInfoGroup latestVersionLocations =
omKeyInfo.getLatestVersionLocations();
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
index e2ce4e9ba5..576bf594a3 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
@@ -115,6 +115,7 @@ import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.SCM_
import static
org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL;
import org.apache.ratis.util.ExitUtils;
+import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
@@ -770,10 +771,9 @@ public class TestKeyManagerImpl {
.setKeyName(keyName)
.setSortDatanodesInPipeline(true)
.build();
-
// lookup for a non-existent key
try {
- keyManager.lookupKey(keyArgs, null);
+ keyManager.lookupKey(keyArgs, resolvedBucket(), null);
fail("Lookup key should fail for non existent key");
} catch (OMException ex) {
if (ex.getResult() != OMException.ResultCodes.KEY_NOT_FOUND) {
@@ -815,7 +815,7 @@ public class TestKeyManagerImpl {
when(mockScmContainerClient.getContainerWithPipelineBatch(
Arrays.asList(containerID))).thenReturn(containerWithPipelines);
- OmKeyInfo key = keyManager.lookupKey(keyArgs, null);
+ OmKeyInfo key = keyManager.lookupKey(keyArgs, resolvedBucket(), null);
assertEquals(key.getKeyName(), keyName);
List<OmKeyLocationInfo> keyLocations =
key.getLatestVersionLocations().getLocationList();
@@ -829,27 +829,37 @@ public class TestKeyManagerImpl {
assertNotEquals(follower1, follower2);
// lookup key, leader as client
- OmKeyInfo key1 = keyManager.lookupKey(keyArgs, leader.getIpAddress());
+ OmKeyInfo key1 = keyManager.lookupKey(keyArgs, resolvedBucket(),
+ leader.getIpAddress());
assertEquals(leader, key1.getLatestVersionLocations()
.getLocationList().get(0).getPipeline().getClosestNode());
// lookup key, follower1 as client
- OmKeyInfo key2 = keyManager.lookupKey(keyArgs, follower1.getIpAddress());
+ OmKeyInfo key2 = keyManager.lookupKey(keyArgs, resolvedBucket(),
+ follower1.getIpAddress());
assertEquals(follower1, key2.getLatestVersionLocations()
.getLocationList().get(0).getPipeline().getClosestNode());
// lookup key, follower2 as client
- OmKeyInfo key3 = keyManager.lookupKey(keyArgs, follower2.getIpAddress());
+ OmKeyInfo key3 = keyManager.lookupKey(keyArgs, resolvedBucket(),
+ follower2.getIpAddress());
assertEquals(follower2, key3.getLatestVersionLocations()
.getLocationList().get(0).getPipeline().getClosestNode());
// lookup key, random node as client
- OmKeyInfo key4 = keyManager.lookupKey(keyArgs,
+ OmKeyInfo key4 = keyManager.lookupKey(keyArgs, resolvedBucket(),
"/d=default-drack/127.0.0.1");
assertEquals(leader, key4.getLatestVersionLocations()
.getLocationList().get(0).getPipeline().getClosestNode());
}
+ @NotNull
+ private ResolvedBucket resolvedBucket() {
+ ResolvedBucket bucket = new ResolvedBucket(VOLUME_NAME, BUCKET_NAME,
+ VOLUME_NAME, BUCKET_NAME, "", BucketLayout.DEFAULT);
+ return bucket;
+ }
+
@Test
public void testLatestLocationVersion() throws IOException {
String keyName = RandomStringUtils.randomAlphabetic(5);
@@ -860,7 +870,7 @@ public class TestKeyManagerImpl {
// lookup for a non-existent key
try {
- keyManager.lookupKey(keyArgs, null);
+ keyManager.lookupKey(keyArgs, resolvedBucket(), null);
fail("Lookup key should fail for non existent key");
} catch (OMException ex) {
if (ex.getResult() != OMException.ResultCodes.KEY_NOT_FOUND) {
@@ -902,14 +912,14 @@ public class TestKeyManagerImpl {
when(mockScmContainerClient.getContainerWithPipelineBatch(
Arrays.asList(1L))).thenReturn(containerWithPipelines);
- OmKeyInfo key = keyManager.lookupKey(keyArgs, null);
+ OmKeyInfo key = keyManager.lookupKey(keyArgs, resolvedBucket(), null);
assertEquals(key.getKeyLocationVersions().size(), 1);
keySession = writeClient.createFile(keyArgs, true, true);
writeClient.commitKey(keyArgs, keySession.getId());
// Test lookupKey (latestLocationVersion == true)
- key = keyManager.lookupKey(keyArgs, null);
+ key = keyManager.lookupKey(keyArgs, resolvedBucket(), null);
assertEquals(key.getKeyLocationVersions().size(), 1);
// Test ListStatus (latestLocationVersion == true)
@@ -934,7 +944,7 @@ public class TestKeyManagerImpl {
.build();
// Test lookupKey (latestLocationVersion == false)
- key = keyManager.lookupKey(keyArgs, null);
+ key = keyManager.lookupKey(keyArgs, resolvedBucket(), null);
assertEquals(key.getKeyLocationVersions().size(), 2);
// Test ListStatus (latestLocationVersion == false)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
index ddb5f2fce1..7fc050e172 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
@@ -343,7 +343,7 @@ public class TestOmMetrics {
assertCounter("NumBlockAllocationFails", 1L, omMetrics);
// inject exception to test for Failure Metrics on the read path
- Mockito.doThrow(exception).when(mockKm).lookupKey(any(), any());
+ Mockito.doThrow(exception).when(mockKm).lookupKey(any(), any(), any());
Mockito.doThrow(exception).when(mockKm).listKeys(
any(), any(), any(), any(), anyInt());
Mockito.doThrow(exception).when(mockKm).listTrash(
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index 409de11d07..096d21d8e9 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -640,9 +640,14 @@ public class TestOmSnapshot {
private OmKeyInfo getOmKeyInfo(String volume, String bucket,
String key) throws IOException {
+ ResolvedBucket resolvedBucket = new ResolvedBucket(volume, bucket,
+ volume, bucket, "", bucketLayout);
return cluster.getOzoneManager().getKeyManager()
- .getKeyInfo(new OmKeyArgs.Builder().setVolumeName(volume)
- .setBucketName(bucket).setKeyName(key).build(), null);
+ .getKeyInfo(new OmKeyArgs.Builder()
+ .setVolumeName(volume)
+ .setBucketName(bucket)
+ .setKeyName(key).build(),
+ resolvedBucket, null);
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
index 4f7fa6ad2f..0fe1cdbe80 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
@@ -65,7 +65,8 @@ public interface KeyManager extends OzoneManagerFS, IOzoneAcl
{
* @return a OmKeyInfo instance client uses to talk to container.
* @throws IOException
*/
- OmKeyInfo lookupKey(OmKeyArgs args, String clientAddress) throws IOException;
+ OmKeyInfo lookupKey(OmKeyArgs args, ResolvedBucket bucketLayout,
+ String clientAddress) throws IOException;
/**
* Return info of an existing key to client side to access to data on
@@ -76,7 +77,8 @@ public interface KeyManager extends OzoneManagerFS, IOzoneAcl
{
* @return a OmKeyInfo instance client uses to talk to container.
* @throws IOException
*/
- OmKeyInfo getKeyInfo(OmKeyArgs args, String clientAddress) throws
IOException;
+ OmKeyInfo getKeyInfo(OmKeyArgs args, ResolvedBucket buctket,
+ String clientAddress) throws IOException;
/**
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 1e62c5c57c..cb92928b67 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -377,12 +377,12 @@ public class KeyManagerImpl implements KeyManager {
return edek;
}
@Override
- public OmKeyInfo lookupKey(OmKeyArgs args, String clientAddress)
- throws IOException {
+ public OmKeyInfo lookupKey(OmKeyArgs args, ResolvedBucket bucket,
+ String clientAddress) throws IOException {
Preconditions.checkNotNull(args);
OmKeyInfo value = captureLatencyNs(metrics.getLookupReadKeyInfoLatencyNs(),
- () -> readKeyInfo(args));
+ () -> readKeyInfo(args, bucket.bucketLayout()));
// If operation is head, do not perform any additional steps based on
flags.
// As head operation does not need any of those details.
@@ -406,7 +406,8 @@ public class KeyManagerImpl implements KeyManager {
return value;
}
- private OmKeyInfo readKeyInfo(OmKeyArgs args) throws IOException {
+ private OmKeyInfo readKeyInfo(OmKeyArgs args, BucketLayout bucketLayout)
+ throws IOException {
String volumeName = args.getVolumeName();
String bucketName = args.getBucketName();
String keyName = args.getKeyName();
@@ -415,9 +416,6 @@ public class KeyManagerImpl implements KeyManager {
metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
bucketName);
try {
- BucketLayout bucketLayout =
- getBucketLayout(metadataManager, args.getVolumeName(),
- args.getBucketName());
keyName = OMClientRequest
.validateAndNormalizeKey(enableFileSystemPaths, keyName,
bucketLayout);
@@ -425,7 +423,7 @@ public class KeyManagerImpl implements KeyManager {
if (bucketLayout.isFileSystemOptimized()) {
value = getOmKeyInfoFSO(volumeName, bucketName, keyName);
} else {
- value = getOmKeyInfo(volumeName, bucketName, keyName);
+ value = getOmKeyInfo(volumeName, bucketName, keyName, bucketLayout);
if (value != null) {
// For Legacy & OBS buckets, any key is a file by default. This is to
// keep getKeyInfo compatible with OFS clients.
@@ -459,11 +457,9 @@ public class KeyManagerImpl implements KeyManager {
}
private OmKeyInfo getOmKeyInfo(String volumeName, String bucketName,
- String keyName) throws IOException {
+ String keyName, BucketLayout bucketLayout) throws IOException {
String keyBytes =
metadataManager.getOzoneKey(volumeName, bucketName, keyName);
- BucketLayout bucketLayout = getBucketLayout(metadataManager, volumeName,
- bucketName);
return metadataManager
.getKeyTable(bucketLayout)
.get(keyBytes);
@@ -477,7 +473,7 @@ public class KeyManagerImpl implements KeyManager {
String keyName) throws IOException {
OzoneFileStatus fileStatus = OMFileRequest.getOMKeyInfoIfExists(
metadataManager, volumeName, bucketName, keyName, scmBlockSize,
- ozoneManager.getDefaultReplicationConfig());
+ ozoneManager.getDefaultReplicationConfig(), false);
if (fileStatus == null) {
return null;
}
@@ -908,7 +904,8 @@ public class KeyManagerImpl implements KeyManager {
if (isBucketFSOptimized(volume, bucket)) {
keyInfo = getOmKeyInfoFSO(volume, bucket, keyName);
} else {
- keyInfo = getOmKeyInfo(volume, bucket, keyName);
+ keyInfo = getOmKeyInfo(volume, bucket, keyName,
+ resolvedBucket.bucketLayout());
}
if (keyInfo == null) {
throw new OMException("Key not found. Key:" + objectKey,
KEY_NOT_FOUND);
@@ -2005,13 +2002,13 @@ public class KeyManagerImpl implements KeyManager {
}
@Override
- public OmKeyInfo getKeyInfo(OmKeyArgs args, String clientAddress)
- throws IOException {
+ public OmKeyInfo getKeyInfo(OmKeyArgs args, ResolvedBucket bucket,
+ String clientAddress) throws IOException {
Preconditions.checkNotNull(args);
OmKeyInfo value = captureLatencyNs(
metrics.getGetKeyInfoReadKeyInfoLatencyNs(),
- () -> readKeyInfo(args));
+ () -> readKeyInfo(args, bucket.bucketLayout()));
// If operation is head, do not perform any additional steps based on
flags.
// As head operation does not need any of those details.
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
index 141ae104f8..5d324cae62 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@ -127,12 +127,12 @@ public class OmMetadataReader implements
IOmMetadataReader, Auditor {
if (isAclEnabled) {
captureLatencyNs(perfMetrics.getLookupAclCheckLatencyNs(),
() -> checkAcls(ResourceType.KEY, StoreType.OZONE,
- ACLType.READ, bucket.realVolume(), bucket.realBucket(),
+ ACLType.READ, bucket,
args.getKeyName())
);
}
metrics.incNumKeyLookups();
- return keyManager.lookupKey(resolvedArgs, getClientAddress());
+ return keyManager.lookupKey(resolvedArgs, bucket, getClientAddress());
} catch (Exception ex) {
metrics.incNumKeyLookupFails();
auditSuccess = false;
@@ -181,13 +181,12 @@ public class OmMetadataReader implements
IOmMetadataReader, Auditor {
captureLatencyNs(perfMetrics.getGetKeyInfoAclCheckLatencyNs(), () ->
checkAcls(ResourceType.KEY,
StoreType.OZONE, ACLType.READ,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName())
+ bucket, args.getKeyName())
);
}
metrics.incNumGetKeyInfo();
- OmKeyInfo keyInfo =
- keyManager.getKeyInfo(resolvedArgs,
+ OmKeyInfo keyInfo = keyManager.getKeyInfo(resolvedArgs, bucket,
OmMetadataReader.getClientAddress());
KeyInfoWithVolumeContext.Builder builder = KeyInfoWithVolumeContext
.newBuilder()
@@ -234,7 +233,7 @@ public class OmMetadataReader implements IOmMetadataReader,
Auditor {
try {
if (isAclEnabled) {
checkAcls(getResourceType(args), StoreType.OZONE, ACLType.READ,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName());
+ bucket, args.getKeyName());
}
metrics.incNumListStatus();
return keyManager.listStatus(args, recursive, startKey,
@@ -303,7 +302,7 @@ public class OmMetadataReader implements IOmMetadataReader,
Auditor {
try {
if (isAclEnabled) {
checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.READ,
- bucket.realVolume(), bucket.realBucket(), args.getKeyName());
+ bucket, args.getKeyName());
}
metrics.incNumLookupFile();
return keyManager.lookupFile(args, getClientAddress());
@@ -460,6 +459,43 @@ public class OmMetadataReader implements
IOmMetadataReader, Auditor {
ozoneManager.getOmRpcServerAddr().getHostName());
}
+ /**
+ * Checks if current caller has acl permissions.
+ *
+ * @param resType - Type of ozone resource. Ex volume, bucket.
+ * @param store - Store type. i.e Ozone, S3.
+ * @param acl - type of access to be checked.
+ * @param resolvedBucket - resolved bucket information.
+ * @param key - key
+ * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied.
+ */
+ void checkAcls(ResourceType resType, StoreType store,
+ ACLType acl, ResolvedBucket resolvedBucket, String key)
+ throws IOException {
+ UserGroupInformation user;
+ if (getS3Auth() != null) {
+ String principal =
+ OzoneAclUtils.accessIdToUserPrincipal(getS3Auth().getAccessId());
+ user = UserGroupInformation.createRemoteUser(principal);
+ } else {
+ user = ProtobufRpcEngine.Server.getRemoteUser();
+ }
+
+ String vol = resolvedBucket.realVolume();
+ String bucket = resolvedBucket.realBucket();
+ InetAddress remoteIp = ProtobufRpcEngine.Server.getRemoteIp();
+ String volumeOwner = ozoneManager.getVolumeOwner(vol, acl, resType);
+ String bucketOwner = resolvedBucket.bucketOwner();
+
+ OzoneAclUtils.checkAllAcls(this, resType, store, acl,
+ vol, bucket, key, volumeOwner, bucketOwner,
+ user != null ? user : getRemoteUser(),
+ remoteIp != null ? remoteIp :
+ ozoneManager.getOmRpcServerAddr().getAddress(),
+ remoteIp != null ? remoteIp.getHostName() :
+ ozoneManager.getOmRpcServerAddr().getHostName());
+ }
+
/**
* CheckAcls for the ozone object.
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index e67f4e505c..9e36a9a2d2 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -4232,7 +4232,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
public ResolvedBucket resolveBucketLink(Pair<String, String> requested,
OMClientRequest omClientRequest)
throws IOException {
- Pair<String, String> resolved;
+ OmBucketInfo resolved;
if (isAclEnabled) {
resolved = resolveBucketLink(requested, new HashSet<>(),
omClientRequest.createUGIForApi(),
@@ -4243,13 +4243,14 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
resolved = resolveBucketLink(requested, new HashSet<>(),
null, null, null, false);
}
- return new ResolvedBucket(requested, resolved);
+ return new ResolvedBucket(requested.getLeft(), requested.getRight(),
+ resolved);
}
public ResolvedBucket resolveBucketLink(Pair<String, String> requested,
boolean allowDanglingBuckets)
throws IOException {
- Pair<String, String> resolved;
+ OmBucketInfo resolved;
if (isAclEnabled) {
UserGroupInformation ugi = getRemoteUser();
if (getS3Auth() != null) {
@@ -4266,7 +4267,8 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
resolved = resolveBucketLink(requested, new HashSet<>(),
null, null, null, allowDanglingBuckets);
}
- return new ResolvedBucket(requested, resolved);
+ return new ResolvedBucket(requested.getLeft(), requested.getRight(),
+ resolved);
}
/**
@@ -4280,7 +4282,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
* @throws IOException (most likely OMException) if ACL check fails, bucket
is
* not found, loop is detected in the links, etc.
*/
- private Pair<String, String> resolveBucketLink(
+ private OmBucketInfo resolveBucketLink(
Pair<String, String> volumeAndBucket,
Set<Pair<String, String>> visited,
UserGroupInformation userGroupInformation,
@@ -4301,7 +4303,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
throw e;
}
if (!info.isLink()) {
- return volumeAndBucket;
+ return info;
}
if (!visited.add(volumeAndBucket)) {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ResolvedBucket.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ResolvedBucket.java
index 8796be405b..9c304ac2f1 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ResolvedBucket.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ResolvedBucket.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
@@ -34,37 +36,70 @@ import java.util.Objects;
*/
public class ResolvedBucket {
- private final Pair<String, String> requested;
- private final Pair<String, String> resolved;
-
- public ResolvedBucket(Pair<String, String> requested,
- Pair<String, String> resolved) {
- this.requested = requested;
- this.resolved = resolved;
+ private final String requestedVolume;
+ private final String requestedBucket;
+ private final String realVolume;
+ private final String realBucket;
+ private final String bucketOwner;
+ private final BucketLayout bucketLayout;
+
+ public ResolvedBucket(String requestedVolume, String requestedBucket,
+ OmBucketInfo resolved) {
+ this.requestedVolume = requestedVolume;
+ this.requestedBucket = requestedBucket;
+ if (resolved != null) {
+ this.realVolume = resolved.getVolumeName();
+ this.realBucket = resolved.getBucketName();
+ this.bucketOwner = resolved.getOwner();
+ this.bucketLayout = resolved.getBucketLayout();
+ } else {
+ this.realVolume = null;
+ this.realBucket = null;
+ this.bucketOwner = null;
+ this.bucketLayout = null;
+ }
}
- public Pair<String, String> requested() {
- return requested;
+ public ResolvedBucket(String requestedVolume, String requestedBucket,
+ String realVolume, String realBucket, String bucketOwner,
+ BucketLayout bucketLayout) {
+ this.requestedVolume = requestedVolume;
+ this.requestedBucket = requestedBucket;
+ this.realVolume = realVolume;
+ this.realBucket = realBucket;
+ this.bucketOwner = bucketOwner;
+ this.bucketLayout = bucketLayout;
}
- public Pair<String, String> resolved() {
- return resolved;
+ public ResolvedBucket(Pair<String, String> requested,
+ Pair<String, String> real, String owner, BucketLayout bucketLayout) {
+ this(requested.getLeft(), requested.getRight(),
+ real.getLeft(), real.getRight(),
+ owner, bucketLayout);
}
public String requestedVolume() {
- return requested.getLeft();
+ return requestedVolume;
}
public String requestedBucket() {
- return requested.getRight();
+ return requestedBucket;
}
public String realVolume() {
- return resolved.getLeft();
+ return realVolume;
}
public String realBucket() {
- return resolved.getRight();
+ return realBucket;
+ }
+
+ public String bucketOwner() {
+ return bucketOwner;
+ }
+
+ public BucketLayout bucketLayout() {
+ return bucketLayout;
}
public OmKeyArgs update(OmKeyArgs args) {
@@ -86,11 +121,12 @@ public class ResolvedBucket {
}
public boolean isLink() {
- return !Objects.equals(requested, resolved);
+ return !Objects.equals(requestedVolume, realVolume)
+ || !Objects.equals(requestedBucket, realBucket);
}
public boolean isDangling() {
- return resolved == null;
+ return realVolume == null || realBucket == null;
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java
index 166bf376a9..723bed9539 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java
@@ -621,7 +621,7 @@ public final class OMFileRequest {
}
/**
- * Gets OmKeyInfo if exists for the given key name in the DB.
+ * Gets OmKeyInfo if exists and validate bucket and volume.
*
* @param omMetadataMgr metadata manager
* @param volumeName volume name
@@ -637,8 +637,34 @@ public final class OMFileRequest {
String volumeName, String bucketName, String keyName,
long scmBlockSize, ReplicationConfig defaultReplication
) throws IOException {
+ return getOMKeyInfoIfExists(omMetadataMgr, volumeName, bucketName, keyName,
+ scmBlockSize, defaultReplication, true);
+ }
+
- OMFileRequest.validateBucket(omMetadataMgr, volumeName, bucketName);
+ /**
+ * Gets OmKeyInfo if exists for the given key name in the DB.
+ *
+ * @param omMetadataMgr metadata manager
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @param keyName key name
+ * @param scmBlockSize scm block size
+ * @param validateBucketAndVolume true if the volume/bucket needs validating.
+ * @return OzoneFileStatus
+ * @throws IOException DB failure
+ */
+ @Nullable
+ public static OzoneFileStatus getOMKeyInfoIfExists(
+ OMMetadataManager omMetadataMgr,
+ String volumeName, String bucketName, String keyName,
+ long scmBlockSize, ReplicationConfig defaultReplication,
+ boolean validateBucketAndVolume
+ ) throws IOException {
+
+ if (validateBucketAndVolume) {
+ OMFileRequest.validateBucket(omMetadataMgr, volumeName, bucketName);
+ }
Path keyPath = Paths.get(keyName);
Iterator<Path> elements = keyPath.iterator();
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
index 0d199eb7f5..68d5762776 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs.Builder;
@@ -397,7 +398,8 @@ public class TestKeyManagerUnit {
.setBucketName("bucketOne")
.setKeyName("keyOne")
.build();
- OmKeyInfo keyInfo = keyManager.getKeyInfo(keyArgs, "test");
+ OmKeyInfo keyInfo = keyManager.getKeyInfo(keyArgs,
+ resolveBucket(keyArgs), "test");
final OmKeyLocationInfo blockLocation1 = keyInfo
.getLatestVersionLocations().getBlocksLatestVersionOnly().get(0);
Assert.assertEquals(blockID1, blockLocation1.getBlockID());
@@ -412,7 +414,8 @@ public class TestKeyManagerUnit {
.setBucketName("bucketOne")
.setKeyName("keyTwo")
.build();
- OmKeyInfo keyInfo2 = keyManager.getKeyInfo(keyArgs, "test");
+ OmKeyInfo keyInfo2 = keyManager.getKeyInfo(keyArgs,
+ resolveBucket(keyArgs), "test");
OmKeyLocationInfo blockLocation2 = keyInfo2
.getLatestVersionLocations().getBlocksLatestVersionOnly().get(0);
Assert.assertEquals(blockID2, blockLocation2.getBlockID());
@@ -428,7 +431,8 @@ public class TestKeyManagerUnit {
.setKeyName("keyTwo")
.setForceUpdateContainerCacheFromSCM(true)
.build();
- keyInfo2 = keyManager.getKeyInfo(keyArgs, "test");
+ keyInfo2 = keyManager.getKeyInfo(keyArgs,
+ resolveBucket(keyArgs), "test");
blockLocation2 = keyInfo2
.getLatestVersionLocations().getBlocksLatestVersionOnly().get(0);
Assert.assertEquals(blockID2, blockLocation2.getBlockID());
@@ -438,6 +442,12 @@ public class TestKeyManagerUnit {
.getContainerWithPipelineBatch(containerIDs);
}
+ private ResolvedBucket resolveBucket(OmKeyArgs keyArgs) {
+ return new ResolvedBucket(keyArgs.getVolumeName(), keyArgs.getBucketName(),
+ keyArgs.getVolumeName(), keyArgs.getBucketName(), "",
+ BucketLayout.DEFAULT);
+ }
+
@Test
public void testLookupFileWithDnFailure() throws IOException {
final DatanodeDetails dnOne = MockDatanodeDetails.randomDatanodeDetails();
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java
index a825b82a31..e22aa77c30 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java
@@ -94,7 +94,8 @@ public class TestBucketRequest {
when(ozoneManager.resolveBucketLink(any(Pair.class)))
.thenAnswer(invocation -> new ResolvedBucket(
- invocation.getArgument(0), invocation.getArgument(0)));
+ invocation.getArgument(0), invocation.getArgument(0),
+ "", BucketLayout.DEFAULT));
}
@After
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java
index 4faf60decb..1d5851960c 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java
@@ -26,7 +26,6 @@ import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.ResolvedBucket;
@@ -102,7 +101,9 @@ public class TestOMDirectoryCreateRequest {
Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
when(ozoneManager.resolveBucketLink(any(KeyArgs.class),
any(OMClientRequest.class)))
- .thenReturn(new ResolvedBucket(Pair.of("", ""), Pair.of("", "")));
+ .thenReturn(new ResolvedBucket("", "",
+ "", "", "",
+ BucketLayout.DEFAULT));
}
@After
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java
index 9b79344741..fa3b559a37 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.ozone.om.request.file;
import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -101,7 +100,9 @@ public class TestOMDirectoryCreateRequestWithFSO {
Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
when(ozoneManager.resolveBucketLink(any(KeyArgs.class),
any(OMClientRequest.class)))
- .thenReturn(new ResolvedBucket(Pair.of("", ""), Pair.of("", "")));
+ .thenReturn(new ResolvedBucket("", "",
+ "", "", "",
+ BucketLayout.DEFAULT));
}
@After
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
index bc8563f6f6..5f4cc63c08 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
@@ -223,15 +223,16 @@ public class TestOMKeyRequest {
random = new Random();
version = 0L;
- Pair<String, String> volumeAndBucket = Pair.of(volumeName, bucketName);
+ ResolvedBucket bucket = new ResolvedBucket(volumeName, bucketName,
+ volumeName, bucketName, "owner", BucketLayout.OBJECT_STORE);
when(ozoneManager.resolveBucketLink(any(KeyArgs.class),
any(OMClientRequest.class)))
- .thenReturn(new ResolvedBucket(volumeAndBucket, volumeAndBucket));
+ .thenReturn(bucket);
when(ozoneManager.resolveBucketLink(any(Pair.class),
any(OMClientRequest.class)))
- .thenReturn(new ResolvedBucket(volumeAndBucket, volumeAndBucket));
+ .thenReturn(bucket);
when(ozoneManager.resolveBucketLink(any(Pair.class)))
- .thenReturn(new ResolvedBucket(volumeAndBucket, volumeAndBucket));
+ .thenReturn(bucket);
OmSnapshotManager omSnapshotManager = new OmSnapshotManager(ozoneManager);
when(ozoneManager.getOmSnapshotManager())
.thenReturn(omSnapshotManager);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java
index afe5a00add..7c9f645014 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java
@@ -34,7 +34,6 @@ import org.junit.Rule;
import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.AuditMessage;
@@ -109,8 +108,9 @@ public class TestS3MultipartRequest {
.thenAnswer(inv -> {
KeyArgs args = (KeyArgs) inv.getArguments()[0];
return new ResolvedBucket(
- Pair.of(args.getVolumeName(), args.getBucketName()),
- Pair.of(args.getVolumeName(), args.getBucketName()));
+ args.getVolumeName(), args.getBucketName(),
+ args.getVolumeName(), args.getBucketName(),
+ "owner", BucketLayout.DEFAULT);
});
OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);
when(lvm.getMetadataLayoutVersion()).thenReturn(0);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
index 2aa96a1e73..8b84341876 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.om.response;
import com.google.common.collect.Iterators;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -35,6 +34,7 @@ import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.ResolvedBucket;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.lock.OzoneLockProvider;
@@ -129,11 +129,10 @@ public class TestCleanupTableInfo {
when(om.getMetadataManager()).thenReturn(metaMgr);
when(om.resolveBucketLink(any(KeyArgs.class), any(OMClientRequest.class)))
.thenAnswer(
- invocationOnMock -> {
- Pair<String, String> pair =
- Pair.of(TEST_VOLUME_NAME, TEST_BUCKET_NAME);
- return new ResolvedBucket(pair, pair);
- }
+ invocationOnMock -> new ResolvedBucket(
+ TEST_VOLUME_NAME, TEST_BUCKET_NAME,
+ TEST_VOLUME_NAME, TEST_BUCKET_NAME,
+ "owner", BucketLayout.DEFAULT)
);
when(om.getAclsEnabled()).thenReturn(false);
when(om.getAuditLogger()).thenReturn(mock(AuditLogger.class));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
