This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 00450a95ef5 HDDS-15803. Fix parameter number warning in
OMKeyRequest.allocateBlock. (#10701)
00450a95ef5 is described below
commit 00450a95ef5ee41dbf849e4dc2877bf88c61f10c
Author: Tsz-Wo Nicholas Sze <[email protected]>
AuthorDate: Thu Jul 9 19:48:04 2026 -0700
HDDS-15803. Fix parameter number warning in OMKeyRequest.allocateBlock.
(#10701)
---
.../ozone/om/request/file/OMFileCreateRequest.java | 16 ++------
.../om/request/key/OMAllocateBlockRequest.java | 17 ++------
.../ozone/om/request/key/OMKeyCreateRequest.java | 25 ++----------
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 46 ++++++++++------------
4 files changed, 32 insertions(+), 72 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index 9788cfbafe1..35e1ac238f7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@ -101,14 +101,12 @@ public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
return getOmRequest().toBuilder().setUserInfo(userInfo).build();
}
- long scmBlockSize = ozoneManager.getScmBlockSize();
-
// NOTE size of a key is not a hard limit on anything, it is a value that
// client should expect, in terms of current size of key. If client sets
// a value, then this value is used, otherwise, we allocate a single
// block which is the current size, if read by the client.
final long requestedSize = keyArgs.getDataSize() > 0 ?
- keyArgs.getDataSize() : scmBlockSize;
+ keyArgs.getDataSize() : ozoneManager.getScmBlockSize();
HddsProtos.ReplicationFactor factor = keyArgs.getFactor();
HddsProtos.ReplicationType type = keyArgs.getType();
@@ -129,16 +127,8 @@ public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
// File system client does not know the final file size in advance but use
0 as
// the placeholder for the data size. Therefore, we should at least
allocate a
// single block and we cannot simply skip the allocate block call
- List< OmKeyLocationInfo > omKeyLocationInfoList =
- allocateBlock(ozoneManager.getScmClient(),
- ozoneManager.getBlockTokenSecretManager(), repConfig,
- new ExcludeList(), requestedSize, scmBlockSize,
- ozoneManager.getPreallocateBlocksMax(),
- ozoneManager.isGrpcBlockTokenEnabled(),
- ozoneManager.getOMServiceId(),
- ozoneManager.getMetrics(),
- keyArgs.getSortDatanodes(),
- userInfo);
+ final List< OmKeyLocationInfo > omKeyLocationInfoList = allocateBlock(
+ repConfig, new ExcludeList(), requestedSize,
keyArgs.getSortDatanodes(), userInfo, ozoneManager);
KeyArgs.Builder newKeyArgs = keyArgs.toBuilder()
.setModificationTime(Time.now()).setType(type).setFactor(factor)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index d7cd3bea5ec..f85cc8b777e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -91,11 +91,8 @@ public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
keyPath = validateAndNormalizeKey(ozoneManager.getEnableFileSystemPaths(),
keyPath, getBucketLayout());
- ExcludeList excludeList = new ExcludeList();
- if (allocateBlockRequest.hasExcludeList()) {
- excludeList =
- ExcludeList.getFromProtoBuf(allocateBlockRequest.getExcludeList());
- }
+ final ExcludeList excludeList = !allocateBlockRequest.hasExcludeList() ?
new ExcludeList()
+ : ExcludeList.getFromProtoBuf(allocateBlockRequest.getExcludeList());
// TODO: Here we are allocating block with out any check for key exist in
// open table or not and also with out any authorization checks.
@@ -111,14 +108,8 @@ public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
// To allocate atleast one block passing requested size and scmBlockSize
// as same value. When allocating block requested size is same as
// scmBlockSize.
- List<OmKeyLocationInfo> omKeyLocationInfoList =
- allocateBlock(ozoneManager.getScmClient(),
- ozoneManager.getBlockTokenSecretManager(), repConfig, excludeList,
- ozoneManager.getScmBlockSize(), ozoneManager.getScmBlockSize(),
- ozoneManager.getPreallocateBlocksMax(),
- ozoneManager.isGrpcBlockTokenEnabled(),
- ozoneManager.getOMServiceId(), ozoneManager.getMetrics(),
- keyArgs.getSortDatanodes(), userInfo);
+ final List<OmKeyLocationInfo> omKeyLocationInfoList =
allocateBlock(repConfig, excludeList,
+ ozoneManager.getScmBlockSize(), keyArgs.getSortDatanodes(), userInfo,
ozoneManager);
// Set modification time and normalize key if required.
KeyArgs.Builder newKeyArgs =
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index d7b14455369..929e46222c0 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -127,16 +127,6 @@ public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
KeyArgs.Builder newKeyArgs = null;
UserInfo userInfo = getUserInfo();
if (!keyArgs.getIsMultipartKey()) {
-
- long scmBlockSize = ozoneManager.getScmBlockSize();
-
- // NOTE size of a key is not a hard limit on anything, it is a value that
- // client should expect, in terms of current size of key. If client sets
- // a value, then this value is used, otherwise, we allocate a single
- // block which is the current size, if read by the client.
- final long requestedSize = keyArgs.getDataSize() > 0 ?
- keyArgs.getDataSize() : scmBlockSize;
-
HddsProtos.ReplicationFactor factor = keyArgs.getFactor();
HddsProtos.ReplicationType type = keyArgs.getType();
@@ -153,7 +143,7 @@ public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
// As for a client for the first time this can be executed on any OM,
// till leader is identified.
- List<OmKeyLocationInfo> omKeyLocationInfoList;
+ final List<OmKeyLocationInfo> omKeyLocationInfoList;
final long effectiveDataSize;
// Skip block allocation if dataSize <= 0. We also consider unspecified
dataSize as
// empty key since the client will not set dataSize if the key is empty
(i.e. dataSize <= 0),
@@ -161,17 +151,10 @@ public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
omKeyLocationInfoList = Collections.emptyList();
effectiveDataSize = 0;
} else {
+ effectiveDataSize = keyArgs.getDataSize();
omKeyLocationInfoList =
captureLatencyNs(perfMetrics.getCreateKeyAllocateBlockLatencyNs(),
- () -> allocateBlock(ozoneManager.getScmClient(),
- ozoneManager.getBlockTokenSecretManager(), repConfig,
- new ExcludeList(), requestedSize, scmBlockSize,
- ozoneManager.getPreallocateBlocksMax(),
- ozoneManager.isGrpcBlockTokenEnabled(),
- ozoneManager.getOMServiceId(),
- ozoneManager.getMetrics(),
- keyArgs.getSortDatanodes(),
- userInfo));
- effectiveDataSize = requestedSize;
+ () -> allocateBlock(repConfig, new ExcludeList(),
effectiveDataSize,
+ keyArgs.getSortDatanodes(), userInfo, ozoneManager));
}
newKeyArgs = keyArgs.toBuilder().setModificationTime(Time.now())
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 7967007263e..115eacd8712 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -46,6 +46,7 @@
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
@@ -56,10 +57,11 @@
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ipc_.Server;
@@ -67,12 +69,10 @@
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OmConfig;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.PrefixManager;
import org.apache.hadoop.ozone.om.ResolvedBucket;
-import org.apache.hadoop.ozone.om.ScmClient;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -100,6 +100,7 @@
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,6 +113,7 @@ public abstract class OMKeyRequest extends OMClientRequest {
// transaction (recursive directory creations) is 2^8 - 1 as only 8
// bits are set aside for this in ObjectID.
private static final long MAX_NUM_OF_RECURSIVE_DIRS = 255;
+ private static final Set<AccessModeProto> READ_WRITE = EnumSet.of(READ,
WRITE);
protected static final Logger LOG =
LoggerFactory.getLogger(OMKeyRequest.class);
@@ -179,22 +181,17 @@ protected KeyArgs
resolveBucketAndCheckOpenKeyAcls(KeyArgs keyArgs,
return resolvedArgs;
}
- /**
- * This methods avoids multiple rpc calls to SCM by allocating multiple
blocks
- * in one rpc call.
- * @throws IOException
- */
- @SuppressWarnings("parameternumber")
- protected List< OmKeyLocationInfo > allocateBlock(ScmClient scmClient,
- OzoneBlockTokenSecretManager secretManager,
+ /** Allocate multiple blocks using one rpc to SCM. */
+ protected List<OmKeyLocationInfo> allocateBlock(
ReplicationConfig replicationConfig, ExcludeList excludeList,
- long requestedSize, long scmBlockSize, int preallocateBlocksMax,
- boolean grpcBlockTokenEnabled, String serviceID, OMMetrics omMetrics,
- boolean shouldSortDatanodes, UserInfo userInfo)
+ long requestedSize, boolean shouldSortDatanodes,
+ UserInfo userInfo, OzoneManager ozoneManager)
throws IOException {
+ final long scmBlockSize = ozoneManager.getScmBlockSize();
+
int dataGroupSize = replicationConfig instanceof ECReplicationConfig
? ((ECReplicationConfig) replicationConfig).getData() : 1;
- int numBlocks = (int) Math.min(preallocateBlocksMax,
+ final int numBlocks = (int)
Math.min(ozoneManager.getPreallocateBlocksMax(),
(requestedSize - 1) / (scmBlockSize * dataGroupSize) + 1);
String clientMachine = "";
@@ -204,15 +201,13 @@ protected List< OmKeyLocationInfo >
allocateBlock(ScmClient scmClient,
List<OmKeyLocationInfo> locationInfos = new ArrayList<>(numBlocks);
String remoteUser = getRemoteUser().getShortUserName();
- List<AllocatedBlock> allocatedBlocks;
+ final List<AllocatedBlock> allocatedBlocks;
try {
- allocatedBlocks = scmClient.getBlockClient()
- .allocateBlock(scmBlockSize, numBlocks, replicationConfig, serviceID,
- excludeList, clientMachine);
+ allocatedBlocks =
ozoneManager.getScmClient().getBlockClient().allocateBlock(
+ scmBlockSize, numBlocks, replicationConfig,
ozoneManager.getOMServiceId(), excludeList, clientMachine);
} catch (SCMException ex) {
- omMetrics.incNumBlockAllocateCallFails();
- if (ex.getResult()
- .equals(SCMException.ResultCodes.SAFE_MODE_EXCEPTION)) {
+ ozoneManager.getMetrics().incNumBlockAllocateCallFails();
+ if (ex.getResult() == SCMException.ResultCodes.SAFE_MODE_EXCEPTION) {
throw new OMException(ex.getMessage(),
OMException.ResultCodes.SCM_IN_SAFE_MODE);
}
@@ -225,9 +220,10 @@ protected List< OmKeyLocationInfo >
allocateBlock(ScmClient scmClient,
.setLength(scmBlockSize)
.setOffset(0)
.setPipeline(allocatedBlock.getPipeline());
- if (grpcBlockTokenEnabled) {
- builder.setToken(secretManager.generateToken(remoteUser, blockID,
- EnumSet.of(READ, WRITE), scmBlockSize));
+ if (ozoneManager.isGrpcBlockTokenEnabled()) {
+ final Token<OzoneBlockTokenIdentifier> token =
ozoneManager.getBlockTokenSecretManager().generateToken(
+ remoteUser, blockID, READ_WRITE, scmBlockSize);
+ builder.setToken(token);
}
locationInfos.add(builder.build());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
