This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4fd7fcd83ef HDDS-15320. Reduce duplication in OMAllocateBlockRequest
OBS/FSO classes (#10327)
4fd7fcd83ef is described below
commit 4fd7fcd83ef29658cd16a4f60308bf94de6fb831
Author: rhalm <[email protected]>
AuthorDate: Fri May 22 21:26:52 2026 +0200
HDDS-15320. Reduce duplication in OMAllocateBlockRequest OBS/FSO classes
(#10327)
---
.../om/request/key/OMAllocateBlockRequest.java | 61 +++++--
.../request/key/OMAllocateBlockRequestWithFSO.java | 184 +++------------------
2 files changed, 68 insertions(+), 177 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index b692cf9d55e..e8b3abfe219 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -21,6 +21,7 @@
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_UNDER_LEASE_RECOVERY;
import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.BUCKET_LOCK;
+import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.nio.file.InvalidPathException;
import java.util.Collections;
@@ -147,7 +148,7 @@ public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
}
@Override
- public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
ExecutionContext context) {
+ public final OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, ExecutionContext context) {
final long trxnLogIndex = context.getIndex();
OzoneManagerProtocolProtos.AllocateBlockRequest allocateBlockRequest =
@@ -190,12 +191,15 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
bucketName);
// Here we don't acquire bucket/volume lock because for a single client
- // allocateBlock is called in serial fashion.
+ // allocateBlock is called in serial fashion. With this approach, it
+ // won't make 'fail-fast' during race condition case on delete/rename op,
+ // assuming that later it will fail at the key commit operation.
- openKeyName = omMetadataManager
- .getOpenKey(volumeName, bucketName, keyName, clientID);
+ openKeyName =
+ getOpenKeyName(volumeName, bucketName, keyName, clientID,
omMetadataManager);
openKeyInfo =
-
omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKeyName);
+ getOpenKeyInfo(omMetadataManager, openKeyName, keyName);
+
if (openKeyInfo == null) {
throw new OMException("Open Key not found " + openKeyName,
KEY_NOT_FOUND);
@@ -241,22 +245,20 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
.build();
// Add to cache.
- omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry(
- new CacheKey<>(openKeyName),
- CacheValue.get(trxnLogIndex, openKeyInfo));
+ addOpenTableCacheEntry(trxnLogIndex, omMetadataManager,
+ openKeyName, keyName, openKeyInfo);
omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder()
.setKeyLocation(blockLocation).build());
- omClientResponse = new OMAllocateBlockResponse(omResponse.build(),
- openKeyInfo, clientID, getBucketLayout());
+ omClientResponse = getOmClientResponse(clientID, omResponse, openKeyInfo,
+ omBucketInfo, omMetadataManager);
LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}",
volumeName, bucketName, openKeyName);
} catch (IOException | InvalidPathException ex) {
omMetrics.incNumBlockAllocateCallFails();
exception = ex;
- omClientResponse = new OMAllocateBlockResponse(createErrorOMResponse(
- omResponse, exception), getBucketLayout());
+ omClientResponse = getOmClientErrorResponse(omResponse, exception);
LOG.error("Allocate Block failed. Volume:{}, Bucket:{}, OpenKey:{}. " +
"Exception:{}", volumeName, bucketName, openKeyName, exception);
} finally {
@@ -276,6 +278,41 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
return omClientResponse;
}
+ protected OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager,
+ String openKeyName, String keyName) throws IOException {
+ return
omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKeyName);
+ }
+
+ protected String getOpenKeyName(String volumeName, String bucketName,
+ String keyName, long clientID, OMMetadataManager omMetadataManager)
+ throws IOException {
+ return omMetadataManager.getOpenKey(volumeName, bucketName, keyName,
clientID);
+ }
+
+ protected void addOpenTableCacheEntry(long trxnLogIndex,
+ OMMetadataManager omMetadataManager, String openKeyName, String keyName,
+ OmKeyInfo openKeyInfo) {
+ omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry(
+ new CacheKey<>(openKeyName),
+ CacheValue.get(trxnLogIndex, openKeyInfo));
+ }
+
+ @Nonnull
+ protected OMClientResponse getOmClientResponse(long clientID,
+ OMResponse.Builder omResponse, OmKeyInfo openKeyInfo,
+ OmBucketInfo omBucketInfo, OMMetadataManager omMetadataManager)
+ throws IOException {
+ return new OMAllocateBlockResponse(omResponse.build(),
+ openKeyInfo, clientID, getBucketLayout());
+ }
+
+ @Nonnull
+ protected OMClientResponse getOmClientErrorResponse(
+ OMResponse.Builder omResponse, Exception exception) {
+ return new OMAllocateBlockResponse(createErrorOMResponse(
+ omResponse, exception), getBucketLayout());
+ }
+
@RequestFeatureValidator(
conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION,
processingPhase = RequestProcessingPhase.PRE_PROCESS,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java
index dba523bed48..a718a8b8c0f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java
@@ -17,200 +17,42 @@
package org.apache.hadoop.ozone.om.request.key;
-import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
-import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_UNDER_LEASE_RECOVERY;
-import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.BUCKET_LOCK;
-
import jakarta.annotation.Nonnull;
import java.io.IOException;
-import java.nio.file.InvalidPathException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.audit.AuditLogger;
-import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.OMMetrics;
-import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmFSOFile;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
-import org.apache.hadoop.ozone.om.helpers.QuotaUtil;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
-import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMAllocateBlockResponseWithFSO;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockResponse;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Handles allocate block request - prefix layout.
*/
public class OMAllocateBlockRequestWithFSO extends OMAllocateBlockRequest {
- private static final Logger LOG =
- LoggerFactory.getLogger(OMAllocateBlockRequestWithFSO.class);
-
public OMAllocateBlockRequestWithFSO(OMRequest omRequest,
BucketLayout bucketLayout) {
super(omRequest, bucketLayout);
}
@Override
- public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
ExecutionContext context) {
- final long trxnLogIndex = context.getIndex();
-
- AllocateBlockRequest allocateBlockRequest =
- getOmRequest().getAllocateBlockRequest();
-
- KeyArgs keyArgs =
- allocateBlockRequest.getKeyArgs();
-
- OzoneManagerProtocolProtos.KeyLocation blockLocation =
- allocateBlockRequest.getKeyLocation();
- Objects.requireNonNull(blockLocation, "blockLocation == null");
-
- String volumeName = keyArgs.getVolumeName();
- String bucketName = keyArgs.getBucketName();
- String keyName = keyArgs.getKeyName();
- long clientID = allocateBlockRequest.getClientID();
-
- OMMetrics omMetrics = ozoneManager.getMetrics();
- omMetrics.incNumBlockAllocateCalls();
-
- AuditLogger auditLogger = ozoneManager.getAuditLogger();
-
- Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs);
- auditMap.put(OzoneConsts.CLIENT_ID, String.valueOf(clientID));
-
- OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
- String openKeyName = null;
-
- OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
- getOmRequest());
- OMClientResponse omClientResponse = null;
-
- OmKeyInfo openKeyInfo = null;
- Exception exception = null;
- OmBucketInfo omBucketInfo = null;
- boolean acquiredLock = false;
-
- try {
- validateBucketAndVolume(omMetadataManager, volumeName,
- bucketName);
-
- // Here we don't acquire bucket/volume lock because for a single client
- // allocateBlock is called in serial fashion. With this approach, it
- // won't make 'fail-fast' during race condition case on delete/rename op,
- // assuming that later it will fail at the key commit operation.
- openKeyName = getOpenKeyName(volumeName, bucketName, keyName, clientID,
- ozoneManager);
- openKeyInfo = getOpenKeyInfo(omMetadataManager, openKeyName, keyName);
- if (openKeyInfo == null) {
- throw new OMException("Open Key not found " + openKeyName,
- KEY_NOT_FOUND);
- }
- if (openKeyInfo.getMetadata().containsKey(OzoneConsts.LEASE_RECOVERY)) {
- throw new OMException("Open Key " + openKeyName + " is under lease
recovery",
- KEY_UNDER_LEASE_RECOVERY);
- }
- if (openKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY)
||
-
openKeyInfo.getMetadata().containsKey(OzoneConsts.OVERWRITTEN_HSYNC_KEY)) {
- throw new OMException("Open Key " + openKeyName + " is already
deleted/overwritten",
- KEY_NOT_FOUND);
- }
- List<OmKeyLocationInfo> newLocationList = Collections.singletonList(
- OmKeyLocationInfo.getFromProtobuf(blockLocation));
-
- mergeOmLockDetails(
- omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
- volumeName, bucketName));
- acquiredLock = getOmLockDetails().isLockAcquired();
- omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
- // check bucket and volume quota
- long preAllocatedKeySize = newLocationList.size()
- * ozoneManager.getScmBlockSize();
- long hadAllocatedKeySize =
- openKeyInfo.getLatestVersionLocations().getLocationList().size()
- * ozoneManager.getScmBlockSize();
- ReplicationConfig repConfig = openKeyInfo.getReplicationConfig();
- long totalAllocatedSpace = QuotaUtil.getReplicatedSize(
- preAllocatedKeySize, repConfig) + QuotaUtil.getReplicatedSize(
- hadAllocatedKeySize, repConfig);
- checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
- totalAllocatedSpace);
- // Append new block
- openKeyInfo.appendNewBlocks(newLocationList, false);
-
- // Set modification time.
- openKeyInfo.setModificationTime(keyArgs.getModificationTime());
-
- // Set the UpdateID to current transactionLogIndex
- openKeyInfo = openKeyInfo.toBuilder()
- .setUpdateID(trxnLogIndex)
- .build();
-
- // Add to cache.
- addOpenTableCacheEntry(trxnLogIndex, omMetadataManager, openKeyName,
keyName,
- openKeyInfo);
-
- omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder()
- .setKeyLocation(blockLocation).build());
- long volumeId = omMetadataManager.getVolumeId(volumeName);
- omClientResponse = getOmClientResponse(clientID, omResponse,
- openKeyInfo, omBucketInfo.copyObject(), volumeId);
- LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}",
- volumeName, bucketName, openKeyName);
- } catch (IOException | InvalidPathException ex) {
- omMetrics.incNumBlockAllocateCallFails();
- exception = ex;
- omClientResponse = new OMAllocateBlockResponseWithFSO(
- createErrorOMResponse(omResponse, exception), getBucketLayout());
- LOG.error("Allocate Block failed. Volume:{}, Bucket:{}, OpenKey:{}. " +
- "Exception:{}", volumeName, bucketName, openKeyName, exception);
- } finally {
- if (acquiredLock) {
- mergeOmLockDetails(
- omMetadataManager.getLock().releaseWriteLock(
- BUCKET_LOCK, volumeName, bucketName));
- }
- if (omClientResponse != null) {
- omClientResponse.setOmLockDetails(getOmLockDetails());
- }
- }
-
- markForAudit(auditLogger, buildAuditMessage(OMAction.ALLOCATE_BLOCK,
auditMap,
- exception, getOmRequest().getUserInfo()));
-
- return omClientResponse;
- }
-
- private OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager,
+ protected OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager,
String openKeyName, String keyName) throws IOException {
String fileName = OzoneFSUtils.getFileName(keyName);
return OMFileRequest.getOmKeyInfoFromFileTable(true,
omMetadataManager, openKeyName, fileName);
}
- private String getOpenKeyName(String volumeName, String bucketName,
- String keyName, long clientID, OzoneManager ozoneManager)
+ @Override
+ protected String getOpenKeyName(String volumeName, String bucketName,
+ String keyName, long clientID, OMMetadataManager omMetadataManager)
throws IOException {
- OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
-
return new OmFSOFile.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
@@ -219,18 +61,30 @@ private String getOpenKeyName(String volumeName, String
bucketName,
.build().getOpenFileName(clientID);
}
- private void addOpenTableCacheEntry(long trxnLogIndex,
+ @Override
+ protected void addOpenTableCacheEntry(long trxnLogIndex,
OMMetadataManager omMetadataManager, String openKeyName, String keyName,
OmKeyInfo openKeyInfo) {
OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, openKeyName,
openKeyInfo, keyName, trxnLogIndex);
}
+ @Override
@Nonnull
- private OMClientResponse getOmClientResponse(long clientID,
+ protected OMClientResponse getOmClientResponse(long clientID,
OMResponse.Builder omResponse, OmKeyInfo openKeyInfo,
- OmBucketInfo omBucketInfo, long volumeId) {
+ OmBucketInfo omBucketInfo, OMMetadataManager omMetadataManager)
+ throws IOException {
+ long volumeId = omMetadataManager.getVolumeId(openKeyInfo.getVolumeName());
return new OMAllocateBlockResponseWithFSO(omResponse.build(), openKeyInfo,
clientID, getBucketLayout(), volumeId, omBucketInfo.getObjectID());
}
+
+ @Override
+ @Nonnull
+ protected OMClientResponse getOmClientErrorResponse(
+ OMResponse.Builder omResponse, Exception exception) {
+ return new OMAllocateBlockResponseWithFSO(
+ createErrorOMResponse(omResponse, exception), getBucketLayout());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
