This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 585f4d5 HDDS-1761. Fix class hierarchy for KeyRequest and FileRequest
classes. (#1052)
585f4d5 is described below
commit 585f4d5c6405c8f85c6ef11a43a14302c261be11
Author: Bharat Viswanadham <[email protected]>
AuthorDate: Mon Jul 15 17:53:19 2019 -0700
HDDS-1761. Fix class hierarchy for KeyRequest and FileRequest classes.
(#1052)
---
.../om/request/file/OMDirectoryCreateRequest.java | 10 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 12 +-
.../om/request/key/OMAllocateBlockRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 263 +------------------
.../ozone/om/request/key/OMKeyDeleteRequest.java | 8 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 15 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 284 +++++++++++++++++++--
8 files changed, 290 insertions(+), 310 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
index 41d99fe..72fdee0 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
-import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.file.OMDirectoryCreateResponse;
@@ -73,8 +72,7 @@ import static
org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryR
/**
* Handle create directory request.
*/
-public class OMDirectoryCreateRequest extends OMClientRequest
- implements OMKeyRequest {
+public class OMDirectoryCreateRequest extends OMKeyRequest {
private static final Logger LOG =
LoggerFactory.getLogger(OMDirectoryCreateRequest.class);
@@ -214,8 +212,8 @@ public class OMDirectoryCreateRequest extends
OMClientRequest
OmBucketInfo omBucketInfo, String volumeName, String bucketName,
String keyName, KeyArgs keyArgs)
throws IOException {
- FileEncryptionInfo encryptionInfo = getFileEncryptionInfo(ozoneManager,
- omBucketInfo);
+ Optional<FileEncryptionInfo> encryptionInfo =
+ getFileEncryptionInfo(ozoneManager, omBucketInfo);
String dirName = OzoneFSUtils.addTrailingSlashIfNeeded(keyName);
return new OmKeyInfo.Builder()
@@ -229,7 +227,7 @@ public class OMDirectoryCreateRequest extends
OMClientRequest
.setDataSize(0)
.setReplicationType(HddsProtos.ReplicationType.RATIS)
.setReplicationFactor(HddsProtos.ReplicationFactor.ONE)
- .setFileEncryptionInfo(encryptionInfo)
+ .setFileEncryptionInfo(encryptionInfo.orNull())
.setAcls(keyArgs.getAclsList())
.build();
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index b4e572f..1c89015 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
+import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +44,6 @@ import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
-import org.apache.hadoop.ozone.om.request.key.OMKeyCreateRequest;
import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@@ -72,8 +72,7 @@ import static
org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryR
/**
* Handles create file request.
*/
-public class OMFileCreateRequest extends OMKeyCreateRequest
- implements OMKeyRequest {
+public class OMFileCreateRequest extends OMKeyRequest {
private static final Logger LOG =
LoggerFactory.getLogger(OMFileCreateRequest.class);
@@ -171,7 +170,7 @@ public class OMFileCreateRequest extends OMKeyCreateRequest
boolean acquiredLock = false;
IOException exception = null;
- FileEncryptionInfo encryptionInfo = null;
+ Optional<FileEncryptionInfo> encryptionInfo = Optional.absent();
OmKeyInfo omKeyInfo = null;
final List<OmKeyLocationInfo> locations = new ArrayList<>();
@@ -263,7 +262,8 @@ public class OMFileCreateRequest extends OMKeyCreateRequest
encryptionInfo = getFileEncryptionInfo(ozoneManager, bucketInfo);
omKeyInfo = prepareKeyInfo(omMetadataManager, keyArgs,
omMetadataManager.getOzoneKey(volumeName, bucketName,
- keyName), keyArgs.getDataSize(), locations, encryptionInfo);
+ keyName), keyArgs.getDataSize(), locations,
+ encryptionInfo.orNull());
} catch (IOException ex) {
exception = ex;
@@ -275,7 +275,7 @@ public class OMFileCreateRequest extends OMKeyCreateRequest
}
return prepareCreateKeyResponse(keyArgs, omKeyInfo, locations,
- encryptionInfo, exception, createFileRequest.getClientID(),
+ encryptionInfo.orNull(), exception, createFileRequest.getClientID(),
transactionLogIndex, volumeName, bucketName, keyName, ozoneManager,
OMAction.CREATE_FILE);
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index 96b1d16..bb94f63 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
-import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMAllocateBlockResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -65,8 +64,7 @@ import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
/**
* Handles allocate block request.
*/
-public class OMAllocateBlockRequest extends OMClientRequest
- implements OMKeyRequest {
+public class OMAllocateBlockRequest extends OMKeyRequest {
private static final Logger LOG =
LoggerFactory.getLogger(OMAllocateBlockRequest.class);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 143abef..678dfc9 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
-import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyCommitResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse;
@@ -61,8 +60,7 @@ import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_L
/**
* Handles CommitKey request.
*/
-public class OMKeyCommitRequest extends OMClientRequest
- implements OMKeyRequest {
+public class OMKeyCommitRequest extends OMKeyRequest {
private static final Logger LOG =
LoggerFactory.getLogger(OMKeyCommitRequest.class);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 93b7afb..3af0359 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -20,12 +20,8 @@ package org.apache.hadoop.ozone.om.request.key;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.stream.Collectors;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
@@ -43,39 +39,24 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
-import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse;
-import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .CreateKeyResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.CreateKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMResponse;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.UniqueId;
-import org.apache.hadoop.utils.db.cache.CacheKey;
-import org.apache.hadoop.utils.db.cache.CacheValue;
import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
-import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey;
-import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateFile;
/**
* Handles CreateKey request.
*/
-public class OMKeyCreateRequest extends OMClientRequest
- implements OMKeyRequest {
+public class OMKeyCreateRequest extends OMKeyRequest {
private static final Logger LOG =
LoggerFactory.getLogger(OMKeyCreateRequest.class);
@@ -175,7 +156,7 @@ public class OMKeyCreateRequest extends OMClientRequest
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
OmKeyInfo omKeyInfo = null;
final List< OmKeyLocationInfo > locations = new ArrayList<>();
- FileEncryptionInfo encryptionInfo = null;
+ Optional<FileEncryptionInfo> encryptionInfo = Optional.absent();
IOException exception = null;
boolean acquireLock = false;
try {
@@ -200,7 +181,7 @@ public class OMKeyCreateRequest extends OMClientRequest
omKeyInfo = prepareKeyInfo(omMetadataManager, keyArgs,
omMetadataManager.getOzoneKey(volumeName, bucketName, keyName),
- keyArgs.getDataSize(), locations, encryptionInfo);
+ keyArgs.getDataSize(), locations, encryptionInfo.orNull());
} catch (IOException ex) {
LOG.error("Key open failed for volume:{} bucket:{} key:{}",
@@ -214,245 +195,9 @@ public class OMKeyCreateRequest extends OMClientRequest
}
return prepareCreateKeyResponse(keyArgs, omKeyInfo, locations,
- encryptionInfo, exception, createKeyRequest.getClientID(),
+ encryptionInfo.orNull(), exception, createKeyRequest.getClientID(),
transactionLogIndex, volumeName, bucketName, keyName, ozoneManager,
OMAction.ALLOCATE_KEY);
}
- /**
- * Prepare the response returned to the client.
- * @param keyArgs
- * @param omKeyInfo
- * @param locations
- * @param encryptionInfo
- * @param exception
- * @param clientID
- * @param transactionLogIndex
- * @param volumeName
- * @param bucketName
- * @param keyName
- * @param ozoneManager
- * @return OMClientResponse
- */
- @SuppressWarnings("parameternumber")
- protected OMClientResponse prepareCreateKeyResponse(@Nonnull KeyArgs keyArgs,
- OmKeyInfo omKeyInfo, @Nonnull List<OmKeyLocationInfo> locations,
- FileEncryptionInfo encryptionInfo, @Nullable IOException exception,
- long clientID, long transactionLogIndex, @Nonnull String volumeName,
- @Nonnull String bucketName, @Nonnull String keyName,
- @Nonnull OzoneManager ozoneManager, @Nonnull OMAction omAction) {
-
- OMResponse.Builder omResponse = OMResponse.newBuilder().setStatus(
- OzoneManagerProtocolProtos.Status.OK);
- OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
-
- Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs);
-
- OMClientResponse omClientResponse = null;
- if (exception == null) {
- if (omKeyInfo == null) {
- // the key does not exist, create a new object, the new blocks are the
- // version 0
- omKeyInfo = createKeyInfo(keyArgs, locations, keyArgs.getFactor(),
- keyArgs.getType(), keyArgs.getDataSize(),
- encryptionInfo);
- }
-
- long openVersion = omKeyInfo.getLatestVersionLocations().getVersion();
-
- // Append blocks
- try {
- omKeyInfo.appendNewBlocks(keyArgs.getKeyLocationsList().stream()
- .map(OmKeyLocationInfo::getFromProtobuf)
- .collect(Collectors.toList()), false);
-
- } catch (IOException ex) {
- exception = ex;
- }
-
- if (exception != null) {
- LOG.error("{} failed for Key: {} in volume/bucket:{}/{}",
- omAction.getAction(), keyName, bucketName, volumeName, exception);
- omClientResponse = createKeyErrorResponse(ozoneManager.getMetrics(),
- omAction, exception, omResponse);
- } else {
- String dbOpenKeyName = omMetadataManager.getOpenKey(volumeName,
- bucketName, keyName, clientID);
-
- // Add to cache entry can be done outside of lock for this openKey.
- // Even if bucket gets deleted, when commitKey we shall identify if
- // bucket gets deleted.
- omMetadataManager.getOpenKeyTable().addCacheEntry(
- new CacheKey<>(dbOpenKeyName),
- new CacheValue<>(Optional.of(omKeyInfo), transactionLogIndex));
-
- LOG.debug("{} for Key: {} in volume/bucket: {}/{}",
- omAction.getAction(), keyName, volumeName, bucketName);
-
-
- if (omAction == OMAction.CREATE_FILE) {
- ozoneManager.getMetrics().incNumCreateFile();
- omResponse.setCreateFileResponse(
- OzoneManagerProtocolProtos.CreateFileResponse.newBuilder()
- .setKeyInfo(omKeyInfo.getProtobuf())
- .setID(clientID)
- .setOpenVersion(openVersion).build());
- omResponse.setCmdType(OzoneManagerProtocolProtos.Type.CreateFile);
- omClientResponse = new OMFileCreateResponse(omKeyInfo, clientID,
- omResponse.build());
- } else {
- ozoneManager.getMetrics().incNumKeyAllocates();
- omResponse.setCreateKeyResponse(CreateKeyResponse.newBuilder()
- .setKeyInfo(omKeyInfo.getProtobuf())
- .setID(clientID).setOpenVersion(openVersion)
- .build());
- omResponse.setCmdType(OzoneManagerProtocolProtos.Type.CreateKey);
- omClientResponse = new OMKeyCreateResponse(omKeyInfo, clientID,
- omResponse.build());
- }
- }
-
- } else {
- LOG.error("{} failed for Key: {} in volume/bucket:{}/{}",
- omAction.getAction(), keyName, volumeName, bucketName, exception);
- omClientResponse = createKeyErrorResponse(ozoneManager.getMetrics(),
- omAction, exception, omResponse);
- }
- // audit log
- auditLog(ozoneManager.getAuditLogger(), buildAuditMessage(omAction,
- auditMap, exception, getOmRequest().getUserInfo()));
- return omClientResponse;
- }
-
- private OMClientResponse createKeyErrorResponse(@Nonnull OMMetrics omMetrics,
- @Nonnull OMAction omAction, @Nonnull IOException exception,
- @Nonnull OMResponse.Builder omResponse) {
- if (omAction == OMAction.CREATE_FILE) {
- omMetrics.incNumCreateFileFails();
- omResponse.setCmdType(CreateFile);
- return new OMFileCreateResponse(null, -1L,
- createErrorOMResponse(omResponse, exception));
- } else {
- omMetrics.incNumKeyAllocateFails();
- omResponse.setCmdType(CreateKey);
- return new OMKeyCreateResponse(null, -1L,
- createErrorOMResponse(omResponse, exception));
- }
- }
-
- /**
- * Prepare OmKeyInfo which will be persisted to openKeyTable.
- * @param omMetadataManager
- * @param keyArgs
- * @param dbKeyName
- * @param size
- * @param locations
- * @param encInfo
- * @return OmKeyInfo
- * @throws IOException
- */
- protected OmKeyInfo prepareKeyInfo(
- @Nonnull OMMetadataManager omMetadataManager,
- @Nonnull KeyArgs keyArgs, @Nonnull String dbKeyName, long size,
- @Nonnull List<OmKeyLocationInfo> locations, FileEncryptionInfo encInfo)
- throws IOException {
- OmKeyInfo keyInfo = null;
- if (keyArgs.getIsMultipartKey()) {
- keyInfo = prepareMultipartKeyInfo(omMetadataManager, keyArgs, size,
- locations, encInfo);
- //TODO args.getMetadata
- } else if (omMetadataManager.getKeyTable().isExist(dbKeyName)) {
- // TODO: Need to be fixed, as when key already exists, we are
- // appending new blocks to existing key.
- keyInfo = omMetadataManager.getKeyTable().get(dbKeyName);
- // the key already exist, the new blocks will be added as new version
- // when locations.size = 0, the new version will have identical blocks
- // as its previous version
- keyInfo.addNewVersion(locations, false);
- keyInfo.setDataSize(size + keyInfo.getDataSize());
- // The modification time is set in preExecute, use the same as
- // modification time when key already exists.
- keyInfo.setModificationTime(keyArgs.getModificationTime());
- }
- return keyInfo;
- }
-
- /**
- * Prepare OmKeyInfo for multi-part upload part key which will be persisted
- * to openKeyTable.
- * @param omMetadataManager
- * @param args
- * @param size
- * @param locations
- * @param encInfo
- * @return OmKeyInfo
- * @throws IOException
- */
- private OmKeyInfo prepareMultipartKeyInfo(
- @Nonnull OMMetadataManager omMetadataManager,
- @Nonnull KeyArgs args, long size,
- @Nonnull List<OmKeyLocationInfo> locations,
- FileEncryptionInfo encInfo) throws IOException {
- HddsProtos.ReplicationFactor factor;
- HddsProtos.ReplicationType type;
-
- Preconditions.checkArgument(args.getMultipartNumber() > 0,
- "PartNumber Should be greater than zero");
- // When key is multipart upload part key, we should take replication
- // type and replication factor from original key which has done
- // initiate multipart upload. If we have not found any such, we throw
- // error no such multipart upload.
- String uploadID = args.getMultipartUploadID();
- Preconditions.checkNotNull(uploadID);
- String multipartKey = omMetadataManager
- .getMultipartKey(args.getVolumeName(), args.getBucketName(),
- args.getKeyName(), uploadID);
- OmKeyInfo partKeyInfo = omMetadataManager.getOpenKeyTable().get(
- multipartKey);
- if (partKeyInfo == null) {
- throw new OMException("No such Multipart upload is with specified " +
- "uploadId " + uploadID,
- OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR);
- } else {
- factor = partKeyInfo.getFactor();
- type = partKeyInfo.getType();
- }
- // For this upload part we don't need to check in KeyTable. As this
- // is not an actual key, it is a part of the key.
- return createKeyInfo(args, locations, factor, type, size, encInfo);
- }
-
- /**
- * Create OmKeyInfo object.
- * @param keyArgs
- * @param locations
- * @param factor
- * @param type
- * @param size
- * @param encInfo
- * @return OmKeyInfo
- */
- private OmKeyInfo createKeyInfo(@Nonnull KeyArgs keyArgs,
- @Nonnull List<OmKeyLocationInfo> locations,
- @Nonnull HddsProtos.ReplicationFactor factor,
- @Nonnull HddsProtos.ReplicationType type, long size,
- FileEncryptionInfo encInfo) {
- OmKeyInfo.Builder builder = new OmKeyInfo.Builder()
- .setVolumeName(keyArgs.getVolumeName())
- .setBucketName(keyArgs.getBucketName())
- .setKeyName(keyArgs.getKeyName())
- .setOmKeyLocationInfos(Collections.singletonList(
- new OmKeyLocationInfoGroup(0, locations)))
- .setCreationTime(keyArgs.getModificationTime())
- .setModificationTime(keyArgs.getModificationTime())
- .setDataSize(size)
- .setReplicationType(type)
- .setReplicationFactor(factor)
- .setFileEncryptionInfo(encInfo);
- if(keyArgs.getAclsList() != null) {
- builder.setAcls(keyArgs.getAclsList());
- }
- return builder.build();
- }
-
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
index 9848817..7d9275e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.util.Map;
import com.google.common.base.Optional;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.OMAction;
@@ -30,7 +32,6 @@ import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyDeleteResponse;
@@ -45,8 +46,6 @@ import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.utils.db.cache.CacheKey;
import org.apache.hadoop.utils.db.cache.CacheValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
.KEY_NOT_FOUND;
@@ -55,8 +54,7 @@ import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_L
/**
* Handles DeleteKey request.
*/
-public class OMKeyDeleteRequest extends OMClientRequest
- implements OMKeyRequest {
+public class OMKeyDeleteRequest extends OMKeyRequest {
private static final Logger LOG =
LoggerFactory.getLogger(OMKeyDeleteRequest.class);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
index 5b6df45..7d53b36 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
@@ -18,8 +18,14 @@
package org.apache.hadoop.ozone.om.request.key;
+import java.io.IOException;
+import java.util.Map;
+
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -27,7 +33,6 @@ import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -44,11 +49,6 @@ import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.db.Table;
import org.apache.hadoop.utils.db.cache.CacheKey;
import org.apache.hadoop.utils.db.cache.CacheValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
@@ -56,8 +56,7 @@ import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_L
/**
* Handles rename key request.
*/
-public class OMKeyRenameRequest extends OMClientRequest
- implements OMKeyRequest {
+public class OMKeyRenameRequest extends OMKeyRequest {
private static final Logger LOG =
LoggerFactory.getLogger(OMKeyRenameRequest.class);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 402b006..266abf5 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -18,7 +18,23 @@
package org.apache.hadoop.ozone.om.request.key;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension
.EncryptedKeyVersion;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -29,38 +45,56 @@ import
org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.ScmClient;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.request.OMClientRequest;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CreateFileResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CreateKeyResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .KeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .OMResponse;
import org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
+import org.apache.hadoop.utils.db.cache.CacheKey;
+import org.apache.hadoop.utils.db.cache.CacheValue;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
.BUCKET_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
.VOLUME_NOT_FOUND;
+import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateFile;
+import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey;
import static org.apache.hadoop.util.Time.monotonicNow;
/**
* Interface for key write requests.
*/
-public interface OMKeyRequest {
+public abstract class OMKeyRequest extends OMClientRequest {
- Logger LOG = LoggerFactory.getLogger(OMKeyRequest.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(OMKeyRequest.class);
+
+ public OMKeyRequest(OMRequest omRequest) {
+ super(omRequest);
+ }
/**
* This methods avoids multiple rpc calls to SCM by allocating multiple
blocks
@@ -68,7 +102,7 @@ public interface OMKeyRequest {
* @throws IOException
*/
@SuppressWarnings("parameternumber")
- default List< OmKeyLocationInfo > allocateBlock(ScmClient scmClient,
+ protected List< OmKeyLocationInfo > allocateBlock(ScmClient scmClient,
OzoneBlockTokenSecretManager secretManager,
HddsProtos.ReplicationType replicationType,
HddsProtos.ReplicationFactor replicationFactor,
@@ -113,7 +147,7 @@ public interface OMKeyRequest {
/* Optimize ugi lookup for RPC operations to avoid a trip through
* UGI.getCurrentUser which is synch'ed.
*/
- default UserGroupInformation getRemoteUser() throws IOException {
+ private UserGroupInformation getRemoteUser() throws IOException {
UserGroupInformation ugi = Server.getRemoteUser();
return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
}
@@ -123,7 +157,7 @@ public interface OMKeyRequest {
* @param user
*
* */
- default EnumSet< HddsProtos.BlockTokenSecretProto.AccessModeProto>
+ private EnumSet< HddsProtos.BlockTokenSecretProto.AccessModeProto>
getAclForUser(String user) {
// TODO: Return correct acl for user.
return EnumSet.allOf(
@@ -137,7 +171,7 @@ public interface OMKeyRequest {
* @param bucketName
* @throws IOException
*/
- default void validateBucketAndVolume(OMMetadataManager omMetadataManager,
+ public void validateBucketAndVolume(OMMetadataManager omMetadataManager,
String volumeName, String bucketName)
throws IOException {
String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
@@ -157,9 +191,9 @@ public interface OMKeyRequest {
}
}
- default FileEncryptionInfo getFileEncryptionInfo(
+ protected Optional<FileEncryptionInfo> getFileEncryptionInfo(
OzoneManager ozoneManager, OmBucketInfo bucketInfo) throws IOException {
- FileEncryptionInfo encInfo = null;
+ Optional<FileEncryptionInfo> encInfo = Optional.absent();
BucketEncryptionKeyInfo ezInfo = bucketInfo.getEncryptionKeyInfo();
if (ezInfo != null) {
if (ozoneManager.getKmsProvider() == null) {
@@ -170,15 +204,16 @@ public interface OMKeyRequest {
final String ezKeyName = ezInfo.getKeyName();
EncryptedKeyVersion edek = generateEDEK(ozoneManager, ezKeyName);
- encInfo = new FileEncryptionInfo(ezInfo.getSuite(), ezInfo.getVersion(),
+ encInfo = Optional.of(new FileEncryptionInfo(ezInfo.getSuite(),
+ ezInfo.getVersion(),
edek.getEncryptedKeyVersion().getMaterial(),
edek.getEncryptedKeyIv(), ezKeyName,
- edek.getEncryptionKeyVersionName());
+ edek.getEncryptionKeyVersionName()));
}
return encInfo;
}
- default EncryptedKeyVersion generateEDEK(OzoneManager ozoneManager,
+ private EncryptedKeyVersion generateEDEK(OzoneManager ozoneManager,
String ezKeyName) throws IOException {
if (ezKeyName == null) {
return null;
@@ -202,4 +237,213 @@ public interface OMKeyRequest {
return edek;
}
+ /**
+ * Prepare the response returned to the client.
+ * @return OMClientResponse
+ */
+ @SuppressWarnings("parameternumber")
+ protected OMClientResponse prepareCreateKeyResponse(@Nonnull KeyArgs keyArgs,
+ OmKeyInfo omKeyInfo, @Nonnull List<OmKeyLocationInfo> locations,
+ FileEncryptionInfo encryptionInfo, @Nullable IOException exception,
+ long clientID, long transactionLogIndex, @Nonnull String volumeName,
+ @Nonnull String bucketName, @Nonnull String keyName,
+ @Nonnull OzoneManager ozoneManager, @Nonnull OMAction omAction) {
+
+ OMResponse.Builder omResponse = OMResponse.newBuilder()
+ .setStatus(OzoneManagerProtocolProtos.Status.OK);
+ OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+
+ Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs);
+
+ OMClientResponse omClientResponse = null;
+ if (exception == null) {
+ if (omKeyInfo == null) {
+ // the key does not exist, create a new object, the new blocks are the
+ // version 0
+ omKeyInfo = createKeyInfo(keyArgs, locations, keyArgs.getFactor(),
+ keyArgs.getType(), keyArgs.getDataSize(),
+ encryptionInfo);
+ }
+
+ long openVersion = omKeyInfo.getLatestVersionLocations().getVersion();
+
+ // Append blocks
+ try {
+ omKeyInfo.appendNewBlocks(keyArgs.getKeyLocationsList().stream()
+ .map(OmKeyLocationInfo::getFromProtobuf)
+ .collect(Collectors.toList()), false);
+
+ } catch (IOException ex) {
+ exception = ex;
+ }
+
+ if (exception != null) {
+ LOG.error("{} failed for Key: {} in volume/bucket:{}/{}",
+ omAction.getAction(), keyName, bucketName, volumeName, exception);
+ omClientResponse = createKeyErrorResponse(ozoneManager.getMetrics(),
+ omAction, exception, omResponse);
+ } else {
+ String dbOpenKeyName = omMetadataManager.getOpenKey(volumeName,
+ bucketName, keyName, clientID);
+
+ // Add to cache entry can be done outside of lock for this openKey.
+ // Even if bucket gets deleted, when commitKey we shall identify if
+ // bucket gets deleted.
+ omMetadataManager.getOpenKeyTable().addCacheEntry(
+ new CacheKey<>(dbOpenKeyName),
+ new CacheValue<>(Optional.of(omKeyInfo), transactionLogIndex));
+
+ LOG.debug("{} for Key: {} in volume/bucket: {}/{}",
+ omAction.getAction(), keyName, volumeName, bucketName);
+
+
+ if (omAction == OMAction.CREATE_FILE) {
+ ozoneManager.getMetrics().incNumCreateFile();
+ omResponse.setCreateFileResponse(CreateFileResponse.newBuilder()
+ .setKeyInfo(omKeyInfo.getProtobuf())
+ .setID(clientID)
+ .setOpenVersion(openVersion).build());
+ omResponse.setCmdType(CreateFile);
+ omClientResponse = new OMFileCreateResponse(omKeyInfo, clientID,
+ omResponse.build());
+ } else {
+ ozoneManager.getMetrics().incNumKeyAllocates();
+ omResponse.setCreateKeyResponse(CreateKeyResponse.newBuilder()
+ .setKeyInfo(omKeyInfo.getProtobuf())
+ .setID(clientID).setOpenVersion(openVersion)
+ .build());
+ omResponse.setCmdType(CreateKey);
+ omClientResponse = new OMKeyCreateResponse(omKeyInfo, clientID,
+ omResponse.build());
+ }
+ }
+
+ } else {
+ LOG.error("{} failed for Key: {} in volume/bucket:{}/{}",
+ omAction.getAction(), keyName, volumeName, bucketName, exception);
+ omClientResponse = createKeyErrorResponse(ozoneManager.getMetrics(),
+ omAction, exception, omResponse);
+ }
+ // audit log
+ auditLog(ozoneManager.getAuditLogger(), buildAuditMessage(omAction,
+ auditMap, exception, getOmRequest().getUserInfo()));
+ return omClientResponse;
+ }
+
+ /**
+ * Create OmKeyInfo object.
+ * @return OmKeyInfo
+ */
+ protected OmKeyInfo createKeyInfo(@Nonnull KeyArgs keyArgs,
+ @Nonnull List<OmKeyLocationInfo> locations,
+ @Nonnull HddsProtos.ReplicationFactor factor,
+ @Nonnull HddsProtos.ReplicationType type, long size,
+ @Nullable FileEncryptionInfo encInfo) {
+ OmKeyInfo.Builder builder = new OmKeyInfo.Builder()
+ .setVolumeName(keyArgs.getVolumeName())
+ .setBucketName(keyArgs.getBucketName())
+ .setKeyName(keyArgs.getKeyName())
+ .setOmKeyLocationInfos(Collections.singletonList(
+ new OmKeyLocationInfoGroup(0, locations)))
+ .setCreationTime(keyArgs.getModificationTime())
+ .setModificationTime(keyArgs.getModificationTime())
+ .setDataSize(size)
+ .setReplicationType(type)
+ .setReplicationFactor(factor)
+ .setFileEncryptionInfo(encInfo);
+ if(keyArgs.getAclsList() != null) {
+ builder.setAcls(keyArgs.getAclsList());
+ }
+ return builder.build();
+ }
+
+ /**
+ * Prepare OmKeyInfo which will be persisted to openKeyTable.
+ * @return OmKeyInfo
+ * @throws IOException
+ */
+ protected OmKeyInfo prepareKeyInfo(
+ @Nonnull OMMetadataManager omMetadataManager,
+ @Nonnull KeyArgs keyArgs, @Nonnull String dbKeyName, long size,
+ @Nonnull List<OmKeyLocationInfo> locations,
+ @Nullable FileEncryptionInfo encInfo)
+ throws IOException {
+ OmKeyInfo keyInfo = null;
+ if (keyArgs.getIsMultipartKey()) {
+ keyInfo = prepareMultipartKeyInfo(omMetadataManager, keyArgs, size,
+ locations, encInfo);
+ //TODO args.getMetadata
+ } else if (omMetadataManager.getKeyTable().isExist(dbKeyName)) {
+ // TODO: Need to be fixed, as when key already exists, we are
+ // appending new blocks to existing key.
+ keyInfo = omMetadataManager.getKeyTable().get(dbKeyName);
+ // the key already exist, the new blocks will be added as new version
+ // when locations.size = 0, the new version will have identical blocks
+ // as its previous version
+ keyInfo.addNewVersion(locations, false);
+ keyInfo.setDataSize(size + keyInfo.getDataSize());
+ // The modification time is set in preExecute, use the same as
+ // modification time when key already exists.
+ keyInfo.setModificationTime(keyArgs.getModificationTime());
+ }
+ return keyInfo;
+ }
+
+ /**
+ * Prepare OmKeyInfo for multi-part upload part key which will be persisted
+ * to openKeyTable.
+ * @return OmKeyInfo
+ * @throws IOException
+ */
+ private OmKeyInfo prepareMultipartKeyInfo(
+ @Nonnull OMMetadataManager omMetadataManager,
+ @Nonnull KeyArgs args, long size,
+ @Nonnull List<OmKeyLocationInfo> locations,
+ FileEncryptionInfo encInfo) throws IOException {
+ HddsProtos.ReplicationFactor factor;
+ HddsProtos.ReplicationType type;
+
+ Preconditions.checkArgument(args.getMultipartNumber() > 0,
+ "PartNumber Should be greater than zero");
+ // When key is multipart upload part key, we should take replication
+ // type and replication factor from original key which has done
+ // initiate multipart upload. If we have not found any such, we throw
+ // error no such multipart upload.
+ String uploadID = args.getMultipartUploadID();
+ Preconditions.checkNotNull(uploadID);
+ String multipartKey = omMetadataManager
+ .getMultipartKey(args.getVolumeName(), args.getBucketName(),
+ args.getKeyName(), uploadID);
+ OmKeyInfo partKeyInfo = omMetadataManager.getOpenKeyTable().get(
+ multipartKey);
+ if (partKeyInfo == null) {
+ throw new OMException("No such Multipart upload is with specified " +
+ "uploadId " + uploadID,
+ OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR);
+ } else {
+ factor = partKeyInfo.getFactor();
+ type = partKeyInfo.getType();
+ }
+ // For this upload part we don't need to check in KeyTable. As this
+ // is not an actual key, it is a part of the key.
+ return createKeyInfo(args, locations, factor, type, size, encInfo);
+ }
+
+
+ private OMClientResponse createKeyErrorResponse(@Nonnull OMMetrics omMetrics,
+ @Nonnull OMAction omAction, @Nonnull IOException exception,
+ @Nonnull OMResponse.Builder omResponse) {
+ if (omAction == OMAction.CREATE_FILE) {
+ omMetrics.incNumCreateFileFails();
+ omResponse.setCmdType(CreateFile);
+ return new OMFileCreateResponse(null, -1L,
+ createErrorOMResponse(omResponse, exception));
+ } else {
+ omMetrics.incNumKeyAllocateFails();
+ omResponse.setCmdType(CreateKey);
+ return new OMKeyCreateResponse(null, -1L,
+ createErrorOMResponse(omResponse, exception));
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
